taosdemo.c 247.8 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

21
#include <stdint.h>
S
slguan 已提交
22
#define _GNU_SOURCE
23 24
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
41
#else
S
TD-1057  
Shengliang Guan 已提交
42 43
  #include <regex.h>
  #include <stdio.h>
44
#endif
H
hzcheng 已提交
45

46
#include <assert.h>
47
#include <stdlib.h>
48 49
#include "cJSON.h"

50
#include "os.h"
H
hzcheng 已提交
51
#include "taos.h"
52
#include "taoserror.h"
H
Hui Li 已提交
53
#include "tutil.h"
S
slguan 已提交
54

55 56 57
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
58 59
extern char configDir[];

60 61 62 63
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

64 65
#define STR_INSERT_INTO     "INSERT INTO "

66 67 68 69 70 71
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};
72

73 74
#define MAX_RECORDS_PER_REQ     32766

75 76
#define HEAD_BUFF_LEN    1024*24  // 16*1024 + (192+32)*2 + insert into ..

77 78
#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
79
#define COND_BUF_LEN        BUFFER_SIZE - 30
80 81
#define MAX_USERNAME_SIZE  64
#define MAX_PASSWORD_SIZE  64
82
#define MAX_DB_NAME_SIZE   64
83
#define MAX_HOSTNAME_SIZE  64
84
#define MAX_TB_NAME_SIZE   64
85
#define MAX_DATA_SIZE      (16*1024)+20     // max record len: 16*1024, timestamp string and ,('') need extra space
86 87 88 89 90 91 92 93 94 95
#define MAX_NUM_DATATYPE   10
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
#define MAX_FILE_NAME_LEN  256

#define   MAX_SAMPLES_ONCE_FROM_FILE   10000
#define   MAX_NUM_DATATYPE 10

#define   MAX_DB_COUNT           8
96
#define   MAX_SUPER_TABLE_COUNT  200
97 98 99
#define   MAX_COLUMN_COUNT       1024
#define   MAX_TAG_COUNT          128

100
#define   MAX_QUERY_SQL_COUNT    100
101
#define   MAX_QUERY_SQL_LENGTH   1024
102 103

#define   MAX_DATABASE_COUNT     256
104
#define INPUT_BUF_LEN   256
105

106
#define DEFAULT_TIMESTAMP_STEP  1
107

108

109 110 111 112 113
typedef enum CREATE_SUB_TALBE_MOD_EN {
  PRE_CREATE_SUBTBL,
  AUTO_CREATE_SUBTBL,
  NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
114

115 116
typedef enum TALBE_EXISTS_EN {
  TBL_NO_EXISTS,
117
  TBL_ALREADY_EXISTS,
118 119 120
  TBL_EXISTS_BUTT
} TALBE_EXISTS_EN;

121
enum enumSYNC_MODE {
122 123
  SYNC_MODE,
  ASYNC_MODE,
124 125
  MODE_BUT
};
126

S
Shuduo Sang 已提交
127
enum enum_TAOS_INTERFACE {
128 129 130
    TAOSC_IFACE,
    REST_IFACE,
    STMT_IFACE,
131 132 133
    INTERFACE_BUT
};

134 135 136 137 138 139
typedef enum enumQUERY_CLASS {
    SPECIFIED_CLASS,
    STABLE_CLASS,
    CLASS_BUT
} QUERY_CLASS;

S
Shuduo Sang 已提交
140
typedef enum enum_PROGRESSIVE_OR_INTERLACE {
141 142 143
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
144
} PROG_OR_INTERLACE_MODE;
145

146
typedef enum enumQUERY_TYPE {
147
  NO_INSERT_TYPE,
148
  INSERT_TYPE,
149
  QUERY_TYPE_BUT
150
} QUERY_TYPE;
151 152 153 154 155 156 157

enum _show_db_index {
  TSDB_SHOW_DB_NAME_INDEX,
  TSDB_SHOW_DB_CREATED_TIME_INDEX,
  TSDB_SHOW_DB_NTABLES_INDEX,
  TSDB_SHOW_DB_VGROUPS_INDEX,
  TSDB_SHOW_DB_REPLICA_INDEX,
158
  TSDB_SHOW_DB_QUORUM_INDEX,
159 160 161 162 163 164 165 166 167 168
  TSDB_SHOW_DB_DAYS_INDEX,
  TSDB_SHOW_DB_KEEP_INDEX,
  TSDB_SHOW_DB_CACHE_INDEX,
  TSDB_SHOW_DB_BLOCKS_INDEX,
  TSDB_SHOW_DB_MINROWS_INDEX,
  TSDB_SHOW_DB_MAXROWS_INDEX,
  TSDB_SHOW_DB_WALLEVEL_INDEX,
  TSDB_SHOW_DB_FSYNC_INDEX,
  TSDB_SHOW_DB_COMP_INDEX,
  TSDB_SHOW_DB_CACHELAST_INDEX,
169
  TSDB_SHOW_DB_PRECISION_INDEX,
170 171 172 173 174 175 176 177 178 179
  TSDB_SHOW_DB_UPDATE_INDEX,
  TSDB_SHOW_DB_STATUS_INDEX,
  TSDB_MAX_SHOW_DB
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
  TSDB_SHOW_STABLES_NAME_INDEX,
  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
  TSDB_SHOW_STABLES_COLUMNS_INDEX,
180 181
  TSDB_SHOW_STABLES_METRIC_INDEX,
  TSDB_SHOW_STABLES_UID_INDEX,
182
  TSDB_SHOW_STABLES_TID_INDEX,
183
  TSDB_SHOW_STABLES_VGID_INDEX,
184
  TSDB_MAX_SHOW_STABLES
185 186
};

187 188 189 190 191 192 193 194 195 196 197 198 199 200
enum _describe_table_index {
  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
  TSDB_MAX_DESCRIBE_METRIC
};

typedef struct {
  char field[TSDB_COL_NAME_LEN + 1];
  char type[16];
  int length;
  char note[128];
} SColDes;
H
hzcheng 已提交
201 202

/* Used by main to communicate with parse_opt. */
203 204
static char *g_dupstr = NULL;

205 206
typedef struct SArguments_S {
  char *   metaFile;
207
  uint32_t test_mode;
208 209
  char *   host;
  uint16_t port;
S
Shuduo Sang 已提交
210
  uint16_t iface;
211 212 213 214 215 216 217
  char *   user;
  char *   password;
  char *   database;
  int      replica;
  char *   tb_prefix;
  char *   sqlFile;
  bool     use_metric;
218
  bool     drop_database;
219
  bool     insert_only;
220
  bool     answer_yes;
221
  bool     debug_print;
222
  bool     verbose_print;
223
  bool     performance_print;
224
  char *   output_file;
225
  bool     async_mode;
226
  char *   datatype[MAX_NUM_DATATYPE + 1];
227 228 229 230
  uint32_t len_of_binary;
  uint32_t num_of_CPR;
  uint32_t num_of_threads;
  uint64_t insert_interval;
231
  int64_t  query_times;
232 233 234
  uint64_t interlace_rows;
  uint64_t num_of_RPR;                  // num_of_records_per_req
  uint64_t max_sql_len;
235 236
  int64_t  num_of_tables;
  int64_t  num_of_DPT;
237
  int      abort;
238 239
  int      disorderRatio;               // 0: no disorder, >0: x%
  int      disorderRange;               // ms or us by database precision
240
  uint32_t method_of_delete;
241
  char **  arg_list;
242 243
  uint64_t totalInsertRows;
  uint64_t totalAffectedRows;
244 245 246
} SArguments;

typedef struct SColumn_S {
247 248 249
  char      field[TSDB_COL_NAME_LEN + 1];
  char      dataType[MAX_TB_NAME_SIZE];
  uint32_t  dataLen;
250
  char      note[128];
251 252 253
} StrColumn;

typedef struct SSuperTable_S {
254
  char         sTblName[MAX_TB_NAME_SIZE+1];
255
  int64_t      childTblCount;
256
  bool         childTblExists;          // 0: no, 1: yes
257 258
  uint64_t     batchCreateTableNum;     // 0: no batch,  > 0: batch table number in one sql
  uint8_t      autoCreateTable;         // 0: create sub table, 1: auto create sub table
259
  char         childTblPrefix[MAX_TB_NAME_SIZE];
260
  char         dataSource[MAX_TB_NAME_SIZE+1];  // rand_gen or sample
261
  uint16_t     insertMode;              // 0: taosc, 1: rest, 2: stmt
262
  int64_t      childTblLimit;
263
  uint64_t     childTblOffset;
264

265
//  int          multiThreadWriteOneTbl;  // 0: no, 1: yes
266
  uint64_t     interlaceRows;           //
267 268
  int          disorderRatio;           // 0: no disorder, >0: x%
  int          disorderRange;           // ms or us by database precision
269
  uint64_t     maxSqlLen;               //
270

271
  uint64_t     insertInterval;          // insert interval, will override global insert interval
272
  int64_t      insertRows;
273
  int64_t      timeStampStep;
274
  char         startTimestamp[MAX_TB_NAME_SIZE];
275
  char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
276 277
  char         sampleFile[MAX_FILE_NAME_LEN+1];
  char         tagsFile[MAX_FILE_NAME_LEN+1];
278

279
  uint32_t     columnCount;
280
  StrColumn    columns[MAX_COLUMN_COUNT];
281
  uint32_t     tagCount;
282 283 284
  StrColumn    tags[MAX_TAG_COUNT];

  char*        childTblName;
285
  char*        colsOfCreateChildTable;
286 287
  uint64_t     lenOfOneRow;
  uint64_t     lenOfTagOfOneRow;
288 289 290 291 292

  char*        sampleDataBuf;
  //int          sampleRowCount;
  //int          sampleUsePos;

293
  uint32_t     tagSource;    // 0: rand, 1: tag sample
294
  char*        tagDataBuf;
295 296
  uint32_t     tagSampleCount;
  uint32_t     tagUsePos;
297

298
  // statistics
299 300
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
301 302 303 304 305
} SSuperTable;

typedef struct {
  char     name[TSDB_DB_NAME_LEN + 1];
  char     create_time[32];
S
Shuduo Sang 已提交
306
  int64_t  ntables;
307
  int32_t  vgroups;
308 309
  int16_t  replica;
  int16_t  quorum;
310
  int16_t  days;
311 312 313 314 315 316 317 318 319 320 321 322 323 324
  char     keeplist[32];
  int32_t  cache; //MB
  int32_t  blocks;
  int32_t  minrows;
  int32_t  maxrows;
  int8_t   wallevel;
  int32_t  fsync;
  int8_t   comp;
  int8_t   cachelast;
  char     precision[8];   // time resolution
  int8_t   update;
  char     status[16];
} SDbInfo;

325
typedef struct SDbCfg_S {
326
//  int       maxtablesPerVnode;
327 328
  uint32_t  minRows;        // 0 means default
  uint32_t  maxRows;        // 0 means default
329 330
  int       comp;
  int       walLevel;
331
  int       cacheLast;
332
  int       fsync;
333 334 335 336 337 338 339
  int       replica;
  int       update;
  int       keep;
  int       days;
  int       cache;
  int       blocks;
  int       quorum;
340
  char      precision[MAX_TB_NAME_SIZE];
341 342 343 344
} SDbCfg;

typedef struct SDataBase_S {
  char         dbName[MAX_DB_NAME_SIZE];
345
  bool         drop;  // 0: use exists, 1: if exists, drop then new create
346
  SDbCfg       dbCfg;
347
  uint64_t     superTblCount;
348 349 350 351
  SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
} SDataBase;

typedef struct SDbs_S {
352
  char         cfgDir[MAX_FILE_NAME_LEN+1];
353
  char         host[MAX_HOSTNAME_SIZE];
354 355
  struct sockaddr_in serv_addr;

356
  uint16_t     port;
357 358
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
359
  char         resultFile[MAX_FILE_NAME_LEN+1];
360 361 362
  bool         use_metric;
  bool         insert_only;
  bool         do_aggreFunc;
363
  bool         asyncMode;
364

365 366 367
  uint32_t     threadCount;
  uint32_t     threadCountByCreateTbl;
  uint32_t     dbCount;
368 369 370
  SDataBase    db[MAX_DB_COUNT];

  // statistics
371 372
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
373

374 375
} SDbs;

376
typedef struct SpecifiedQueryInfo_S {
377 378 379
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint64_t     concurrent;
  uint64_t     sqlCount;
380
  uint32_t     asyncMode; // 0: sync, 1: async
381 382
  uint64_t     subscribeInterval; // ms
  uint64_t     queryTimes;
383
  bool         subscribeRestart;
384
  int          subscribeKeepProgress;
385
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
386
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
387
  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
388
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
389
  uint64_t     totalQueried;
390
} SpecifiedQueryInfo;
391

392
typedef struct SuperQueryInfo_S {
393
  char         sTblName[MAX_TB_NAME_SIZE+1];
394 395
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint32_t     threadCnt;
396
  uint32_t     asyncMode; // 0: sync, 1: async
397
  uint64_t     subscribeInterval; // ms
398
  bool         subscribeRestart;
399
  int          subscribeKeepProgress;
400
  uint64_t     queryTimes;
401
  int64_t      childTblCount;
402
  char         childTblPrefix[MAX_TB_NAME_SIZE];
403
  uint64_t     sqlCount;
404
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
405
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
406
  int          resubAfterConsume;
407
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
408

409
  char*        childTblName;
410
  uint64_t     totalQueried;
411
} SuperQueryInfo;
412 413

typedef struct SQueryMetaInfo_S {
414
  char         cfgDir[MAX_FILE_NAME_LEN+1];
415
  char         host[MAX_HOSTNAME_SIZE];
416
  uint16_t     port;
417
  struct       sockaddr_in serv_addr;
418 419
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
420
  char         dbName[MAX_DB_NAME_SIZE+1];
421
  char         queryMode[MAX_TB_NAME_SIZE];  // taosc, rest
422

423
  SpecifiedQueryInfo  specifiedQueryInfo;
424
  SuperQueryInfo      superQueryInfo;
425
  uint64_t     totalQueried;
426 427 428
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
429
  TAOS *    taos;
430
  TAOS_STMT *stmt;
431 432 433 434 435 436 437
  int       threadID;
  char      db_name[MAX_DB_NAME_SIZE+1];
  uint32_t  time_precision;
  char      fp[4096];
  char      tb_prefix[MAX_TB_NAME_SIZE];
  uint64_t  start_table_from;
  uint64_t  end_table_to;
S
Shuduo Sang 已提交
438
  int64_t   ntables;
439 440 441 442
  uint64_t  data_of_rate;
  int64_t   start_time;
  char*     cols;
  bool      use_metric;
443
  SSuperTable* superTblInfo;
S
Shuduo Sang 已提交
444
  char      *buffer;    // sql cmd buffer
445 446

  // for async insert
447 448
  tsem_t    lock_sem;
  int64_t   counter;
449 450
  uint64_t  st;
  uint64_t  et;
451
  uint64_t  lastTs;
452

453
  // sample data
454
  int64_t   samplePos;
455
  // statistics
456 457
  uint64_t  totalInsertRows;
  uint64_t  totalAffectedRows;
458 459

  // insert delay statistics
460 461 462 463 464
  uint64_t  cntDelay;
  uint64_t  totalDelay;
  uint64_t  avgDelay;
  uint64_t  maxDelay;
  uint64_t  minDelay;
465

466
  // seq of query or subscribe
467
  uint64_t  querySeq;   // sequence number of sql command
468

469 470
} threadInfo;

471
#ifdef WINDOWS
472 473
#define _CRT_RAND_S

474
#include <windows.h>
475 476 477 478 479
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
480 481
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
482 483
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
484 485 486 487

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

488
static void setupForAnsiEscape(void) {
489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
507
  }
508 509
}

510
static void resetAfterAnsiEscape(void) {
511
  // Reset colors
512
  printf("\x1b[0m");
513 514 515 516 517 518

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
519

520
static int taosRandom()
521 522 523 524 525 526
{
    int number;
    rand_s(&number);

    return number;
}
527
#else   // Not windows
528
static void setupForAnsiEscape(void) {}
529

530
static void resetAfterAnsiEscape(void) {
531 532 533
  // Reset colors
  printf("\x1b[0m");
}
534

535 536
#include <time.h>

537
static int taosRandom()
538
{
539
  return rand();
540 541
}

542
#endif // ifdef Windows
543

544
static void prompt();
545
static int createDatabasesAndStables();
546
static void createChildTables();
547
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet);
548 549
static int postProceSql(char *host, struct sockaddr_in *pServAddr,
        uint16_t port, char* sqlstr, char *resultFile);
550 551 552 553 554 555 556

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
557
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)",
558 559
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

560
SArguments g_args = {
561 562 563 564
                     NULL,            // metaFile
                     0,               // test_mode
                     "127.0.0.1",     // host
                     6030,            // port
S
Shuduo Sang 已提交
565
                     TAOSC_IFACE,     // iface
566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597
                     "root",          // user
                     #ifdef _TD_POWER_
                     "powerdb",      // password
                     #else
                     "taosdata",      // password
                     #endif
                     "test",          // database
                     1,               // replica
                     "t",             // tb_prefix
                     NULL,            // sqlFile
                     true,            // use_metric
                     true,            // drop_database
                     true,            // insert_only
                     false,           // debug_print
                     false,           // verbose_print
                     false,           // performance statistic print
                     false,           // answer_yes;
                     "./output.txt",  // output_file
                     0,               // mode : sync or async
                     {
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     },
                     16,              // len_of_binary
                     4,               // num_of_CPR
                     10,              // num_of_connections/thread
                     0,               // insert_interval
                     1,               // query_times
                     0,               // interlace_rows;
                     30000,           // num_of_RPR
sangshuduo's avatar
sangshuduo 已提交
598
                     (1024*1024),         // max_sql_len
599 600 601 602 603 604 605
                     10000,           // num_of_tables
                     10000,           // num_of_DPT
                     0,               // abort
                     0,               // disorderRatio
                     1000,            // disorderRange
                     1,               // method_of_delete
                     NULL             // arg_list
606 607 608 609 610
};



static SDbs            g_Dbs;
611
static int64_t         g_totalChildTables = 0;
612 613 614 615
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

#define debugPrint(fmt, ...) \
616
    do { if (g_args.debug_print || g_args.verbose_print) \
617
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
618

619
#define verbosePrint(fmt, ...) \
620 621
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
622

623 624 625 626
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)

627 628 629 630
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)


631
///////////////////////////////////////////////////
H
hzcheng 已提交
632

633 634
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

635 636 637 638 639 640 641 642
#ifndef TAOSDEMO_COMMIT_SHA1
#define TAOSDEMO_COMMIT_SHA1 "unknown"
#endif

#ifndef TD_VERNUMBER
#define TD_VERNUMBER    "unknown"
#endif

643
#ifndef TAOSDEMO_STATUS
644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660
#define TAOSDEMO_STATUS "unknown"
#endif

static void printVersion() {
    char tdengine_ver[] = TD_VERNUMBER;
    char taosdemo_ver[] = TAOSDEMO_COMMIT_SHA1;
    char taosdemo_status[] = TAOSDEMO_STATUS;

    if (strlen(taosdemo_status) == 0) {
        printf("taosdemo verison %s-%s\n",
                tdengine_ver, taosdemo_ver);
    } else {
        printf("taosdemo verison %s-%s, status:%s\n",
                tdengine_ver, taosdemo_ver, taosdemo_status);
    }
}

661
static void printHelp() {
662
  char indent[10] = "        ";
663
  printf("%s%s%s%s\n", indent, "-f", indent,
664
          "The meta file to the execution procedure. Default is './meta.json'.");
665
  printf("%s%s%s%s\n", indent, "-u", indent,
666 667
          "The TDengine user name to use when connecting to the server. Default is 'root'.");
#ifdef _TD_POWER_
668
  printf("%s%s%s%s\n", indent, "-P", indent,
669
          "The password to use when connecting to the server. Default is 'powerdb'.");
670
  printf("%s%s%s%s\n", indent, "-c", indent,
671 672
          "Configuration directory. Default is '/etc/power/'.");
#else
673
  printf("%s%s%s%s\n", indent, "-P", indent,
674
          "The password to use when connecting to the server. Default is 'taosdata'.");
675
  printf("%s%s%s%s\n", indent, "-c", indent,
676
          "Configuration directory. Default is '/etc/taos/'.");
677
#endif
678
  printf("%s%s%s%s\n", indent, "-h", indent,
679
          "The host to connect to TDengine. Default is localhost.");
680
  printf("%s%s%s%s\n", indent, "-p", indent,
681
          "The TCP/IP port number to use for the connection. Default is 0.");
682 683
  printf("%s%s%s%s\n", indent, "-I", indent,
          "The interface (taosc, rest, and stmt) taosdemo uses. Default is 'taosc'.");
684
  printf("%s%s%s%s\n", indent, "-d", indent,
685
          "Destination database. Default is 'test'.");
686
  printf("%s%s%s%s\n", indent, "-a", indent,
687
          "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
688
  printf("%s%s%s%s\n", indent, "-m", indent,
689 690
          "Table prefix name. Default is 't'.");
  printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
691
  printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
692
  printf("%s%s%s%s\n", indent, "-o", indent,
693
          "Direct output to the named file. Default is './output.txt'.");
694
  printf("%s%s%s%s\n", indent, "-q", indent,
695
          "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC.");
696
  printf("%s%s%s%s\n", indent, "-b", indent,
sangshuduo's avatar
sangshuduo 已提交
697
          "The data_type of columns, default: INT,INT,INT,INT.");
698
  printf("%s%s%s%s\n", indent, "-w", indent,
699
          "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
700 701 702
  printf("%s%s%s%s%d\n", indent, "-l", indent,
          "The number of columns per record. Default is 4. Max values is ",
       MAX_NUM_DATATYPE);
703
  printf("%s%s%s%s\n", indent, "-T", indent,
704
          "The number of threads. Default is 10.");
705
  printf("%s%s%s%s\n", indent, "-i", indent,
706
          "The sleep time (ms) between insertion. Default is 0.");
707
  printf("%s%s%s%s\n", indent, "-r", indent,
708
          "The number of records per request. Default is 30000.");
709
  printf("%s%s%s%s\n", indent, "-t", indent,
710
          "The number of tables. Default is 10000.");
711
  printf("%s%s%s%s\n", indent, "-n", indent,
712 713 714
          "The number of records per table. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
  printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
715
  printf("%s%s%s%s\n", indent, "-O", indent,
716
          "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order.");
717
  printf("%s%s%s%s\n", indent, "-R", indent,
718
          "Out of order data's range, ms, default is 1000.");
719
  printf("%s%s%s%s\n", indent, "-g", indent,
720
          "Print debug info.");
721
  printf("%s%s%s\n", indent, "-V, --version\t",
722
          "Print version info.");
723 724
  printf("%s%s%s%s\n", indent, "--help\t", indent,
          "Print command line arguments list info.");
725
/*    printf("%s%s%s%s\n", indent, "-D", indent,
726 727 728 729
          "if elete database if exists. 0: no, 1: yes, default is 1");
          */
}

730 731 732 733 734 735 736 737 738 739 740 741 742 743 744
static bool isStringNumber(char *input)
{
  int len = strlen(input);
  if (0 == len) {
    return false;
  }

  for (int i = 0; i < len; i++) {
    if (!isdigit(input[i]))
      return false;
  }

  return true;
}

745
static void parse_args(int argc, char *argv[], SArguments *arguments) {
746 747 748 749 750

  for (int i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-f") == 0) {
      arguments->metaFile = argv[++i];
    } else if (strcmp(argv[i], "-c") == 0) {
H
Haojun Liao 已提交
751 752 753 754 755
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-c need a valid path following!\n");
        exit(EXIT_FAILURE);
      }
756
      tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN);
757
    } else if (strcmp(argv[i], "-h") == 0) {
H
Haojun Liao 已提交
758 759 760 761 762
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-h need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
763 764
      arguments->host = argv[++i];
    } else if (strcmp(argv[i], "-p") == 0) {
765
      if ((argc == i+1) ||
766
        (!isStringNumber(argv[i+1]))) {
767 768 769 770
        printHelp();
        errorPrint("%s", "\n\t-p need a number following!\n");
        exit(EXIT_FAILURE);
      }
771
      arguments->port = atoi(argv[++i]);
772 773 774 775 776 777 778 779
    } else if (strcmp(argv[i], "-I") == 0) {
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-I need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
      ++i;
      if (0 == strcasecmp(argv[i], "taosc")) {
S
Shuduo Sang 已提交
780
          arguments->iface = TAOSC_IFACE;
781
      } else if (0 == strcasecmp(argv[i], "rest")) {
S
Shuduo Sang 已提交
782
          arguments->iface = REST_IFACE;
783
      } else if (0 == strcasecmp(argv[i], "stmt")) {
S
Shuduo Sang 已提交
784
          arguments->iface = STMT_IFACE;
785 786 787 788
      } else {
        errorPrint("%s", "\n\t-I need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
789
    } else if (strcmp(argv[i], "-u") == 0) {
H
Haojun Liao 已提交
790 791 792 793 794
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-u need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
795 796
      arguments->user = argv[++i];
    } else if (strcmp(argv[i], "-P") == 0) {
H
Haojun Liao 已提交
797 798 799 800 801
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-P need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
802 803
      arguments->password = argv[++i];
    } else if (strcmp(argv[i], "-o") == 0) {
H
Haojun Liao 已提交
804 805 806 807 808
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-o need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
809 810
      arguments->output_file = argv[++i];
    } else if (strcmp(argv[i], "-s") == 0) {
H
Haojun Liao 已提交
811 812 813 814 815
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-s need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
816 817
      arguments->sqlFile = argv[++i];
    } else if (strcmp(argv[i], "-q") == 0) {
818 819
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
820
        printHelp();
821
        errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n");
822 823
        exit(EXIT_FAILURE);
      }
824
      arguments->async_mode = atoi(argv[++i]);
825
    } else if (strcmp(argv[i], "-T") == 0) {
826 827
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
828 829 830 831
        printHelp();
        errorPrint("%s", "\n\t-T need a number following!\n");
        exit(EXIT_FAILURE);
      }
832 833
      arguments->num_of_threads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
834 835
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
836 837 838 839
        printHelp();
        errorPrint("%s", "\n\t-i need a number following!\n");
        exit(EXIT_FAILURE);
      }
840
      arguments->insert_interval = atoi(argv[++i]);
841
    } else if (strcmp(argv[i], "-qt") == 0) {
842
      if ((argc == i+1)
843
              || (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
844
        printHelp();
845
        errorPrint("%s", "\n\t-qt need a number following!\n");
H
Haojun Liao 已提交
846 847
        exit(EXIT_FAILURE);
      }
848
      arguments->query_times = atoi(argv[++i]);
849
    } else if (strcmp(argv[i], "-B") == 0) {
850 851
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
852 853 854 855
        printHelp();
        errorPrint("%s", "\n\t-B need a number following!\n");
        exit(EXIT_FAILURE);
      }
856
      arguments->interlace_rows = atoi(argv[++i]);
857
    } else if (strcmp(argv[i], "-r") == 0) {
858 859
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
860 861 862 863
        printHelp();
        errorPrint("%s", "\n\t-r need a number following!\n");
        exit(EXIT_FAILURE);
      }
864 865
      arguments->num_of_RPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-t") == 0) {
866 867 868 869 870 871
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-t need a number following!\n");
        exit(EXIT_FAILURE);
      }
872 873
      arguments->num_of_tables = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
874 875 876 877 878 879
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-n need a number following!\n");
        exit(EXIT_FAILURE);
      }
880 881
      arguments->num_of_DPT = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-d") == 0) {
H
Haojun Liao 已提交
882 883 884 885 886
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-d need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
887 888
      arguments->database = argv[++i];
    } else if (strcmp(argv[i], "-l") == 0) {
889 890 891 892 893 894
      if (argc == i+1) {
        if (!isStringNumber(argv[i+1])) {
            printHelp();
            errorPrint("%s", "\n\t-l need a number following!\n");
            exit(EXIT_FAILURE);
        }
895
      }
896
      arguments->num_of_CPR = atoi(argv[++i]);
897 898 899 900 901 902 903 904 905 906 907

      if (arguments->num_of_CPR > MAX_NUM_DATATYPE) {
          printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE);
          prompt();
          arguments->num_of_CPR = MAX_NUM_DATATYPE;
      }

      for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) {
          arguments->datatype[col] = NULL;
      }

908
    } else if (strcmp(argv[i], "-b") == 0) {
909 910 911 912 913
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-b need valid string following!\n");
        exit(EXIT_FAILURE);
      }
914 915
      ++i;
      if (strstr(argv[i], ",") == NULL) {
916
        // only one col
917
        if (strcasecmp(argv[i], "INT")
918 919 920 921 922 923 924 925
                && strcasecmp(argv[i], "FLOAT")
                && strcasecmp(argv[i], "TINYINT")
                && strcasecmp(argv[i], "BOOL")
                && strcasecmp(argv[i], "SMALLINT")
                && strcasecmp(argv[i], "BIGINT")
                && strcasecmp(argv[i], "DOUBLE")
                && strcasecmp(argv[i], "BINARY")
                && strcasecmp(argv[i], "NCHAR")) {
926
          printHelp();
927
          errorPrint("%s", "-b: Invalid data_type!\n");
928
          exit(EXIT_FAILURE);
929
        }
930
        arguments->datatype[0] = argv[i];
931
      } else {
932
        // more than one col
933
        int index = 0;
934 935
        g_dupstr = strdup(argv[i]);
        char *running = g_dupstr;
936
        char *token = strsep(&running, ",");
937
        while(token != NULL) {
938
          if (strcasecmp(token, "INT")
939 940 941 942 943 944 945 946
                  && strcasecmp(token, "FLOAT")
                  && strcasecmp(token, "TINYINT")
                  && strcasecmp(token, "BOOL")
                  && strcasecmp(token, "SMALLINT")
                  && strcasecmp(token, "BIGINT")
                  && strcasecmp(token, "DOUBLE")
                  && strcasecmp(token, "BINARY")
                  && strcasecmp(token, "NCHAR")) {
947
            printHelp();
948
            free(g_dupstr);
949
            errorPrint("%s", "-b: Invalid data_type!\n");
950
            exit(EXIT_FAILURE);
H
hzcheng 已提交
951
          }
952
          arguments->datatype[index++] = token;
953 954
          token = strsep(&running, ",");
          if (index >= MAX_NUM_DATATYPE) break;
S
TD-1057  
Shengliang Guan 已提交
955
        }
956
        arguments->datatype[index] = NULL;
957
      }
958
    } else if (strcmp(argv[i], "-w") == 0) {
959
      if ((argc == i+1) ||
960
        (!isStringNumber(argv[i+1]))) {
961 962 963 964
        printHelp();
        errorPrint("%s", "\n\t-w need a number following!\n");
        exit(EXIT_FAILURE);
      }
965 966
      arguments->len_of_binary = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-m") == 0) {
H
Haojun Liao 已提交
967
      if ((argc == i+1) ||
968
        (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
969 970 971 972
        printHelp();
        errorPrint("%s", "\n\t-m need a number following!\n");
        exit(EXIT_FAILURE);
      }
973
      arguments->tb_prefix = argv[++i];
974 975
    } else if (strcmp(argv[i], "-N") == 0) {
      arguments->use_metric = false;
976
    } else if (strcmp(argv[i], "-x") == 0) {
977
      arguments->insert_only = false;
978
    } else if (strcmp(argv[i], "-y") == 0) {
979
      arguments->answer_yes = true;
980 981
    } else if (strcmp(argv[i], "-g") == 0) {
      arguments->debug_print = true;
982 983
    } else if (strcmp(argv[i], "-gg") == 0) {
      arguments->verbose_print = true;
984 985
    } else if (strcmp(argv[i], "-pp") == 0) {
      arguments->performance_print = true;
986
    } else if (strcmp(argv[i], "-O") == 0) {
987 988 989 990 991 992
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-O need a number following!\n");
        exit(EXIT_FAILURE);
      }
993

994
      arguments->disorderRatio = atoi(argv[++i]);
995

996
      if (arguments->disorderRatio > 50) {
997
        arguments->disorderRatio = 50;
998
      }
999

1000
      if (arguments->disorderRatio < 0) {
1001
        arguments->disorderRatio = 0;
1002
      }
1003

1004
    } else if (strcmp(argv[i], "-R") == 0) {
1005 1006 1007 1008 1009 1010
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-R need a number following!\n");
        exit(EXIT_FAILURE);
      }
1011

1012
      arguments->disorderRange = atoi(argv[++i]);
1013 1014 1015
      if (arguments->disorderRange < 0)
        arguments->disorderRange = 1000;

1016
    } else if (strcmp(argv[i], "-a") == 0) {
1017 1018 1019 1020 1021 1022
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-a need a number following!\n");
        exit(EXIT_FAILURE);
      }
1023 1024 1025
      arguments->replica = atoi(argv[++i]);
      if (arguments->replica > 3 || arguments->replica < 1) {
          arguments->replica = 1;
1026
      }
1027 1028
    } else if (strcmp(argv[i], "-D") == 0) {
      arguments->method_of_delete = atoi(argv[++i]);
1029 1030 1031
      if (arguments->method_of_delete > 3) {
        errorPrint("%s", "\n\t-D need a valud (0~3) number following!\n");
        exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1032
      }
1033 1034 1035 1036
    } else if ((strcmp(argv[i], "--version") == 0) ||
        (strcmp(argv[i], "-V") == 0)){
      printVersion();
      exit(0);
1037 1038 1039 1040 1041
    } else if (strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else {
      printHelp();
1042
      errorPrint("%s", "ERROR: wrong options\n");
1043
      exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1044 1045
    }
  }
1046

1047
  if (((arguments->debug_print) && (arguments->metaFile == NULL))
1048
          || arguments->verbose_print) {
1049
    printf("###################################################################\n");
1050
    printf("# meta file:                         %s\n", arguments->metaFile);
1051
    printf("# Server IP:                         %s:%hu\n",
1052 1053
            arguments->host == NULL ? "localhost" : arguments->host,
            arguments->port );
1054 1055
    printf("# User:                              %s\n", arguments->user);
    printf("# Password:                          %s\n", arguments->password);
1056 1057
    printf("# Use metric:                        %s\n",
            arguments->use_metric ? "true" : "false");
1058
    if (*(arguments->datatype)) {
1059 1060 1061 1062 1063 1064 1065
        printf("# Specified data type:               ");
        for (int i = 0; i < MAX_NUM_DATATYPE; i++)
            if (arguments->datatype[i])
               printf("%s,", arguments->datatype[i]);
            else
                break;
        printf("\n");
1066
    }
1067
    printf("# Insertion interval:                %"PRIu64"\n",
1068
            arguments->insert_interval);
1069
    printf("# Number of records per req:         %"PRIu64"\n",
1070
            arguments->num_of_RPR);
1071
    printf("# Max SQL length:                    %"PRIu64"\n",
1072
            arguments->max_sql_len);
1073
    printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
1074
    printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
1075
    printf("# Number of Tables:                  %"PRId64"\n",
1076
            arguments->num_of_tables);
1077
    printf("# Number of Data per Table:          %"PRId64"\n",
1078
            arguments->num_of_DPT);
1079 1080 1081 1082 1083
    printf("# Database name:                     %s\n", arguments->database);
    printf("# Table prefix:                      %s\n", arguments->tb_prefix);
    if (arguments->disorderRatio) {
      printf("# Data order:                        %d\n", arguments->disorderRatio);
      printf("# Data out of order rate:            %d\n", arguments->disorderRange);
1084

1085 1086 1087 1088
    }
    printf("# Delete method:                     %d\n", arguments->method_of_delete);
    printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
    printf("# Print debug info:                  %d\n", arguments->debug_print);
1089
    printf("# Print verbose info:                %d\n", arguments->verbose_print);
1090
    printf("###################################################################\n");
1091 1092

    prompt();
1093
  }
1094
}
H
hzcheng 已提交
1095

1096 1097
static bool getInfoFromJsonFile(char* file);
static void init_rand_data();
1098
static void tmfclose(FILE *fp) {
1099 1100 1101 1102
  if (NULL != fp) {
    fclose(fp);
  }
}
1103

1104
static void tmfree(char *buf) {
1105 1106
  if (NULL != buf) {
    free(buf);
H
Hui Li 已提交
1107
  }
1108
}
H
Hui Li 已提交
1109

1110
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
1111 1112 1113
  int i;
  TAOS_RES *res = NULL;
  int32_t   code = -1;
1114

S
Shuduo Sang 已提交
1115
  for (i = 0; i < 5 /* retry */; i++) {
1116 1117 1118
    if (NULL != res) {
      taos_free_result(res);
      res = NULL;
H
hzcheng 已提交
1119
    }
1120

1121 1122 1123 1124
    res = taos_query(taos, command);
    code = taos_errno(res);
    if (0 == code) {
      break;
1125
    }
H
hzcheng 已提交
1126 1127
  }

1128
  if (code != 0) {
1129 1130
    if (!quiet) {
      debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
1131
      errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res));
1132
    }
1133 1134 1135
    taos_free_result(res);
    //taos_close(taos);
    return -1;
1136
  }
H
hzcheng 已提交
1137

1138 1139 1140 1141
  if (INSERT_TYPE == type) {
    int affectedRows = taos_affected_rows(res);
    taos_free_result(res);
    return affectedRows;
H
hzcheng 已提交
1142
  }
1143

1144
  taos_free_result(res);
1145 1146
  return 0;
}
H
hzcheng 已提交
1147

1148 1149
static void appendResultBufToFile(char *resultBuf, char *resultFile)
{
1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160
    FILE *fp = NULL;
    if (resultFile[0] != 0) {
        fp = fopen(resultFile, "at");
        if (fp == NULL) {
            errorPrint(
                    "%s() LN%d, failed to open result file: %s, result will not save to file\n",
                    __func__, __LINE__, resultFile);
            return;
        }
        fprintf(fp, "%s", resultBuf);
        tmfclose(fp);
1161
    }
1162 1163 1164 1165 1166 1167 1168 1169
}

static void appendResultToFile(TAOS_RES *res, char* resultFile) {
  TAOS_ROW    row = NULL;
  int         num_rows = 0;
  int         num_fields = taos_field_count(res);
  TAOS_FIELD *fields     = taos_fetch_fields(res);

1170 1171
  char* databuf = (char*) calloc(1, 100*1024*1024);
  if (databuf == NULL) {
1172 1173
    errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n",
            __func__, __LINE__);
1174 1175
    return ;
  }
H
hzcheng 已提交
1176

1177 1178
  int   totalLen = 0;
  char  temp[16000];
H
hzcheng 已提交
1179

1180
  // fetch the records row by row
1181
  while((row = taos_fetch_row(res))) {
1182
    if (totalLen >= 100*1024*1024 - 32000) {
1183
      appendResultBufToFile(databuf, resultFile);
1184 1185
      totalLen = 0;
      memset(databuf, 0, 100*1024*1024);
H
hzcheng 已提交
1186
    }
1187 1188 1189 1190 1191 1192
    num_rows++;
    int len = taos_print_row(temp, row, fields, num_fields);
    len += sprintf(temp + len, "\n");
    //printf("query result:%s\n", temp);
    memcpy(databuf + totalLen, temp, len);
    totalLen += len;
H
hzcheng 已提交
1193 1194
  }

1195 1196
  verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n",
          __func__, __LINE__, databuf, resultFile);
1197
  appendResultBufToFile(databuf, resultFile);
1198 1199
  free(databuf);
}
H
hzcheng 已提交
1200

1201
static void selectAndGetResult(
1202 1203
        threadInfo *pThreadInfo, char *command)
{
1204 1205 1206 1207 1208 1209 1210 1211 1212
  if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) {
    TAOS_RES *res = taos_query(pThreadInfo->taos, command);
    if (res == NULL || taos_errno(res) != 0) {
        errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n",
            __func__, __LINE__, command, taos_errstr(res));
        taos_free_result(res);
        return;
    }

1213 1214
    if ((strlen(pThreadInfo->fp))) {
      appendResultToFile(res, pThreadInfo->fp);
1215
    }
1216
    taos_free_result(res);
1217

1218 1219 1220
  } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
      int retCode = postProceSql(
              g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port,
1221
              command,
1222
              pThreadInfo->fp);
1223 1224 1225 1226 1227 1228 1229 1230
      if (0 != retCode) {
        printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID);
      }

  } else {
      errorPrint("%s() LN%d, unknown query mode: %s\n",
        __func__, __LINE__, g_queryInfo.queryMode);
  }
1231
}
H
hzcheng 已提交
1232

1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271
static int32_t rand_bool(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 2;
}

static int32_t rand_tinyint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 128;
}

static int32_t rand_smallint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 32767;
}

static int32_t rand_int(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor];
}

static int64_t rand_bigint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randbigint[cursor];
}

static float rand_float(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
1272
  return randfloat[cursor];
1273 1274
}

1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291
#if 0
static const char charNum[] = "0123456789";

static void nonrand_string(char *, int) __attribute__ ((unused));   // reserve for debugging purpose
static void nonrand_string(char *str, int size)
{
  str[0] = 0;
  if (size > 0) {
    int n;
    for (n = 0; n < size; n++) {
      str[n] = charNum[n % 10];
    }
    str[n] = 0;
  }
}
#endif

1292
static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1293

1294
static void rand_string(char *str, int size) {
1295 1296 1297 1298
  str[0] = 0;
  if (size > 0) {
    //--size;
    int n;
1299
    for (n = 0; n < size; n++) {
1300
      int key = abs(rand_tinyint()) % (int)(sizeof(charset) - 1);
1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316
      str[n] = charset[key];
    }
    str[n] = 0;
  }
}

static double rand_double() {
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randdouble[cursor];

}

static void init_rand_data() {
  for (int i = 0; i < MAX_PREPARED_RAND; i++){
1317 1318 1319 1320
    randint[i] = (int)(taosRandom() % 65535);
    randbigint[i] = (int64_t)(taosRandom() % 2147483648);
    randfloat[i] = (float)(taosRandom() / 1000.0);
    randdouble[i] = (double)(taosRandom() / 1000000.0);
1321 1322 1323
  }
}

1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1344
static int printfInsertMeta() {
1345
    SHOW_PARSE_RESULT_START();
1346

1347
  printf("interface:                  \033[33m%s\033[0m\n",
S
Shuduo Sang 已提交
1348
          (g_args.iface==TAOSC_IFACE)?"taosc":(g_args.iface==REST_IFACE)?"rest":"stmt");
1349 1350
  printf("host:                       \033[33m%s:%u\033[0m\n",
          g_Dbs.host, g_Dbs.port);
1351 1352
  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
1353
  printf("configDir:                  \033[33m%s\033[0m\n", configDir);
1354 1355
  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
1356 1357
  printf("thread num of create table: \033[33m%d\033[0m\n",
          g_Dbs.threadCountByCreateTbl);
1358
  printf("top insert interval:        \033[33m%"PRIu64"\033[0m\n",
1359
          g_args.insert_interval);
1360
  printf("number of records per req:  \033[33m%"PRIu64"\033[0m\n",
1361
          g_args.num_of_RPR);
1362
  printf("max sql length:             \033[33m%"PRIu64"\033[0m\n",
1363
          g_args.max_sql_len);
1364 1365

  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
1366

1367 1368
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    printf("database[\033[33m%d\033[0m]:\n", i);
1369 1370
    printf("  database[%d] name:      \033[33m%s\033[0m\n",
            i, g_Dbs.db[i].dbName);
1371
    if (0 == g_Dbs.db[i].drop) {
1372 1373 1374
      printf("  drop:                  \033[33mno\033[0m\n");
    } else {
      printf("  drop:                  \033[33myes\033[0m\n");
1375 1376 1377
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
1378 1379
      printf("  blocks:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.blocks);
1380 1381
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
1382 1383
      printf("  cache:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.cache);
1384 1385
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
1386 1387
      printf("  days:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.days);
1388 1389
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
1390 1391
      printf("  keep:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.keep);
1392 1393
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
1394 1395
      printf("  replica:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.replica);
1396 1397
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
1398 1399
      printf("  update:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.update);
1400 1401
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
1402 1403
      printf("  minRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.minRows);
1404 1405
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
1406 1407
      printf("  maxRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.maxRows);
1408 1409 1410 1411 1412
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
1413 1414
      printf("  walLevel:              \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.walLevel);
1415 1416
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
1417 1418
      printf("  fsync:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.fsync);
1419 1420
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
1421 1422
      printf("  quorum:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.quorum);
1423 1424
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1425
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
1426
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1427
        printf("  precision:             \033[33m%s\033[0m\n",
1428
            g_Dbs.db[i].dbCfg.precision);
1429
      } else {
1430
        printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
1431
                g_Dbs.db[i].dbCfg.precision);
1432
        return -1;
1433 1434 1435
      }
    }

1436
    printf("  super table count:     \033[33m%"PRIu64"\033[0m\n",
1437
        g_Dbs.db[i].superTblCount);
1438 1439
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      printf("  super table[\033[33m%"PRIu64"\033[0m]:\n", j);
1440

1441
      printf("      stbName:           \033[33m%s\033[0m\n",
1442
          g_Dbs.db[i].superTbls[j].sTblName);
1443 1444 1445 1446 1447 1448 1449 1450

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
      }
1451

1452 1453 1454 1455 1456 1457 1458
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
      }
1459

1460
      printf("      childTblCount:     \033[33m%"PRId64"\033[0m\n",
1461
              g_Dbs.db[i].superTbls[j].childTblCount);
1462
      printf("      childTblPrefix:    \033[33m%s\033[0m\n",
1463
              g_Dbs.db[i].superTbls[j].childTblPrefix);
1464
      printf("      dataSource:        \033[33m%s\033[0m\n",
1465
              g_Dbs.db[i].superTbls[j].dataSource);
1466
      printf("      insertMode:        \033[33m%s\033[0m\n",
1467 1468
          (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc":
          (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt");
1469
      if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
1470
        printf("      childTblLimit:     \033[33m%"PRId64"\033[0m\n",
1471
                g_Dbs.db[i].superTbls[j].childTblLimit);
1472
      }
1473
      if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) {
1474
        printf("      childTblOffset:    \033[33m%"PRIu64"\033[0m\n",
1475
                g_Dbs.db[i].superTbls[j].childTblOffset);
1476
      }
1477
      printf("      insertRows:        \033[33m%"PRId64"\033[0m\n",
1478
              g_Dbs.db[i].superTbls[j].insertRows);
1479
/*
1480
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1481
        printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
1482
      }else {
1483
        printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
1484
      }
1485
      */
1486
      printf("      interlaceRows:     \033[33m%"PRIu64"\033[0m\n",
1487
              g_Dbs.db[i].superTbls[j].interlaceRows);
1488 1489

      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1490
        printf("      stable insert interval:   \033[33m%"PRIu64"\033[0m\n",
1491
            g_Dbs.db[i].superTbls[j].insertInterval);
1492 1493
      }

1494
      printf("      disorderRange:     \033[33m%d\033[0m\n",
1495
              g_Dbs.db[i].superTbls[j].disorderRange);
1496
      printf("      disorderRatio:     \033[33m%d\033[0m\n",
1497
              g_Dbs.db[i].superTbls[j].disorderRatio);
1498
      printf("      maxSqlLen:         \033[33m%"PRIu64"\033[0m\n",
1499
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1500
      printf("      timeStampStep:     \033[33m%"PRId64"\033[0m\n",
1501
              g_Dbs.db[i].superTbls[j].timeStampStep);
1502
      printf("      startTimestamp:    \033[33m%s\033[0m\n",
1503
              g_Dbs.db[i].superTbls[j].startTimestamp);
1504
      printf("      sampleFormat:      \033[33m%s\033[0m\n",
1505
              g_Dbs.db[i].superTbls[j].sampleFormat);
1506
      printf("      sampleFile:        \033[33m%s\033[0m\n",
1507
              g_Dbs.db[i].superTbls[j].sampleFile);
1508
      printf("      tagsFile:          \033[33m%s\033[0m\n",
1509
              g_Dbs.db[i].superTbls[j].tagsFile);
1510
      printf("      columnCount:       \033[33m%d\033[0m\n",
1511
              g_Dbs.db[i].superTbls[j].columnCount);
1512 1513
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1514
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
1515 1516 1517
                       "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "nchar", 5))) {
1518
          printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
1519 1520
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1521
        } else {
1522
          printf("column[%d]:\033[33m%s\033[0m ", k,
1523
                  g_Dbs.db[i].superTbls[j].columns[k].dataType);
1524 1525 1526
        }
      }
      printf("\n");
1527 1528

      printf("      tagCount:            \033[33m%d\033[0m\n        ",
1529
              g_Dbs.db[i].superTbls[j].tagCount);
1530 1531
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1532
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
1533 1534 1535
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1536
          printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
1537 1538
                  g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1539
        } else {
1540
          printf("tag[%d]:\033[33m%s\033[0m ", k,
1541
                  g_Dbs.db[i].superTbls[j].tags[k].dataType);
1542
        }
1543 1544 1545 1546 1547
      }
      printf("\n");
    }
    printf("\n");
  }
1548 1549

  SHOW_PARSE_RESULT_END();
1550 1551

  return 0;
1552 1553 1554
}

static void printfInsertMetaToFile(FILE* fp) {
1555 1556

  SHOW_PARSE_RESULT_START_TO_FILE(fp);
1557

1558 1559
  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
  fprintf(fp, "user:                       %s\n", g_Dbs.user);
1560
  fprintf(fp, "configDir:                  %s\n", configDir);
1561 1562 1563
  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
1564 1565
  fprintf(fp, "number of records per req:  %"PRIu64"\n", g_args.num_of_RPR);
  fprintf(fp, "max sql length:             %"PRIu64"\n", g_args.max_sql_len);
1566
  fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
1567

1568 1569
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    fprintf(fp, "database[%d]:\n", i);
1570
    fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
1571
    if (0 == g_Dbs.db[i].drop) {
1572
      fprintf(fp, "  drop:                  no\n");
1573
    }else {
1574
      fprintf(fp, "  drop:                  yes\n");
1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1614
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
1615
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1616 1617
        fprintf(fp, "  precision:             %s\n",
                g_Dbs.db[i].dbCfg.precision);
1618
      } else {
1619 1620
        fprintf(fp, "  precision error:       %s\n",
                g_Dbs.db[i].dbCfg.precision);
1621 1622 1623
      }
    }

1624 1625
    fprintf(fp, "  super table count:     %"PRIu64"\n",
            g_Dbs.db[i].superTblCount);
1626 1627
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      fprintf(fp, "  super table[%d]:\n", j);
1628

1629 1630
      fprintf(fp, "      stbName:           %s\n",
              g_Dbs.db[i].superTbls[j].sTblName);
1631 1632 1633

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "no");
1634 1635
      } else if (AUTO_CREATE_SUBTBL
              == g_Dbs.db[i].superTbls[j].autoCreateTable) {
1636 1637 1638 1639
        fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
      } else {
        fprintf(fp, "      autoCreateTable:   %s\n",  "error");
      }
1640

1641 1642
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "no");
1643 1644
      } else if (TBL_ALREADY_EXISTS
              == g_Dbs.db[i].superTbls[j].childTblExists) {
1645 1646 1647 1648
        fprintf(fp, "      childTblExists:    %s\n",  "yes");
      } else {
        fprintf(fp, "      childTblExists:    %s\n",  "error");
      }
1649

1650
      fprintf(fp, "      childTblCount:     %"PRId64"\n",
1651 1652 1653 1654 1655 1656
              g_Dbs.db[i].superTbls[j].childTblCount);
      fprintf(fp, "      childTblPrefix:    %s\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      fprintf(fp, "      dataSource:        %s\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      fprintf(fp, "      insertMode:        %s\n",
1657 1658
          (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc":
          (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt");
1659
      fprintf(fp, "      insertRows:        %"PRId64"\n",
1660
              g_Dbs.db[i].superTbls[j].insertRows);
1661
      fprintf(fp, "      interlace rows:    %"PRIu64"\n",
1662
              g_Dbs.db[i].superTbls[j].interlaceRows);
1663
      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1664
        fprintf(fp, "      stable insert interval:   %"PRIu64"\n",
1665
                g_Dbs.db[i].superTbls[j].insertInterval);
1666
      }
1667
/*
1668
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1669
        fprintf(fp, "      multiThreadWriteOneTbl:  no\n");
1670
      }else {
1671
        fprintf(fp, "      multiThreadWriteOneTbl:  yes\n");
1672
      }
1673
      */
1674
      fprintf(fp, "      interlaceRows:     %"PRIu64"\n",
1675
              g_Dbs.db[i].superTbls[j].interlaceRows);
1676 1677 1678 1679
      fprintf(fp, "      disorderRange:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      fprintf(fp, "      disorderRatio:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
1680
      fprintf(fp, "      maxSqlLen:         %"PRIu64"\n",
1681
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1682

1683 1684 1685 1686
      fprintf(fp, "      timeStampStep:     %"PRId64"\n",
              g_Dbs.db[i].superTbls[j].timeStampStep);
      fprintf(fp, "      startTimestamp:    %s\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
1687 1688 1689 1690 1691 1692
      fprintf(fp, "      sampleFormat:      %s\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      fprintf(fp, "      sampleFile:        %s\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      fprintf(fp, "      tagsFile:          %s\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
1693

1694 1695
      fprintf(fp, "      columnCount:       %d\n        ",
              g_Dbs.db[i].superTbls[j].columnCount);
1696 1697
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1698
        if ((0 == strncasecmp(
1699 1700
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "binary", strlen("binary")))
1701 1702
                || (0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
1703
                        "nchar", strlen("nchar")))) {
1704 1705 1706
          fprintf(fp, "column[%d]:%s(%d) ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1707
        } else {
1708 1709
          fprintf(fp, "column[%d]:%s ",
                  k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
1710 1711 1712
        }
      }
      fprintf(fp, "\n");
1713 1714 1715

      fprintf(fp, "      tagCount:            %d\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1716 1717
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1718
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
1719 1720 1721
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1722 1723
          fprintf(fp, "tag[%d]:%s(%d) ",
                  k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
1724
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1725 1726
        } else {
          fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
1727
        }
1728 1729 1730 1731 1732
      }
      fprintf(fp, "\n");
    }
    fprintf(fp, "\n");
  }
1733

1734
  SHOW_PARSE_RESULT_END_TO_FILE(fp);
1735 1736 1737
}

static void printfQueryMeta() {
1738

1739
  SHOW_PARSE_RESULT_START();
1740

1741
  printf("host:                    \033[33m%s:%u\033[0m\n",
1742
          g_queryInfo.host, g_queryInfo.port);
1743 1744 1745 1746
  printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
  printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

  printf("\n");
1747 1748 1749 1750

  if ((SUBSCRIBE_TEST == g_args.test_mode) || (QUERY_TEST == g_args.test_mode)) {
    printf("specified table query info:                   \n");
    printf("sqlCount:       \033[33m%"PRIu64"\033[0m\n",
1751
      g_queryInfo.specifiedQueryInfo.sqlCount);
1752 1753 1754
    if (g_queryInfo.specifiedQueryInfo.sqlCount > 0) {
      printf("specified tbl query times:\n");
      printf("                \033[33m%"PRIu64"\033[0m\n",
1755
      g_queryInfo.specifiedQueryInfo.queryTimes);
1756 1757 1758 1759 1760 1761
      printf("query interval: \033[33m%"PRIu64" ms\033[0m\n",
        g_queryInfo.specifiedQueryInfo.queryInterval);
      printf("top query times:\033[33m%"PRIu64"\033[0m\n", g_args.query_times);
      printf("concurrent:     \033[33m%"PRIu64"\033[0m\n",
      g_queryInfo.specifiedQueryInfo.concurrent);
      printf("mod:            \033[33m%s\033[0m\n",
1762
        (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync");
1763
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1764
        g_queryInfo.specifiedQueryInfo.subscribeInterval);
1765
      printf("restart:        \033[33m%d\033[0m\n",
1766
        g_queryInfo.specifiedQueryInfo.subscribeRestart);
1767
      printf("keepProgress:   \033[33m%d\033[0m\n",
1768
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
1769

1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785
      for (uint64_t i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        printf("  sql[%"PRIu64"]: \033[33m%s\033[0m\n",
            i, g_queryInfo.specifiedQueryInfo.sql[i]);
      }
      printf("\n");
    }

    printf("super table query info:\n");
    printf("sqlCount:       \033[33m%"PRIu64"\033[0m\n",
      g_queryInfo.superQueryInfo.sqlCount);

    if (g_queryInfo.superQueryInfo.sqlCount > 0) {
      printf("query interval: \033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryInterval);
      printf("threadCnt:      \033[33m%d\033[0m\n",
        g_queryInfo.superQueryInfo.threadCnt);
1786
      printf("childTblCount:  \033[33m%"PRId64"\033[0m\n",
1787 1788 1789 1790 1791 1792 1793
        g_queryInfo.superQueryInfo.childTblCount);
      printf("stable name:    \033[33m%s\033[0m\n",
        g_queryInfo.superQueryInfo.sTblName);
      printf("stb query times:\033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryTimes);

      printf("mod:            \033[33m%s\033[0m\n",
1794
        (g_queryInfo.superQueryInfo.asyncMode)?"async":"sync");
1795
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1796
        g_queryInfo.superQueryInfo.subscribeInterval);
1797
      printf("restart:        \033[33m%d\033[0m\n",
1798
        g_queryInfo.superQueryInfo.subscribeRestart);
1799
      printf("keepProgress:   \033[33m%d\033[0m\n",
1800
        g_queryInfo.superQueryInfo.subscribeKeepProgress);
1801

1802 1803 1804 1805 1806 1807
      for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
        printf("  sql[%d]: \033[33m%s\033[0m\n",
            i, g_queryInfo.superQueryInfo.sql[i]);
      }
      printf("\n");
    }
1808
  }
1809

1810
  SHOW_PARSE_RESULT_END();
1811 1812
}

1813
static char* formatTimestamp(char* buf, int64_t val, int precision) {
1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844
  time_t tt;
  if (precision == TSDB_TIME_PRECISION_MICRO) {
    tt = (time_t)(val / 1000000);
  } else {
    tt = (time_t)(val / 1000);
  }

/* comment out as it make testcases like select_with_tags.sim fail.
  but in windows, this may cause the call to localtime crash if tt < 0,
  need to find a better solution.
  if (tt < 0) {
    tt = 0;
  }
  */

#ifdef WINDOWS
  if (tt < 0) tt = 0;
#endif

  struct tm* ptm = localtime(&tt);
  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);

  if (precision == TSDB_TIME_PRECISION_MICRO) {
    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
  } else {
    sprintf(buf + pos, ".%03d", (int)(val % 1000));
  }

  return buf;
}

1845
static void xDumpFieldToFile(FILE* fp, const char* val,
1846
        TAOS_FIELD* field, int32_t length, int precision) {
1847

1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882
  if (val == NULL) {
    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
    return;
  }

  char buf[TSDB_MAX_BYTES_PER_ROW];
  switch (field->type) {
    case TSDB_DATA_TYPE_BOOL:
      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
      break;
    case TSDB_DATA_TYPE_TINYINT:
      fprintf(fp, "%d", *((int8_t *)val));
      break;
    case TSDB_DATA_TYPE_SMALLINT:
      fprintf(fp, "%d", *((int16_t *)val));
      break;
    case TSDB_DATA_TYPE_INT:
      fprintf(fp, "%d", *((int32_t *)val));
      break;
    case TSDB_DATA_TYPE_BIGINT:
      fprintf(fp, "%" PRId64, *((int64_t *)val));
      break;
    case TSDB_DATA_TYPE_FLOAT:
      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
      break;
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      memcpy(buf, val, length);
      buf[length] = 0;
      fprintf(fp, "\'%s\'", buf);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
1883
      formatTimestamp(buf, *(int64_t*)val, precision);
1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898
      fprintf(fp, "'%s'", buf);
      break;
    default:
      break;
  }
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  FILE* fp = fopen(fname, "at");
  if (fp == NULL) {
1899
    errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913
    return -1;
  }

  int num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int precision = taos_result_precision(tres);

  for (int col = 0; col < num_fields; col++) {
    if (col > 0) {
      fprintf(fp, ",");
    }
    fprintf(fp, "%s", fields[col].name);
  }
  fputc('\n', fp);
1914

1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934
  int numOfRows = 0;
  do {
    int32_t* length = taos_fetch_lengths(tres);
    for (int i = 0; i < num_fields; i++) {
      if (i > 0) {
        fputc(',', fp);
      }
      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
    }
    fputc('\n', fp);

    numOfRows++;
    row = taos_fetch_row(tres);
  } while( row != NULL);

  fclose(fp);

  return numOfRows;
}

1935 1936
static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
  TAOS_RES * res;
1937 1938
  TAOS_ROW row = NULL;
  int count = 0;
1939 1940

  res = taos_query(taos, "show databases;");
1941
  int32_t code = taos_errno(res);
1942

1943
  if (code != 0) {
1944
    errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
1945 1946 1947 1948 1949
    return -1;
  }

  TAOS_FIELD *fields = taos_fetch_fields(res);

1950
  while((row = taos_fetch_row(res)) != NULL) {
1951
    // sys database name : 'log'
1952
    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log",
1953
                fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) {
1954 1955
      continue;
    }
1956 1957 1958

    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
    if (dbInfos[count] == NULL) {
1959
      errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
1960 1961 1962
      return -1;
    }

1963
    tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
1964
            fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
1965
    formatTimestamp(dbInfos[count]->create_time,
1966 1967
            *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
            TSDB_TIME_PRECISION_MILLI);
S
Shuduo Sang 已提交
1968
    dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
1969
    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
1970 1971
    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
1972
    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
1973

1974
    tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
1975
            fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
1976 1977 1978 1979 1980 1981 1982
    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
1983
    dbInfos[count]->cachelast =
1984
      (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));
1985

1986
    tstrncpy(dbInfos[count]->precision,
1987 1988
            (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
            fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
1989
    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
1990
    tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
1991
            fields[TSDB_SHOW_DB_STATUS_INDEX].bytes);
1992

1993 1994
    count++;
    if (count > MAX_DATABASE_COUNT) {
1995
      errorPrint("%s() LN%d, The database count overflow than %d\n",
1996
         __func__, __LINE__, MAX_DATABASE_COUNT);
1997 1998 1999 2000 2001 2002 2003
      break;
    }
  }

  return count;
}

2004
static void printfDbInfoForQueryToFile(
2005
        char* filename, SDbInfo* dbInfos, int index) {
2006

2007
  if (filename[0] == 0)
2008
      return;
2009 2010 2011

  FILE *fp = fopen(filename, "at");
  if (fp == NULL) {
2012
    errorPrint( "failed to open file: %s\n", filename);
2013
    return;
2014 2015 2016 2017 2018
  }

  fprintf(fp, "================ database[%d] ================\n", index);
  fprintf(fp, "name: %s\n", dbInfos->name);
  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
S
Shuduo Sang 已提交
2019
  fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables);
2020
  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);
2021 2022
  fprintf(fp, "replica: %d\n", dbInfos->replica);
  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
2023 2024
  fprintf(fp, "days: %d\n", dbInfos->days);
  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);
2025 2026 2027 2028 2029 2030 2031
  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
  fprintf(fp, "comp: %d\n", dbInfos->comp);
2032 2033
  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);
  fprintf(fp, "precision: %s\n", dbInfos->precision);
2034
  fprintf(fp, "update: %d\n", dbInfos->update);
2035
  fprintf(fp, "status: %s\n", dbInfos->status);
2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049
  fprintf(fp, "\n");

  fclose(fp);
}

static void printfQuerySystemInfo(TAOS * taos) {
  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
  TAOS_RES* res;

  time_t t;
  struct tm* lt;
  time(&t);
  lt = localtime(&t);
2050
  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
2051 2052
          lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
          lt->tm_sec);
2053

2054 2055
  // show variables
  res = taos_query(taos, "show variables;");
2056
  //appendResultToFile(res, filename);
2057 2058 2059 2060 2061
  xDumpResultToFile(filename, res);

  // show dnodes
  res = taos_query(taos, "show dnodes;");
  xDumpResultToFile(filename, res);
2062
  //appendResultToFile(res, filename);
2063

2064 2065 2066 2067
  // show databases
  res = taos_query(taos, "show databases;");
  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
  if (dbInfos == NULL) {
2068
    errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
2069 2070 2071
    return;
  }
  int dbCount = getDbFromServer(taos, dbInfos);
2072
  if (dbCount <= 0) {
2073 2074
      free(dbInfos);
      return;
2075
  }
2076 2077

  for (int i = 0; i < dbCount; i++) {
2078
    // printf database info
2079
    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
2080

2081 2082 2083 2084
    // show db.vgroups
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);
2085

2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096
    // show db.stables
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);

    free(dbInfos[i]);
  }

  free(dbInfos);
}

2097 2098
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
        char* sqlstr, char *resultFile)
2099
{
2100
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
2101

2102
    char *url = "/rest/sql";
2103

2104 2105 2106 2107
    int bytes, sent, received, req_str_len, resp_len;
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
2108

2109
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;
2110

2111 2112 2113 2114 2115
    request_buf = malloc(req_buf_len);
    if (NULL == request_buf) {
      errorPrint("%s", "ERROR, cannot allocate memory.\n");
      exit(EXIT_FAILURE);
    }
2116

2117 2118
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};
2119

2120 2121 2122 2123 2124 2125 2126 2127
    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
2128

2129 2130 2131 2132
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);
2133

2134
    char base64_buf[INPUT_BUF_LEN];
2135
#ifdef WINDOWS
2136
    WSADATA wsaData;
2137
    WSAStartup(MAKEWORD(2, 2), &wsaData);
2138 2139
    SOCKET sockfd;
#else
2140
    int sockfd;
2141
#endif
2142 2143
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (sockfd < 0) {
2144
#ifdef WINDOWS
2145
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
2146
#endif
2147 2148 2149 2150
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
        free(request_buf);
        ERROR_EXIT("ERROR opening socket");
    }
2151

2152
    int retConn = connect(sockfd, (struct sockaddr *)pServAddr, sizeof(struct sockaddr));
2153 2154 2155 2156 2157
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
        free(request_buf);
        ERROR_EXIT("ERROR connecting");
    }
2158

2159
    memset(base64_buf, 0, INPUT_BUF_LEN);
2160

2161 2162 2163 2164 2165 2166 2167 2168
    for (int n = 0, m = 0; n < userpass_buf_len;) {
      uint32_t oct_a = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t oct_b = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t oct_c = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;
2169

2170 2171 2172 2173 2174
      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }
2175

2176 2177
    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';
2178

2179 2180 2181
    debugPrint("%s() LN%d: auth string base64 encoded: %s\n",
            __func__, __LINE__, base64_buf);
    char *auth = base64_buf;
2182

2183 2184 2185 2186 2187 2188 2189 2190 2191
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
2192

2193 2194 2195
    req_str_len = strlen(request_buf);
    sent = 0;
    do {
2196
#ifdef WINDOWS
2197
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
2198
#else
2199
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
2200
#endif
2201 2202 2203 2204 2205 2206
        if (bytes < 0)
            ERROR_EXIT("ERROR writing message to socket");
        if (bytes == 0)
            break;
        sent+=bytes;
    } while(sent < req_str_len);
2207

2208 2209 2210 2211
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
    received = 0;
    do {
2212
#ifdef WINDOWS
2213
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
2214
#else
2215
        bytes = read(sockfd, response_buf + received, resp_len - received);
2216
#endif
2217 2218 2219 2220 2221 2222 2223 2224
        if (bytes < 0) {
            free(request_buf);
            ERROR_EXIT("ERROR reading response from socket");
        }
        if (bytes == 0)
            break;
        received += bytes;
    } while(received < resp_len);
2225

2226 2227 2228 2229
    if (received == resp_len) {
        free(request_buf);
        ERROR_EXIT("ERROR storing complete response from socket");
    }
H
Haojun Liao 已提交
2230

2231 2232
    response_buf[RESP_BUF_LEN - 1] = '\0';
    printf("Response:\n%s\n", response_buf);
H
Haojun Liao 已提交
2233

2234 2235 2236 2237
    if (resultFile) {
       appendResultBufToFile(response_buf, resultFile);
    }

2238
    free(request_buf);
2239
#ifdef WINDOWS
2240
    closesocket(sockfd);
2241 2242
    WSACleanup();
#else
2243
    close(sockfd);
2244
#endif
2245

2246
    return 0;
2247 2248
}

2249
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
2250 2251
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
2252
    errorPrint("%s() LN%d, calloc failed! size:%d\n",
2253
        __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
2254 2255
    return NULL;
  }
2256

2257
  int    dataLen = 0;
2258
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2259
          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);
2260

2261 2262 2263
  return dataBuf;
}

2264
static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) {
2265 2266 2267 2268 2269 2270 2271 2272 2273
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
    printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
    return NULL;
  }

  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
  for (int i = 0; i < stbInfo->tagCount; i++) {
2274
    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
2275
            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
2276
      if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
2277
        printf("binary or nchar length overflow, max size:%u\n",
2278
                (uint32_t)TSDB_MAX_BINARY_LEN);
2279 2280 2281
        tmfree(dataBuf);
        return NULL;
      }
2282

2283 2284
      int tagBufLen = stbInfo->tags[i].dataLen + 1;
      char* buf = (char*)calloc(tagBufLen, 1);
2285 2286 2287 2288 2289
      if (NULL == buf) {
        printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
        tmfree(dataBuf);
        return NULL;
      }
2290 2291 2292 2293 2294 2295 2296

      if (tableSeq % 2) {
        tstrncpy(buf, "beijing", tagBufLen);
      } else {
        tstrncpy(buf, "shanghai", tagBufLen);
      }
      //rand_string(buf, stbInfo->tags[i].dataLen);
2297
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2298
              "\'%s\', ", buf);
2299
      tmfree(buf);
2300
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2301
                "int", strlen("int"))) {
2302
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2303
              "%d, ", tableSeq);
2304
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2305
                "bigint", strlen("bigint"))) {
2306
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2307
              "%"PRId64", ", rand_bigint());
2308
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2309
                "float", strlen("float"))) {
2310
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2311
              "%f, ", rand_float());
2312
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2313
                "double", strlen("double"))) {
2314
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2315
              "%f, ", rand_double());
2316
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2317
                "smallint", strlen("smallint"))) {
2318
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2319
              "%d, ", rand_smallint());
2320
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2321
                "tinyint", strlen("tinyint"))) {
2322
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2323
              "%d, ", rand_tinyint());
2324
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2325
                "bool", strlen("bool"))) {
2326
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2327
              "%d, ", rand_bool());
2328
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2329
                "timestamp", strlen("timestamp"))) {
2330
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2331
              "%"PRId64", ", rand_bigint());
2332 2333 2334 2335 2336 2337
    }  else {
      printf("No support data type: %s\n", stbInfo->tags[i].dataType);
      tmfree(dataBuf);
      return NULL;
    }
  }
2338

2339
  dataLen -= 2;
2340
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");
2341 2342 2343
  return dataBuf;
}

2344
static int calcRowLen(SSuperTable*  superTbls) {
2345 2346
  int colIndex;
  int  lenOfOneRow = 0;
2347

2348 2349
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2350

2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
2367
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2368
      lenOfOneRow += 42;
2369
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
2383

2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
2400
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2401 2402 2403 2404 2405 2406 2407 2408
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2409

2410 2411 2412 2413
  return 0;
}


2414
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
2415
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
2416
        int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) {
2417

2418
  char command[BUFFER_SIZE] = "\0";
2419 2420
  char limitBuf[100] = "\0";

2421
  TAOS_RES * res;
2422 2423 2424
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2425 2426

  if (offset >= 0) {
2427
    snprintf(limitBuf, 100, " limit %"PRId64" offset %"PRIu64"",
2428
            limit, offset);
2429 2430
  }

2431
  //get all child table name use cmd: select tbname from superTblName;
2432
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s",
2433
          dbName, sTblName, limitBuf);
2434

2435
  res = taos_query(taos, command);
2436 2437 2438 2439
  int32_t code = taos_errno(res);
  if (code != 0) {
    taos_free_result(res);
    taos_close(taos);
2440
    errorPrint("%s() LN%d, failed to run command %s\n",
2441
           __func__, __LINE__, command);
2442 2443 2444
    exit(-1);
  }

2445 2446
  int64_t childTblCount = (limit < 0)?10000:limit;
  int64_t count = 0;
2447 2448 2449
  if (childTblName == NULL) {
    childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (NULL ==  childTblName) {
2450 2451 2452 2453
    taos_free_result(res);
        taos_close(taos);
        errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__);
        exit(-1);
2454 2455 2456
    }
  }

2457
  char* pTblName = childTblName;
2458
  while((row = taos_fetch_row(res)) != NULL) {
2459
    int32_t* len = taos_fetch_lengths(res);
2460
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2461 2462 2463
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2464
      char *tmp = realloc(childTblName,
2465
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2466 2467 2468
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2469
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
2470
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2471 2472
      } else {
        // exit, if allocate more memory failed
2473
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
2474
               __func__, __LINE__, dbName, sTblName);
2475 2476 2477 2478 2479 2480
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2481
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2482
  }
2483

2484 2485 2486 2487 2488 2489 2490
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2491
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
2492
        char* sTblName, char** childTblNameOfSuperTbl,
2493
        int64_t* childTblCountOfSuperTbl) {
2494

2495 2496
    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
2497
            -1, 0);
2498 2499
}

2500
static int getSuperTableFromServer(TAOS * taos, char* dbName,
2501
        SSuperTable*  superTbls) {
2502

2503
  char command[BUFFER_SIZE] = "\0";
2504
  TAOS_RES * res;
2505 2506
  TAOS_ROW row = NULL;
  int count = 0;
2507

2508
  //get schema use cmd: describe superTblName;
2509
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2510
  res = taos_query(taos, command);
2511 2512 2513
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2514
    taos_free_result(res);
2515 2516 2517 2518 2519 2520
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
2521
  while((row = taos_fetch_row(res)) != NULL) {
2522 2523 2524
    if (0 == count) {
      count++;
      continue;
2525
    }
2526 2527

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2528
      tstrncpy(superTbls->tags[tagIndex].field,
2529 2530
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
2531
      tstrncpy(superTbls->tags[tagIndex].dataType,
2532 2533
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
2534 2535 2536
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
2537 2538
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2539
      tagIndex++;
2540
    } else {
2541
      tstrncpy(superTbls->columns[columnIndex].field,
2542 2543
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
2544
      tstrncpy(superTbls->columns[columnIndex].dataType,
2545 2546
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
2547 2548 2549
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
2550 2551
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2563
/*
2564
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
2565
    //get all child table name use cmd: select tbname from superTblName;
2566 2567 2568
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2569
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2570 2571
      return -1;
    }
2572
    getAllChildNameOfSuperTable(taos, dbName,
2573 2574 2575
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2576
  }
2577
  */
2578 2579 2580
  return 0;
}

H
Haojun Liao 已提交
2581
static int createSuperTable(
2582 2583
        TAOS * taos, char* dbName,
        SSuperTable*  superTbl) {
H
Haojun Liao 已提交
2584

2585
  char command[BUFFER_SIZE] = "\0";
2586

2587 2588 2589 2590 2591
  char cols[STRING_LEN] = "\0";
  int colIndex;
  int len = 0;

  int  lenOfOneRow = 0;
2592 2593 2594

  if (superTbl->columnCount == 0) {
    errorPrint("%s() LN%d, super table column count is %d\n",
2595
            __func__, __LINE__, superTbl->columnCount);
2596 2597 2598 2599 2600
    return -1;
  }

  for (colIndex = 0; colIndex < superTbl->columnCount; colIndex++) {
    char* dataType = superTbl->columns[colIndex].dataType;
2601

2602
    if (strcasecmp(dataType, "BINARY") == 0) {
2603
      len += snprintf(cols + len, STRING_LEN - len,
2604 2605
          ", col%d %s(%d)", colIndex, "BINARY",
          superTbl->columns[colIndex].dataLen);
2606
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2607
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
2608
      len += snprintf(cols + len, STRING_LEN - len,
2609 2610
          ", col%d %s(%d)", colIndex, "NCHAR",
          superTbl->columns[colIndex].dataLen);
2611
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
      lenOfOneRow += 22;
2630
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2631 2632
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
      lenOfOneRow += 42;
2633
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2634 2635 2636 2637
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
      lenOfOneRow += 21;
    } else {
      taos_close(taos);
2638
      errorPrint("%s() LN%d, config error data type : %s\n",
2639
         __func__, __LINE__, dataType);
2640 2641 2642 2643
      exit(-1);
    }
  }

2644 2645
  superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp
  //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbl[j].sTblName, g_Dbs.db[i].superTbl[j].columnCount, lenOfOneRow);
2646 2647

  // save for creating child table
2648 2649 2650
  superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1);
  if (NULL == superTbl->colsOfCreateChildTable) {
    errorPrint("%s() LN%d, Failed when calloc, size:%d",
2651
           __func__, __LINE__, len+1);
2652 2653 2654 2655
    taos_close(taos);
    exit(-1);
  }

2656
  snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
2657
  verbosePrint("%s() LN%d: %s\n",
2658
      __func__, __LINE__, superTbl->colsOfCreateChildTable);
2659

2660 2661
  if (superTbl->tagCount == 0) {
    errorPrint("%s() LN%d, super table tag count is %d\n",
2662
            __func__, __LINE__, superTbl->tagCount);
2663 2664
    return -1;
  }
2665

2666 2667 2668
  char tags[STRING_LEN] = "\0";
  int tagIndex;
  len = 0;
2669

2670 2671 2672 2673
  int lenOfTagOfOneRow = 0;
  len += snprintf(tags + len, STRING_LEN - len, "(");
  for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) {
    char* dataType = superTbl->tags[tagIndex].dataType;
2674

2675 2676
    if (strcasecmp(dataType, "BINARY") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
2677
              "BINARY", superTbl->tags[tagIndex].dataLen);
2678 2679 2680
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
2681
              "NCHAR", superTbl->tags[tagIndex].dataLen);
2682 2683 2684
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2685
              "INT");
2686 2687 2688
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2689
              "BIGINT");
2690 2691 2692
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2693
              "SMALLINT");
2694 2695 2696
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2697
              "TINYINT");
2698 2699 2700
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2701
              "BOOL");
2702 2703 2704
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2705
              "FLOAT");
2706 2707 2708
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2709
              "DOUBLE");
2710 2711 2712
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42;
    } else {
      taos_close(taos);
2713
      errorPrint("%s() LN%d, config error tag type : %s\n",
2714
         __func__, __LINE__, dataType);
2715
      exit(-1);
2716 2717
    }
  }
2718 2719 2720 2721 2722 2723 2724

  len -= 2;
  len += snprintf(tags + len, STRING_LEN - len, ")");

  superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow;

  snprintf(command, BUFFER_SIZE,
2725 2726
          "create table if not exists %s.%s (ts timestamp%s) tags %s",
          dbName, superTbl->sTblName, cols, tags);
2727 2728 2729
  verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command);

  if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2730 2731 2732
      errorPrint( "create supertable %s failed!\n\n",
              superTbl->sTblName);
      return -1;
2733 2734
  }
  debugPrint("create supertable %s success!\n\n", superTbl->sTblName);
2735 2736 2737
  return 0;
}

2738
static int createDatabasesAndStables() {
2739 2740 2741 2742
  TAOS * taos = NULL;
  int    ret = 0;
  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
  if (taos == NULL) {
2743
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2744
    return -1;
2745 2746 2747
  }
  char command[BUFFER_SIZE] = "\0";

2748
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2749 2750
    if (g_Dbs.db[i].drop) {
      sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
2751
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);
2752
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2753 2754 2755 2756
        taos_close(taos);
        return -1;
      }

2757
      int dataLen = 0;
2758
      dataLen += snprintf(command + dataLen,
2759
          BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2760

2761 2762
      if (g_Dbs.db[i].dbCfg.blocks > 0) {
        dataLen += snprintf(command + dataLen,
2763
            BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
2764 2765 2766
      }
      if (g_Dbs.db[i].dbCfg.cache > 0) {
        dataLen += snprintf(command + dataLen,
2767
            BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
2768 2769 2770
      }
      if (g_Dbs.db[i].dbCfg.days > 0) {
        dataLen += snprintf(command + dataLen,
2771
            BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
2772 2773 2774
      }
      if (g_Dbs.db[i].dbCfg.keep > 0) {
        dataLen += snprintf(command + dataLen,
2775
            BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
2776 2777 2778
      }
      if (g_Dbs.db[i].dbCfg.quorum > 1) {
        dataLen += snprintf(command + dataLen,
2779
            BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
2780 2781 2782
      }
      if (g_Dbs.db[i].dbCfg.replica > 0) {
        dataLen += snprintf(command + dataLen,
2783
            BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
2784 2785 2786
      }
      if (g_Dbs.db[i].dbCfg.update > 0) {
        dataLen += snprintf(command + dataLen,
2787
            BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
2788 2789 2790 2791 2792 2793 2794
      }
      //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
      //  dataLen += snprintf(command + dataLen,
      //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
      //}
      if (g_Dbs.db[i].dbCfg.minRows > 0) {
        dataLen += snprintf(command + dataLen,
2795
            BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
2796 2797 2798
      }
      if (g_Dbs.db[i].dbCfg.maxRows > 0) {
        dataLen += snprintf(command + dataLen,
2799
            BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
2800 2801 2802
      }
      if (g_Dbs.db[i].dbCfg.comp > 0) {
        dataLen += snprintf(command + dataLen,
2803
            BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
2804 2805 2806
      }
      if (g_Dbs.db[i].dbCfg.walLevel > 0) {
        dataLen += snprintf(command + dataLen,
2807
            BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
2808 2809 2810
      }
      if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
        dataLen += snprintf(command + dataLen,
2811
            BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
2812 2813 2814
      }
      if (g_Dbs.db[i].dbCfg.fsync > 0) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2815
                " fsync %d", g_Dbs.db[i].dbCfg.fsync);
2816 2817
      }
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
2818 2819
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                      "us", strlen("us")))) {
2820
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2821
                " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
2822 2823 2824 2825 2826 2827 2828 2829 2830
      }

      debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
        taos_close(taos);
        errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
        return -1;
      }
      printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
2831 2832
    }

2833
    debugPrint("%s() LN%d supertbl count:%"PRIu64"\n",
2834
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
2835 2836 2837

    int validStbCount = 0;

S
Shuduo Sang 已提交
2838
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
2839 2840 2841 2842 2843 2844 2845
      sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
              g_Dbs.db[i].superTbls[j].sTblName);
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);

      ret = queryDbExec(taos, command, NO_INSERT_TYPE, true);

      if ((ret != 0) || (g_Dbs.db[i].drop)) {
2846
        ret = createSuperTable(taos, g_Dbs.db[i].dbName,
2847
                &g_Dbs.db[i].superTbls[j]);
2848

2849
        if (0 != ret) {
S
Shuduo Sang 已提交
2850
          errorPrint("create super table %"PRIu64" failed!\n\n", j);
2851
          continue;
2852 2853 2854
        }
      }

2855
      ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
2856
                &g_Dbs.db[i].superTbls[j]);
2857 2858
      if (0 != ret) {
        errorPrint("\nget super table %s.%s info failed!\n\n",
2859
                g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName);
2860
        continue;
2861
      }
2862 2863

      validStbCount ++;
2864
    }
2865 2866

    g_Dbs.db[i].superTblCount = validStbCount;
2867 2868 2869 2870 2871 2872
  }

  taos_close(taos);
  return 0;
}

2873 2874
static void* createTable(void *sarg)
{
2875 2876
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
2877

S
Shuduo Sang 已提交
2878
  uint64_t  lastPrintTime = taosGetTimestampMs();
2879

2880
  int buff_len;
2881
  buff_len = BUFFER_SIZE / 8;
2882

2883
  char *buffer = calloc(buff_len, 1);
2884
  if (buffer == NULL) {
2885
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
2886 2887
    exit(-1);
  }
2888 2889 2890

  int len = 0;
  int batchNum = 0;
2891

2892
  verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n",
2893 2894
          __func__, __LINE__,
          pThreadInfo->start_table_from, pThreadInfo->end_table_to);
2895

2896
  for (uint64_t i = pThreadInfo->start_table_from;
2897
          i <= pThreadInfo->end_table_to; i++) {
2898
    if (0 == g_Dbs.use_metric) {
2899
      snprintf(buffer, buff_len,
2900
              "create table if not exists %s.%s%"PRIu64" %s;",
2901 2902 2903
              pThreadInfo->db_name,
              g_args.tb_prefix, i,
              pThreadInfo->cols);
2904
    } else {
2905 2906
      if (superTblInfo == NULL) {
        errorPrint("%s() LN%d, use metric, but super table info is NULL\n",
2907
                  __func__, __LINE__);
2908
        free(buffer);
2909 2910 2911 2912 2913 2914
        exit(-1);
      } else {
        if (0 == len) {
          batchNum = 0;
          memset(buffer, 0, buff_len);
          len += snprintf(buffer + len,
2915
                  buff_len - len, "create table ");
2916 2917 2918
        }
        char* tagsValBuf = NULL;
        if (0 == superTblInfo->tagSource) {
2919
          tagsValBuf = generateTagVaulesForStb(superTblInfo, i);
2920 2921
        } else {
          tagsValBuf = getTagValueFromTagSample(
2922 2923
                  superTblInfo,
                  i % superTblInfo->tagSampleCount);
2924 2925 2926 2927 2928 2929
        }
        if (NULL == tagsValBuf) {
          free(buffer);
          return NULL;
        }
        len += snprintf(buffer + len,
2930
                buff_len - len,
2931
                "if not exists %s.%s%"PRIu64" using %s.%s tags %s ",
2932 2933 2934
                pThreadInfo->db_name, superTblInfo->childTblPrefix,
                i, pThreadInfo->db_name,
                superTblInfo->sTblName, tagsValBuf);
2935 2936 2937
        free(tagsValBuf);
        batchNum++;
        if ((batchNum < superTblInfo->batchCreateTableNum)
2938 2939
                && ((buff_len - len)
                    >= (superTblInfo->lenOfTagOfOneRow + 256))) {
2940 2941
          continue;
        }
2942 2943 2944 2945
      }
    }

    len = 0;
2946
    verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
2947
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)){
2948
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2949
      free(buffer);
2950 2951 2952
      return NULL;
    }

S
Shuduo Sang 已提交
2953
    uint64_t  currentPrintTime = taosGetTimestampMs();
2954
    if (currentPrintTime - lastPrintTime > 30*1000) {
2955
      printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n",
2956
              pThreadInfo->threadID, pThreadInfo->start_table_from, i);
2957 2958 2959
      lastPrintTime = currentPrintTime;
    }
  }
2960

2961
  if (0 != len) {
2962
    verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer);
2963
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)) {
2964
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2965
    }
2966
  }
2967

2968 2969 2970 2971
  free(buffer);
  return NULL;
}

2972
static int startMultiThreadCreateChildTable(
S
Shuduo Sang 已提交
2973
        char* cols, int threads, uint64_t startFrom, int64_t ntables,
2974
        char* db_name, SSuperTable* superTblInfo) {
2975

2976
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
2977
  threadInfo *infos = calloc(1, threads * sizeof(threadInfo));
2978 2979 2980 2981 2982 2983 2984 2985 2986 2987

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

S
Shuduo Sang 已提交
2988
  int64_t a = ntables / threads;
2989 2990 2991 2992 2993
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

S
Shuduo Sang 已提交
2994
  int64_t b = 0;
2995
  b = ntables % threads;
2996

S
Shuduo Sang 已提交
2997
  for (int i = 0; i < threads; i++) {
2998 2999
    threadInfo *t_info = infos + i;
    t_info->threadID = i;
H
Hui Li 已提交
3000
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
3001
    t_info->superTblInfo = superTblInfo;
3002
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
3003
    t_info->taos = taos_connect(
3004 3005 3006 3007 3008
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
3009
    if (t_info->taos == NULL) {
3010
      errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n",
3011
         __func__, __LINE__, taos_errstr(NULL));
3012
      free(pids);
3013
      free(infos);
3014 3015
      return -1;
    }
3016 3017 3018 3019 3020

    t_info->start_table_from = startFrom;
    t_info->ntables = i<b?a+1:a;
    t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
    startFrom = t_info->end_table_to + 1;
3021
    t_info->use_metric = true;
3022
    t_info->cols = cols;
3023
    t_info->minDelay = UINT64_MAX;
3024 3025
    pthread_create(pids + i, NULL, createTable, t_info);
  }
3026

3027 3028 3029 3030 3031 3032 3033 3034 3035 3036
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
    threadInfo *t_info = infos + i;
    taos_close(t_info->taos);
  }

  free(pids);
3037
  free(infos);
3038 3039

  return 0;
3040 3041
}

3042
static void createChildTables() {
3043 3044
    char tblColsBuf[MAX_SQL_SIZE];
    int len;
3045

3046
  for (int i = 0; i < g_Dbs.dbCount; i++) {
3047 3048
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
3049
          // with super table
S
Shuduo Sang 已提交
3050
        for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
3051
          if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
3052
                || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
3053 3054
            continue;
          }
3055

3056
          verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
3057
                  g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
S
Shuduo Sang 已提交
3058
          uint64_t startFrom = 0;
3059 3060
          g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;

3061
          verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n",
3062
                  __func__, __LINE__, g_totalChildTables, startFrom);
3063
          startMultiThreadCreateChildTable(
3064 3065 3066 3067 3068
                g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
                g_Dbs.threadCountByCreateTbl,
                startFrom,
                g_Dbs.db[i].superTbls[j].childTblCount,
                g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
3069
        }
3070 3071
      }
    } else {
3072 3073
      // normal table
      len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
H
Haojun Liao 已提交
3074
      for (int j = 0; j < g_args.num_of_CPR; j++) {
3075 3076 3077 3078 3079 3080 3081 3082 3083 3084
          if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
                  || (strncasecmp(g_args.datatype[j],
                      "NCHAR", strlen("NCHAR")) == 0)) {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                      ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary);
          } else {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                      ", COL%d %s", j, g_args.datatype[j]);
          }
          len = strlen(tblColsBuf);
3085
      }
3086

3087
      snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");
3088

3089
      verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n",
3090 3091
              __func__, __LINE__,
              g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
3092
      startMultiThreadCreateChildTable(
3093 3094 3095 3096 3097 3098
            tblColsBuf,
            g_Dbs.threadCountByCreateTbl,
            0,
            g_args.num_of_tables,
            g_Dbs.db[i].dbName,
            NULL);
3099
    }
3100 3101 3102 3103 3104 3105
  }
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3106
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
3107 3108 3109
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
3110

3111 3112
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
3113
    printf("Failed to open tags file: %s, reason:%s\n",
3114
            superTblInfo->tagsFile, strerror(errno));
3115 3116 3117 3118 3119 3120 3121
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
3122

3123 3124 3125 3126 3127 3128 3129 3130 3131
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

3132
  while((readLen = tgetline(&line, &n, fp)) != -1) {
3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
3145
      char *tmp = realloc(tagDataBuf,
3146
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
3147 3148 3149
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
3150
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
3151
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

3171
#if 0
3172 3173 3174 3175
int readSampleFromJsonFileToMem(SSuperTable  * superTblInfo) {
  // TODO
  return 0;
}
3176
#endif
3177 3178 3179 3180

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3181
static int readSampleFromCsvFileToMem(
3182
        SSuperTable* superTblInfo) {
3183 3184 3185 3186
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
3187

3188 3189
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
3190 3191 3192
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
              superTblInfo->sampleFile, strerror(errno));
      return -1;
3193
  }
3194

3195
  assert(superTblInfo->sampleDataBuf);
3196
  memset(superTblInfo->sampleDataBuf, 0,
3197
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
3198
  while(1) {
3199
    readLen = tgetline(&line, &n, fp);
3200 3201
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
3202
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
3203
                superTblInfo->sampleFile, strerror(errno));
3204
        fclose(fp);
3205 3206 3207 3208
        return -1;
      }
      continue;
    }
3209

3210 3211 3212 3213 3214 3215 3216 3217 3218
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
3219
      printf("sample row len[%d] overflow define schema len[%"PRIu64"], so discard this row\n",
3220
              (int32_t)readLen, superTblInfo->lenOfOneRow);
3221 3222 3223
      continue;
    }

3224
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
3225
          line, readLen);
3226 3227 3228 3229 3230 3231 3232
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

3233
  fclose(fp);
3234 3235 3236 3237
  tmfree(line);
  return 0;
}

3238
static bool getColumnAndTagTypeFromInsertJsonFile(
3239
        cJSON* stbInfo, SSuperTable* superTbls) {
3240
  bool  ret = false;
3241

3242
  // columns
3243 3244
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
3245
    printf("ERROR: failed to read json, columns not found\n");
3246 3247 3248 3249 3250 3251
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
3252

3253
  int columnSize = cJSON_GetArraySize(columns);
3254
  if ((columnSize + 1/* ts */) > MAX_COLUMN_COUNT) {
3255
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
3256
            __func__, __LINE__, MAX_COLUMN_COUNT);
3257 3258 3259 3260 3261 3262
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
3263

3264
  //superTbls->columnCount = columnSize;
3265 3266 3267 3268 3269 3270 3271
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
3272
      count = countObj->valueint;
3273
    } else if (countObj && countObj->type != cJSON_Number) {
3274
      errorPrint("%s() LN%d, failed to read json, column count not found\n",
3275
          __func__, __LINE__);
3276 3277 3278 3279 3280
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3281
    // column info
3282 3283
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
3284 3285 3286
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d: failed to read json, column type not found\n",
3287
          __func__, __LINE__);
3288 3289
      goto PARSE_OVER;
    }
3290 3291
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3292

3293 3294
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3295
      columnCase.dataLen = dataLen->valueint;
3296
    } else if (dataLen && dataLen->type != cJSON_Number) {
3297
      debugPrint("%s() LN%d: failed to read json, column len not found\n",
3298
          __func__, __LINE__);
3299 3300 3301 3302
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
3303

3304
    for (int n = 0; n < count; ++n) {
3305
      tstrncpy(superTbls->columns[index].dataType,
3306
              columnCase.dataType, MAX_TB_NAME_SIZE);
3307
      superTbls->columns[index].dataLen = columnCase.dataLen;
3308 3309
      index++;
    }
3310
  }
3311

3312 3313
  if ((index + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n",
3314
            __func__, __LINE__, MAX_COLUMN_COUNT);
3315 3316 3317
    goto PARSE_OVER;
  }

3318
  superTbls->columnCount = index;
3319

3320 3321
  count = 1;
  index = 0;
3322
  // tags
3323 3324
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
3325
    errorPrint("%s() LN%d, failed to read json, tags not found\n",
3326
        __func__, __LINE__);
3327 3328
    goto PARSE_OVER;
  }
3329

3330 3331
  int tagSize = cJSON_GetArraySize(tags);
  if (tagSize > MAX_TAG_COUNT) {
3332
    errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n",
3333
        __func__, __LINE__, MAX_TAG_COUNT);
3334 3335
    goto PARSE_OVER;
  }
3336 3337

  //superTbls->tagCount = tagSize;
3338 3339 3340
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
3341

3342 3343 3344
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
3345
      count = countObj->valueint;
3346
    } else if (countObj && countObj->type != cJSON_Number) {
3347
      printf("ERROR: failed to read json, column count not found\n");
3348 3349 3350 3351 3352
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3353
    // column info
3354 3355
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
3356 3357 3358
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d, failed to read json, tag type not found\n",
3359
          __func__, __LINE__);
3360 3361
      goto PARSE_OVER;
    }
3362
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3363

3364 3365
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3366
      columnCase.dataLen = dataLen->valueint;
3367
    } else if (dataLen && dataLen->type != cJSON_Number) {
3368
      errorPrint("%s() LN%d, failed to read json, column len not found\n",
3369
          __func__, __LINE__);
3370 3371 3372
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
3373 3374
    }

3375
    for (int n = 0; n < count; ++n) {
3376
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType,
3377
          MAX_TB_NAME_SIZE);
3378
      superTbls->tags[index].dataLen = columnCase.dataLen;
3379 3380
      index++;
    }
3381
  }
3382 3383

  if (index > MAX_TAG_COUNT) {
3384
    errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n",
3385
        __func__, __LINE__, MAX_TAG_COUNT);
3386 3387 3388
    goto PARSE_OVER;
  }

3389 3390
  superTbls->tagCount = index;

3391 3392
  if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n",
3393
        __func__, __LINE__, MAX_COLUMN_COUNT);
3394 3395
    goto PARSE_OVER;
  }
3396 3397
  ret = true;

3398
PARSE_OVER:
3399 3400 3401 3402 3403 3404 3405 3406
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3407
    tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3408 3409 3410 3411
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3412
    tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE);
3413
  } else if (!host) {
3414
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3415
  } else {
3416
    printf("ERROR: failed to read json, host not found\n");
3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_Dbs.port = port->valueint;
  } else if (!port) {
    g_Dbs.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3429
    tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE);
3430
  } else if (!user) {
3431
    tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE);
3432 3433 3434 3435
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3436
    tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE);
3437
  } else if (!password) {
3438
    tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE);
3439 3440 3441 3442
  }

  cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
  if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
3443
    tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
3444
  } else if (!resultfile) {
3445
    tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
3446 3447 3448 3449 3450 3451 3452 3453
  }

  cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
  if (threads && threads->type == cJSON_Number) {
    g_Dbs.threadCount = threads->valueint;
  } else if (!threads) {
    g_Dbs.threadCount = 1;
  } else {
3454
    printf("ERROR: failed to read json, threads not found\n");
3455
    goto PARSE_OVER;
3456 3457
  }

3458 3459 3460 3461
  cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
  if (threads2 && threads2->type == cJSON_Number) {
    g_Dbs.threadCountByCreateTbl = threads2->valueint;
  } else if (!threads2) {
sangshuduo's avatar
sangshuduo 已提交
3462
    g_Dbs.threadCountByCreateTbl = 1;
3463
  } else {
3464
    errorPrint("%s() LN%d, failed to read json, threads2 not found\n",
3465
            __func__, __LINE__);
3466
    goto PARSE_OVER;
3467
  }
3468

3469 3470
  cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
  if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
3471 3472 3473 3474 3475
    if (gInsertInterval->valueint <0) {
      errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n",
            __func__, __LINE__);
      goto PARSE_OVER;
    }
3476 3477 3478 3479
    g_args.insert_interval = gInsertInterval->valueint;
  } else if (!gInsertInterval) {
    g_args.insert_interval = 0;
  } else {
3480
    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
3481
        __func__, __LINE__);
3482 3483
    goto PARSE_OVER;
  }
3484

3485 3486
  cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows");
  if (interlaceRows && interlaceRows->type == cJSON_Number) {
3487 3488 3489 3490 3491 3492
    if (interlaceRows->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;

    }
3493 3494
    g_args.interlace_rows = interlaceRows->valueint;
  } else if (!interlaceRows) {
3495
    g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3496
  } else {
3497
    errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
3498
        __func__, __LINE__);
3499
    goto PARSE_OVER;
3500
  }
3501

3502 3503
  cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
  if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
3504 3505 3506 3507 3508
    if (maxSqlLen->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
    }
3509 3510
    g_args.max_sql_len = maxSqlLen->valueint;
  } else if (!maxSqlLen) {
sangshuduo's avatar
sangshuduo 已提交
3511
    g_args.max_sql_len = (1024*1024);
3512
  } else {
3513
    errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
3514
        __func__, __LINE__);
3515 3516 3517 3518 3519
    goto PARSE_OVER;
  }

  cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
  if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
3520
    if (numRecPerReq->valueint <= 0) {
3521 3522 3523
      errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
3524
    } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) {
3525 3526 3527 3528 3529
      printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n",
              numRecPerReq->valueint, MAX_RECORDS_PER_REQ);
      printf("        number of records per request value will be set to %d\n\n",
              MAX_RECORDS_PER_REQ);
      prompt();
3530
      numRecPerReq->valueint = MAX_RECORDS_PER_REQ;
3531
    }
3532 3533
    g_args.num_of_RPR = numRecPerReq->valueint;
  } else if (!numRecPerReq) {
3534
    g_args.num_of_RPR = MAX_RECORDS_PER_REQ;
3535
  } else {
3536
    errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n",
3537
        __func__, __LINE__);
3538 3539 3540
    goto PARSE_OVER;
  }

3541
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3542
  if (answerPrompt
3543 3544
          && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3545 3546 3547 3548 3549 3550 3551 3552
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
    } else {
      g_args.answer_yes = false;
    }
  } else if (!answerPrompt) {
3553
    g_args.answer_yes = true;   // default is no, mean answer_yes.
3554
  } else {
3555
    errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n");
3556
    goto PARSE_OVER;
3557
  }
3558

3559 3560 3561 3562 3563 3564 3565 3566 3567 3568
  // rows per table need be less than insert batch
  if (g_args.interlace_rows > g_args.num_of_RPR) {
      printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
              g_args.interlace_rows, g_args.num_of_RPR);
      printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
              g_args.num_of_RPR);
      prompt();
      g_args.interlace_rows = g_args.num_of_RPR;
  }

3569 3570
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (!dbs || dbs->type != cJSON_Array) {
3571
    printf("ERROR: failed to read json, databases not found\n");
3572 3573 3574 3575 3576
    goto PARSE_OVER;
  }

  int dbSize = cJSON_GetArraySize(dbs);
  if (dbSize > MAX_DB_COUNT) {
3577
    errorPrint(
3578 3579
            "ERROR: failed to read json, databases size overflow, max database is %d\n",
            MAX_DB_COUNT);
3580 3581 3582 3583 3584 3585 3586 3587
    goto PARSE_OVER;
  }

  g_Dbs.dbCount = dbSize;
  for (int i = 0; i < dbSize; ++i) {
    cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
    if (dbinfos == NULL) continue;

3588
    // dbinfo
3589 3590
    cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
    if (!dbinfo || dbinfo->type != cJSON_Object) {
3591
      printf("ERROR: failed to read json, dbinfo not found\n");
3592 3593
      goto PARSE_OVER;
    }
3594

3595 3596
    cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
    if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
3597
      printf("ERROR: failed to read json, db name not found\n");
3598 3599
      goto PARSE_OVER;
    }
3600
    tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE);
3601 3602 3603

    cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
    if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
3604 3605
      if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) {
        g_Dbs.db[i].drop = true;
3606
      } else {
3607
        g_Dbs.db[i].drop = false;
3608
      }
3609
    } else if (!drop) {
3610
      g_Dbs.db[i].drop = g_args.drop_database;
3611
    } else {
3612
      errorPrint("%s() LN%d, failed to read json, drop input mistake\n",
3613
              __func__, __LINE__);
3614 3615 3616 3617
      goto PARSE_OVER;
    }

    cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
3618
    if (precision && precision->type == cJSON_String
3619
            && precision->valuestring != NULL) {
3620
      tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
3621
              MAX_DB_NAME_SIZE);
3622
    } else if (!precision) {
3623
      //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
3624 3625
      memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE);
    } else {
3626
      printf("ERROR: failed to read json, precision not found\n");
3627 3628 3629 3630 3631 3632 3633 3634 3635
      goto PARSE_OVER;
    }

    cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
    if (update && update->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.update = update->valueint;
    } else if (!update) {
      g_Dbs.db[i].dbCfg.update = -1;
    } else {
3636
      printf("ERROR: failed to read json, update not found\n");
3637 3638 3639 3640 3641 3642 3643 3644 3645
      goto PARSE_OVER;
    }

    cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
    if (replica && replica->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.replica = replica->valueint;
    } else if (!replica) {
      g_Dbs.db[i].dbCfg.replica = -1;
    } else {
3646
      printf("ERROR: failed to read json, replica not found\n");
3647 3648 3649 3650 3651 3652 3653 3654 3655
      goto PARSE_OVER;
    }

    cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
    if (keep && keep->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.keep = keep->valueint;
    } else if (!keep) {
      g_Dbs.db[i].dbCfg.keep = -1;
    } else {
3656 3657
     printf("ERROR: failed to read json, keep not found\n");
     goto PARSE_OVER;
3658
    }
3659

3660 3661 3662 3663 3664 3665
    cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
    if (days && days->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.days = days->valueint;
    } else if (!days) {
      g_Dbs.db[i].dbCfg.days = -1;
    } else {
3666 3667
     printf("ERROR: failed to read json, days not found\n");
     goto PARSE_OVER;
3668
    }
3669

3670 3671 3672 3673 3674 3675
    cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
    if (cache && cache->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cache = cache->valueint;
    } else if (!cache) {
      g_Dbs.db[i].dbCfg.cache = -1;
    } else {
3676 3677
     printf("ERROR: failed to read json, cache not found\n");
     goto PARSE_OVER;
3678
    }
3679

3680 3681 3682 3683 3684 3685
    cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
    if (blocks && blocks->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
    } else if (!blocks) {
      g_Dbs.db[i].dbCfg.blocks = -1;
    } else {
3686 3687
     printf("ERROR: failed to read json, block not found\n");
     goto PARSE_OVER;
3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703
    }

    //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
    //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
    //} else if (!maxtablesPerVnode) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
    //} else {
    // printf("failed to read json, maxtablesPerVnode not found");
    // goto PARSE_OVER;
    //}

    cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
    if (minRows && minRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
    } else if (!minRows) {
3704
      g_Dbs.db[i].dbCfg.minRows = 0;    // 0 means default
3705
    } else {
3706 3707
     printf("ERROR: failed to read json, minRows not found\n");
     goto PARSE_OVER;
3708 3709 3710 3711 3712 3713
    }

    cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
    if (maxRows && maxRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
    } else if (!maxRows) {
3714
      g_Dbs.db[i].dbCfg.maxRows = 0;    // 0 means default
3715
    } else {
3716 3717
     printf("ERROR: failed to read json, maxRows not found\n");
     goto PARSE_OVER;
3718 3719 3720 3721 3722 3723 3724 3725
    }

    cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
    if (comp && comp->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.comp = comp->valueint;
    } else if (!comp) {
      g_Dbs.db[i].dbCfg.comp = -1;
    } else {
3726 3727
     printf("ERROR: failed to read json, comp not found\n");
     goto PARSE_OVER;
3728 3729 3730 3731 3732 3733 3734 3735
    }

    cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
    if (walLevel && walLevel->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
    } else if (!walLevel) {
      g_Dbs.db[i].dbCfg.walLevel = -1;
    } else {
3736 3737
     printf("ERROR: failed to read json, walLevel not found\n");
     goto PARSE_OVER;
3738 3739
    }

3740 3741 3742 3743 3744 3745
    cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
    if (cacheLast && cacheLast->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
    } else if (!cacheLast) {
      g_Dbs.db[i].dbCfg.cacheLast = -1;
    } else {
3746 3747
     printf("ERROR: failed to read json, cacheLast not found\n");
     goto PARSE_OVER;
3748 3749
    }

3750 3751 3752 3753
    cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
    if (quorum && quorum->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
    } else if (!quorum) {
3754
      g_Dbs.db[i].dbCfg.quorum = 1;
3755
    } else {
3756 3757
     printf("failed to read json, quorum input mistake");
     goto PARSE_OVER;
3758 3759 3760 3761 3762 3763 3764 3765
    }

    cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
    if (fsync && fsync->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
    } else if (!fsync) {
      g_Dbs.db[i].dbCfg.fsync = -1;
    } else {
3766
      errorPrint("%s() LN%d, failed to read json, fsync input mistake\n",
3767
              __func__, __LINE__);
3768
      goto PARSE_OVER;
3769
    }
3770

3771
    // super_talbes
3772 3773
    cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
    if (!stables || stables->type != cJSON_Array) {
3774
      errorPrint("%s() LN%d, failed to read json, super_tables not found\n",
3775
              __func__, __LINE__);
3776
      goto PARSE_OVER;
3777 3778
    }

3779 3780
    int stbSize = cJSON_GetArraySize(stables);
    if (stbSize > MAX_SUPER_TABLE_COUNT) {
3781
      errorPrint(
3782 3783
              "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n",
              __func__, __LINE__, MAX_SUPER_TABLE_COUNT);
3784 3785 3786 3787 3788 3789 3790
      goto PARSE_OVER;
    }

    g_Dbs.db[i].superTblCount = stbSize;
    for (int j = 0; j < stbSize; ++j) {
      cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
      if (stbInfo == NULL) continue;
3791 3792

      // dbinfo
3793 3794
      cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
      if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) {
3795
        errorPrint("%s() LN%d, failed to read json, stb name not found\n",
3796
                __func__, __LINE__);
3797 3798
        goto PARSE_OVER;
      }
3799
      tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE);
3800

3801 3802
      cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
      if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
3803
        printf("ERROR: failed to read json, childtable_prefix not found\n");
3804 3805
        goto PARSE_OVER;
      }
3806
      tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE);
3807 3808

      cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null
3809
      if (autoCreateTbl
3810 3811
              && autoCreateTbl->type == cJSON_String
              && autoCreateTbl->valuestring != NULL) {
3812 3813 3814 3815 3816 3817 3818 3819 3820 3821
        if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
        } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        } else {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        }
      } else if (!autoCreateTbl) {
        g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      } else {
3822
        printf("ERROR: failed to read json, auto_create_table not found\n");
3823 3824
        goto PARSE_OVER;
      }
3825

3826 3827 3828 3829
      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
      } else if (!batchCreateTbl) {
3830
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
3831
      } else {
3832
        printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
3833
        goto PARSE_OVER;
3834
      }
3835 3836

      cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
3837
      if (childTblExists
3838 3839
              && childTblExists->type == cJSON_String
              && childTblExists->valuestring != NULL) {
3840 3841
        if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3))
            && (g_Dbs.db[i].drop == false)) {
3842
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
3843
        } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2)
3844
              || (g_Dbs.db[i].drop == true))) {
3845 3846 3847 3848 3849 3850 3851
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        } else {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        }
      } else if (!childTblExists) {
        g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
      } else {
3852
        errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n",
3853
                __func__, __LINE__);
3854 3855
        goto PARSE_OVER;
      }
3856

3857 3858
      cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
      if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
3859
        errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n",
3860
                __func__, __LINE__);
3861 3862 3863 3864 3865
        goto PARSE_OVER;
      }
      g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

      cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
3866
      if (dataSource && dataSource->type == cJSON_String
3867
              && dataSource->valuestring != NULL) {
3868
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
3869
                dataSource->valuestring, MAX_DB_NAME_SIZE);
3870
      } else if (!dataSource) {
3871
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE);
3872
      } else {
3873
        errorPrint("%s() LN%d, failed to read json, data_source not found\n",
3874
            __func__, __LINE__);
3875 3876 3877
        goto PARSE_OVER;
      }

3878
      cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt
3879
      if (insertMode && insertMode->type == cJSON_String
3880
              && insertMode->valuestring != NULL) {
3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891
        if (0 == strcasecmp(insertMode->valuestring, "taosc")) {
            g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE;
        } else if (0 == strcasecmp(insertMode->valuestring, "rest")) {
            g_Dbs.db[i].superTbls[j].insertMode = REST_IFACE;
        } else if (0 == strcasecmp(insertMode->valuestring, "stmt")) {
            g_Dbs.db[i].superTbls[j].insertMode = STMT_IFACE;
        } else {
            errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n",
                    __func__, __LINE__, insertMode->valuestring);
            goto PARSE_OVER;
        }
3892
      } else if (!insertMode) {
3893
        g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE;
3894
      } else {
3895
        errorPrint("%s", "failed to read json, insert_mode not found\n");
3896 3897 3898
        goto PARSE_OVER;
      }

3899
      cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
3900 3901
      if ((childTbl_limit) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3902
        if (childTbl_limit->type != cJSON_Number) {
3903 3904
            printf("ERROR: failed to read json, childtable_limit\n");
            goto PARSE_OVER;
3905 3906 3907
        }
        g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
      } else {
3908
        g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid.
3909 3910 3911
      }

      cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
3912 3913
      if ((childTbl_offset) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3914
        if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) {
3915 3916
            printf("ERROR: failed to read json, childtable_offset\n");
            goto PARSE_OVER;
3917 3918 3919 3920 3921 3922
        }
        g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblOffset = 0;
      }

3923 3924
      cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
      if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
3925
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
3926
                ts->valuestring, MAX_DB_NAME_SIZE);
3927
      } else if (!ts) {
3928
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
3929
                "now", MAX_DB_NAME_SIZE);
3930
      } else {
3931
        printf("ERROR: failed to read json, start_timestamp not found\n");
3932 3933
        goto PARSE_OVER;
      }
3934

3935 3936 3937 3938
      cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
      if (timestampStep && timestampStep->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
      } else if (!timestampStep) {
3939
        g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
3940
      } else {
3941
        printf("ERROR: failed to read json, timestamp_step not found\n");
3942 3943 3944 3945
        goto PARSE_OVER;
      }

      cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
3946
      if (sampleFormat && sampleFormat->type
3947
              == cJSON_String && sampleFormat->valuestring != NULL) {
3948
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
3949
                sampleFormat->valuestring, MAX_DB_NAME_SIZE);
3950
      } else if (!sampleFormat) {
3951
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE);
3952
      } else {
3953
        printf("ERROR: failed to read json, sample_format not found\n");
3954
        goto PARSE_OVER;
3955
      }
3956

3957
      cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
3958 3959
      if (sampleFile && sampleFile->type == cJSON_String
          && sampleFile->valuestring != NULL) {
3960
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
3961
                sampleFile->valuestring, MAX_FILE_NAME_LEN);
3962 3963 3964
      } else if (!sampleFile) {
        memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
      } else {
3965
        printf("ERROR: failed to read json, sample_file not found\n");
3966
        goto PARSE_OVER;
3967
      }
3968

3969 3970
      cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
      if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) {
3971
        tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
3972
                tagsFile->valuestring, MAX_FILE_NAME_LEN);
3973 3974 3975 3976 3977 3978 3979 3980 3981
        if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
          g_Dbs.db[i].superTbls[j].tagSource = 0;
        } else {
          g_Dbs.db[i].superTbls[j].tagSource = 1;
        }
      } else if (!tagsFile) {
        memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
        g_Dbs.db[i].superTbls[j].tagSource = 0;
      } else {
3982
        printf("ERROR: failed to read json, tags_file not found\n");
3983 3984
        goto PARSE_OVER;
      }
3985

sangshuduo's avatar
sangshuduo 已提交
3986 3987 3988
      cJSON* stbMaxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
      if (stbMaxSqlLen && stbMaxSqlLen->type == cJSON_Number) {
        int32_t len = stbMaxSqlLen->valueint;
3989 3990
        if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
          len = TSDB_MAX_ALLOWED_SQL_LEN;
3991 3992
        } else if (len < 5) {
          len = 5;
3993
        }
3994 3995
        g_Dbs.db[i].superTbls[j].maxSqlLen = len;
      } else if (!maxSqlLen) {
3996
        g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len;
3997
      } else {
sangshuduo's avatar
sangshuduo 已提交
3998
        errorPrint("%s() LN%d, failed to read json, stbMaxSqlLen input mistake\n",
3999
            __func__, __LINE__);
4000
        goto PARSE_OVER;
4001
      }
4002
/*
4003 4004 4005 4006 4007
      cJSON *multiThreadWriteOneTbl =
          cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
      if (multiThreadWriteOneTbl
              && multiThreadWriteOneTbl->type == cJSON_String
              && multiThreadWriteOneTbl->valuestring != NULL) {
4008 4009 4010 4011
        if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
        } else {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
4012
        }
4013 4014 4015
      } else if (!multiThreadWriteOneTbl) {
        g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
      } else {
4016
        printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
4017 4018
        goto PARSE_OVER;
      }
4019
*/
sangshuduo's avatar
sangshuduo 已提交
4020 4021 4022
      cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows");
      if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) {
        if (stbInterlaceRows->valueint < 0) {
4023 4024 4025 4026
          errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n",
            __func__, __LINE__);
          goto PARSE_OVER;
        }
sangshuduo's avatar
sangshuduo 已提交
4027
        g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint;
4028 4029
        // rows per table need be less than insert batch
        if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) {
4030
          printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
4031
                  i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR);
4032
          printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
4033
                  g_args.num_of_RPR);
4034
          prompt();
4035 4036
          g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR;
        }
sangshuduo's avatar
sangshuduo 已提交
4037
      } else if (!stbInterlaceRows) {
4038
        g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
4039
      } else {
4040
        errorPrint(
4041 4042
                "%s() LN%d, failed to read json, interlace rows input mistake\n",
                __func__, __LINE__);
4043
        goto PARSE_OVER;
4044
      }
4045 4046 4047

      cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
      if (disorderRatio && disorderRatio->type == cJSON_Number) {
4048 4049 4050 4051 4052 4053
        if (disorderRatio->valueint > 50)
          disorderRatio->valueint = 50;

        if (disorderRatio->valueint < 0)
          disorderRatio->valueint = 0;

4054 4055 4056 4057
        g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
      } else if (!disorderRatio) {
        g_Dbs.db[i].superTbls[j].disorderRatio = 0;
      } else {
4058
        printf("ERROR: failed to read json, disorderRatio not found\n");
4059
        goto PARSE_OVER;
4060
      }
4061 4062 4063 4064 4065 4066 4067

      cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
      if (disorderRange && disorderRange->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
      } else if (!disorderRange) {
        g_Dbs.db[i].superTbls[j].disorderRange = 1000;
      } else {
4068
        printf("ERROR: failed to read json, disorderRange not found\n");
4069 4070
        goto PARSE_OVER;
      }
4071

4072 4073
      cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
      if (insertRows && insertRows->type == cJSON_Number) {
4074 4075 4076 4077 4078
        if (insertRows->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4079 4080 4081 4082
        g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
      } else if (!insertRows) {
        g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
      } else {
4083
        errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
4084
                __func__, __LINE__);
4085 4086 4087 4088 4089 4090
        goto PARSE_OVER;
      }

      cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
      if (insertInterval && insertInterval->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
4091 4092 4093 4094 4095
        if (insertInterval->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4096
      } else if (!insertInterval) {
4097
        verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n",
4098
                __func__, __LINE__, g_args.insert_interval);
4099 4100
        g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
      } else {
4101
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
4102
                __func__, __LINE__);
4103 4104 4105
        goto PARSE_OVER;
      }

4106
      int retVal = getColumnAndTagTypeFromInsertJsonFile(
4107
              stbInfo, &g_Dbs.db[i].superTbls[j]);
4108 4109
      if (false == retVal) {
        goto PARSE_OVER;
4110 4111
      }
    }
4112 4113 4114 4115
  }

  ret = true;

4116
PARSE_OVER:
4117 4118 4119 4120 4121 4122 4123 4124
  return ret;
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
4125
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
4126 4127 4128 4129
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
4130
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE);
4131
  } else if (!host) {
4132
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
4133
  } else {
4134
    printf("ERROR: failed to read json, host not found\n");
4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
4147
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE);
4148
  } else if (!user) {
4149
    tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ;
4150 4151 4152 4153
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
4154
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE);
4155
  } else if (!password) {
4156
    tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);;
4157
  }
H
hzcheng 已提交
4158

4159
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
4160
  if (answerPrompt && answerPrompt->type == cJSON_String
4161
          && answerPrompt->valuestring != NULL) {
4162 4163 4164 4165
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
4166
    } else {
4167
      g_args.answer_yes = false;
H
hzcheng 已提交
4168
    }
4169 4170 4171
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
4172
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
4173
    goto PARSE_OVER;
4174
  }
4175

4176 4177
  cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
  if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
4178
    if (gQueryTimes->valueint <= 0) {
4179 4180
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4181 4182
      goto PARSE_OVER;
    }
4183 4184 4185 4186
    g_args.query_times = gQueryTimes->valueint;
  } else if (!gQueryTimes) {
    g_args.query_times = 1;
  } else {
4187
    errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4188
        __func__, __LINE__);
4189 4190 4191
    goto PARSE_OVER;
  }

4192 4193
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
4194
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE);
4195
  } else if (!dbs) {
4196
    printf("ERROR: failed to read json, databases not found\n");
4197
    goto PARSE_OVER;
H
hzcheng 已提交
4198
  }
4199 4200 4201

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
4202
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
4203
  } else if (!queryMode) {
4204
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
4205
  } else {
4206
    printf("ERROR: failed to read json, query_mode not found\n");
4207
    goto PARSE_OVER;
H
hzcheng 已提交
4208
  }
4209

4210
  // specified_table_query
4211 4212
  cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!specifiedQuery) {
4213
    g_queryInfo.specifiedQueryInfo.concurrent = 1;
4214 4215
    g_queryInfo.specifiedQueryInfo.sqlCount = 0;
  } else if (specifiedQuery->type != cJSON_Object) {
4216
    printf("ERROR: failed to read json, super_table_query not found\n");
4217
    goto PARSE_OVER;
4218
  } else {
4219 4220 4221 4222 4223
    cJSON* queryInterval = cJSON_GetObjectItem(specifiedQuery, "query_interval");
    if (queryInterval && queryInterval->type == cJSON_Number) {
      g_queryInfo.specifiedQueryInfo.queryInterval = queryInterval->valueint;
    } else if (!queryInterval) {
      g_queryInfo.specifiedQueryInfo.queryInterval = 0;
4224 4225
    }

4226
    cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery,
4227
        "query_times");
4228
    if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
4229
      if (specifiedQueryTimes->valueint <= 0) {
4230 4231
        errorPrint(
                "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4232
          __func__, __LINE__, specifiedQueryTimes->valueint);
4233 4234 4235
        goto PARSE_OVER;

      }
4236 4237 4238 4239
      g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint;
    } else if (!specifiedQueryTimes) {
      g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times;
    } else {
4240
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4241
          __func__, __LINE__);
4242
      goto PARSE_OVER;
4243
    }
4244

4245
    cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent");
4246
    if (concurrent && concurrent->type == cJSON_Number) {
4247
      if (concurrent->valueint <= 0) {
4248 4249
        errorPrint(
                "%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n",
4250 4251
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount,
4252
              g_queryInfo.specifiedQueryInfo.concurrent);
4253 4254
        goto PARSE_OVER;
      }
4255
      g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint;
4256
    } else if (!concurrent) {
4257
      g_queryInfo.specifiedQueryInfo.concurrent = 1;
4258
    }
4259

4260 4261 4262 4263 4264 4265 4266
    cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode");
    if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String
        && specifiedAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE;
4267
      } else {
4268
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4269
            __func__, __LINE__);
4270 4271 4272
        goto PARSE_OVER;
      }
    } else {
4273
      g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
4274
    }
4275

4276
    cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval");
4277
    if (interval && interval->type == cJSON_Number) {
4278
      g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint;
4279
    } else if (!interval) {
4280 4281
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4282
      g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000;
4283
    }
4284

4285
    cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart");
4286
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
4287
      if (0 == strcmp("yes", restart->valuestring)) {
4288
        g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4289
      } else if (0 == strcmp("no", restart->valuestring)) {
4290
        g_queryInfo.specifiedQueryInfo.subscribeRestart = false;
4291
      } else {
4292
        printf("ERROR: failed to read json, subscribe restart error\n");
4293 4294 4295
        goto PARSE_OVER;
      }
    } else {
4296
      g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4297
    }
4298

4299
    cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress");
4300
    if (keepProgress
4301 4302
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
4303
      if (0 == strcmp("yes", keepProgress->valuestring)) {
4304
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1;
4305
      } else if (0 == strcmp("no", keepProgress->valuestring)) {
4306
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4307
      } else {
4308
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4309 4310 4311
        goto PARSE_OVER;
      }
    } else {
4312
      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4313 4314
    }

4315
    // sqls
4316
    cJSON* superSqls = cJSON_GetObjectItem(specifiedQuery, "sqls");
4317
    if (!superSqls) {
4318
      g_queryInfo.specifiedQueryInfo.sqlCount = 0;
4319
    } else if (superSqls->type != cJSON_Array) {
4320
      errorPrint("%s() LN%d, failed to read json, super sqls not found\n",
4321
          __func__, __LINE__);
4322
      goto PARSE_OVER;
4323
    } else {
4324 4325
      int superSqlSize = cJSON_GetArraySize(superSqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4326
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
4327
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4328 4329
        goto PARSE_OVER;
      }
4330

4331
      g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize;
4332 4333 4334
      for (int j = 0; j < superSqlSize; ++j) {
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
        if (sql == NULL) continue;
4335

4336 4337
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
4338
          printf("ERROR: failed to read json, sql not found\n");
4339 4340
          goto PARSE_OVER;
        }
4341 4342
        tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j],
                sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
H
hzcheng 已提交
4343

4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355
        cJSON* resubAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume");
        if (resubAfterConsume
                && resubAfterConsume->type == cJSON_Number) {
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j]
                = resubAfterConsume->valueint;
        } else if (!resubAfterConsume) {
            //printf("failed to read json, subscribe interval no found\n");
            //goto PARSE_OVER;
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = 1;
        }

4356
        cJSON *result = cJSON_GetObjectItem(sql, "result");
4357 4358 4359 4360
        if ((NULL != result) && (result->type == cJSON_String)
                && (result->valuestring != NULL)) {
          tstrncpy(g_queryInfo.specifiedQueryInfo.result[j],
                  result->valuestring, MAX_FILE_NAME_LEN);
4361
        } else if (NULL == result) {
4362 4363
          memset(g_queryInfo.specifiedQueryInfo.result[j],
                  0, MAX_FILE_NAME_LEN);
4364
        } else {
4365
          printf("ERROR: failed to read json, super query result file not found\n");
4366
          goto PARSE_OVER;
4367
        }
4368 4369 4370
      }
    }
  }
4371

4372
  // super_table_query
4373 4374
  cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!superQuery) {
4375
    g_queryInfo.superQueryInfo.threadCnt = 1;
4376 4377
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
4378
    printf("ERROR: failed to read json, sub_table_query not found\n");
4379 4380
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
4381
  } else {
4382
    cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval");
4383
    if (subrate && subrate->type == cJSON_Number) {
4384
      g_queryInfo.superQueryInfo.queryInterval = subrate->valueint;
4385
    } else if (!subrate) {
4386
      g_queryInfo.superQueryInfo.queryInterval = 0;
4387 4388 4389 4390
    }

    cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
    if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
4391 4392
      if (superQueryTimes->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4393
          __func__, __LINE__, superQueryTimes->valueint);
4394 4395
        goto PARSE_OVER;
      }
4396 4397 4398 4399
      g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
    } else if (!superQueryTimes) {
      g_queryInfo.superQueryInfo.queryTimes = g_args.query_times;
    } else {
4400
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4401
          __func__, __LINE__);
4402
      goto PARSE_OVER;
4403
    }
4404

4405
    cJSON* threads = cJSON_GetObjectItem(superQuery, "threads");
4406
    if (threads && threads->type == cJSON_Number) {
4407 4408 4409 4410 4411 4412
      if (threads->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, threads input mistake\n",
          __func__, __LINE__);
        goto PARSE_OVER;

      }
4413
      g_queryInfo.superQueryInfo.threadCnt = threads->valueint;
4414
    } else if (!threads) {
4415
      g_queryInfo.superQueryInfo.threadCnt = 1;
4416
    }
4417

4418
    //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count");
4419
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
4420
    //  g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint;
4421
    //} else if (!subTblCnt) {
4422
    //  g_queryInfo.superQueryInfo.childTblCount = 0;
4423
    //}
4424

4425
    cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname");
4426 4427 4428
    if (stblname && stblname->type == cJSON_String
        && stblname->valuestring != NULL) {
      tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring,
4429
          MAX_TB_NAME_SIZE);
4430
    } else {
4431
      errorPrint("%s() LN%d, failed to read json, super table name input error\n",
4432
          __func__, __LINE__);
4433 4434
      goto PARSE_OVER;
    }
4435

4436 4437 4438 4439 4440 4441 4442
    cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode");
    if (superAsyncMode && superAsyncMode->type == cJSON_String
        && superAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE;
4443
      } else {
4444
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4445
            __func__, __LINE__);
4446 4447 4448
        goto PARSE_OVER;
      }
    } else {
4449
      g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
4450
    }
4451

4452 4453 4454 4455 4456 4457 4458 4459 4460
    cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval");
    if (superInterval && superInterval->type == cJSON_Number) {
      if (superInterval->valueint < 0) {
        errorPrint("%s() LN%d, failed to read json, interval input mistake\n",
            __func__, __LINE__);
        goto PARSE_OVER;
      }
      g_queryInfo.superQueryInfo.subscribeInterval = superInterval->valueint;
    } else if (!superInterval) {
4461 4462
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4463
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
4464
    }
4465

4466
    cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart");
4467 4468
    if (subrestart && subrestart->type == cJSON_String
        && subrestart->valuestring != NULL) {
4469
      if (0 == strcmp("yes", subrestart->valuestring)) {
4470
        g_queryInfo.superQueryInfo.subscribeRestart = true;
4471
      } else if (0 == strcmp("no", subrestart->valuestring)) {
4472
        g_queryInfo.superQueryInfo.subscribeRestart = false;
4473
      } else {
4474
        printf("ERROR: failed to read json, subscribe restart error\n");
4475 4476 4477
        goto PARSE_OVER;
      }
    } else {
4478
      g_queryInfo.superQueryInfo.subscribeRestart = true;
4479
    }
4480

4481 4482 4483 4484 4485
    cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
    if (superkeepProgress &&
            superkeepProgress->type == cJSON_String
            && superkeepProgress->valuestring != NULL) {
      if (0 == strcmp("yes", superkeepProgress->valuestring)) {
4486
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
4487
      } else if (0 == strcmp("no", superkeepProgress->valuestring)) {
4488
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4489
      } else {
4490
        printf("ERROR: failed to read json, subscribe super table keepProgress error\n");
4491 4492 4493
        goto PARSE_OVER;
      }
    } else {
4494
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4495
    }
4496

4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508
    cJSON* superResubAfterConsume =
            cJSON_GetObjectItem(superQuery, "resubAfterConsume");
    if (superResubAfterConsume
            && superResubAfterConsume->type == cJSON_Number) {
        g_queryInfo.superQueryInfo.resubAfterConsume =
            superResubAfterConsume->valueint;
    } else if (!superResubAfterConsume) {
        //printf("failed to read json, subscribe interval no found\n");
        ////goto PARSE_OVER;
        g_queryInfo.superQueryInfo.resubAfterConsume = 1;
    }

4509
    // sqls
4510
    cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls");
4511
    if (!subsqls) {
4512
      g_queryInfo.superQueryInfo.sqlCount = 0;
4513
    } else if (subsqls->type != cJSON_Array) {
4514
      errorPrint("%s() LN%d: failed to read json, super sqls not found\n",
4515
          __func__, __LINE__);
4516
      goto PARSE_OVER;
4517
    } else {
4518 4519
      int superSqlSize = cJSON_GetArraySize(subsqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4520
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
4521
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4522 4523
        goto PARSE_OVER;
      }
4524

4525
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
4526
      for (int j = 0; j < superSqlSize; ++j) {
4527 4528
        cJSON* sql = cJSON_GetArrayItem(subsqls, j);
        if (sql == NULL) continue;
4529

4530
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
4531 4532 4533
        if (!sqlStr || sqlStr->type != cJSON_String
            || sqlStr->valuestring == NULL) {
          errorPrint("%s() LN%d, failed to read json, sql not found\n",
4534
              __func__, __LINE__);
4535 4536
          goto PARSE_OVER;
        }
4537
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring,
4538
            MAX_QUERY_SQL_LENGTH);
4539 4540

        cJSON *result = cJSON_GetObjectItem(sql, "result");
4541 4542 4543
        if (result != NULL && result->type == cJSON_String
            && result->valuestring != NULL){
          tstrncpy(g_queryInfo.superQueryInfo.result[j],
4544
              result->valuestring, MAX_FILE_NAME_LEN);
4545
        } else if (NULL == result) {
4546
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4547
        }  else {
4548
          errorPrint("%s() LN%d, failed to read json, sub query result file not found\n",
4549
              __func__, __LINE__);
4550
          goto PARSE_OVER;
4551
        }
4552 4553
      }
    }
H
hzcheng 已提交
4554 4555
  }

4556
  ret = true;
H
hzcheng 已提交
4557

4558
PARSE_OVER:
4559 4560
  return ret;
}
H
hzcheng 已提交
4561

4562
static bool getInfoFromJsonFile(char* file) {
4563
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
4564

4565 4566 4567 4568 4569
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
4570

4571
  bool  ret = false;
4572
  int   maxLen = 6400000;
4573 4574 4575 4576 4577 4578 4579
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
4580
  }
H
Hui Li 已提交
4581

4582 4583 4584
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
4585
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
4586 4587
    goto PARSE_OVER;
  }
H
Hui Li 已提交
4588

4589 4590 4591
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
4592
      g_args.test_mode = INSERT_TEST;
4593
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
4594
      g_args.test_mode = QUERY_TEST;
4595
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
4596
      g_args.test_mode = SUBSCRIBE_TEST;
4597
    } else {
4598
      printf("ERROR: failed to read json, filetype not support\n");
4599 4600 4601
      goto PARSE_OVER;
    }
  } else if (!filetype) {
4602
    g_args.test_mode = INSERT_TEST;
4603
  } else {
4604
    printf("ERROR: failed to read json, filetype not found\n");
4605 4606
    goto PARSE_OVER;
  }
H
hzcheng 已提交
4607

4608
  if (INSERT_TEST == g_args.test_mode) {
4609
    ret = getMetaFromInsertJsonFile(root);
4610
  } else if ((QUERY_TEST == g_args.test_mode)
4611
          || (SUBSCRIBE_TEST == g_args.test_mode)) {
4612 4613
    ret = getMetaFromQueryJsonFile(root);
  } else {
4614
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n",
4615
            __func__, __LINE__);
4616
    goto PARSE_OVER;
4617
  }
4618

4619
PARSE_OVER:
4620 4621
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
4622
  fclose(fp);
4623 4624
  return ret;
}
H
hzcheng 已提交
4625

4626
static void prepareSampleData() {
4627
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4628 4629 4630
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
        (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]);
S
Shuaiqiang Chang 已提交
4631
      }
4632 4633 4634 4635
    }
  }
}

4636
static void postFreeResource() {
4637
  tmfclose(g_fpOfInsertResult);
4638
  for (int i = 0; i < g_Dbs.dbCount; i++) {
S
Shuduo Sang 已提交
4639
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
4640 4641 4642
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
4643
      }
4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4656
    }
4657 4658
  }
}
S
Shuaiqiang Chang 已提交
4659

4660 4661 4662
static int getRowDataFromSample(
        char* dataBuf, int64_t maxLen, int64_t timestamp,
      SSuperTable* superTblInfo, int64_t* sampleUsePos) {
4663
  if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
4664
/*    int ret = readSampleFromCsvFileToMem(superTblInfo);
4665
    if (0 != ret) {
4666
      tmfree(superTblInfo->sampleDataBuf);
4667
      superTblInfo->sampleDataBuf = NULL;
4668 4669
      return -1;
    }
4670
*/
4671 4672
    *sampleUsePos = 0;
  }
S
Shuaiqiang Chang 已提交
4673

4674
  int    dataLen = 0;
4675

4676
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
4677
          "(%" PRId64 ", ", timestamp);
4678
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
4679
          "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
4680
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
S
Shuaiqiang Chang 已提交
4681

4682
  (*sampleUsePos)++;
4683

4684 4685
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4686

4687 4688 4689 4690
static int64_t generateStbRowData(
        SSuperTable* stbInfo,
        char* recBuf, int64_t timestamp
        ) {
4691
  int64_t   dataLen = 0;
4692
  char  *pstr = recBuf;
4693
  int64_t maxLen = MAX_DATA_SIZE;
4694

4695 4696
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
          "(%" PRId64 ",", timestamp);
4697

4698
  for (int i = 0; i < stbInfo->columnCount; i++) {
4699 4700 4701 4702
    if ((0 == strncasecmp(stbInfo->columns[i].dataType,
                    "BINARY", strlen("BINARY")))
            || (0 == strncasecmp(stbInfo->columns[i].dataType,
                    "NCHAR", strlen("NCHAR")))) {
4703
      if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
4704
        errorPrint( "binary or nchar length overflow, max size:%u\n",
4705
                (uint32_t)TSDB_MAX_BINARY_LEN);
4706
        return -1;
4707
      }
4708

4709 4710
      char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
      if (NULL == buf) {
4711
        errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
4712
        return -1;
4713 4714
      }
      rand_string(buf, stbInfo->columns[i].dataLen);
4715
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf);
4716
      tmfree(buf);
4717
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4718
                "INT", 3)) {
4719
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4720
              "%d,", rand_int());
4721
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4722
                "BIGINT", 6)) {
4723
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4724
              "%"PRId64",", rand_bigint());
4725
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4726
                "FLOAT", 5)) {
4727
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4728
              "%f,", rand_float());
4729
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4730
                "DOUBLE", 6)) {
4731
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4732
              "%f,", rand_double());
4733
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4734
                "SMALLINT", 8)) {
4735
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4736
          "%d,", rand_smallint());
4737
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4738
          "TINYINT", strlen("TINYINT"))) {
4739
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4740
          "%d,", rand_tinyint());
4741
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4742
          "BOOL", strlen("BOOL"))) {
4743
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4744
          "%d,", rand_bool());
4745
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4746
          "TIMESTAMP", strlen("TIMESTAMP"))) {
4747
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4748
          "%"PRId64",", rand_bigint());
4749
    }  else {
4750
      errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType);
4751
      return -1;
4752
    }
S
Shuaiqiang Chang 已提交
4753
  }
4754

4755
  dataLen -= 1;
4756
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")");
4757

4758 4759 4760
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

  return strlen(recBuf);
4761
}
S
Shuaiqiang Chang 已提交
4762

4763
static int64_t generateData(char *recBuf, char **data_type,
4764
        int64_t timestamp, int lenOfBinary) {
4765 4766
  memset(recBuf, 0, MAX_DATA_SIZE);
  char *pstr = recBuf;
4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780
  pstr += sprintf(pstr, "(%" PRId64, timestamp);
  int c = 0;

  for (; c < MAX_NUM_DATATYPE; c++) {
    if (data_type[c] == NULL) {
      break;
    }
  }

  if (0 == c) {
    perror("data type error!");
    exit(-1);
  }

H
Haojun Liao 已提交
4781
  for (int i = 0; i < c; i++) {
4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792
    if (strcasecmp(data_type[i % c], "TINYINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_tinyint() );
    } else if (strcasecmp(data_type[i % c], "SMALLINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_smallint());
    } else if (strcasecmp(data_type[i % c], "INT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_int());
    } else if (strcasecmp(data_type[i % c], "BIGINT") == 0) {
      pstr += sprintf(pstr, ",%" PRId64, rand_bigint());
    } else if (strcasecmp(data_type[i % c], "FLOAT") == 0) {
      pstr += sprintf(pstr, ",%10.4f", rand_float());
    } else if (strcasecmp(data_type[i % c], "DOUBLE") == 0) {
4793
      double t = rand_double();
4794 4795
      pstr += sprintf(pstr, ",%20.8f", t);
    } else if (strcasecmp(data_type[i % c], "BOOL") == 0) {
4796
      bool b = rand_bool() & 1;
4797 4798
      pstr += sprintf(pstr, ",%s", b ? "true" : "false");
    } else if (strcasecmp(data_type[i % c], "BINARY") == 0) {
4799 4800
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4801
      pstr += sprintf(pstr, ",\"%s\"", s);
4802
      free(s);
4803
    } else if (strcasecmp(data_type[i % c], "NCHAR") == 0) {
4804 4805
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4806
      pstr += sprintf(pstr, ",\"%s\"", s);
4807 4808 4809
      free(s);
    }

4810
    if (strlen(recBuf) > MAX_DATA_SIZE) {
4811 4812 4813 4814 4815 4816 4817
      perror("column length too long, abort");
      exit(-1);
    }
  }

  pstr += sprintf(pstr, ")");

4818 4819
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

4820
  return (int32_t)strlen(recBuf);
4821 4822
}

4823
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
4824 4825
  char* sampleDataBuf = NULL;

4826
  sampleDataBuf = calloc(
4827
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
4828
  if (sampleDataBuf == NULL) {
4829
      errorPrint("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n",
4830 4831 4832 4833
              __func__, __LINE__,
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE,
              strerror(errno));
      return -1;
4834
  }
4835

4836 4837 4838 4839
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
4840 4841 4842 4843 4844
      errorPrint("%s() LN%d, read sample from csv file failed.\n",
          __func__, __LINE__);
      tmfree(sampleDataBuf);
      superTblInfo->sampleDataBuf = NULL;
      return -1;
H
Hui Li 已提交
4845
  }
4846

4847 4848 4849
  return 0;
}

S
Shuduo Sang 已提交
4850
static int64_t execInsert(threadInfo *pThreadInfo, uint64_t k)
4851 4852
{
  int affectedRows;
4853
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4854

4855
  verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
S
Shuduo Sang 已提交
4856
            __func__, __LINE__, pThreadInfo->buffer);
4857
  if (superTblInfo) {
4858
    if (superTblInfo->insertMode == TAOSC_IFACE) {
S
Shuduo Sang 已提交
4859 4860 4861
      affectedRows = queryDbExec(
              pThreadInfo->taos,
              pThreadInfo->buffer, INSERT_TYPE, false);
4862
    } else if (superTblInfo->insertMode == REST_IFACE) {
4863
      if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port,
S
Shuduo Sang 已提交
4864
                  pThreadInfo->buffer, NULL /* not set result file */)) {
4865
        affectedRows = -1;
4866
        printf("========restful return fail, threadID[%d]\n",
4867
            pThreadInfo->threadID);
4868 4869 4870
      } else {
        affectedRows = k;
      }
4871
    } else if (superTblInfo->insertMode == STMT_IFACE) {
S
Shuduo Sang 已提交
4872 4873 4874 4875 4876 4877
      debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt);
      if (0 != taos_stmt_execute(pThreadInfo->stmt)) {
        errorPrint("%s() LN%d, failied to execute insert statement\n",
                    __func__, __LINE__);
        exit(-1);
      }
S
Shuduo Sang 已提交
4878 4879

      affectedRows = k;
4880
    } else {
4881
      errorPrint("%s() LN%d: unknown insert mode: %d\n",
4882 4883
        __func__, __LINE__, superTblInfo->insertMode);
      affectedRows = 0;
4884
    }
4885
  } else {
S
Shuduo Sang 已提交
4886
    affectedRows = queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, INSERT_TYPE, false);
4887 4888 4889 4890 4891
  }

  return affectedRows;
}

4892 4893
static void getTableName(char *pTblName,
        threadInfo* pThreadInfo, uint64_t tableSeq)
4894 4895
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4896 4897
  if ((superTblInfo)
          && (AUTO_CREATE_SUBTBL != superTblInfo->autoCreateTable)) {
4898
    if (superTblInfo->childTblLimit > 0) {
4899 4900 4901
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName +
            (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
4902
    } else {
4903

4904
        verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n",
4905 4906 4907 4908 4909
                pThreadInfo->threadID, __func__, __LINE__,
                pThreadInfo->start_table_from,
                pThreadInfo->ntables, tableSeq);
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
4910 4911
    }
  } else {
4912
    snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"",
4913
        g_args.tb_prefix, tableSeq);
4914 4915 4916
  }
}

4917 4918 4919 4920 4921
static int64_t generateDataTailWithoutStb(
        uint64_t batch, char* buffer,
        int64_t remainderBufLen, int64_t insertRows,
        uint64_t startFrom, int64_t startTime,
        /* int64_t *pSamplePos, */int64_t *dataLen) {
4922

4923
  uint64_t len = 0;
4924 4925
  char *pstr = buffer;

4926
  verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch);
4927

4928
  int64_t k = 0;
4929
  for (k = 0; k < batch;) {
4930
    char data[MAX_DATA_SIZE];
4931 4932
    memset(data, 0, MAX_DATA_SIZE);

4933
    int64_t retLen = 0;
4934

4935 4936
    char **data_type = g_args.datatype;
    int lenOfBinary = g_args.len_of_binary;
4937

4938 4939 4940 4941 4942 4943 4944 4945 4946
    int64_t randTail = DEFAULT_TIMESTAMP_STEP * k;

    if (g_args.disorderRatio != 0) {
        int rand_num = taosRandom() % 100;
        if (rand_num < g_args.disorderRatio) {
          randTail = (randTail +
                  (taosRandom() % g_args.disorderRange + 1)) * (-1);

          debugPrint("rand data generated, back %"PRId64"\n", randTail);
4947
        }
4948 4949 4950
    } else {
        randTail = DEFAULT_TIMESTAMP_STEP * k;
    }
4951

4952 4953 4954
    retLen = generateData(data, data_type,
            startTime + randTail,
            lenOfBinary);
4955

4956
    if (len > remainderBufLen)
4957
        break;
4958

4959 4960 4961 4962 4963 4964 4965
    pstr += sprintf(pstr, "%s", data);
    k++;
    len += retLen;
    remainderBufLen -= retLen;

    verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n",
            __func__, __LINE__, len, k, buffer);
4966

4967
    startFrom ++;
4968

4969 4970 4971 4972
    if (startFrom >= insertRows) {
      break;
    }
  }
4973

4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014
  *dataLen = len;
  return k;
}

static int64_t generateStbDataTail(
        SSuperTable* superTblInfo,
        uint64_t batch, char* buffer,
        int64_t remainderBufLen, int64_t insertRows,
        uint64_t startFrom, int64_t startTime,
        int64_t *pSamplePos, int64_t *dataLen) {
  uint64_t len = 0;

  char *pstr = buffer;

  verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch);

  int64_t k = 0;
  for (k = 0; k < batch;) {
    char data[MAX_DATA_SIZE];
    memset(data, 0, MAX_DATA_SIZE);

    int64_t retLen = 0;

    if (0 == strncasecmp(superTblInfo->dataSource,
                "sample", strlen("sample"))) {
          retLen = getRowDataFromSample(
                  data,
                  remainderBufLen,
                  startTime + superTblInfo->timeStampStep * k,
                  superTblInfo,
                  pSamplePos);
    } else if (0 == strncasecmp(superTblInfo->dataSource,
                "rand", strlen("rand"))) {
        int64_t randTail = superTblInfo->timeStampStep * k;
        if (superTblInfo->disorderRatio > 0) {
            int rand_num = taosRandom() % 100;
            if(rand_num < superTblInfo->disorderRatio) {
                randTail = (randTail +
                        (taosRandom() % superTblInfo->disorderRange + 1)) * (-1);
                debugPrint("rand data generated, back %"PRId64"\n", randTail);
            }
5015
        }
5016

5017 5018 5019
        int64_t d = startTime + randTail;
        retLen = generateStbRowData(superTblInfo, data, d);
    }
5020

5021
    if (retLen > remainderBufLen) {
5022
        break;
5023 5024
    }

5025 5026 5027 5028 5029
    pstr += snprintf(pstr , retLen + 1, "%s", data);
    k++;
    len += retLen;
    remainderBufLen -= retLen;

5030
    verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n",
5031
            __func__, __LINE__, len, k, buffer);
5032 5033

    startFrom ++;
5034

5035
    if (startFrom >= insertRows) {
5036
      break;
5037
    }
5038
  }
5039

5040 5041 5042
  *dataLen = len;
  return k;
}
5043

5044 5045 5046

static int generateSQLHeadWithoutStb(char *tableName,
        char *dbName,
5047
        char *buffer, int remainderBufLen)
5048 5049
{
  int len;
5050 5051 5052

  char headBuf[HEAD_BUFF_LEN];

5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078
  len = snprintf(
          headBuf,
          HEAD_BUFF_LEN,
          "%s.%s values",
          dbName,
          tableName);

  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

  return len;
}

static int generateStbSQLHead(
        SSuperTable* superTblInfo,
        char *tableName, int32_t tableSeq,
        char *dbName,
        char *buffer, int remainderBufLen)
{
  int len;

  char headBuf[HEAD_BUFF_LEN];

  if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
5079 5080
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
5081
            tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq);
5082
      } else {
5083 5084 5085
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
5086 5087
      }
      if (NULL == tagsValBuf) {
5088
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n",
5089
            __func__, __LINE__);
5090 5091 5092
        return -1;
      }

5093 5094
      len = snprintf(
          headBuf,
5095 5096
                  HEAD_BUFF_LEN,
                  "%s.%s using %s.%s tags %s values",
5097
                  dbName,
5098
                  tableName,
5099
                  dbName,
5100 5101
                  superTblInfo->sTblName,
                  tagsValBuf);
5102 5103
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
5104 5105
      len = snprintf(
          headBuf,
5106 5107
                  HEAD_BUFF_LEN,
                  "%s.%s values",
5108
                  dbName,
5109
                  tableName);
5110
    } else {
5111 5112
      len = snprintf(
          headBuf,
5113 5114
                  HEAD_BUFF_LEN,
                  "%s.%s values",
5115
                  dbName,
5116
                  tableName);
5117 5118
  }

5119 5120 5121 5122 5123
  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

5124 5125 5126
  return len;
}

5127 5128
static int64_t generateStbInterlaceData(
        SSuperTable *superTblInfo,
5129 5130
        char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes,
        uint64_t tableSeq,
5131
        threadInfo *pThreadInfo, char *buffer,
5132
        int64_t insertRows,
5133
        int64_t startTime,
5134
        uint64_t *pRemainderBufLen)
5135
{
5136
  assert(buffer);
5137 5138
  char *pstr = buffer;

5139 5140 5141
  int headLen = generateStbSQLHead(
          superTblInfo,
          tableName, tableSeq, pThreadInfo->db_name,
5142
          pstr, *pRemainderBufLen);
5143 5144 5145 5146 5147

  if (headLen <= 0) {
    return 0;
  }
  // generate data buffer
5148
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n",
5149
            pThreadInfo->threadID, __func__, __LINE__, i, buffer);
5150 5151 5152 5153

  pstr += headLen;
  *pRemainderBufLen -= headLen;

5154
  int64_t dataLen = 0;
5155

5156
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%"PRIu64" batchPerTbl = %"PRIu64"\n",
5157 5158
            pThreadInfo->threadID, __func__, __LINE__,
            i, batchPerTblTimes, batchPerTbl);
5159

5160
  if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
5161
      startTime = taosGetTimestamp(pThreadInfo->time_precision);
5162
  }
5163

5164
  int64_t  k = generateStbDataTail(
5165 5166 5167 5168
            superTblInfo,
            batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
            startTime,
            &(pThreadInfo->samplePos), &dataLen);
5169 5170 5171 5172

  if (k == batchPerTbl) {
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
5173
  } else {
5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208
    debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n",
            __func__, __LINE__, k, batchPerTbl);
    pstr -= headLen;
    pstr[0] = '\0';
    k = 0;
  }

  return k;
}

static int64_t generateInterlaceDataWithoutStb(
        char *tableName, uint64_t batchPerTbl,
        uint64_t tableSeq,
        char *dbName, char *buffer,
        int64_t insertRows,
        uint64_t *pRemainderBufLen)
{
  assert(buffer);
  char *pstr = buffer;

  int headLen = generateSQLHeadWithoutStb(
          tableName, dbName,
            pstr, *pRemainderBufLen);

  if (headLen <= 0) {
    return 0;
  }

  pstr += headLen;
  *pRemainderBufLen -= headLen;

  int64_t dataLen = 0;

  int64_t startTime = 1500000000000;
  int64_t  k = generateDataTailWithoutStb(
5209 5210
            batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
            startTime,
5211
            &dataLen);
5212 5213

  if (k == batchPerTbl) {
5214 5215 5216
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
  } else {
5217
    debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n",
5218
            __func__, __LINE__, k, batchPerTbl);
5219 5220
    pstr -= headLen;
    pstr[0] = '\0';
5221
    k = 0;
5222 5223 5224 5225 5226
  }

  return k;
}

5227 5228
static int64_t generateStbProgressiveData(
        SSuperTable *superTblInfo,
5229
        char *tableName,
5230
        int64_t tableSeq,
5231
        char *dbName, char *buffer,
5232
        int64_t insertRows,
5233
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos,
5234
        int64_t *pRemainderBufLen)
5235 5236
{
  assert(buffer != NULL);
5237
  char *pstr = buffer;
5238

5239
  memset(buffer, 0, *pRemainderBufLen);
5240

5241
  int64_t headLen = generateStbSQLHead(
5242
              superTblInfo,
5243
          tableName, tableSeq, dbName,
5244
          buffer, *pRemainderBufLen);
5245 5246 5247 5248

  if (headLen <= 0) {
    return 0;
  }
5249
  pstr += headLen;
5250
  *pRemainderBufLen -= headLen;
5251

5252
  int64_t dataLen;
5253

5254
  return generateStbDataTail(superTblInfo,
5255 5256
          g_args.num_of_RPR, pstr, *pRemainderBufLen,
          insertRows, startFrom,
5257 5258
          startTime,
          pSamplePos, &dataLen);
5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286
}

static int64_t generateProgressiveDataWithoutStb(
        char *tableName,
        int64_t tableSeq,
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos,
        int64_t *pRemainderBufLen)
{
  assert(buffer != NULL);
  char *pstr = buffer;

  memset(buffer, 0, *pRemainderBufLen);

  int64_t headLen = generateSQLHeadWithoutStb(
          tableName, pThreadInfo->db_name,
          buffer, *pRemainderBufLen);

  if (headLen <= 0) {
    return 0;
  }
  pstr += headLen;
  *pRemainderBufLen -= headLen;

  int64_t dataLen;

  return generateDataTailWithoutStb(
5287 5288 5289
          g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom,
          startTime,
          /*pSamplePos, */&dataLen);
5290
}
5291

5292 5293 5294 5295 5296 5297 5298 5299 5300
static void printStatPerThread(threadInfo *pThreadInfo)
{
  fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n",
          pThreadInfo->threadID,
          pThreadInfo->totalInsertRows,
          pThreadInfo->totalAffectedRows,
          (double)(pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay/1000.0)));
}

5301
// sync write interlace data
5302
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
5303
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
5304 5305
         pThreadInfo->threadID, __func__, __LINE__);

5306
  int64_t insertRows;
5307
  uint64_t interlaceRows;
5308 5309 5310
  uint64_t maxSqlLen;
  int64_t nTimeStampStep;
  uint64_t insert_interval;
5311 5312

  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5313

5314 5315 5316 5317 5318 5319 5320 5321 5322
  if (superTblInfo) {
    insertRows = superTblInfo->insertRows;

    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
5323 5324 5325
    maxSqlLen = superTblInfo->maxSqlLen;
    nTimeStampStep = superTblInfo->timeStampStep;
    insert_interval = superTblInfo->insertInterval;
5326 5327 5328
  } else {
    insertRows = g_args.num_of_DPT;
    interlaceRows = g_args.interlace_rows;
5329 5330 5331
    maxSqlLen = g_args.max_sql_len;
    nTimeStampStep = DEFAULT_TIMESTAMP_STEP;
    insert_interval = g_args.insert_interval;
5332
  }
5333

5334 5335 5336 5337 5338
  debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n",
          pThreadInfo->threadID, __func__, __LINE__,
          pThreadInfo->start_table_from,
          pThreadInfo->ntables, insertRows);

5339
  if (interlaceRows > insertRows)
H
Haojun Liao 已提交
5340
    interlaceRows = insertRows;
5341

5342 5343
  if (interlaceRows > g_args.num_of_RPR)
    interlaceRows = g_args.num_of_RPR;
5344

5345 5346
  uint64_t batchPerTbl = interlaceRows;
  uint64_t batchPerTblTimes;
5347

5348 5349 5350
  if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) {
    batchPerTblTimes =
        g_args.num_of_RPR / interlaceRows;
5351
  } else {
5352
    batchPerTblTimes = 1;
5353 5354
  }

S
Shuduo Sang 已提交
5355 5356
  pThreadInfo->buffer = calloc(maxSqlLen, 1);
  if (NULL == pThreadInfo->buffer) {
5357
    errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5358
              __func__, __LINE__, maxSqlLen, strerror(errno));
5359 5360 5361 5362 5363 5364
    return NULL;
  }

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

5365 5366
  uint64_t st = 0;
  uint64_t et = UINT64_MAX;
5367

5368 5369 5370
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5371

5372
  uint64_t tableSeq = pThreadInfo->start_table_from;
5373 5374
  int64_t startTime = pThreadInfo->start_time;

5375
  uint64_t generatedRecPerTbl = 0;
5376
  bool flagSleep = true;
5377
  uint64_t sleepTimeTotal = 0;
5378

5379
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
5380
    if ((flagSleep) && (insert_interval)) {
5381 5382
        st = taosGetTimestampMs();
        flagSleep = false;
5383 5384
    }
    // generate data
S
Shuduo Sang 已提交
5385
    memset(pThreadInfo->buffer, 0, maxSqlLen);
5386
    uint64_t remainderBufLen = maxSqlLen;
5387

S
Shuduo Sang 已提交
5388
    char *pstr = pThreadInfo->buffer;
5389

5390 5391
    int len = snprintf(pstr,
            strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO);
5392 5393 5394
    pstr += len;
    remainderBufLen -= len;

5395
    uint64_t recOfBatch = 0;
5396

5397
    for (uint64_t i = 0; i < batchPerTblTimes; i ++) {
5398 5399
        char tableName[TSDB_TABLE_NAME_LEN];

5400
      getTableName(tableName, pThreadInfo, tableSeq);
5401 5402
      if (0 == strlen(tableName)) {
        errorPrint("[%d] %s() LN%d, getTableName return null\n",
5403
            pThreadInfo->threadID, __func__, __LINE__);
S
Shuduo Sang 已提交
5404
        free(pThreadInfo->buffer);
5405 5406
        return NULL;
      }
5407

5408
      uint64_t oldRemainderLen = remainderBufLen;
5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427

      int64_t generated;
      if (superTblInfo) {
        generated = generateStbInterlaceData(
                superTblInfo,
                tableName, batchPerTbl, i, batchPerTblTimes,
                tableSeq,
                pThreadInfo, pstr,
                insertRows,
                startTime,
                &remainderBufLen);
      } else {
        generated = generateInterlaceDataWithoutStb(
                tableName, batchPerTbl,
                tableSeq,
                pThreadInfo->db_name, pstr,
                insertRows,
                &remainderBufLen);
      }
5428

5429 5430
      debugPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5431
      if (generated < 0) {
5432
        errorPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
5433
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5434
        goto free_of_interlace;
5435 5436
      } else if (generated == 0) {
        break;
5437 5438
      }

5439
      tableSeq ++;
5440
      recOfBatch += batchPerTbl;
5441
      pstr += (oldRemainderLen - remainderBufLen);
5442
//      startTime += batchPerTbl * superTblInfo->timeStampStep;
5443
      pThreadInfo->totalInsertRows += batchPerTbl;
5444
      verbosePrint("[%d] %s() LN%d batchPerTbl=%"PRId64" recOfBatch=%"PRId64"\n",
5445 5446
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
5447

5448
      if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
5449 5450 5451
            // turn to first table
            tableSeq = pThreadInfo->start_table_from;
            generatedRecPerTbl += batchPerTbl;
5452

5453 5454
            startTime = pThreadInfo->start_time
              + generatedRecPerTbl * nTimeStampStep;
5455

5456 5457 5458
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;
5459

5460 5461 5462
            int remainRows = insertRows - generatedRecPerTbl;
            if ((remainRows > 0) && (batchPerTbl > remainRows))
              batchPerTbl = remainRows;
5463

5464 5465 5466
            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
      }
5467

5468
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n",
5469 5470
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);
5471 5472

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
5473 5474 5475
        break;
    }

5476
    verbosePrint("[%d] %s() LN%d recOfBatch=%"PRIu64" totalInsertRows=%"PRIu64"\n",
5477 5478
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
5479
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
S
Shuduo Sang 已提交
5480
           pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer);
5481

5482
    startTs = taosGetTimestampMs();
5483

5484 5485 5486 5487 5488 5489 5490
    if (recOfBatch == 0) {
      errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n",
              pThreadInfo->threadID, __func__, __LINE__,
              recOfBatch);
      errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n");
      goto free_of_interlace;
    }
S
Shuduo Sang 已提交
5491
    int64_t affectedRows = execInsert(pThreadInfo, recOfBatch);
5492

5493
    endTs = taosGetTimestampMs();
5494 5495
    uint64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n",
5496
            __func__, __LINE__, delay);
5497 5498 5499
    verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5500 5501 5502 5503 5504 5505

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

5506 5507
    if (recOfBatch != affectedRows) {
        errorPrint("[%d] %s() LN%d execInsert insert %"PRIu64", affected rows: %"PRId64"\n%s\n",
5508
                pThreadInfo->threadID, __func__, __LINE__,
S
Shuduo Sang 已提交
5509
                recOfBatch, affectedRows, pThreadInfo->buffer);
5510
        goto free_of_interlace;
5511
    }
5512

5513
    pThreadInfo->totalAffectedRows += affectedRows;
5514

5515 5516
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
5517
      printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n",
5518 5519 5520
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5521 5522
      lastPrintTime = currentPrintTime;
    }
5523

5524
    if ((insert_interval) && flagSleep) {
5525
      et = taosGetTimestampMs();
5526

5527
      if (insert_interval > (et - st) ) {
S
Shuduo Sang 已提交
5528 5529
        uint64_t sleepTime = insert_interval - (et -st);
        performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n",
5530
                    __func__, __LINE__, sleepTime);
5531 5532 5533
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
5534
    }
5535 5536
  }

5537
free_of_interlace:
S
Shuduo Sang 已提交
5538
  tmfree(pThreadInfo->buffer);
5539
  printStatPerThread(pThreadInfo);
5540 5541 5542
  return NULL;
}

5543
// sync insertion progressive data
5544
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
5545
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
5546

5547
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5548
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
5549 5550 5551 5552 5553 5554
  int64_t timeStampStep =
      superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
  int64_t insertRows =
        (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
  verbosePrint("%s() LN%d insertRows=%"PRId64"\n",
            __func__, __LINE__, insertRows);
5555

S
Shuduo Sang 已提交
5556 5557
  pThreadInfo->buffer = calloc(maxSqlLen, 1);
  if (NULL == pThreadInfo->buffer) {
5558
    errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5559 5560
              maxSqlLen,
              strerror(errno));
5561 5562
    return NULL;
  }
5563

5564 5565 5566
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5567

5568
/*  int insert_interval =
5569
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5570 5571
  uint64_t st = 0;
  uint64_t et = 0xffffffff;
5572
  */
5573

5574 5575
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
5576

5577
  pThreadInfo->samplePos = 0;
5578

5579 5580 5581
  for (uint64_t tableSeq = pThreadInfo->start_table_from;
          tableSeq <= pThreadInfo->end_table_to;
          tableSeq ++) {
5582
    int64_t start_time = pThreadInfo->start_time;
5583

5584
    for (uint64_t i = 0; i < insertRows;) {
5585 5586
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
5587
      verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n",
5588 5589
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);
5590

5591
      int64_t remainderBufLen = maxSqlLen;
S
Shuduo Sang 已提交
5592
      char *pstr = pThreadInfo->buffer;
5593

5594 5595
      int len = snprintf(pstr,
              strlen(STR_INSERT_INTO) + 1, "%s", STR_INSERT_INTO);
5596 5597 5598 5599

      pstr += len;
      remainderBufLen -= len;

5600 5601 5602 5603 5604 5605 5606 5607 5608 5609
      int64_t generated;
      if (superTblInfo) {
        generated = generateStbProgressiveData(
                superTblInfo,
              tableName, tableSeq, pThreadInfo->db_name, pstr, insertRows,
            i, start_time,
            &(pThreadInfo->samplePos),
            &remainderBufLen);
      } else {
        generated = generateProgressiveDataWithoutStb(
5610 5611 5612 5613
              tableName, tableSeq, pThreadInfo, pstr, insertRows,
            i, start_time,
            &(pThreadInfo->samplePos),
            &remainderBufLen);
5614
      }
5615 5616 5617
      if (generated > 0)
        i += generated;
      else
5618
        goto free_of_progressive;
5619

5620
      start_time +=  generated * timeStampStep;
5621
      pThreadInfo->totalInsertRows += generated;
5622

5623
      startTs = taosGetTimestampMs();
5624

S
Shuduo Sang 已提交
5625
      int64_t affectedRows = execInsert(pThreadInfo, generated);
5626

5627
      endTs = taosGetTimestampMs();
5628
      uint64_t delay = endTs - startTs;
5629
      performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n",
5630
              __func__, __LINE__, delay);
5631 5632 5633
      verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5634

5635 5636 5637 5638
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
5639

5640 5641 5642 5643 5644
      if (affectedRows < 0) {
        errorPrint("%s() LN%d, affected rows: %"PRId64"\n",
                __func__, __LINE__, affectedRows);
        goto free_of_progressive;
      }
5645 5646 5647

      pThreadInfo->totalAffectedRows += affectedRows;

5648 5649
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
5650
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
5651 5652 5653
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5654 5655 5656
        lastPrintTime = currentPrintTime;
      }

5657
      if (i >= insertRows)
5658 5659 5660
        break;
    }   // num_of_DPT

5661 5662
    if ((g_args.verbose_print) &&
      (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) &&
5663 5664
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
5665
          verbosePrint("%s() LN%d samplePos=%"PRId64"\n",
5666
                  __func__, __LINE__, pThreadInfo->samplePos);
5667
    }
5668
  } // tableSeq
5669

5670
free_of_progressive:
S
Shuduo Sang 已提交
5671
  tmfree(pThreadInfo->buffer);
5672
  printStatPerThread(pThreadInfo);
5673
  return NULL;
H
Hui Li 已提交
5674 5675
}

5676 5677
static void* syncWrite(void *sarg) {

5678 5679
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5680

5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692
  int interlaceRows;

  if (superTblInfo) {
    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    interlaceRows = g_args.interlace_rows;
  }
5693

5694
  if (interlaceRows > 0) {
5695
    // interlace mode
5696
    return syncWriteInterlace(pThreadInfo);
5697 5698
  } else {
    // progressive mode
5699
    return syncWriteProgressive(pThreadInfo);
5700 5701 5702
  }
}

5703
static void callBack(void *param, TAOS_RES *res, int code) {
5704 5705
  threadInfo* pThreadInfo = (threadInfo*)param;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5706

5707 5708
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5709
  if (insert_interval) {
5710 5711 5712
    pThreadInfo->et = taosGetTimestampMs();
    if ((pThreadInfo->et - pThreadInfo->st) < insert_interval) {
      taosMsleep(insert_interval - (pThreadInfo->et - pThreadInfo->st)); // ms
5713
    }
H
Hui Li 已提交
5714
  }
5715

5716
  char *buffer = calloc(1, pThreadInfo->superTblInfo->maxSqlLen);
5717
  char data[MAX_DATA_SIZE];
5718
  char *pstr = buffer;
5719 5720
  pstr += sprintf(pstr, "insert into %s.%s%"PRId64" values",
          pThreadInfo->db_name, pThreadInfo->tb_prefix,
5721
          pThreadInfo->start_table_from);
5722 5723 5724 5725 5726 5727 5728
//  if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
  if (pThreadInfo->counter >= g_args.num_of_RPR) {
    pThreadInfo->start_table_from++;
    pThreadInfo->counter = 0;
  }
  if (pThreadInfo->start_table_from > pThreadInfo->end_table_to) {
    tsem_post(&pThreadInfo->lock_sem);
5729 5730 5731 5732
    free(buffer);
    taos_free_result(res);
    return;
  }
5733

5734
  for (int i = 0; i < g_args.num_of_RPR; i++) {
5735
    int rand_num = taosRandom() % 100;
5736
    if (0 != pThreadInfo->superTblInfo->disorderRatio
5737
            && rand_num < pThreadInfo->superTblInfo->disorderRatio) {
5738 5739 5740
      int64_t d = pThreadInfo->lastTs
          - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1);
      generateStbRowData(pThreadInfo->superTblInfo, data, d);
5741
    } else {
5742 5743
      generateStbRowData(pThreadInfo->superTblInfo,
              data, pThreadInfo->lastTs += 1000);
H
Hui Li 已提交
5744
    }
5745
    pstr += sprintf(pstr, "%s", data);
5746
    pThreadInfo->counter++;
H
Hui Li 已提交
5747

5748
    if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
5749
      break;
H
Hui Li 已提交
5750 5751
    }
  }
5752

5753
  if (insert_interval) {
5754
    pThreadInfo->st = taosGetTimestampMs();
5755
  }
5756
  taos_query_a(pThreadInfo->taos, buffer, callBack, pThreadInfo);
5757
  free(buffer);
H
Hui Li 已提交
5758

5759
  taos_free_result(res);
H
Hui Li 已提交
5760 5761
}

5762
static void *asyncWrite(void *sarg) {
5763 5764
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5765

5766 5767 5768
  pThreadInfo->st = 0;
  pThreadInfo->et = 0;
  pThreadInfo->lastTs = pThreadInfo->start_time;
5769

5770
  int insert_interval =
5771
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5772
  if (insert_interval) {
5773
    pThreadInfo->st = taosGetTimestampMs();
H
Hui Li 已提交
5774
  }
5775
  taos_query_a(pThreadInfo->taos, "show databases", callBack, pThreadInfo);
5776

5777
  tsem_wait(&(pThreadInfo->lock_sem));
H
Hui Li 已提交
5778 5779 5780 5781

  return NULL;
}

5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807
static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in *serv_addr)
{
  uint16_t rest_port = port + TSDB_PORT_HTTP;
  struct hostent *server = gethostbyname(host);
  if ((server == NULL) || (server->h_addr == NULL)) {
    errorPrint("%s", "ERROR, no such host");
    return -1;
  }

  debugPrint("h_name: %s\nh_addr=%p\nh_addretype: %s\nh_length: %d\n",
            server->h_name,
            server->h_addr,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

  memset(serv_addr, 0, sizeof(struct sockaddr_in));
  serv_addr->sin_family = AF_INET;
  serv_addr->sin_port = htons(rest_port);
#ifdef WINDOWS
  serv_addr->sin_addr.s_addr = inet_addr(host);
#else
  memcpy(&(serv_addr->sin_addr.s_addr), server->h_addr, server->h_length);
#endif
  return 0;
}

5808
static void startMultiThreadInsertData(int threads, char* db_name,
5809
        char* precision,SSuperTable* superTblInfo) {
5810

5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826
  //TAOS* taos;
  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
  //  if (NULL == taos) {
  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
  //    exit(-1);
  //  }
  //}

  int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
  if (0 != precision[0]) {
    if (0 == strncasecmp(precision, "ms", 2)) {
      timePrec = TSDB_TIME_PRECISION_MILLI;
    }  else if (0 == strncasecmp(precision, "us", 2)) {
      timePrec = TSDB_TIME_PRECISION_MICRO;
    }  else {
5827
      errorPrint("Not support precision: %s\n", precision);
5828 5829 5830 5831
      exit(-1);
    }
  }

5832
  int64_t start_time;
5833 5834
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
5835
        start_time = taosGetTimestamp(timePrec);
5836
    } else {
5837
      if (TSDB_CODE_SUCCESS != taosParseTime(
5838 5839 5840 5841 5842
        superTblInfo->startTimestamp,
        &start_time,
        strlen(superTblInfo->startTimestamp),
        timePrec, 0)) {
          ERROR_EXIT("failed to parse time!\n");
5843
      }
5844
    }
5845
  } else {
5846
     start_time = 1500000000000;
5847 5848
  }

5849
  int64_t start = taosGetTimestampMs();
5850

5851
  // read sample data from file first
5852
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5853
              "sample", strlen("sample")))) {
5854
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5855
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
5856
              __func__, __LINE__);
5857 5858 5859 5860
      exit(-1);
    }
  }

5861
  // read sample data from file first
5862
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5863
              "sample", strlen("sample")))) {
5864
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5865
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
5866
              __func__, __LINE__);
5867 5868 5869 5870
      exit(-1);
    }
  }

5871
  TAOS* taos0 = taos_connect(
5872 5873
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
5874
  if (NULL == taos0) {
5875
    errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
5876
                __func__, __LINE__, taos_errstr(NULL));
5877 5878 5879
    exit(-1);
  }

S
Shuduo Sang 已提交
5880
  int64_t ntables = 0;
S
Shuduo Sang 已提交
5881
  uint64_t startFrom;
5882

5883
  if (superTblInfo) {
5884 5885
    int64_t limit;
    uint64_t offset;
5886

H
Haojun Liao 已提交
5887
    if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) &&
5888
            ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) {
H
Haojun Liao 已提交
5889
      printf("WARNING: offset and limit will not be used since the child tables not exists!\n");
5890 5891
    }

5892
    if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) {
5893 5894
      if ((superTblInfo->childTblLimit < 0)
          || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit)
5895
            > (superTblInfo->childTblCount))) {
5896 5897 5898 5899 5900 5901
        superTblInfo->childTblLimit =
            superTblInfo->childTblCount - superTblInfo->childTblOffset;
      }

      offset = superTblInfo->childTblOffset;
      limit = superTblInfo->childTblLimit;
5902
    } else {
5903 5904
      limit = superTblInfo->childTblCount;
      offset = 0;
5905 5906
    }

5907 5908 5909
    ntables = limit;
    startFrom = offset;

5910 5911 5912 5913
    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
        && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
            > superTblInfo->childTblCount)) {
      printf("WARNING: specified offset + limit > child table count!\n");
5914
      prompt();
5915 5916 5917
    }

    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
5918
            && (0 == superTblInfo->childTblLimit)) {
5919
      printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
5920
      prompt();
5921 5922
    }

5923
    superTblInfo->childTblName = (char*)calloc(1,
5924
        limit * TSDB_TABLE_NAME_LEN);
5925
    if (superTblInfo->childTblName == NULL) {
5926
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
5927
      taos_close(taos0);
5928 5929 5930
      exit(-1);
    }

5931
    int64_t childTblCount;
5932
    getChildNameOfSuperTableWithLimitAndOffset(
5933
        taos0,
5934 5935 5936 5937
        db_name, superTblInfo->sTblName,
        &superTblInfo->childTblName, &childTblCount,
        limit,
        offset);
5938 5939 5940
  } else {
    ntables = g_args.num_of_tables;
    startFrom = 0;
5941
  }
5942

5943
  taos_close(taos0);
5944

S
Shuduo Sang 已提交
5945
  int64_t a = ntables / threads;
5946 5947 5948 5949 5950
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

S
Shuduo Sang 已提交
5951
  int64_t b = 0;
5952 5953 5954 5955
  if (threads != 0) {
    b = ntables % threads;
  }

5956
  if ((superTblInfo)
5957 5958 5959 5960
      && (superTblInfo->insertMode == REST_IFACE)) {
      if (convertHostToServAddr(g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) {
        exit(-1);
      }
5961 5962
  }

5963 5964 5965 5966 5967 5968 5969 5970 5971
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  assert(pids != NULL);

  threadInfo *infos = calloc(1, threads * sizeof(threadInfo));
  assert(infos != NULL);

  memset(pids, 0, threads * sizeof(pthread_t));
  memset(infos, 0, threads * sizeof(threadInfo));

H
Hui Li 已提交
5972
  for (int i = 0; i < threads; i++) {
5973
    threadInfo *t_info = infos + i;
H
Hui Li 已提交
5974 5975
    t_info->threadID = i;
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
5976
    t_info->time_precision = timePrec;
5977 5978 5979
    t_info->superTblInfo = superTblInfo;

    t_info->start_time = start_time;
5980
    t_info->minDelay = UINT64_MAX;
5981

5982
    if ((NULL == superTblInfo) ||
5983
            (superTblInfo->insertMode != REST_IFACE)) {
5984
      //t_info->taos = taos;
5985
      t_info->taos = taos_connect(
5986 5987
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
5988
      if (NULL == t_info->taos) {
5989
        errorPrint(
5990 5991
                "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n",
                __func__, __LINE__,
5992
                taos_errstr(NULL));
5993
        free(infos);
5994 5995
        exit(-1);
      }
5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008

      if ((superTblInfo) && (superTblInfo->insertMode == STMT_IFACE)) {
        t_info->stmt = taos_stmt_init(t_info->taos);
        if (NULL == t_info->stmt) {
            errorPrint(
                "%s() LN%d, failed init stmt, reason: %s\n",
                __func__, __LINE__,
                taos_errstr(NULL));
            free(pids);
            free(infos);
            exit(-1);
        }
      }
6009 6010 6011 6012
    } else {
      t_info->taos = NULL;
    }

6013
/*    if ((NULL == superTblInfo)
6014
            || (0 == superTblInfo->multiThreadWriteOneTbl)) {
6015
            */
6016 6017 6018 6019
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
6020
/*    } else {
6021 6022
      t_info->start_table_from = 0;
      t_info->ntables = superTblInfo->childTblCount;
6023 6024
      t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint();
    }
6025
*/
6026
    tsem_init(&(t_info->lock_sem), 0, 0);
6027
    if (ASYNC_MODE == g_Dbs.asyncMode) {
6028
      pthread_create(pids + i, NULL, asyncWrite, t_info);
6029 6030
    } else {
      pthread_create(pids + i, NULL, syncWrite, t_info);
6031
    }
H
Hui Li 已提交
6032
  }
6033

H
Hui Li 已提交
6034 6035 6036 6037
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

6038 6039 6040 6041
  uint64_t totalDelay = 0;
  uint64_t maxDelay = 0;
  uint64_t minDelay = UINT64_MAX;
  uint64_t cntDelay = 1;
6042 6043
  double  avgDelay = 0;

H
Hui Li 已提交
6044
  for (int i = 0; i < threads; i++) {
6045 6046
    threadInfo *t_info = infos + i;

S
TD-1057  
Shengliang Guan 已提交
6047
    tsem_destroy(&(t_info->lock_sem));
6048 6049 6050 6051

    if (t_info->stmt) {
      taos_stmt_close(t_info->stmt);
    }
6052 6053
    taos_close(t_info->taos);

6054
    debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n",
6055 6056 6057
            __func__, __LINE__,
            t_info->threadID, t_info->totalInsertRows,
            t_info->totalAffectedRows);
6058
    if (superTblInfo) {
6059 6060
        superTblInfo->totalAffectedRows += t_info->totalAffectedRows;
        superTblInfo->totalInsertRows += t_info->totalInsertRows;
6061
    } else {
6062 6063
        g_args.totalAffectedRows += t_info->totalAffectedRows;
        g_args.totalInsertRows += t_info->totalInsertRows;
6064
    }
6065 6066 6067 6068

    totalDelay  += t_info->totalDelay;
    cntDelay   += t_info->cntDelay;
    if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay;
6069
    if (t_info->minDelay < minDelay) minDelay = t_info->minDelay;
H
Hui Li 已提交
6070
  }
6071
  cntDelay -= 1;
H
Hui Li 已提交
6072

6073
  if (cntDelay == 0)    cntDelay = 1;
6074 6075
  avgDelay = (double)totalDelay / cntDelay;

6076 6077
  int64_t end = taosGetTimestampMs();
  int64_t t = end - start;
6078 6079

  if (superTblInfo) {
6080
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
6081 6082 6083 6084
          t / 1000.0, superTblInfo->totalInsertRows,
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
6085 6086 6087 6088

    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
6089 6090 6091 6092
          t / 1000.0, superTblInfo->totalInsertRows,
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
6093
    }
6094
  } else {
6095
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
6096 6097 6098 6099
          t / 1000.0, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          (double)g_args.totalInsertRows / (t / 1000.0));
6100 6101 6102
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
6103 6104 6105 6106
          t * 1000.0, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          (double)g_args.totalInsertRows / (t / 1000.0));
6107
    }
6108
  }
6109

6110
  fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
6111
          avgDelay, maxDelay, minDelay);
6112 6113
  if (g_fpOfInsertResult) {
    fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
6114
          avgDelay, maxDelay, minDelay);
6115
  }
6116

6117 6118
  //taos_close(taos);

H
Hui Li 已提交
6119
  free(pids);
6120
  free(infos);
H
Hui Li 已提交
6121 6122
}

6123
static void *readTable(void *sarg) {
6124
#if 1
6125
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
6126 6127
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
6128
  uint64_t sTime = rinfo->start_time;
H
hzcheng 已提交
6129 6130
  char *tb_prefix = rinfo->tb_prefix;
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
6131
  if (NULL == fp) {
6132
    errorPrint( "fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
H
Hui Li 已提交
6133 6134
    return NULL;
  }
6135

6136
  int64_t num_of_DPT;
6137
/*  if (rinfo->superTblInfo) {
6138 6139
    num_of_DPT = rinfo->superTblInfo->insertRows; //  nrecords_per_table;
  } else {
6140
  */
6141
      num_of_DPT = g_args.num_of_DPT;
6142
//  }
6143

S
Shuduo Sang 已提交
6144 6145
  int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
  int64_t totalData = num_of_DPT * num_of_tables;
6146
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6147 6148 6149 6150 6151

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
S
Shuduo Sang 已提交
6152
  printf("%"PRId64" records:\n", totalData);
H
hzcheng 已提交
6153 6154
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

6155
  for (int j = 0; j < n; j++) {
H
hzcheng 已提交
6156
    double totalT = 0;
6157
    uint64_t count = 0;
6158 6159
    for (int64_t i = 0; i < num_of_tables; i++) {
      sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64,
6160
              aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
6161

6162
      double t = taosGetTimestampMs();
S
Shuaiqiang Chang 已提交
6163 6164
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
6165

S
Shuaiqiang Chang 已提交
6166
      if (code != 0) {
6167
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6168
        taos_free_result(pSql);
H
hzcheng 已提交
6169
        taos_close(taos);
6170
        fclose(fp);
6171
        return NULL;
H
hzcheng 已提交
6172 6173
      }

6174
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6175 6176 6177
        count++;
      }

6178
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6179 6180
      totalT += t;

S
Shuaiqiang Chang 已提交
6181
      taos_free_result(pSql);
H
hzcheng 已提交
6182 6183
    }

6184
    fprintf(fp, "|%10s  |   %"PRId64"   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
6185
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
6186
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
6187
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000);
H
hzcheng 已提交
6188 6189 6190
  }
  fprintf(fp, "\n");
  fclose(fp);
6191
#endif
H
hzcheng 已提交
6192 6193 6194
  return NULL;
}

6195
static void *readMetric(void *sarg) {
6196
#if 1
6197
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
6198 6199 6200
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
6201 6202 6203 6204
  if (NULL == fp) {
    printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
    return NULL;
  }
6205

6206
  int64_t num_of_DPT = rinfo->superTblInfo->insertRows;
S
Shuduo Sang 已提交
6207 6208
  int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
  int64_t totalData = num_of_DPT * num_of_tables;
6209
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6210 6211 6212 6213 6214

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
S
Shuduo Sang 已提交
6215 6216
  printf("%"PRId64" records:\n", totalData);
  fprintf(fp, "Querying On %"PRId64" records:\n", totalData);
H
hzcheng 已提交
6217 6218

  for (int j = 0; j < n; j++) {
6219
    char condition[COND_BUF_LEN] = "\0";
B
Bomin Zhang 已提交
6220
    char tempS[64] = "\0";
H
hzcheng 已提交
6221

6222
    int64_t m = 10 < num_of_tables ? 10 : num_of_tables;
H
hzcheng 已提交
6223

6224
    for (int64_t i = 1; i <= m; i++) {
H
hzcheng 已提交
6225
      if (i == 1) {
6226
        sprintf(tempS, "t1 = %"PRId64"", i);
H
hzcheng 已提交
6227
      } else {
6228
        sprintf(tempS, " or t1 = %"PRId64" ", i);
H
hzcheng 已提交
6229
      }
6230
      strncat(condition, tempS, COND_BUF_LEN - 1);
H
hzcheng 已提交
6231

L
liu0x54 已提交
6232
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
6233 6234 6235 6236

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

6237
      double t = taosGetTimestampMs();
H
hzcheng 已提交
6238

S
Shuaiqiang Chang 已提交
6239 6240 6241 6242
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
6243
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6244
        taos_free_result(pSql);
H
hzcheng 已提交
6245
        taos_close(taos);
6246
        fclose(fp);
6247
        return NULL;
H
hzcheng 已提交
6248 6249
      }
      int count = 0;
6250
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6251 6252
        count++;
      }
6253
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6254

6255
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
6256 6257
              num_of_tables * num_of_DPT / (t * 1000.0), t);
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0);
H
hzcheng 已提交
6258

S
Shuaiqiang Chang 已提交
6259
      taos_free_result(pSql);
H
hzcheng 已提交
6260 6261 6262 6263
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
6264
#endif
H
hzcheng 已提交
6265 6266 6267
  return NULL;
}

6268 6269 6270 6271 6272 6273 6274
static void prompt()
{
  if (!g_args.answer_yes) {
    printf("         Press enter key to continue or Ctrl-C to stop\n\n");
    (void)getchar();
  }
}
H
Hui Li 已提交
6275

6276
static int insertTestProcess() {
6277

6278 6279 6280
  setupForAnsiEscape();
  int ret = printfInsertMeta();
  resetAfterAnsiEscape();
6281

6282 6283 6284
  if (ret == -1)
    exit(EXIT_FAILURE);

6285
  debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
6286 6287
  g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
  if (NULL == g_fpOfInsertResult) {
6288
    errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
6289 6290
    return -1;
  }
6291

6292 6293
  if (g_fpOfInsertResult)
    printfInsertMetaToFile(g_fpOfInsertResult);
6294

6295
  prompt();
6296

6297 6298 6299
  init_rand_data();

  // create database and super tables
6300
  if(createDatabasesAndStables() != 0) {
6301 6302
    if (g_fpOfInsertResult)
      fclose(g_fpOfInsertResult);
6303 6304
    return -1;
  }
6305 6306

  // pretreatement
6307
  prepareSampleData();
6308

6309 6310 6311 6312
  double start;
  double end;

  // create child tables
6313
  start = taosGetTimestampMs();
6314
  createChildTables();
6315
  end = taosGetTimestampMs();
6316

6317
  if (g_totalChildTables > 0) {
6318
    fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6319
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6320 6321
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
6322
            "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6323
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6324
    }
6325
  }
6326

6327
  taosMsleep(1000);
6328
  // create sub threads for inserting data
6329
  //start = taosGetTimestampMs();
6330
  for (int i = 0; i < g_Dbs.dbCount; i++) {
6331 6332
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
S
Shuduo Sang 已提交
6333
        for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6334 6335 6336 6337 6338

          SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];

          if (superTblInfo && (superTblInfo->insertRows > 0)) {
            startMultiThreadInsertData(
6339 6340 6341 6342
              g_Dbs.threadCount,
              g_Dbs.db[i].dbName,
              g_Dbs.db[i].dbCfg.precision,
              superTblInfo);
6343
          }
6344
        }
6345
      }
6346
    } else {
6347
        startMultiThreadInsertData(
6348 6349 6350
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
6351
          NULL);
H
Hui Li 已提交
6352
    }
6353
  }
6354
  //end = taosGetTimestampMs();
6355

6356
  //int64_t    totalInsertRows = 0;
6357
  //int64_t    totalAffectedRows = 0;
6358
  //for (int i = 0; i < g_Dbs.dbCount; i++) {
6359
  //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6360
  //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
6361 6362
  //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
  //}
6363
  //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
6364
  postFreeResource();
6365

6366 6367 6368
  return 0;
}

6369 6370
static void *specifiedTableQuery(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6371

6372
  if (pThreadInfo->taos == NULL) {
6373 6374
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
6375 6376 6377 6378
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
6379 6380
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6381
            pThreadInfo->threadID, taos_errstr(NULL));
6382 6383
      return NULL;
    } else {
6384
      pThreadInfo->taos = taos;
6385 6386 6387
    }
  }

6388 6389
  char sqlStr[MAX_DB_NAME_SIZE + 5];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6390 6391
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6392 6393 6394 6395
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
    return NULL;
  }
6396

6397 6398
  uint64_t st = 0;
  uint64_t et = 0;
6399

6400
  uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes;
6401

6402 6403 6404
  uint64_t totalQueried = 0;
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
6405

6406 6407 6408 6409 6410 6411
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
        sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }

6412
  while(queryTimes --) {
6413
    if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) <
6414
            (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) {
6415
      taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms
6416 6417
    }

6418 6419 6420
    st = taosGetTimestampMs();

    selectAndGetResult(pThreadInfo,
6421
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]);
6422

6423 6424 6425 6426
    et = taosGetTimestampMs();
    printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n",
              taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0);

6427 6428
    totalQueried ++;
    g_queryInfo.specifiedQueryInfo.totalQueried ++;
6429

6430 6431
    uint64_t  currentPrintTime = taosGetTimestampMs();
    uint64_t  endTs = taosGetTimestampMs();
6432
    if (currentPrintTime - lastPrintTime > 30*1000) {
6433
      debugPrint("%s() LN%d, endTs=%"PRIu64"ms, startTs=%"PRIu64"ms\n",
6434
          __func__, __LINE__, endTs, startTs);
6435
      printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.6f\n",
6436 6437
                    pThreadInfo->threadID,
                    totalQueried,
6438
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6439
      lastPrintTime = currentPrintTime;
6440
    }
H
Hui Li 已提交
6441
  }
6442 6443
  return NULL;
}
H
Hui Li 已提交
6444

6445
static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) {
6446 6447
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
6448 6449
  sprintf(subTblName, "%s.%s",
          g_queryInfo.dbName,
6450
          g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
6451 6452

  //printf("inSql: %s\n", inSql);
6453

6454 6455
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
6456
    return;
H
Hui Li 已提交
6457
  }
6458

6459
  tstrncpy(outSql, inSql, pos - inSql + 1);
6460
  //printf("1: %s\n", outSql);
6461
  strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1);
6462
  //printf("2: %s\n", outSql);
6463
  strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1);
6464
  //printf("3: %s\n", outSql);
H
Hui Li 已提交
6465 6466
}

6467
static void *superTableQuery(void *sarg) {
6468
  char sqlstr[MAX_QUERY_SQL_LENGTH];
6469
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6470

6471
  if (pThreadInfo->taos == NULL) {
6472 6473 6474 6475 6476 6477 6478 6479
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6480
            pThreadInfo->threadID, taos_errstr(NULL));
6481 6482
      return NULL;
    } else {
6483
      pThreadInfo->taos = taos;
6484 6485 6486
    }
  }

6487 6488
  uint64_t st = 0;
  uint64_t et = (int64_t)g_queryInfo.superQueryInfo.queryInterval;
6489

6490 6491 6492
  uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes;
  uint64_t totalQueried = 0;
  uint64_t  startTs = taosGetTimestampMs();
6493

6494
  uint64_t  lastPrintTime = taosGetTimestampMs();
6495
  while(queryTimes --) {
6496 6497 6498
    if (g_queryInfo.superQueryInfo.queryInterval
            && (et - st) < (int64_t)g_queryInfo.superQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.superQueryInfo.queryInterval - (et - st)); // ms
6499
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to);
6500
    }
H
Hui Li 已提交
6501

6502
    st = taosGetTimestampMs();
6503
    for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) {
6504
      for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
6505
        memset(sqlstr,0,sizeof(sqlstr));
6506
        replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i);
6507
        if (g_queryInfo.superQueryInfo.result[j][0] != 0) {
6508
          sprintf(pThreadInfo->fp, "%s-%d",
6509
                  g_queryInfo.superQueryInfo.result[j],
6510
                  pThreadInfo->threadID);
6511
        }
6512
        selectAndGetResult(pThreadInfo, sqlstr);
6513 6514 6515 6516 6517 6518 6519

        totalQueried++;
        g_queryInfo.superQueryInfo.totalQueried ++;

        int64_t  currentPrintTime = taosGetTimestampMs();
        int64_t  endTs = taosGetTimestampMs();
        if (currentPrintTime - lastPrintTime > 30*1000) {
6520
          printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.3f\n",
6521 6522
                    pThreadInfo->threadID,
                    totalQueried,
6523
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6524
          lastPrintTime = currentPrintTime;
6525
        }
6526
      }
H
Hui Li 已提交
6527
    }
6528
    et = taosGetTimestampMs();
6529
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%"PRIu64" - %"PRIu64"] once queries duration:%.4fs\n\n",
6530
            taosGetSelfPthreadId(),
6531 6532
            pThreadInfo->start_table_from,
            pThreadInfo->end_table_to,
6533
            (double)(et - st)/1000.0);
6534
  }
6535

6536 6537 6538
  return NULL;
}

6539
static int queryTestProcess() {
6540 6541 6542 6543

  setupForAnsiEscape();
  printfQueryMeta();
  resetAfterAnsiEscape();
6544 6545 6546 6547 6548 6549

  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
6550
          g_queryInfo.port);
6551
  if (taos == NULL) {
6552 6553
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6554 6555 6556
    exit(-1);
  }

6557
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6558 6559
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6560 6561 6562
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6563
  }
6564

6565
  prompt();
6566

6567 6568 6569
  if (g_args.debug_print || g_args.verbose_print) {
    printfQuerySystemInfo(taos);
  }
6570

6571 6572 6573 6574 6575 6576
  if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
    if (convertHostToServAddr(
        g_queryInfo.host, g_queryInfo.port, &g_queryInfo.serv_addr) != 0)
      exit(-1);
  }

6577 6578 6579
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
6580
  int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent;
6581
  uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount;
6582

6583
  uint64_t startTs = taosGetTimestampMs();
6584

6585 6586 6587 6588 6589 6590
  if ((nSqlCount > 0) && (nConcurrent > 0)) {

    pids  = malloc(nConcurrent * nSqlCount * sizeof(pthread_t));
    infos = malloc(nConcurrent * nSqlCount * sizeof(threadInfo));

    if ((NULL == pids) || (NULL == infos)) {
6591
      taos_close(taos);
6592
      ERROR_EXIT("memory allocation failed for create threads\n");
6593
    }
6594

6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615
    for (uint64_t i = 0; i < nSqlCount; i++) {
        for (int j = 0; j < nConcurrent; j++) {
            uint64_t seq = i * nConcurrent + j;
            threadInfo *t_info = infos + seq;
            t_info->threadID = seq;
            t_info->querySeq = i;

            if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {

                char sqlStr[MAX_TB_NAME_SIZE*2];
                sprintf(sqlStr, "use %s", g_queryInfo.dbName);
                verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
                if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) {
                    taos_close(taos);
                    free(infos);
                    free(pids);
                    errorPrint( "use database %s failed!\n\n",
                        g_queryInfo.dbName);
                    return -1;
                }
            }
6616

6617
            t_info->taos = NULL;// TODO: workaround to use separate taos connection;
6618

6619 6620
            pthread_create(pids + seq, NULL, specifiedTableQuery,
                t_info);
6621
        }
6622
    }
6623
  } else {
6624
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
6625
  }
6626

6627 6628
  taos_close(taos);

6629 6630 6631
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
6632 6633 6634 6635
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
    pidsOfSub  = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t));
    infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo));
6636 6637

    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
6638 6639
      free(infos);
      free(pids);
6640

6641
      ERROR_EXIT("memory allocation failed for create threads\n");
6642
    }
6643

S
Shuduo Sang 已提交
6644
    int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
6645
    int threads = g_queryInfo.superQueryInfo.threadCnt;
6646

S
Shuduo Sang 已提交
6647
    int64_t a = ntables / threads;
6648 6649 6650 6651
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
6652

S
Shuduo Sang 已提交
6653
    int64_t b = 0;
6654 6655 6656
    if (threads != 0) {
      b = ntables % threads;
    }
6657

6658
    uint64_t startFrom = 0;
6659
    for (int i = 0; i < threads; i++) {
6660 6661
      threadInfo *t_info = infosOfSub + i;
      t_info->threadID = i;
6662

6663 6664 6665 6666
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
6667
      t_info->taos = NULL; // TODO: workaround to use separate taos connection;
6668
      pthread_create(pidsOfSub + i, NULL, superTableQuery, t_info);
6669 6670
    }

6671
    g_queryInfo.superQueryInfo.threadCnt = threads;
6672
  } else {
6673
    g_queryInfo.superQueryInfo.threadCnt = 0;
6674
  }
6675

6676 6677 6678 6679 6680 6681
  if ((nSqlCount > 0) && (nConcurrent > 0)) {
    for (int i = 0; i < nConcurrent; i++) {
      for (int j = 0; j < nSqlCount; j++) {
        pthread_join(pids[i * nSqlCount + j], NULL);
      }
    }
6682
  }
S
Shuaiqiang Chang 已提交
6683

6684
  tmfree((char*)pids);
6685
  tmfree((char*)infos);
6686

6687
  for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
6688
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
6689
  }
H
Hui Li 已提交
6690

6691
  tmfree((char*)pidsOfSub);
6692
  tmfree((char*)infosOfSub);
6693

6694
//  taos_close(taos);// TODO: workaround to use separate taos connection;
6695
  uint64_t endTs = taosGetTimestampMs();
6696

6697
  uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried +
6698 6699
    g_queryInfo.superQueryInfo.totalQueried;

6700
  fprintf(stderr, "==== completed total queries: %"PRIu64", the QPS of all threads: %10.3f====\n",
6701
          totalQueried,
6702
          (double)(totalQueried/((endTs-startTs)/1000.0)));
6703 6704 6705
  return 0;
}

6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720
static void stable_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
  if (res == NULL || taos_errno(res) != 0) {
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
    return;
  }

  if (param)
    appendResultToFile(res, ((threadInfo *)param)->fp);
  // tao_unscribe() will free result.
}

static void specified_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
6721
  if (res == NULL || taos_errno(res) != 0) {
6722 6723
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
6724 6725
    return;
  }
6726

6727
  if (param)
6728
    appendResultToFile(res, ((threadInfo *)param)->fp);
6729
  // tao_unscribe() will free result.
H
hzcheng 已提交
6730 6731
}

6732
static TAOS_SUB* subscribeImpl(
6733 6734 6735 6736
        QUERY_CLASS class,
        threadInfo *pThreadInfo,
        char *sql, char* topic, bool restart, uint64_t interval)
{
6737
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
6738

6739 6740 6741 6742
  if ((SPECIFIED_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) {
    tsub = taos_subscribe(
            pThreadInfo->taos,
6743
            restart,
6744
            topic, sql, specified_sub_callback, (void*)pThreadInfo,
6745
            g_queryInfo.specifiedQueryInfo.subscribeInterval);
6746 6747 6748 6749 6750 6751 6752
  } else if ((STABLE_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) {
    tsub = taos_subscribe(
            pThreadInfo->taos,
            restart,
            topic, sql, stable_sub_callback, (void*)pThreadInfo,
            g_queryInfo.superQueryInfo.subscribeInterval);
6753
  } else {
6754 6755
    tsub = taos_subscribe(
            pThreadInfo->taos,
6756
            restart,
6757
            topic, sql, NULL, NULL, interval);
6758
  }
6759

6760 6761 6762
  if (tsub == NULL) {
    printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
    return NULL;
6763
  }
6764

6765 6766
  return tsub;
}
H
hzcheng 已提交
6767

6768 6769
static void *superSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6770
  char subSqlstr[MAX_QUERY_SQL_LENGTH];
6771
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT] = {0};
6772
  uint64_t tsubSeq;
H
hzcheng 已提交
6773

6774 6775
  if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
      errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n",
6776 6777 6778 6779 6780
              pThreadInfo->ntables,
              MAX_QUERY_SQL_COUNT);
      exit(-1);
  }

6781
  if (pThreadInfo->taos == NULL) {
6782 6783 6784 6785 6786 6787 6788 6789
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6790
            pThreadInfo->threadID, taos_errstr(NULL));
6791 6792
      return NULL;
    } else {
6793
      pThreadInfo->taos = taos;
6794 6795 6796
    }
  }

6797 6798
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6799 6800
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6801 6802
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
6803 6804
    return NULL;
  }
6805

S
Shengliang Guan 已提交
6806 6807 6808
  char topic[32] = {0};
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6809 6810

      tsubSeq = i - pThreadInfo->start_table_from;
6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829
      verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n",
              __func__, __LINE__,
              pThreadInfo->threadID,
              pThreadInfo->start_table_from,
              pThreadInfo->end_table_to, i);
      sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"",
              i, pThreadInfo->querySeq);
      memset(subSqlstr, 0, sizeof(subSqlstr));
      replaceChildTblName(
              g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq],
              subSqlstr, i);
      if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
        sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
      }

      debugPrint("%s() LN%d, [%d] subSqlstr: %s\n",
              __func__, __LINE__, pThreadInfo->threadID, subSqlstr);
6830
      tsub[tsubSeq] = subscribeImpl(
6831 6832 6833 6834
              STABLE_CLASS,
              pThreadInfo, subSqlstr, topic,
              g_queryInfo.superQueryInfo.subscribeRestart,
              g_queryInfo.superQueryInfo.subscribeInterval);
6835
      if (NULL == tsub[tsubSeq]) {
6836
        taos_close(pThreadInfo->taos);
6837 6838
        return NULL;
      }
S
Shengliang Guan 已提交
6839
  }
6840 6841

  // start loop to consume result
6842 6843 6844 6845
  int consumed[MAX_QUERY_SQL_COUNT];
  for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) {
    consumed[i] = 0;
  }
6846
  TAOS_RES* res = NULL;
6847

6848 6849
  uint64_t st = 0, et = 0;

6850
  while(1) {
6851 6852
    for (uint64_t i = pThreadInfo->start_table_from;
            i <= pThreadInfo->end_table_to; i++) {
6853 6854 6855 6856
      tsubSeq = i - pThreadInfo->start_table_from;
      if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
          continue;
      }
6857

6858 6859
      st = taosGetTimestampMs();
      performancePrint("st: %"PRIu64" et: %"PRIu64" st-et: %"PRIu64"\n", st, et, (st - et));
6860
      res = taos_consume(tsub[tsubSeq]);
6861 6862 6863
      et = taosGetTimestampMs();
      performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st));

6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880
      if (res) {
          if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
              sprintf(pThreadInfo->fp, "%s-%d",
                      g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
              appendResultToFile(res, pThreadInfo->fp);
          }
          if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
              sprintf(pThreadInfo->fp, "%s-%d",
                      g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
              appendResultToFile(res, pThreadInfo->fp);
          }
          consumed[tsubSeq] ++;

          if ((g_queryInfo.superQueryInfo.subscribeKeepProgress)
                  && (consumed[tsubSeq] >=
6881
                      g_queryInfo.superQueryInfo.resubAfterConsume)) {
6882 6883 6884 6885
              printf("keepProgress:%d, resub super table query: %"PRIu64"\n",
                      g_queryInfo.superQueryInfo.subscribeKeepProgress,
                      pThreadInfo->querySeq);
              taos_unsubscribe(tsub[tsubSeq],
6886
                    g_queryInfo.superQueryInfo.subscribeKeepProgress);
6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899
              consumed[tsubSeq]= 0;
              tsub[tsubSeq] = subscribeImpl(
                      STABLE_CLASS,
                      pThreadInfo, subSqlstr, topic,
                      g_queryInfo.superQueryInfo.subscribeRestart,
                      g_queryInfo.superQueryInfo.subscribeInterval
                      );
              if (NULL == tsub[tsubSeq]) {
                  taos_close(pThreadInfo->taos);
                  return NULL;
              }
          }
      }
H
hzcheng 已提交
6900 6901
    }
  }
6902
  taos_free_result(res);
6903

S
Shengliang Guan 已提交
6904 6905
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6906 6907
    tsubSeq = i - pThreadInfo->start_table_from;
    taos_unsubscribe(tsub[tsubSeq], 0);
6908
  }
6909

6910
  taos_close(pThreadInfo->taos);
H
hzcheng 已提交
6911 6912 6913
  return NULL;
}

6914 6915
static void *specifiedSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6916
  TAOS_SUB*  tsub = NULL;
6917

6918
  if (pThreadInfo->taos == NULL) {
6919 6920 6921 6922 6923 6924 6925 6926
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6927
            pThreadInfo->threadID, taos_errstr(NULL));
6928 6929
      return NULL;
    } else {
6930
      pThreadInfo->taos = taos;
6931 6932 6933
    }
  }

6934 6935
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6936
  debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
6937 6938
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6939 6940
    return NULL;
  }
6941

S
Shengliang Guan 已提交
6942
  char topic[32] = {0};
6943 6944 6945 6946 6947 6948 6949 6950 6951 6952
  sprintf(topic, "taosdemo-subscribe-%"PRIu64"", pThreadInfo->querySeq);
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
      sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }
  tsub = subscribeImpl(
                      SPECIFIED_CLASS, pThreadInfo,
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
          topic,
6953
          g_queryInfo.specifiedQueryInfo.subscribeRestart,
6954 6955 6956 6957
          g_queryInfo.specifiedQueryInfo.subscribeInterval);
  if (NULL == tsub) {
      taos_close(pThreadInfo->taos);
      return NULL;
S
Shengliang Guan 已提交
6958
  }
6959

6960 6961
  // start loop to consume result
  TAOS_RES* res = NULL;
6962

6963
  int consumed;
6964

6965
  while(1) {
6966
      if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) {
6967 6968
        continue;
      }
6969

6970
      res = taos_consume(tsub);
6971
      if (res) {
6972 6973 6974 6975 6976 6977 6978
          if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
              sprintf(pThreadInfo->fp, "%s-%d",
                      g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
              appendResultToFile(res, pThreadInfo->fp);
          }

6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996
          consumed ++;
          if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress)
                && (consumed >=
                    g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
              printf("keepProgress:%d, resub specified query: %"PRIu64"\n",
                    g_queryInfo.specifiedQueryInfo.subscribeKeepProgress,
                    pThreadInfo->querySeq);
              consumed = 0;
              taos_unsubscribe(tsub,
                      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
              tsub = subscribeImpl(
                      SPECIFIED_CLASS,
                      pThreadInfo,
                      g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
                      topic,
                      g_queryInfo.specifiedQueryInfo.subscribeRestart,
                      g_queryInfo.specifiedQueryInfo.subscribeInterval);
              if (NULL == tsub) {
6997 6998
                taos_close(pThreadInfo->taos);
                return NULL;
6999 7000
              }
          }
7001 7002 7003
      }
  }
  taos_free_result(res);
7004
  taos_unsubscribe(tsub, 0);
7005
  taos_close(pThreadInfo->taos);
7006

H
hzcheng 已提交
7007 7008 7009
  return NULL;
}

7010
static int subscribeTestProcess() {
7011
  setupForAnsiEscape();
7012
  printfQueryMeta();
7013
  resetAfterAnsiEscape();
7014

7015
  prompt();
7016

7017 7018 7019 7020 7021
  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
7022
          g_queryInfo.port);
7023
  if (taos == NULL) {
7024 7025
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
7026 7027 7028
    exit(-1);
  }

7029
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
7030 7031
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
7032 7033 7034
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
7035 7036
  }

7037 7038
  taos_close(taos); // TODO: workaround to use separate taos connection;

7039 7040
  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
7041 7042 7043 7044 7045

  pthread_t  *pidsOfStable  = NULL;
  threadInfo *infosOfStable = NULL;

  //==== create threads for query for specified table
S
Shengliang Guan 已提交
7046
  if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) {
7047
    debugPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n",
7048
              __func__, __LINE__,
S
Shengliang Guan 已提交
7049 7050 7051 7052 7053 7054 7055 7056
              g_queryInfo.specifiedQueryInfo.sqlCount);
  } else {
    if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) {
        errorPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n",
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount);
        exit(-1);
    }
7057

7058 7059 7060 7061 7062 7063 7064 7065
    pids  = malloc(
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(pthread_t));
    infos = malloc(
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(threadInfo));
S
Shengliang Guan 已提交
7066 7067 7068 7069
    if ((NULL == pids) || (NULL == infos)) {
        errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
        exit(-1);
    }
7070

7071 7072 7073 7074 7075 7076 7077 7078 7079
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
            uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
            threadInfo *t_info = infos + seq;
            t_info->threadID = seq;
            t_info->querySeq = i;
            t_info->taos = NULL;  // TODO: workaround to use separate taos connection;
            pthread_create(pids + seq, NULL, specifiedSubscribe, t_info);
        }
S
Shengliang Guan 已提交
7080
    }
H
hzcheng 已提交
7081
  }
7082

7083 7084 7085 7086 7087 7088 7089
  //==== create threads for super table query
  if (g_queryInfo.superQueryInfo.sqlCount <= 0) {
    printf("%s() LN%d, super table query sqlCount %"PRIu64".\n",
              __func__, __LINE__,
              g_queryInfo.superQueryInfo.sqlCount);
  } else {
    if ((g_queryInfo.superQueryInfo.sqlCount > 0)
7090
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
7091 7092 7093
        pidsOfStable  = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
7094
            sizeof(pthread_t));
7095 7096 7097
        infosOfStable = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
7098
            sizeof(threadInfo));
7099 7100
        if ((NULL == pidsOfStable) || (NULL == infosOfStable)) {
            errorPrint("%s() LN%d, malloc failed for create threads\n",
7101
              __func__, __LINE__);
7102 7103 7104
            // taos_close(taos);
            exit(-1);
        }
7105

7106 7107
        int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
        int threads = g_queryInfo.superQueryInfo.threadCnt;
7108

7109 7110 7111 7112 7113
        int64_t a = ntables / threads;
        if (a < 1) {
            threads = ntables;
            a = 1;
        }
7114

7115 7116 7117 7118
        int64_t b = 0;
        if (threads != 0) {
            b = ntables % threads;
        }
7119

7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136
        for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            uint64_t startFrom = 0;
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                threadInfo *t_info = infosOfStable + seq;
                t_info->threadID = seq;
                t_info->querySeq = i;

                t_info->start_table_from = startFrom;
                t_info->ntables = j<b?a+1:a;
                t_info->end_table_to = j<b?startFrom+a:startFrom+a-1;
                startFrom = t_info->end_table_to + 1;
                t_info->taos = NULL; // TODO: workaround to use separate taos connection;
                pthread_create(pidsOfStable + seq,
                        NULL, superSubscribe, t_info);
            }
        }
7137

7138
        g_queryInfo.superQueryInfo.threadCnt = threads;
7139

7140 7141 7142 7143 7144 7145
        for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                pthread_join(pidsOfStable[seq], NULL);
            }
        }
7146
    }
H
hzcheng 已提交
7147
  }
7148

7149 7150 7151 7152 7153
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
    for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
        uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
        pthread_join(pids[seq], NULL);
    }
7154
  }
H
hzcheng 已提交
7155

7156
  tmfree((char*)pids);
7157
  tmfree((char*)infos);
H
hzcheng 已提交
7158

7159 7160
  tmfree((char*)pidsOfStable);
  tmfree((char*)infosOfStable);
7161
//   taos_close(taos);
7162
  return 0;
H
hzcheng 已提交
7163 7164
}

7165
static void initOfInsertMeta() {
7166
  memset(&g_Dbs, 0, sizeof(SDbs));
7167

7168
  // set default values
7169
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7170
  g_Dbs.port = 6030;
7171 7172
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
7173
  g_Dbs.threadCount = 2;
7174 7175

  g_Dbs.use_metric = g_args.use_metric;
L
Liu Tao 已提交
7176 7177
}

7178
static void initOfQueryMeta() {
7179
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
7180

7181
  // set default values
7182
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7183
  g_queryInfo.port = 6030;
7184 7185
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
L
Liu Tao 已提交
7186 7187
}

7188
static void setParaFromArg(){
7189
  if (g_args.host) {
7190
    tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE);
7191
  } else {
7192
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7193
  }
L
Liu Tao 已提交
7194

7195
  if (g_args.user) {
7196
    tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE);
7197
  }
7198 7199

  if (g_args.password) {
7200
    tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE);
7201 7202
  }

7203 7204
  if (g_args.port) {
    g_Dbs.port = g_args.port;
7205
  }
L
Liu Tao 已提交
7206

7207 7208 7209
  g_Dbs.threadCount = g_args.num_of_threads;
  g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;

7210 7211
  g_Dbs.dbCount = 1;
  g_Dbs.db[0].drop = 1;
7212

7213
  tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE);
7214
  g_Dbs.db[0].dbCfg.replica = g_args.replica;
7215
  tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
7216

7217
  tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
7218 7219 7220 7221 7222

  g_Dbs.use_metric = g_args.use_metric;
  g_Dbs.insert_only = g_args.insert_only;

  g_Dbs.do_aggreFunc = true;
L
Liu Tao 已提交
7223

7224 7225
  char dataString[STRING_LEN];
  char **data_type = g_args.datatype;
7226

7227
  memset(dataString, 0, STRING_LEN);
L
Liu Tao 已提交
7228

7229 7230
  if (strcasecmp(data_type[0], "BINARY") == 0
          || strcasecmp(data_type[0], "BOOL") == 0
7231
          || strcasecmp(data_type[0], "NCHAR") == 0 ) {
7232
    g_Dbs.do_aggreFunc = false;
L
Liu Tao 已提交
7233
  }
H
hzcheng 已提交
7234

7235 7236 7237 7238 7239
  if (g_args.use_metric) {
    g_Dbs.db[0].superTblCount = 1;
    tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE);
    g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
    g_Dbs.threadCount = g_args.num_of_threads;
7240
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
7241
    g_Dbs.asyncMode = g_args.async_mode;
7242

7243 7244 7245 7246
    g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
    g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
    g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
7247
    tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix,
7248 7249
            g_args.tb_prefix, MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
S
Shuduo Sang 已提交
7250
    g_Dbs.db[0].superTbls[0].insertMode = g_args.iface;
7251
    tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp,
7252
            "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
7253
    g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
7254

7255
    g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
7256
    g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len;
7257

7258 7259 7260 7261 7262
    g_Dbs.db[0].superTbls[0].columnCount = 0;
    for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
      if (data_type[i] == NULL) {
        break;
      }
7263

7264
      tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
7265
              data_type[i], MAX_TB_NAME_SIZE);
7266
      g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;
7267 7268
      g_Dbs.db[0].superTbls[0].columnCount++;
    }
7269

7270 7271 7272
    if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
      g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
    } else {
7273 7274 7275 7276
      for (int i = g_Dbs.db[0].superTbls[0].columnCount;
              i < g_args.num_of_CPR; i++) {
        tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
                "INT", MAX_TB_NAME_SIZE);
7277
        g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;
7278 7279 7280
        g_Dbs.db[0].superTbls[0].columnCount++;
      }
    }
7281

7282 7283
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType,
            "INT", MAX_TB_NAME_SIZE);
7284
    g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;
7285

7286 7287
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType,
            "BINARY", MAX_TB_NAME_SIZE);
7288 7289
    g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;
    g_Dbs.db[0].superTbls[0].tagCount = 2;
7290
  } else {
7291 7292
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
    g_Dbs.db[0].superTbls[0].tagCount = 0;
H
hzcheng 已提交
7293
  }
7294 7295 7296 7297 7298 7299
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
7300

7301 7302 7303
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
7304 7305 7306
    exit(-1);
  }

7307 7308 7309 7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

7331
static void querySqlFile(TAOS* taos, char* sqlFile)
7332 7333 7334 7335 7336 7337
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
7338

7339 7340 7341 7342 7343 7344
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

7345
  double t = taosGetTimestampMs();
7346

7347
  while((read_len = tgetline(&line, &line_len, fp)) != -1) {
7348 7349 7350 7351 7352 7353 7354 7355 7356 7357 7358 7359
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
7360
    }
7361 7362

    memcpy(cmd + cmd_len, line, read_len);
7363
    verbosePrint("%s() LN%d cmd: %s\n", __func__, __LINE__, cmd);
7364 7365 7366
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) {
        errorPrint("%s() LN%d, queryDbExec %s failed!\n",
               __func__, __LINE__, cmd);
7367 7368 7369 7370 7371
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
7372 7373
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
7374 7375
  }

7376
  t = taosGetTimestampMs() - t;
7377
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
7378

7379 7380 7381 7382
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
7383 7384
}

7385
static void testMetaFile() {
7386
    if (INSERT_TEST == g_args.test_mode) {
7387 7388
      if (g_Dbs.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
7389

7390
      insertTestProcess();
7391

7392
    } else if (QUERY_TEST == g_args.test_mode) {
7393 7394
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7395

7396
      queryTestProcess();
7397

7398
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
7399 7400
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7401

7402
      subscribeTestProcess();
7403

7404 7405 7406
    }  else {
      ;
    }
7407
}
7408

7409
static void queryResult() {
7410
  // query data
7411

7412 7413 7414 7415 7416
  pthread_t read_id;
  threadInfo *rInfo = malloc(sizeof(threadInfo));
  assert(rInfo);
  rInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
  rInfo->start_table_from = 0;
7417

7418 7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439 7440 7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453
  //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
  if (g_args.use_metric) {
    rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
    rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
    rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
    tstrncpy(rInfo->tb_prefix,
          g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE);
  } else {
    rInfo->ntables = g_args.num_of_tables;
    rInfo->end_table_to = g_args.num_of_tables -1;
    tstrncpy(rInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE);
  }

  rInfo->taos = taos_connect(
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
          g_Dbs.port);
  if (rInfo->taos == NULL) {
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
    free(rInfo);
    exit(-1);
  }

  tstrncpy(rInfo->fp, g_Dbs.resultFile, MAX_FILE_NAME_LEN);

  if (!g_Dbs.use_metric) {
    pthread_create(&read_id, NULL, readTable, rInfo);
  } else {
    pthread_create(&read_id, NULL, readMetric, rInfo);
  }
  pthread_join(read_id, NULL);
  taos_close(rInfo->taos);
  free(rInfo);
7454 7455
}

7456 7457
static void testCmdLine() {

7458 7459 7460 7461 7462 7463 7464 7465 7466 7467
  if (strlen(configDir)) {
    wordexp_t full_path;
    if (wordexp(configDir, &full_path, 0) != 0) {
      errorPrint( "Invalid path %s\n", configDir);
      return;
    }
    taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
    wordfree(&full_path);
  }

7468 7469
  g_args.test_mode = INSERT_TEST;
  insertTestProcess();
7470

7471
  if (false == g_Dbs.insert_only)
7472
    queryResult();
7473 7474
}

7475 7476 7477
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

7478
  debugPrint("meta file: %s\n", g_args.metaFile);
7479 7480 7481

  if (g_args.metaFile) {
    initOfInsertMeta();
7482
    initOfQueryMeta();
7483 7484 7485 7486 7487 7488 7489 7490 7491 7492 7493 7494 7495

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
7496 7497 7498 7499
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
7500
          g_Dbs.port);
7501
      querySqlFile(qtaos, g_args.sqlFile);
7502 7503 7504 7505 7506
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
7507 7508 7509

    if (g_dupstr)
        free(g_dupstr);
H
hzcheng 已提交
7510
  }
7511 7512

  return 0;
H
hzcheng 已提交
7513
}
7514