taosdemo.c 208.5 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

S
slguan 已提交
21
#define _GNU_SOURCE
22 23
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
#else  
  #include <regex.h>
  #include <stdio.h>
#endif  
H
hzcheng 已提交
44

45
#include <assert.h>
46
#include <stdlib.h>
47 48
#include "cJSON.h"

49
#include "os.h"
H
hzcheng 已提交
50
#include "taos.h"
51
#include "taoserror.h"
H
Hui Li 已提交
52
#include "tutil.h"
S
slguan 已提交
53

54 55 56
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
57 58
extern char configDir[];

59 60 61 62
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

63 64 65 66 67 68
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92

#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
#define MAX_DB_NAME_SIZE   64
#define MAX_TB_NAME_SIZE   64
#define MAX_DATA_SIZE      16000
#define MAX_NUM_DATATYPE   10
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
#define MAX_FILE_NAME_LEN  256

#define   MAX_SAMPLES_ONCE_FROM_FILE   10000
#define   MAX_NUM_DATATYPE 10

#define   MAX_DB_COUNT           8
#define   MAX_SUPER_TABLE_COUNT  8
#define   MAX_COLUMN_COUNT       1024
#define   MAX_TAG_COUNT          128

#define   MAX_QUERY_SQL_COUNT    10
#define   MAX_QUERY_SQL_LENGTH   256

#define   MAX_DATABASE_COUNT     256
93
#define INPUT_BUF_LEN   256
94

95 96
#define DEFAULT_TIMESTAMP_STEP  10

97 98 99 100 101 102 103 104
typedef enum CREATE_SUB_TALBE_MOD_EN {
  PRE_CREATE_SUBTBL,
  AUTO_CREATE_SUBTBL,
  NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
  
typedef enum TALBE_EXISTS_EN {
  TBL_NO_EXISTS,
105
  TBL_ALREADY_EXISTS,
106 107 108 109 110 111 112 113
  TBL_EXISTS_BUTT
} TALBE_EXISTS_EN;

enum MODE {
  SYNC, 
  ASYNC,
  MODE_BUT
};
114

115 116 117 118 119 120
typedef enum enum_INSERT_MODE {
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
} INSERT_MODE;

121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
enum QUERY_TYPE {
  NO_INSERT_TYPE,
  INSERT_TYPE, 
  QUERY_TYPE_BUT
} ;  

enum _show_db_index {
  TSDB_SHOW_DB_NAME_INDEX,
  TSDB_SHOW_DB_CREATED_TIME_INDEX,
  TSDB_SHOW_DB_NTABLES_INDEX,
  TSDB_SHOW_DB_VGROUPS_INDEX,
  TSDB_SHOW_DB_REPLICA_INDEX,
  TSDB_SHOW_DB_QUORUM_INDEX,  
  TSDB_SHOW_DB_DAYS_INDEX,
  TSDB_SHOW_DB_KEEP_INDEX,
  TSDB_SHOW_DB_CACHE_INDEX,
  TSDB_SHOW_DB_BLOCKS_INDEX,
  TSDB_SHOW_DB_MINROWS_INDEX,
  TSDB_SHOW_DB_MAXROWS_INDEX,
  TSDB_SHOW_DB_WALLEVEL_INDEX,
  TSDB_SHOW_DB_FSYNC_INDEX,
  TSDB_SHOW_DB_COMP_INDEX,
  TSDB_SHOW_DB_CACHELAST_INDEX,
  TSDB_SHOW_DB_PRECISION_INDEX,  
  TSDB_SHOW_DB_UPDATE_INDEX,
  TSDB_SHOW_DB_STATUS_INDEX,
  TSDB_MAX_SHOW_DB
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
  TSDB_SHOW_STABLES_NAME_INDEX,
  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
  TSDB_SHOW_STABLES_COLUMNS_INDEX,
  TSDB_SHOW_STABLES_METRIC_INDEX,  
  TSDB_SHOW_STABLES_UID_INDEX,  
  TSDB_SHOW_STABLES_TID_INDEX,
  TSDB_SHOW_STABLES_VGID_INDEX,  
  TSDB_MAX_SHOW_STABLES
160 161
};

162 163 164 165 166 167 168 169 170 171 172 173 174 175
enum _describe_table_index {
  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
  TSDB_MAX_DESCRIBE_METRIC
};

typedef struct {
  char field[TSDB_COL_NAME_LEN + 1];
  char type[16];
  int length;
  char note[128];
} SColDes;
H
hzcheng 已提交
176 177

/* Used by main to communicate with parse_opt. */
178 179
typedef struct SArguments_S {
  char *   metaFile;
180
  int      test_mode;
181 182 183 184 185 186 187 188 189 190
  char *   host;
  uint16_t port;
  char *   user;
  char *   password;
  char *   database;
  int      replica;
  char *   tb_prefix;
  char *   sqlFile;
  bool     use_metric;
  bool     insert_only;
191
  bool     answer_yes;
192
  bool     debug_print;
193
  bool     verbose_print;
194
  bool     performance_print;
195 196 197 198 199 200
  char *   output_file;
  int      mode;
  char *   datatype[MAX_NUM_DATATYPE + 1];
  int      len_of_binary;
  int      num_of_CPR;
  int      num_of_threads;
201
  int      insert_interval;
202
  int      rows_per_tbl;
203
  int      num_of_RPR;
204
  int      max_sql_len;
205 206 207
  int      num_of_tables;
  int      num_of_DPT;
  int      abort;
208 209
  int      disorderRatio;
  int      disorderRange;
210 211
  int      method_of_delete;
  char **  arg_list;
212 213
  int64_t  totalInsertRows;
  int64_t  totalAffectedRows;
214 215 216 217 218 219 220 221 222 223
} SArguments;

typedef struct SColumn_S {
  char  field[TSDB_COL_NAME_LEN + 1];
  char  dataType[MAX_TB_NAME_SIZE];
  int   dataLen;
  char  note[128];  
} StrColumn;

typedef struct SSuperTable_S {
224
  char         sTblName[MAX_TB_NAME_SIZE+1];
225 226 227 228 229 230
  int          childTblCount;
  bool         superTblExists;    // 0: no, 1: yes
  bool         childTblExists;    // 0: no, 1: yes  
  int          batchCreateTableNum;  // 0: no batch,  > 0: batch table number in one sql
  int8_t       autoCreateTable;                  // 0: create sub table, 1: auto create sub table
  char         childTblPrefix[MAX_TB_NAME_SIZE];
231
  char         dataSource[MAX_TB_NAME_SIZE+1];  // rand_gen or sample
232
  char         insertMode[MAX_TB_NAME_SIZE];  // taosc, restful
233 234
  int           childTblLimit;
  int           childTblOffset;
235 236 237 238 239 240

  int          multiThreadWriteOneTbl;   // 0: no, 1: yes
  int          rowsPerTbl;               // 
  int          disorderRatio;            // 0: no disorder, >0: x%
  int          disorderRange;            // ms or us by database precision
  int          maxSqlLen;                // 
241 242

  int          insertInterval;          // insert interval, will override global insert interval
243 244 245 246
  int64_t      insertRows;               // 0: no limit
  int          timeStampStep;
  char         startTimestamp[MAX_TB_NAME_SIZE];  // 
  char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
247 248
  char         sampleFile[MAX_FILE_NAME_LEN+1];
  char         tagsFile[MAX_FILE_NAME_LEN+1];
249 250 251 252 253 254 255

  int          columnCount;
  StrColumn    columns[MAX_COLUMN_COUNT];
  int          tagCount;
  StrColumn    tags[MAX_TAG_COUNT];

  char*        childTblName;
256
  char*        colsOfCreateChildTable;
257 258 259 260 261 262 263 264 265 266 267 268 269 270
  int          lenOfOneRow;
  int          lenOfTagOfOneRow;

  char*        sampleDataBuf;
  int          sampleDataBufSize;
  //int          sampleRowCount;
  //int          sampleUsePos;

  int          tagSource;    // 0: rand, 1: tag sample
  char*        tagDataBuf;
  int          tagSampleCount;
  int          tagUsePos;

  // statistics  
271
  int64_t    totalInsertRows;
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302
  int64_t    totalAffectedRows;
} SSuperTable;

typedef struct {
  char     name[TSDB_DB_NAME_LEN + 1];
  char     create_time[32];
  int32_t  ntables;
  int32_t  vgroups;  
  int16_t  replica;
  int16_t  quorum;
  int16_t  days;  
  char     keeplist[32];
  int32_t  cache; //MB
  int32_t  blocks;
  int32_t  minrows;
  int32_t  maxrows;
  int8_t   wallevel;
  int32_t  fsync;
  int8_t   comp;
  int8_t   cachelast;
  char     precision[8];   // time resolution
  int8_t   update;
  char     status[16];
} SDbInfo;

typedef struct SDbCfg_S { 
//  int       maxtablesPerVnode;
  int       minRows; 
  int       maxRows;
  int       comp;
  int       walLevel;
303
  int       cacheLast;
304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
  int       fsync;  
  int       replica;
  int       update;
  int       keep;
  int       days;
  int       cache;
  int       blocks;
  int       quorum;
  char      precision[MAX_TB_NAME_SIZE];  
} SDbCfg;

typedef struct SDataBase_S {
  char         dbName[MAX_DB_NAME_SIZE];
  int          drop;  // 0: use exists, 1: if exists, drop then new create
  SDbCfg       dbCfg;
  int          superTblCount;
  SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
} SDataBase;

typedef struct SDbs_S {
324
  char         cfgDir[MAX_FILE_NAME_LEN+1];
325 326 327 328
  char         host[MAX_DB_NAME_SIZE];
  uint16_t     port;
  char         user[MAX_DB_NAME_SIZE];
  char         password[MAX_DB_NAME_SIZE];
329
  char         resultFile[MAX_FILE_NAME_LEN+1];
330 331 332 333 334 335 336 337 338 339 340
  bool         use_metric;
  bool         insert_only;
  bool         do_aggreFunc;
  bool         queryMode;
  
  int          threadCount;
  int          threadCountByCreateTbl;
  int          dbCount;
  SDataBase    db[MAX_DB_COUNT];

  // statistics
341
  int64_t    totalInsertRows;
342
  int64_t    totalAffectedRows;
343

344 345 346 347 348 349 350 351 352 353
} SDbs;

typedef struct SuperQueryInfo_S {
  int          rate;  // 0: unlimit  > 0   loop/s
  int          concurrent;
  int          sqlCount;
  int          subscribeMode; // 0: sync, 1: async
  int          subscribeInterval; // ms
  int          subscribeRestart;
  int          subscribeKeepProgress;
354 355
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];  
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
356 357 358 359
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
} SuperQueryInfo;

typedef struct SubQueryInfo_S {
360
  char         sTblName[MAX_TB_NAME_SIZE+1];
361 362 363 364 365 366 367 368 369
  int          rate;  // 0: unlimit  > 0   loop/s
  int          threadCnt;  
  int          subscribeMode; // 0: sync, 1: async
  int          subscribeInterval; // ms
  int          subscribeRestart;
  int          subscribeKeepProgress;
  int          childTblCount;
  char         childTblPrefix[MAX_TB_NAME_SIZE];
  int          sqlCount;
370 371
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];  
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
372 373 374 375 376 377
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
  
  char*        childTblName;
} SubQueryInfo;

typedef struct SQueryMetaInfo_S {
378
  char         cfgDir[MAX_FILE_NAME_LEN+1];
379 380 381 382
  char         host[MAX_DB_NAME_SIZE];
  uint16_t     port;
  char         user[MAX_DB_NAME_SIZE];
  char         password[MAX_DB_NAME_SIZE];
383
  char         dbName[MAX_DB_NAME_SIZE+1];
384 385 386 387 388 389 390 391 392
  char         queryMode[MAX_TB_NAME_SIZE];  // taosc, restful

  SuperQueryInfo  superQueryInfo;
  SubQueryInfo    subQueryInfo;  
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
  TAOS *taos;
  int threadID;
393
  char db_name[MAX_DB_NAME_SIZE+1];
394 395
  char fp[4096];
  char tb_prefix[MAX_TB_NAME_SIZE];
396 397 398
  int start_table_from;
  int end_table_to;
  int ntables;
399
  int data_of_rate;
400
  uint64_t start_time;  
401 402 403 404 405 406 407
  char* cols;  
  bool  use_metric;  
  SSuperTable* superTblInfo;

  // for async insert
  tsem_t lock_sem;
  int64_t  counter;  
408 409
  uint64_t  st;
  uint64_t  et;
410 411
  int64_t  lastTs;

412 413
  // sample data
  int samplePos;
414
  // statistics
415
  int64_t totalInsertRows;
416
  int64_t totalAffectedRows;
417 418 419 420 421 422 423 424

  // insert delay statistics
  int64_t cntDelay;
  int64_t totalDelay;
  int64_t avgDelay;
  int64_t maxDelay;
  int64_t minDelay;
  
425 426
} threadInfo;

427
#ifdef WINDOWS
428 429
#define _CRT_RAND_S

430
#include <windows.h>
431 432 433 434 435
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
436 437
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
438 439
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
440 441 442 443

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

444
static void setupForAnsiEscape(void) {
445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
  }    
}

466
static void resetAfterAnsiEscape(void) {
467 468 469 470 471 472 473 474
  // Reset colors
  printf("\x1b[0m");    

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
475

476
static int taosRandom()
477 478 479 480 481 482
{
    int number;
    rand_s(&number);

    return number;
}
483
#else
484
static void setupForAnsiEscape(void) {}
485

486
static void resetAfterAnsiEscape(void) {
487 488 489
  // Reset colors
  printf("\x1b[0m");
}
490

491
static int taosRandom()
492 493 494 495
{
    return random();
}

496 497
#endif

498 499 500
static int createDatabases();
static void createChildTables();
static int queryDbExec(TAOS *taos, char *command, int type);
501 502 503 504 505 506 507 508 509 510

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", 
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

511
SArguments g_args = {
512 513
                     NULL,            // metaFile
                     0,               // test_mode
514 515 516 517 518 519 520 521 522 523 524 525 526 527 528
                     "127.0.0.1",     // host
                     6030,            // port
                     "root",          // user
                     #ifdef _TD_POWER_ 
                     "powerdb",      // password
                     #else
                     "taosdata",      // password
                     #endif
                     "test",          // database
                     1,               // replica
                     "t",             // tb_prefix
                     NULL,            // sqlFile
                     false,           // use_metric
                     false,           // insert_only
                     false,           // debug_print
529
                     false,           // verbose_print
530
                     false,           // performance statistic print 
531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549
                     false,           // answer_yes;
                     "./output.txt",  // output_file
                     0,               // mode : sync or async
                     {
                     "TINYINT",           // datatype
                     "SMALLINT",
                     "INT",
                     "BIGINT",
                     "FLOAT",
                     "DOUBLE",
                     "BINARY",
                     "NCHAR",
                     "BOOL",
                     "TIMESTAMP"
                     },
                     16,              // len_of_binary
                     10,              // num_of_CPR
                     10,              // num_of_connections/thread
                     0,               // insert_interval
550
                     0,               // rows_per_tbl;
551
                     100,             // num_of_RPR
552
                     TSDB_PAYLOAD_SIZE,  // max_sql_len
553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569
                     10000,           // num_of_tables
                     10000,           // num_of_DPT
                     0,               // abort
                     0,               // disorderRatio
                     1000,            // disorderRange
                     1,               // method_of_delete
                     NULL             // arg_list
};



static SDbs            g_Dbs;
static int             g_totalChildTables = 0;
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

#define debugPrint(fmt, ...) \
570
    do { if (g_args.debug_print || g_args.verbose_print) \
571
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
572

573
#define verbosePrint(fmt, ...) \
574 575
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
576

577 578 579 580
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)

581 582 583 584
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)


585
///////////////////////////////////////////////////
H
hzcheng 已提交
586

587 588
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

589
static void printHelp() {
590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643
  char indent[10] = "        ";
  printf("%s%s%s%s\n", indent, "-f", indent, 
          "The meta file to the execution procedure. Default is './meta.json'.");
  printf("%s%s%s%s\n", indent, "-u", indent, 
          "The TDengine user name to use when connecting to the server. Default is 'root'.");
#ifdef _TD_POWER_
  printf("%s%s%s%s\n", indent, "-P", indent, 
          "The password to use when connecting to the server. Default is 'powerdb'.");
  printf("%s%s%s%s\n", indent, "-c", indent, 
          "Configuration directory. Default is '/etc/power/'.");
#else
  printf("%s%s%s%s\n", indent, "-P", indent, 
          "The password to use when connecting to the server. Default is 'taosdata'.");
  printf("%s%s%s%s\n", indent, "-c", indent, 
          "Configuration directory. Default is '/etc/taos/'.");
#endif  
  printf("%s%s%s%s\n", indent, "-h", indent, 
          "The host to connect to TDengine. Default is localhost.");
  printf("%s%s%s%s\n", indent, "-p", indent, 
          "The TCP/IP port number to use for the connection. Default is 0.");
  printf("%s%s%s%s\n", indent, "-d", indent, 
          "Destination database. Default is 'test'.");
  printf("%s%s%s%s\n", indent, "-a", indent, 
          "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
  printf("%s%s%s%s\n", indent, "-m", indent, 
          "Table prefix name. Default is 't'.");
  printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
  printf("%s%s%s%s\n", indent, "-M", indent, "Use metric flag.");
  printf("%s%s%s%s\n", indent, "-o", indent, 
          "Direct output to the named file. Default is './output.txt'.");
  printf("%s%s%s%s\n", indent, "-q", indent, 
          "Query mode--0: SYNC, 1: ASYNC. Default is SYNC.");
  printf("%s%s%s%s\n", indent, "-b", indent, 
          "The data_type of columns, default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP.");
  printf("%s%s%s%s\n", indent, "-w", indent, 
          "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
  printf("%s%s%s%s\n", indent, "-l", indent, 
          "The number of columns per record. Default is 10.");
  printf("%s%s%s%s\n", indent, "-T", indent, 
          "The number of threads. Default is 10.");
  printf("%s%s%s%s\n", indent, "-i", indent, 
          "The sleep time (ms) between insertion. Default is 0.");
  printf("%s%s%s%s\n", indent, "-r", indent, 
          "The number of records per request. Default is 100.");
  printf("%s%s%s%s\n", indent, "-t", indent, 
          "The number of tables. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-n", indent, 
          "The number of records per table. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
  printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
  printf("%s%s%s%s\n", indent, "-O", indent, 
          "Insert mode--0: In order, > 0: disorder ratio. Default is in order.");
  printf("%s%s%s%s\n", indent, "-R", indent, 
          "Out of order data's range, ms, default is 1000.");
644 645
  printf("%s%s%s%s\n", indent, "-g", indent, 
          "Print debug info.");
646 647 648 649 650
/*    printf("%s%s%s%s\n", indent, "-D", indent, 
          "if elete database if exists. 0: no, 1: yes, default is 1");
          */
}

651
static void parse_args(int argc, char *argv[], SArguments *arguments) {
652
  char **sptr;
653 654 655 656 657 658 659 660
  wordexp_t full_path;

  for (int i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-f") == 0) {
      arguments->metaFile = argv[++i];
    } else if (strcmp(argv[i], "-c") == 0) {
        char *configPath = argv[++i];
      if (wordexp(configPath, &full_path, 0) != 0) {
661
          errorPrint( "Invalid path %s\n", configPath);
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683
          return;
      }
      taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
      wordfree(&full_path);
    } else if (strcmp(argv[i], "-h") == 0) {
      arguments->host = argv[++i];
    } else if (strcmp(argv[i], "-p") == 0) {
      arguments->port = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-u") == 0) {
      arguments->user = argv[++i];
    } else if (strcmp(argv[i], "-P") == 0) {
      arguments->password = argv[++i];
    } else if (strcmp(argv[i], "-o") == 0) {
      arguments->output_file = argv[++i];
    } else if (strcmp(argv[i], "-s") == 0) {
      arguments->sqlFile = argv[++i];
    } else if (strcmp(argv[i], "-q") == 0) {
      arguments->mode = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-T") == 0) {
      arguments->num_of_threads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
      arguments->insert_interval = atoi(argv[++i]);
684 685
    } else if (strcmp(argv[i], "-B") == 0) {
      arguments->rows_per_tbl = atoi(argv[++i]);
686 687 688 689 690 691 692 693 694 695 696
    } else if (strcmp(argv[i], "-r") == 0) {
      arguments->num_of_RPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-t") == 0) {
      arguments->num_of_tables = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
      arguments->num_of_DPT = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-d") == 0) {
      arguments->database = argv[++i];
    } else if (strcmp(argv[i], "-l") == 0) {
      arguments->num_of_CPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-b") == 0) {
697
      sptr = arguments->datatype;
698 699
      ++i;
      if (strstr(argv[i], ",") == NULL) {
700
        // only one col
701 702 703 704 705 706 707 708 709 710
        if (strcasecmp(argv[i], "INT")
                && strcasecmp(argv[i], "FLOAT")
                && strcasecmp(argv[i], "TINYINT")
                && strcasecmp(argv[i], "BOOL")
                && strcasecmp(argv[i], "SMALLINT")
                && strcasecmp(argv[i], "BIGINT")
                && strcasecmp(argv[i], "DOUBLE")
                && strcasecmp(argv[i], "BINARY") 
                && strcasecmp(argv[i], "NCHAR")) {
          printHelp();
711
          ERROR_EXIT( "Invalid data_type!\n");
712
          exit(EXIT_FAILURE);
713
        }
714
        sptr[0] = argv[i];
715
      } else {
716
        // more than one col
717
        int index = 0;
718
        char *dupstr = strdup(argv[i]);
719 720 721
        char *running = dupstr;
        char *token = strsep(&running, ",");
        while (token != NULL) {
722 723 724 725 726 727 728 729 730 731
          if (strcasecmp(token, "INT")
                  && strcasecmp(token, "FLOAT")
                  && strcasecmp(token, "TINYINT")
                  && strcasecmp(token, "BOOL")
                  && strcasecmp(token, "SMALLINT")
                  && strcasecmp(token, "BIGINT")
                  && strcasecmp(token, "DOUBLE")
                  && strcasecmp(token, "BINARY")
                  && strcasecmp(token, "NCHAR")) {
            printHelp();
732
            ERROR_EXIT("Invalid data_type!\n");
733
            exit(EXIT_FAILURE);
H
hzcheng 已提交
734
          }
735 736 737
          sptr[index++] = token;
          token = strsep(&running, ",");
          if (index >= MAX_NUM_DATATYPE) break;
S
TD-1057  
Shengliang Guan 已提交
738
        }
739
        sptr[index] = NULL;
740
      }
741 742 743 744 745
    } else if (strcmp(argv[i], "-w") == 0) {
      arguments->len_of_binary = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-m") == 0) {
      arguments->tb_prefix = argv[++i];
    } else if (strcmp(argv[i], "-M") == 0) {
746
      arguments->use_metric = true;
747
    } else if (strcmp(argv[i], "-x") == 0) {
748
      arguments->insert_only = true;
749
    } else if (strcmp(argv[i], "-y") == 0) {
750
      arguments->answer_yes = true;
751 752
    } else if (strcmp(argv[i], "-g") == 0) {
      arguments->debug_print = true;
753 754
    } else if (strcmp(argv[i], "-gg") == 0) {
      arguments->verbose_print = true;
755 756
    } else if (strcmp(argv[i], "-pp") == 0) {
      arguments->performance_print = true;
757 758 759 760 761 762 763 764 765
    } else if (strcmp(argv[i], "-c") == 0) {
      strcpy(configDir, argv[++i]);
    } else if (strcmp(argv[i], "-O") == 0) {
      arguments->disorderRatio = atoi(argv[++i]);
      if (arguments->disorderRatio > 1 
              || arguments->disorderRatio < 0) {
        arguments->disorderRatio = 0;
      } else if (arguments->disorderRatio == 1) {
        arguments->disorderRange = 10;
766
      }
767 768 769 770 771 772
    } else if (strcmp(argv[i], "-R") == 0) {
      arguments->disorderRange = atoi(argv[++i]);
      if (arguments->disorderRange == 1 
              && (arguments->disorderRange > 50 
              || arguments->disorderRange <= 0)) {
        arguments->disorderRange = 10;
773
      }
774 775 776 777
    } else if (strcmp(argv[i], "-a") == 0) {
      arguments->replica = atoi(argv[++i]);
      if (arguments->replica > 3 || arguments->replica < 1) {
          arguments->replica = 1;
778
      }
779 780 781 782 783
    } else if (strcmp(argv[i], "-D") == 0) {
      arguments->method_of_delete = atoi(argv[++i]);
      if (arguments->method_of_delete < 0
              || arguments->method_of_delete > 3) {
        arguments->method_of_delete = 0;
S
TD-1057  
Shengliang Guan 已提交
784
      }
785 786 787 788 789
    } else if (strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else {
      printHelp();
790
      ERROR_EXIT("ERROR: wrong options\n");
791
      exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
792 793
    }
  }
794

795 796
  if (((arguments->debug_print) && (arguments->metaFile == NULL))
          || arguments->verbose_print) {
797
    printf("###################################################################\n");
798
    printf("# meta file:                         %s\n", arguments->metaFile);
799 800 801 802 803 804
    printf("# Server IP:                         %s:%hu\n", 
            arguments->host == NULL ? "localhost" : arguments->host,
            arguments->port );
    printf("# User:                              %s\n", arguments->user);
    printf("# Password:                          %s\n", arguments->password);
    printf("# Use metric:                        %s\n", arguments->use_metric ? "true" : "false");
805 806 807 808 809 810 811 812 813
    if (*(arguments->datatype)) {
        printf("# Specified data type:               ");
        for (int i = 0; i < MAX_NUM_DATATYPE; i++)
            if (arguments->datatype[i])
               printf("%s,", arguments->datatype[i]);
            else
                break;
        printf("\n");
    }
814
    printf("# Insertion interval:                %d\n", arguments->insert_interval);
815
    printf("# Number of records per req:         %d\n", arguments->num_of_RPR);
816
    printf("# Max SQL length:                    %d\n", arguments->max_sql_len);
817
    printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
818 819 820 821 822 823 824 825 826 827 828 829 830
    printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
    printf("# Number of Tables:                  %d\n", arguments->num_of_tables);
    printf("# Number of Data per Table:          %d\n", arguments->num_of_DPT);
    printf("# Database name:                     %s\n", arguments->database);
    printf("# Table prefix:                      %s\n", arguments->tb_prefix);
    if (arguments->disorderRatio) {
      printf("# Data order:                        %d\n", arguments->disorderRatio);
      printf("# Data out of order rate:            %d\n", arguments->disorderRange);
  
    }
    printf("# Delete method:                     %d\n", arguments->method_of_delete);
    printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
    printf("# Print debug info:                  %d\n", arguments->debug_print);
831
    printf("# Print verbose info:                %d\n", arguments->verbose_print);
832 833 834 835 836 837
    printf("###################################################################\n");
    if (!arguments->answer_yes) {
        printf("Press enter key to continue\n\n");
        (void) getchar();
    }
  }
838
}
H
hzcheng 已提交
839

840 841 842 843
static bool getInfoFromJsonFile(char* file);
//static int generateOneRowDataForStb(SSuperTable* stbInfo);
//static int getDataIntoMemForStb(SSuperTable* stbInfo);
static void init_rand_data();
844
static void tmfclose(FILE *fp) {
845 846 847 848
  if (NULL != fp) {
    fclose(fp);
  }
}
849

850
static void tmfree(char *buf) {
851 852
  if (NULL != buf) {
    free(buf);
H
Hui Li 已提交
853
  }
854
}
H
Hui Li 已提交
855

856
static int queryDbExec(TAOS *taos, char *command, int type) {
857 858 859
  int i;
  TAOS_RES *res = NULL;
  int32_t   code = -1;
860

861 862 863 864
  for (i = 0; i < 5; i++) {
    if (NULL != res) {
      taos_free_result(res);
      res = NULL;
H
hzcheng 已提交
865
    }
866

867 868 869 870 871
    res = taos_query(taos, command);
    code = taos_errno(res);
    if (0 == code) {
      break;
    }    
H
hzcheng 已提交
872 873
  }

874
  if (code != 0) {
875
    debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
876
    errorPrint( "Failed to run %s, reason: %s\n", command, taos_errstr(res));
877 878 879
    taos_free_result(res);
    //taos_close(taos);
    return -1;
880
  }
H
hzcheng 已提交
881

882 883 884 885
  if (INSERT_TYPE == type) {
    int affectedRows = taos_affected_rows(res);
    taos_free_result(res);
    return affectedRows;
H
hzcheng 已提交
886
  }
887
  
888
  taos_free_result(res);
889 890
  return 0;
}
H
hzcheng 已提交
891

892 893 894 895 896 897 898 899 900 901
static void getResult(TAOS_RES *res, char* resultFileName) {  
  TAOS_ROW    row = NULL;
  int         num_rows = 0;
  int         num_fields = taos_field_count(res);
  TAOS_FIELD *fields     = taos_fetch_fields(res);

  FILE *fp = NULL;
  if (resultFileName[0] != 0) {
    fp = fopen(resultFileName, "at");
    if (fp == NULL) {
902
      errorPrint("%s() LN%d, failed to open result file: %s, result will not save to file\n", __func__, __LINE__, resultFileName);
903 904 905 906 907
    }
  }
  
  char* databuf = (char*) calloc(1, 100*1024*1024);
  if (databuf == NULL) {
908
    errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n", __func__, __LINE__);
909 910
    if (fp)
        fclose(fp);
911 912
    return ;
  }
H
hzcheng 已提交
913

914 915
  int   totalLen = 0;
  char  temp[16000];
H
hzcheng 已提交
916

917 918 919 920 921 922
  // fetch the records row by row
  while ((row = taos_fetch_row(res))) {
    if (totalLen >= 100*1024*1024 - 32000) {
      if (fp) fprintf(fp, "%s", databuf);
      totalLen = 0;
      memset(databuf, 0, 100*1024*1024);
H
hzcheng 已提交
923
    }
924 925 926 927 928 929
    num_rows++;
    int len = taos_print_row(temp, row, fields, num_fields);
    len += sprintf(temp + len, "\n");
    //printf("query result:%s\n", temp);
    memcpy(databuf + totalLen, temp, len);
    totalLen += len;
H
hzcheng 已提交
930 931
  }

932 933 934 935
  if (fp) fprintf(fp, "%s", databuf);
  tmfclose(fp);
  free(databuf);
}
H
hzcheng 已提交
936

937 938 939 940 941 942
static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) {
  TAOS_RES *res = taos_query(taos, command);
  if (res == NULL || taos_errno(res) != 0) {
    printf("failed to sql:%s, reason:%s\n", command, taos_errstr(res));
    taos_free_result(res);
    return;
H
hzcheng 已提交
943
  }
H
Hui Li 已提交
944
  
945 946 947
  getResult(res, resultFileName);
  taos_free_result(res);
}
H
hzcheng 已提交
948

949
static double getCurrentTime() {
950 951 952 953
  struct timeval tv;
  if (gettimeofday(&tv, NULL) != 0) {
    perror("Failed to get current time in ms");
    return 0.0;
H
hzcheng 已提交
954
  }
S
Shuduo Sang 已提交
955

956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
  return tv.tv_sec + tv.tv_usec / 1E6;
}

static int32_t rand_bool(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 2;
}

static int32_t rand_tinyint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 128;
}

static int32_t rand_smallint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 32767;
}

static int32_t rand_int(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor];
}

static int64_t rand_bigint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randbigint[cursor];
  
}

static float rand_float(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randfloat[cursor];    
}

static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1003
static void rand_string(char *str, int size) {
1004 1005 1006 1007
  str[0] = 0;
  if (size > 0) {
    //--size;
    int n;
1008
    for (n = 0; n < size - 1; n++) {
1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
      int key = rand_tinyint() % (int)(sizeof(charset) - 1);
      str[n] = charset[key];
    }
    str[n] = 0;
  }
}

static double rand_double() {
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randdouble[cursor];

}

static void init_rand_data() {
  for (int i = 0; i < MAX_PREPARED_RAND; i++){
1026 1027 1028 1029
    randint[i] = (int)(taosRandom() % 65535);
    randbigint[i] = (int64_t)(taosRandom() % 2147483648);
    randfloat[i] = (float)(taosRandom() / 1000.0);
    randdouble[i] = (double)(taosRandom() / 1000000.0);
1030 1031 1032
  }
}

1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1053
static int printfInsertMeta() {
1054 1055
    SHOW_PARSE_RESULT_START();

1056 1057 1058 1059 1060 1061
  printf("host:                       \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port);
  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
  printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl);
1062 1063
  printf("insert interval:            \033[33m%d\033[0m\n", g_args.insert_interval);
  printf("number of records per req:  \033[33m%d\033[0m\n", g_args.num_of_RPR);
1064
  printf("max sql length:             \033[33m%d\033[0m\n", g_args.max_sql_len);
1065 1066 1067 1068

  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    printf("database[\033[33m%d\033[0m]:\n", i);
1069
    printf("  database[%d] name:      \033[33m%s\033[0m\n", i, g_Dbs.db[i].dbName);
1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
    if (0 == g_Dbs.db[i].drop) {
      printf("  drop:                  \033[33mno\033[0m\n");     
    }else {
      printf("  drop:                  \033[33myes\033[0m\n"); 
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      printf("  blocks:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      printf("  cache:                 \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      printf("  days:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      printf("  keep:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      printf("  replica:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      printf("  update:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      printf("  minRows:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      printf("  maxRows:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      printf("  walLevel:              \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      printf("  fsync:                 \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      printf("  quorum:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1113 1114
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1115 1116
        printf("  precision:             \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision);
      } else {
1117 1118 1119
        printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
                g_Dbs.db[i].dbCfg.precision);
        return -1;
1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143
      }
    }

    printf("  super table count:     \033[33m%d\033[0m\n", g_Dbs.db[i].superTblCount);
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      printf("  super table[\033[33m%d\033[0m]:\n", j);
    
      printf("      stbName:           \033[33m%s\033[0m\n",  g_Dbs.db[i].superTbls[j].sTblName);   

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
      }
      
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
      }
1144 1145 1146 1147 1148 1149 1150 1151

      printf("      childTblCount:     \033[33m%d\033[0m\n",  g_Dbs.db[i].superTbls[j].childTblCount);
      printf("      childTblPrefix:    \033[33m%s\033[0m\n",  g_Dbs.db[i].superTbls[j].childTblPrefix);
      printf("      dataSource:        \033[33m%s\033[0m\n",  g_Dbs.db[i].superTbls[j].dataSource);
      printf("      insertMode:        \033[33m%s\033[0m\n",  g_Dbs.db[i].superTbls[j].insertMode);
      if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
        printf("      childTblLimit:     \033[33m%d\033[0m\n",  g_Dbs.db[i].superTbls[j].childTblLimit);
      }
1152
      if (g_Dbs.db[i].superTbls[j].childTblOffset >= 0) {
1153 1154 1155
        printf("      childTblOffset:    \033[33m%d\033[0m\n",  g_Dbs.db[i].superTbls[j].childTblOffset);
      }
      printf("      insertRows:        \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows);
1156 1157

      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1158
        printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
1159
      }else {
1160
        printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
1161
      }
1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181
      printf("      rowsPerTbl:        \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].rowsPerTbl);
      printf("      disorderRange:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      printf("      disorderRatio:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
      printf("      maxSqlLen:         \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].maxSqlLen);
      printf("      timeStampStep:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].timeStampStep);
      printf("      startTimestamp:    \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
      printf("      sampleFormat:      \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      printf("      sampleFile:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      printf("      tagsFile:          \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
      printf("      columnCount:       \033[33m%d\033[0m\n        ",
              g_Dbs.db[i].superTbls[j].columnCount);
1182 1183
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1184 1185 1186 1187
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "nchar", 5))) {
1188
          printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
1189 1190
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1191
        } else {
1192 1193
          printf("column[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType);
1194 1195 1196
        }
      }
      printf("\n");
1197 1198 1199

      printf("      tagCount:            \033[33m%d\033[0m\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1200 1201
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1202 1203 1204
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) {
          printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
1205 1206
                  g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1207
        } else {
1208 1209
          printf("tag[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].tags[k].dataType);
1210
        }
1211 1212 1213 1214 1215
      }
      printf("\n");
    }
    printf("\n");
  }
1216 1217

  SHOW_PARSE_RESULT_END();
1218 1219

  return 0;
1220 1221 1222
}

static void printfInsertMetaToFile(FILE* fp) {
1223 1224
    SHOW_PARSE_RESULT_START_TO_FILE(fp);

1225 1226 1227 1228 1229 1230 1231 1232 1233 1234
  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
  fprintf(fp, "user:                       %s\n", g_Dbs.user);
  fprintf(fp, "password:                   %s\n", g_Dbs.password);
  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);

  fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    fprintf(fp, "database[%d]:\n", i);
1235
    fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
    if (0 == g_Dbs.db[i].drop) {
      fprintf(fp, "  drop:                  no\n");     
    }else {
      fprintf(fp, "  drop:                  yes\n"); 
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1279 1280
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313
        fprintf(fp, "  precision:             %s\n", g_Dbs.db[i].dbCfg.precision);
      } else {
        fprintf(fp, "  precision error:       %s\n", g_Dbs.db[i].dbCfg.precision);
      }
    }

    fprintf(fp, "  super table count:     %d\n", g_Dbs.db[i].superTblCount);
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      fprintf(fp, "  super table[%d]:\n", j);
    
      fprintf(fp, "      stbName:           %s\n",  g_Dbs.db[i].superTbls[j].sTblName);   

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
      } else {
        fprintf(fp, "      autoCreateTable:   %s\n",  "error");
      }
      
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "yes");
      } else {
        fprintf(fp, "      childTblExists:    %s\n",  "error");
      }
      
      fprintf(fp, "      childTblCount:     %d\n",  g_Dbs.db[i].superTbls[j].childTblCount);      
      fprintf(fp, "      childTblPrefix:    %s\n",  g_Dbs.db[i].superTbls[j].childTblPrefix);      
      fprintf(fp, "      dataSource:        %s\n",  g_Dbs.db[i].superTbls[j].dataSource);      
      fprintf(fp, "      insertMode:        %s\n",  g_Dbs.db[i].superTbls[j].insertMode);      
      fprintf(fp, "      insertRows:        %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); 
1314
      fprintf(fp, "      insert interval:   %d\n", g_Dbs.db[i].superTbls[j].insertInterval);
1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334

      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
        fprintf(fp, "      multiThreadWriteOneTbl:  no\n");     
      }else {
        fprintf(fp, "      multiThreadWriteOneTbl:  yes\n"); 
      }
      fprintf(fp, "      rowsPerTbl:        %d\n",  g_Dbs.db[i].superTbls[j].rowsPerTbl);     
      fprintf(fp, "      disorderRange:     %d\n",  g_Dbs.db[i].superTbls[j].disorderRange);     
      fprintf(fp, "      disorderRatio:     %d\n",  g_Dbs.db[i].superTbls[j].disorderRatio);
      fprintf(fp, "      maxSqlLen:         %d\n",  g_Dbs.db[i].superTbls[j].maxSqlLen);     
      
      fprintf(fp, "      timeStampStep:     %d\n",  g_Dbs.db[i].superTbls[j].timeStampStep);      
      fprintf(fp, "      startTimestamp:    %s\n",  g_Dbs.db[i].superTbls[j].startTimestamp);             
      fprintf(fp, "      sampleFormat:      %s\n",  g_Dbs.db[i].superTbls[j].sampleFormat);
      fprintf(fp, "      sampleFile:        %s\n",  g_Dbs.db[i].superTbls[j].sampleFile); 
      fprintf(fp, "      tagsFile:          %s\n",  g_Dbs.db[i].superTbls[j].tagsFile);   
    
      fprintf(fp, "      columnCount:       %d\n        ",  g_Dbs.db[i].superTbls[j].columnCount);
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1335 1336 1337 1338 1339 1340 1341 1342
        if ((0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "nchar", strlen("nchar")))) {
          fprintf(fp, "column[%d]:%s(%d) ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1343 1344 1345 1346 1347
        } else {
          fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
        }
      }
      fprintf(fp, "\n");
1348 1349 1350

      fprintf(fp, "      tagCount:            %d\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1351 1352
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1353 1354 1355 1356 1357 1358
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
          fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1359 1360
        } else {
          fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
1361
        }
1362 1363 1364 1365 1366
      }
      fprintf(fp, "\n");
    }
    fprintf(fp, "\n");
  }
1367
  SHOW_PARSE_RESULT_END_TO_FILE(fp);
1368 1369 1370
}

static void printfQueryMeta() {
1371 1372 1373
  SHOW_PARSE_RESULT_START();
  printf("host:                    \033[33m%s:%u\033[0m\n",
          g_queryInfo.host, g_queryInfo.port);
1374 1375 1376 1377 1378 1379 1380 1381 1382 1383
  printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
  printf("password:                \033[33m%s\033[0m\n", g_queryInfo.password);
  printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

  printf("\n");
  printf("specified table query info:                   \n");  
  printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.rate);
  printf("concurrent:     \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.concurrent);
  printf("sqlCount:       \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.sqlCount); 

1384
  if (SUBSCRIBE_TEST == g_args.test_mode) {
1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
    printf("mod:            \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeMode);
    printf("interval:       \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeInterval);
    printf("restart:        \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeRestart);
    printf("keepProgress:   \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeKeepProgress);
  }

  for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
    printf("  sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.superQueryInfo.sql[i]);
  }
  printf("\n");
  printf("super table query info:                   \n");  
  printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.rate);
  printf("threadCnt:      \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.threadCnt);
  printf("childTblCount:  \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.childTblCount);
  printf("stable name:    \033[33m%s\033[0m\n", g_queryInfo.subQueryInfo.sTblName);

1401
  if (SUBSCRIBE_TEST == g_args.test_mode) {
1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412
    printf("mod:            \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeMode);
    printf("interval:       \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeInterval);
    printf("restart:        \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeRestart);
    printf("keepProgress:   \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeKeepProgress);
  }
  
  printf("sqlCount:       \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.sqlCount);  
  for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) {
    printf("  sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.subQueryInfo.sql[i]);
  }  
  printf("\n");
1413 1414

    SHOW_PARSE_RESULT_END();
1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501
}


static char* xFormatTimestamp(char* buf, int64_t val, int precision) {
  time_t tt;
  if (precision == TSDB_TIME_PRECISION_MICRO) {
    tt = (time_t)(val / 1000000);
  } else {
    tt = (time_t)(val / 1000);
  }

/* comment out as it make testcases like select_with_tags.sim fail.
  but in windows, this may cause the call to localtime crash if tt < 0,
  need to find a better solution.
  if (tt < 0) {
    tt = 0;
  }
  */

#ifdef WINDOWS
  if (tt < 0) tt = 0;
#endif

  struct tm* ptm = localtime(&tt);
  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);

  if (precision == TSDB_TIME_PRECISION_MICRO) {
    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
  } else {
    sprintf(buf + pos, ".%03d", (int)(val % 1000));
  }

  return buf;
}

static void xDumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_t length, int precision) {
  if (val == NULL) {
    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
    return;
  }

  char buf[TSDB_MAX_BYTES_PER_ROW];
  switch (field->type) {
    case TSDB_DATA_TYPE_BOOL:
      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
      break;
    case TSDB_DATA_TYPE_TINYINT:
      fprintf(fp, "%d", *((int8_t *)val));
      break;
    case TSDB_DATA_TYPE_SMALLINT:
      fprintf(fp, "%d", *((int16_t *)val));
      break;
    case TSDB_DATA_TYPE_INT:
      fprintf(fp, "%d", *((int32_t *)val));
      break;
    case TSDB_DATA_TYPE_BIGINT:
      fprintf(fp, "%" PRId64, *((int64_t *)val));
      break;
    case TSDB_DATA_TYPE_FLOAT:
      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
      break;
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      memcpy(buf, val, length);
      buf[length] = 0;
      fprintf(fp, "\'%s\'", buf);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
      xFormatTimestamp(buf, *(int64_t*)val, precision);
      fprintf(fp, "'%s'", buf);
      break;
    default:
      break;
  }
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  FILE* fp = fopen(fname, "at");
  if (fp == NULL) {
1502
    errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546
    return -1;
  }

  int num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int precision = taos_result_precision(tres);

  for (int col = 0; col < num_fields; col++) {
    if (col > 0) {
      fprintf(fp, ",");
    }
    fprintf(fp, "%s", fields[col].name);
  }
  fputc('\n', fp);
  
  int numOfRows = 0;
  do {
    int32_t* length = taos_fetch_lengths(tres);
    for (int i = 0; i < num_fields; i++) {
      if (i > 0) {
        fputc(',', fp);
      }
      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
    }
    fputc('\n', fp);

    numOfRows++;
    row = taos_fetch_row(tres);
  } while( row != NULL);

  fclose(fp);

  return numOfRows;
}

static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {  
  TAOS_RES * res;  
  TAOS_ROW row = NULL;
  int count = 0;
  
  res = taos_query(taos, "show databases;");  
  int32_t code = taos_errno(res);
  
  if (code != 0) {
1547
    errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558
    return -1;
  }

  TAOS_FIELD *fields = taos_fetch_fields(res);

  while ((row = taos_fetch_row(res)) != NULL) {
    // sys database name : 'log'
    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) continue;

    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
    if (dbInfos[count] == NULL) {
1559
      errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
1560 1561 1562
      return -1;
    }

1563 1564 1565 1566 1567
    tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
            fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
    xFormatTimestamp(dbInfos[count]->create_time,
            *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
            TSDB_TIME_PRECISION_MILLI);
1568 1569 1570 1571 1572 1573
    dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);  
    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);  

1574 1575
    tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
            fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);      
1576 1577 1578 1579 1580 1581 1582 1583 1584
    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
    dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));

1585 1586 1587
    tstrncpy(dbInfos[count]->precision, 
            (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
            fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);  
1588
    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
1589 1590 1591
    tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
            fields[TSDB_SHOW_DB_STATUS_INDEX].bytes); 

1592 1593
    count++;
    if (count > MAX_DATABASE_COUNT) {
1594
      errorPrint( "The database count overflow than %d\n", MAX_DATABASE_COUNT);      
1595 1596 1597 1598 1599 1600 1601 1602
      break;
    }
  }

  return count;
}

static void printfDbInfoForQueryToFile(char* filename, SDbInfo* dbInfos, int index) {
1603
  if (filename[0] == 0)
1604
      return;
1605 1606 1607

  FILE *fp = fopen(filename, "at");
  if (fp == NULL) {
1608
    errorPrint( "failed to open file: %s\n", filename);
1609
    return;
1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645
  }

  fprintf(fp, "================ database[%d] ================\n", index);
  fprintf(fp, "name: %s\n", dbInfos->name);
  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
  fprintf(fp, "ntables: %d\n", dbInfos->ntables);
  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);  
  fprintf(fp, "replica: %d\n", dbInfos->replica);
  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
  fprintf(fp, "days: %d\n", dbInfos->days);    
  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);      
  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
  fprintf(fp, "comp: %d\n", dbInfos->comp);
  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);  
  fprintf(fp, "precision: %s\n", dbInfos->precision);  
  fprintf(fp, "update: %d\n", dbInfos->update);
  fprintf(fp, "status: %s\n", dbInfos->status); 
  fprintf(fp, "\n");

  fclose(fp);
}

static void printfQuerySystemInfo(TAOS * taos) {
  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
  TAOS_RES* res;

  time_t t;
  struct tm* lt;
  time(&t);
  lt = localtime(&t);
1646 1647 1648 1649
  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
          lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
          lt->tm_sec);

1650 1651 1652 1653 1654 1655 1656 1657 1658
  // show variables
  res = taos_query(taos, "show variables;");
  //getResult(res, filename);
  xDumpResultToFile(filename, res);

  // show dnodes
  res = taos_query(taos, "show dnodes;");
  xDumpResultToFile(filename, res);
  //getResult(res, filename);
1659

1660 1661 1662 1663
  // show databases
  res = taos_query(taos, "show databases;");
  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
  if (dbInfos == NULL) {
1664
    errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
1665 1666 1667
    return;
  }
  int dbCount = getDbFromServer(taos, dbInfos);
1668 1669 1670 1671
  if (dbCount <= 0) {
      free(dbInfos);
      return;
  }
1672 1673 1674 1675

  for (int i = 0; i < dbCount; i++) {
    // printf database info 
    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
1676

1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693
    // show db.vgroups
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);
  
    // show db.stables
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);

    free(dbInfos[i]);
  }

  free(dbInfos);
  
}

1694
static int postProceSql(char* host, uint16_t port, char* sqlstr)
1695
{
1696
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
1697 1698 1699 1700 1701

    char *url = "/rest/sql";

    struct hostent *server;
    struct sockaddr_in serv_addr;
1702
    int bytes, sent, received, req_str_len, resp_len;
1703 1704 1705
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
1706

1707 1708 1709 1710
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;

    request_buf = malloc(req_buf_len);
    if (NULL == request_buf)
1711
        ERROR_EXIT("ERROR, cannot allocate memory.");
1712

1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};

    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
1724

1725 1726 1727 1728 1729 1730
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);

    char base64_buf[INPUT_BUF_LEN];
1731
#ifdef WINDOWS
1732
    WSADATA wsaData;
1733
    WSAStartup(MAKEWORD(2, 2), &wsaData);
1734 1735 1736 1737
    SOCKET sockfd;
#else
    int sockfd;
#endif
1738
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
1739
    if (sockfd < 0) {
1740
#ifdef WINDOWS
1741
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
1742
#endif
1743
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
1744
        free(request_buf);
1745
        ERROR_EXIT("ERROR opening socket");
1746
    }
1747 1748

    server = gethostbyname(host);
1749 1750
    if (server == NULL) {
        free(request_buf);
1751
        ERROR_EXIT("ERROR, no such host");
1752
    }
1753

1754
    debugPrint("h_name: %s\nh_addretype: %s\nh_length: %d\n",
1755 1756 1757 1758
            server->h_name,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

1759
    memset(&serv_addr, 0, sizeof(serv_addr));
1760
    serv_addr.sin_family = AF_INET;
1761
    serv_addr.sin_port = htons(rest_port);
1762 1763 1764
#ifdef WINDOWS
    serv_addr.sin_addr.s_addr = inet_addr(host);
#else
1765
    memcpy(&serv_addr.sin_addr.s_addr,server->h_addr,server->h_length);
1766
#endif
1767

1768 1769 1770
    int retConn = connect(sockfd,(struct sockaddr *)&serv_addr,sizeof(serv_addr));
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
1771
        free(request_buf);
1772
        ERROR_EXIT("ERROR connecting");
1773
    }
1774

1775 1776 1777
    memset(base64_buf, 0, INPUT_BUF_LEN);

    for (int n = 0, m = 0; n < userpass_buf_len;) {
1778
      uint32_t oct_a = n < userpass_buf_len ?
1779
        (unsigned char) userpass_buf[n++]:0;
1780
      uint32_t oct_b = n < userpass_buf_len ?
1781
        (unsigned char) userpass_buf[n++]:0;
1782
      uint32_t oct_c = n < userpass_buf_len ?
1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;

      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }

    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';

    debugPrint("%s() LN%d: auth string base64 encoded: %s\n", __func__, __LINE__, base64_buf);
    char *auth = base64_buf;

1798 1799 1800
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
1801 1802 1803 1804 1805
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
1806
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
1807

1808
    req_str_len = strlen(request_buf);
1809 1810
    sent = 0;
    do {
1811 1812 1813
#ifdef WINDOWS
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
#else
1814
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
1815
#endif
1816
        if (bytes < 0)
1817
            ERROR_EXIT("ERROR writing message to socket");
1818 1819 1820
        if (bytes == 0)
            break;
        sent+=bytes;
1821
    } while (sent < req_str_len);
1822

1823 1824
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
1825 1826
    received = 0;
    do {
1827 1828 1829
#ifdef WINDOWS
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
#else
1830
        bytes = read(sockfd, response_buf + received, resp_len - received);
1831
#endif
1832 1833
        if (bytes < 0) {
            free(request_buf);
1834
            ERROR_EXIT("ERROR reading response from socket");
1835
        }
1836 1837
        if (bytes == 0)
            break;
1838 1839
        received += bytes;
    } while (received < resp_len);
1840

1841 1842
    if (received == resp_len) {
        free(request_buf);
1843
        ERROR_EXIT("ERROR storing complete response from socket");
1844
    }
1845

1846
    response_buf[RESP_BUF_LEN - 1] = '\0';
1847 1848 1849
    printf("Response:\n%s\n", response_buf);

    free(request_buf);
1850 1851 1852 1853
#ifdef WINDOWS
    closesocket(sockfd);
    WSACleanup();
#else
1854
    close(sockfd);
1855
#endif
1856 1857 1858 1859

    return 0;
}

1860
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
1861 1862
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
1863
    errorPrint("%s() LN%d, calloc failed! size:%d\n", __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
1864 1865
    return NULL;
  }
1866

1867
  int    dataLen = 0;
1868 1869 1870
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);

1871 1872 1873
  return dataBuf;
}

1874
static char* generateTagVaulesForStb(SSuperTable* stbInfo) {
1875 1876 1877 1878 1879 1880 1881 1882 1883
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
    printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
    return NULL;
  }

  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
  for (int i = 0; i < stbInfo->tagCount; i++) {
1884 1885
    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
1886
      if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
1887 1888
        printf("binary or nchar length overflow, max size:%u\n",
                (uint32_t)TSDB_MAX_BINARY_LEN);
1889 1890 1891
        tmfree(dataBuf);
        return NULL;
      }
1892

1893 1894 1895 1896 1897 1898 1899
      char* buf = (char*)calloc(stbInfo->tags[i].dataLen+1, 1);
      if (NULL == buf) {
        printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
        tmfree(dataBuf);
        return NULL;
      }
      rand_string(buf, stbInfo->tags[i].dataLen);
1900 1901
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "\'%s\', ", buf);
1902
      tmfree(buf);
1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "int", strlen("int"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_int());
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bigint", strlen("bigint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "float", strlen("float"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_float());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "double", strlen("double"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_double());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "smallint", strlen("smallint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_smallint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "tinyint", strlen("tinyint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_tinyint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bool", strlen("bool"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_bool());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "timestamp", strlen("timestamp"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
1935 1936 1937 1938 1939 1940
    }  else {
      printf("No support data type: %s\n", stbInfo->tags[i].dataType);
      tmfree(dataBuf);
      return NULL;
    }
  }
1941

1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985
  dataLen -= 2;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");  
  return dataBuf;
}

static int calcRowLen(SSuperTable*  superTbls) {  
  int colIndex;
  int  lenOfOneRow = 0;
  
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
    
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) { 
      lenOfOneRow += 42;
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) { 
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
1986

1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) { 
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
    
  return 0;
}


2017 2018 2019 2020
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
        int* childTblCountOfSuperTbl, int limit, int offset) {

2021
  char command[BUFFER_SIZE] = "\0";
2022 2023
  char limitBuf[100] = "\0";

2024 2025 2026 2027
  TAOS_RES * res;  
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2028 2029 2030 2031 2032

  if (offset >= 0) {
    snprintf(limitBuf, 100, " limit %d offset %d", limit, offset);
  }

2033
  //get all child table name use cmd: select tbname from superTblName;  
2034 2035
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s", dbName, sTblName, limitBuf);

2036 2037 2038 2039 2040 2041 2042 2043 2044
  res = taos_query(taos, command);  
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
    taos_free_result(res);
    taos_close(taos);
    exit(-1);
  }

2045
  int childTblCount = (limit < 0)?10000:limit;
2046
  int count = 0;
2047
//  childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
2048 2049 2050
  char* pTblName = childTblName;
  while ((row = taos_fetch_row(res)) != NULL) {
    int32_t* len = taos_fetch_lengths(res);
2051
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2052 2053 2054
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2055 2056
      char *tmp = realloc(childTblName,
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2057 2058 2059
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2060 2061
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2062 2063
      } else {
        // exit, if allocate more memory failed
2064 2065
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
               __func__, __LINE__, dbName, sTblName);
2066 2067 2068 2069 2070 2071
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2072
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2073
  }
2074

2075 2076 2077 2078 2079 2080 2081
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2082 2083
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
        char* sTblName, char** childTblNameOfSuperTbl,
2084
        int* childTblCountOfSuperTbl) {
2085 2086 2087 2088 2089 2090

    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
            -1, -1);
}

2091 2092
static int getSuperTableFromServer(TAOS * taos, char* dbName,
        SSuperTable*  superTbls) {
2093 2094 2095 2096
  char command[BUFFER_SIZE] = "\0";
  TAOS_RES * res;  
  TAOS_ROW row = NULL;
  int count = 0;
2097

2098
  //get schema use cmd: describe superTblName;
2099
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2100
  res = taos_query(taos, command);
2101 2102 2103
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2104
    taos_free_result(res);
2105 2106 2107 2108 2109 2110 2111 2112 2113 2114
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
  while ((row = taos_fetch_row(res)) != NULL) {
    if (0 == count) {
      count++;
      continue;
2115
    }
2116 2117

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128
      tstrncpy(superTbls->tags[tagIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->tags[tagIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2129
      tagIndex++;
2130
    } else {
2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141
      tstrncpy(superTbls->columns[columnIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->columns[columnIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2153
/*
2154 2155
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
    //get all child table name use cmd: select tbname from superTblName;  
2156 2157 2158
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2159
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2160 2161
      return -1;
    }
2162
    getAllChildNameOfSuperTable(taos, dbName,
2163 2164 2165
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2166
  }
2167
  */
2168 2169 2170
  return 0;
}

2171
static int createSuperTable(TAOS * taos, char* dbName, SSuperTable*  superTbls, bool use_metric) {
2172
  char command[BUFFER_SIZE] = "\0";
2173

2174 2175 2176 2177 2178 2179 2180
  char cols[STRING_LEN] = "\0";
  int colIndex;
  int len = 0;

  int  lenOfOneRow = 0;
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2181
 
2182
    if (strcasecmp(dataType, "BINARY") == 0) {
2183 2184 2185
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "BINARY",
          superTbls->columns[colIndex].dataLen);
2186 2187
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
2188 2189 2190
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "NCHAR",
          superTbls->columns[colIndex].dataLen);
2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
      lenOfOneRow += 22;
2210
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2211 2212
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
      lenOfOneRow += 42;
2213
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
      lenOfOneRow += 21;
    } else {
      taos_close(taos);
      printf("config error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp
  //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName, g_Dbs.db[i].superTbls[j].columnCount, lenOfOneRow);

  // save for creating child table
2227 2228
  superTbls->colsOfCreateChildTable = (char*)calloc(len+20, 1);
  if (NULL == superTbls->colsOfCreateChildTable) {
2229 2230 2231 2232
    printf("Failed when calloc, size:%d", len+1);
    taos_close(taos);
    exit(-1);
  }
2233
  snprintf(superTbls->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
2234
  verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, superTbls->colsOfCreateChildTable);
2235 2236 2237 2238 2239 2240 2241 2242 2243 2244

  if (use_metric) {
    char tags[STRING_LEN] = "\0";
    int tagIndex;
    len = 0;

    int lenOfTagOfOneRow = 0;
    len += snprintf(tags + len, STRING_LEN - len, "(");
    for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
      char* dataType = superTbls->tags[tagIndex].dataType;
2245

2246
      if (strcasecmp(dataType, "BINARY") == 0) {
2247 2248
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
                "BINARY", superTbls->tags[tagIndex].dataLen);
2249 2250
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
      } else if (strcasecmp(dataType, "NCHAR") == 0) {
2251 2252
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
                "NCHAR", superTbls->tags[tagIndex].dataLen);
2253 2254
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
      } else if (strcasecmp(dataType, "INT") == 0)  {
2255 2256
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "INT");
2257 2258
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
      } else if (strcasecmp(dataType, "BIGINT") == 0)  {
2259 2260
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "BIGINT");
2261 2262
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
      } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
2263 2264
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "SMALLINT");
2265 2266
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
      } else if (strcasecmp(dataType, "TINYINT") == 0)  {
2267 2268
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "TINYINT");
2269 2270
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
      } else if (strcasecmp(dataType, "BOOL") == 0)  {
2271 2272
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "BOOL");
2273 2274
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
      } else if (strcasecmp(dataType, "FLOAT") == 0) {
2275 2276
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "FLOAT");
2277 2278
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
      } else if (strcasecmp(dataType, "DOUBLE") == 0) { 
2279 2280
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "DOUBLE");
2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
      } else {
        taos_close(taos);
        printf("config error tag type : %s\n", dataType);
        exit(-1);
      }
    }
    len -= 2;
    len += snprintf(tags + len, STRING_LEN - len, ")");

    superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2292 2293 2294 2295

    snprintf(command, BUFFER_SIZE,
            "create table if not exists %s.%s (ts timestamp%s) tags %s",
            dbName, superTbls->sTblName, cols, tags);
2296
    verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command);
2297

2298
    if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) {
2299
        errorPrint( "create supertable %s failed!\n\n",
2300
                superTbls->sTblName);
2301
        return -1;
2302
    }
2303
    debugPrint("create supertable %s success!\n\n", superTbls->sTblName);
2304 2305 2306 2307
  }
  return 0;
}

2308
static int createDatabases() {
2309 2310 2311 2312
  TAOS * taos = NULL;
  int    ret = 0;
  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
  if (taos == NULL) {
2313
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2314
    return -1;
2315 2316 2317 2318 2319 2320
  }
  char command[BUFFER_SIZE] = "\0";

  for (int i = 0; i < g_Dbs.dbCount; i++) {   
    if (g_Dbs.db[i].drop) {
      sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
2321
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);
2322 2323 2324 2325 2326
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) {
        taos_close(taos);
        return -1;
      }
    }
2327

2328
    int dataLen = 0;
2329
    dataLen += snprintf(command + dataLen, 
2330
        BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2331 2332

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
2333
      dataLen += snprintf(command + dataLen,
2334
          BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
2335 2336
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
2337
      dataLen += snprintf(command + dataLen,
2338
          BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
2339 2340
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
2341
      dataLen += snprintf(command + dataLen,
2342
          BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
2343 2344
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
2345
      dataLen += snprintf(command + dataLen,
2346
          BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
2347
    }
2348 2349
    if (g_Dbs.db[i].dbCfg.quorum > 1) {
      dataLen += snprintf(command + dataLen,
2350
          BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
2351 2352
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
2353
      dataLen += snprintf(command + dataLen,
2354
          BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
2355 2356
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
2357
      dataLen += snprintf(command + dataLen,
2358
          BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
2359 2360
    }
    //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
2361 2362
    //  dataLen += snprintf(command + dataLen,
    //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
2363 2364
    //}
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
2365
      dataLen += snprintf(command + dataLen,
2366
          BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
2367 2368
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
2369
      dataLen += snprintf(command + dataLen,
2370
          BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
2371 2372
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
2373
      dataLen += snprintf(command + dataLen,
2374
          BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
2375 2376
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
2377
      dataLen += snprintf(command + dataLen,
2378
          BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
2379
    }
2380
    if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
2381
      dataLen += snprintf(command + dataLen,
2382
          BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
2383
    }
2384
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
2385 2386
      dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
              " fsync %d", g_Dbs.db[i].dbCfg.fsync);
2387
    }
2388
    if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
2389 2390
            || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                    "us", strlen("us")))) {
2391
      dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2392
              " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
2393
    }
2394

2395
    debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
2396 2397
    if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) {
      taos_close(taos);
2398
      errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
2399 2400 2401 2402
      return -1;
    }
    printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);

2403 2404
    debugPrint("%s() %d supertbl count:%d\n",
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
2405 2406
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      // describe super table, if exists
2407 2408
      sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
              g_Dbs.db[i].superTbls[j].sTblName);
2409
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);
2410 2411
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) {
        g_Dbs.db[i].superTbls[j].superTblExists = TBL_NO_EXISTS;
2412 2413
        ret = createSuperTable(taos, g_Dbs.db[i].dbName,
                &g_Dbs.db[i].superTbls[j], g_Dbs.use_metric);
2414 2415
      } else {      
        g_Dbs.db[i].superTbls[j].superTblExists = TBL_ALREADY_EXISTS;
2416 2417 2418

        if (g_Dbs.db[i].superTbls[j].childTblExists != TBL_ALREADY_EXISTS) {
            ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
2419
                &g_Dbs.db[i].superTbls[j]);
2420
        }
2421 2422 2423
      }

      if (0 != ret) {
2424
        printf("\ncreate super table %d failed!\n\n", j);
2425 2426 2427 2428 2429 2430 2431 2432 2433 2434
        taos_close(taos);
        return -1;
      }
    }    
  }

  taos_close(taos);
  return 0;
}

2435
static void* createTable(void *sarg) 
2436 2437 2438 2439 2440 2441
{  
  threadInfo *winfo = (threadInfo *)sarg; 
  SSuperTable* superTblInfo = winfo->superTblInfo;

  int64_t  lastPrintTime = taosGetTimestampMs();

2442 2443 2444 2445 2446 2447
  int buff_len;
  if (superTblInfo)
    buff_len = superTblInfo->maxSqlLen;
  else
    buff_len = BUFFER_SIZE;

2448
  char *buffer = calloc(buff_len, 1);
2449
  if (buffer == NULL) {
2450
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
2451 2452
    exit(-1);
  }
2453 2454 2455

  int len = 0;
  int batchNum = 0;
2456 2457 2458 2459 2460 2461

  verbosePrint("%s() LN%d: Creating table from %d to %d\n", 
          __func__, __LINE__,
          winfo->start_table_from, winfo->end_table_to);

  for (int i = winfo->start_table_from; i <= winfo->end_table_to; i++) {
2462
    if (0 == g_Dbs.use_metric) {
2463
      snprintf(buffer, buff_len, 
2464 2465
              "create table if not exists %s.%s%d %s;",
              winfo->db_name,
2466 2467
              g_args.tb_prefix, i,
              winfo->cols);
2468 2469 2470
    } else {
      if (0 == len) {  
        batchNum = 0;
2471
        memset(buffer, 0, buff_len);
2472
        len += snprintf(buffer + len,
2473
                buff_len - len, "create table ");
2474
      }
2475

2476 2477 2478 2479
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
        tagsValBuf = generateTagVaulesForStb(superTblInfo);
      } else {
2480 2481 2482
        tagsValBuf = getTagValueFromTagSample(
                superTblInfo,
                i % superTblInfo->tagSampleCount);
2483 2484 2485 2486 2487 2488
      }
      if (NULL == tagsValBuf) {
        free(buffer);
        return NULL;
      }
      
2489 2490 2491 2492 2493 2494
      len += snprintf(buffer + len,
              superTblInfo->maxSqlLen - len,
              "if not exists %s.%s%d using %s.%s tags %s ",
              winfo->db_name, superTblInfo->childTblPrefix,
              i, winfo->db_name,
              superTblInfo->sTblName, tagsValBuf);
2495 2496 2497
      free(tagsValBuf);
      batchNum++;

2498 2499 2500
      if ((batchNum < superTblInfo->batchCreateTableNum)
              && ((superTblInfo->maxSqlLen - len) 
                  >= (superTblInfo->lenOfTagOfOneRow + 256))) {
2501 2502 2503 2504 2505
        continue;
      }
    }

    len = 0;
2506
    verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
2507
    if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE)){
2508
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2509 2510 2511 2512 2513 2514
      free(buffer);
      return NULL;
    }

    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
2515
      printf("thread[%d] already create %d - %d tables\n",
2516
              winfo->threadID, winfo->start_table_from, i);
2517 2518 2519 2520 2521
      lastPrintTime = currentPrintTime;
    }
  }
  
  if (0 != len) {
2522
    verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer);
2523
    if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE)) {
2524
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2525
    }
2526
  }
2527

2528 2529 2530 2531
  free(buffer);
  return NULL;
}

2532
static int startMultiThreadCreateChildTable(
2533
        char* cols, int threads, int startFrom, int ntables,
2534
        char* db_name, SSuperTable* superTblInfo) {
2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  threadInfo *infos = malloc(threads * sizeof(threadInfo));

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

  int a = ntables / threads;
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

  int b = 0;
  b = ntables % threads;
2555
 
2556 2557 2558
  for (int i = 0; i < threads; i++) {
    threadInfo *t_info = infos + i;
    t_info->threadID = i;
H
Hui Li 已提交
2559
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
2560
    t_info->superTblInfo = superTblInfo;
2561
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
2562 2563 2564 2565 2566 2567
    t_info->taos = taos_connect(
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
2568
    if (t_info->taos == NULL) {
2569
      errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2570 2571
      free(pids);
      free(infos);  
2572 2573
      return -1;
    }
2574 2575 2576 2577 2578

    t_info->start_table_from = startFrom;
    t_info->ntables = i<b?a+1:a;
    t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
    startFrom = t_info->end_table_to + 1;
2579 2580
    t_info->use_metric = 1;
    t_info->cols = cols;
2581
    t_info->minDelay = INT16_MAX;
2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595
    pthread_create(pids + i, NULL, createTable, t_info);
  }
  
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
    threadInfo *t_info = infos + i;
    taos_close(t_info->taos);
  }

  free(pids);
  free(infos);  
2596 2597

  return 0;
2598 2599 2600
}


2601 2602 2603 2604
static void createChildTables() {
    char tblColsBuf[MAX_SQL_SIZE];
    int len;

2605
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2606
    if (g_Dbs.db[i].superTblCount > 0) {
2607
        // with super table
2608 2609
      for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
        if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
2610
              || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
2611 2612 2613
          continue;
        }

2614
        verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
2615
                g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
2616 2617
        int startFrom = 0;
        g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;
2618 2619 2620

        verbosePrint("%s() LN%d: create %d child tables from %d\n", __func__, __LINE__,
              g_totalChildTables, startFrom);  
2621
        startMultiThreadCreateChildTable(
2622
              g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
2623
              g_Dbs.threadCountByCreateTbl,
2624 2625
              startFrom,
              g_totalChildTables,
2626
              g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
2627 2628
      }
    } else {
2629 2630
        // normal table
        len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
2631 2632 2633
        int j = 0;
        while (g_args.datatype[j]) {
            if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
2634 2635
                    || (strncasecmp(g_args.datatype[j],
                        "NCHAR", strlen("NCHAR")) == 0)) {
2636
                len = snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
2637
                        ", COL%d %s(60)", j, g_args.datatype[j]);
2638
            } else {
2639
                len = snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
2640
                        ", COL%d %s", j, g_args.datatype[j]);
2641 2642
            }
            len = strlen(tblColsBuf);
2643
            j++;
2644 2645
        }

2646
        len = snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");
2647

2648 2649
        verbosePrint("%s() LN%d: dbName: %s num of tb: %d schema: %s\n",
                __func__, __LINE__,
2650
                g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
2651
        startMultiThreadCreateChildTable(
2652
              tblColsBuf,
2653
              g_Dbs.threadCountByCreateTbl,
2654
              0,
2655
              g_args.num_of_tables,
2656 2657 2658
              g_Dbs.db[i].dbName,
              NULL);
    }
2659 2660 2661 2662 2663 2664
  }
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
2665
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
2666 2667 2668
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
2669

2670 2671
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
2672 2673
    printf("Failed to open tags file: %s, reason:%s\n",
            superTblInfo->tagsFile, strerror(errno));
2674 2675 2676 2677 2678 2679 2680
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
2681

2682 2683 2684 2685 2686 2687 2688 2689 2690
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

2691
  while ((readLen = tgetline(&line, &n, fp)) != -1) {
2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
2704 2705
      char *tmp = realloc(tagDataBuf,
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
2706 2707 2708
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
2709 2710
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

int readSampleFromJsonFileToMem(SSuperTable  * superTblInfo) {
  // TODO
  return 0;
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
2738 2739
static int readSampleFromCsvFileToMem(
        SSuperTable* superTblInfo) {
2740 2741 2742 2743
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
2744 2745 2746
 
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
2747
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
2748 2749 2750
              superTblInfo->sampleFile, strerror(errno));
      return -1;
  }
2751

2752
  assert(superTblInfo->sampleDataBuf);
2753 2754
  memset(superTblInfo->sampleDataBuf, 0,
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
2755
  while (1) {
2756
    readLen = tgetline(&line, &n, fp);
2757 2758
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
2759
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
2760
                superTblInfo->sampleFile, strerror(errno));
2761
        fclose(fp);
2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775
        return -1;
      }
      continue;
    }
  
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
2776 2777
      printf("sample row len[%d] overflow define schema len[%d], so discard this row\n",
              (int32_t)readLen, superTblInfo->lenOfOneRow);
2778 2779 2780
      continue;
    }

sangshuduo's avatar
sangshuduo 已提交
2781
    verbosePrint("readLen=%ld stb->lenOfOneRow=%d getRows=%d\n", (long)readLen,
2782
            superTblInfo->lenOfOneRow, getRows);
2783

2784 2785
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
          line, readLen);
2786 2787 2788 2789 2790 2791 2792
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

2793
  fclose(fp);
2794 2795 2796 2797
  tmfree(line);
  return 0;
}

2798 2799
static bool getColumnAndTagTypeFromInsertJsonFile(
        cJSON* stbInfo, SSuperTable* superTbls) {
2800
  bool  ret = false;
2801

2802 2803 2804
  // columns 
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
2805
    printf("ERROR: failed to read json, columns not found\n");
2806 2807 2808 2809 2810 2811
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
2812
 
2813 2814
  int columnSize = cJSON_GetArraySize(columns);
  if (columnSize > MAX_COLUMN_COUNT) {
2815 2816
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
            __func__, __LINE__, MAX_COLUMN_COUNT);
2817 2818 2819 2820 2821 2822
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
2823

2824
  //superTbls->columnCount = columnSize;
2825 2826 2827 2828 2829 2830 2831
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
2832
      count = countObj->valueint;
2833
    } else if (countObj && countObj->type != cJSON_Number) {
2834
      errorPrint("%s() LN%d, failed to read json, column count not found\n", __func__, __LINE__);
2835 2836 2837 2838 2839 2840 2841 2842 2843
      goto PARSE_OVER;
    } else {
      count = 1;
    }

    // column info 
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
    if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) {
2844
      errorPrint("%s() LN%d: failed to read json, column type not found\n", __func__, __LINE__);
2845 2846
      goto PARSE_OVER;
    }
2847 2848
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
2849

2850 2851
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
2852
      columnCase.dataLen = dataLen->valueint;
2853
    } else if (dataLen && dataLen->type != cJSON_Number) {
2854
      debugPrint("%s() LN%d: failed to read json, column len not found\n", __func__, __LINE__);
2855 2856 2857 2858
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
2859

2860
    for (int n = 0; n < count; ++n) {
2861 2862 2863
      tstrncpy(superTbls->columns[index].dataType,
              columnCase.dataType, MAX_TB_NAME_SIZE);
      superTbls->columns[index].dataLen = columnCase.dataLen;
2864 2865
      index++;
    }
2866 2867 2868
  }
  superTbls->columnCount = index;
 
2869 2870 2871 2872 2873
  count = 1;
  index = 0;
  // tags 
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
2874
    debugPrint("%s() LN%d, failed to read json, tags not found\n", __func__, __LINE__);
2875 2876 2877 2878 2879
    goto PARSE_OVER;
  }
  
  int tagSize = cJSON_GetArraySize(tags);
  if (tagSize > MAX_TAG_COUNT) {
2880
    debugPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", __func__, __LINE__, MAX_TAG_COUNT);
2881 2882
    goto PARSE_OVER;
  }
2883 2884

  //superTbls->tagCount = tagSize;
2885 2886 2887
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
2888

2889 2890 2891
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
2892
      count = countObj->valueint;
2893
    } else if (countObj && countObj->type != cJSON_Number) {
2894
      printf("ERROR: failed to read json, column count not found\n");
2895 2896 2897 2898 2899 2900 2901 2902 2903
      goto PARSE_OVER;
    } else {
      count = 1;
    }

    // column info 
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
    if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) {
2904
      printf("ERROR: failed to read json, tag type not found\n");
2905 2906
      goto PARSE_OVER;
    }
2907
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
2908

2909 2910
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
2911
      columnCase.dataLen = dataLen->valueint;
2912
    } else if (dataLen && dataLen->type != cJSON_Number) {
2913
      printf("ERROR: failed to read json, column len not found\n");
2914 2915 2916
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
2917 2918
    }

2919
    for (int n = 0; n < count; ++n) {
2920
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, MAX_TB_NAME_SIZE);
2921
      superTbls->tags[index].dataLen = columnCase.dataLen;
2922 2923
      index++;
    }
2924
  }
2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940
  superTbls->tagCount = index;

  ret = true;

PARSE_OVER:
  //free(content);
  //cJSON_Delete(root);
  //fclose(fp);
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
2941
    tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
2942 2943 2944 2945
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
2946
    tstrncpy(g_Dbs.host, host->valuestring, MAX_DB_NAME_SIZE);
2947
  } else if (!host) {
2948
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE);
2949
  } else {
2950
    printf("ERROR: failed to read json, host not found\n");
2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_Dbs.port = port->valueint;
  } else if (!port) {
    g_Dbs.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
2963
    tstrncpy(g_Dbs.user, user->valuestring, MAX_DB_NAME_SIZE);   
2964
  } else if (!user) {
2965
    tstrncpy(g_Dbs.user, "root", MAX_DB_NAME_SIZE);
2966 2967 2968 2969
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
2970
    tstrncpy(g_Dbs.password, password->valuestring, MAX_DB_NAME_SIZE);
2971
  } else if (!password) {
2972
    tstrncpy(g_Dbs.password, "taosdata", MAX_DB_NAME_SIZE);
2973 2974 2975 2976
  }

  cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
  if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
2977
    tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
2978
  } else if (!resultfile) {
2979
    tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
2980 2981 2982 2983 2984 2985 2986 2987
  }

  cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
  if (threads && threads->type == cJSON_Number) {
    g_Dbs.threadCount = threads->valueint;
  } else if (!threads) {
    g_Dbs.threadCount = 1;
  } else {
2988
    printf("ERROR: failed to read json, threads not found\n");
2989 2990
    goto PARSE_OVER;
  }  
2991
 
2992 2993 2994 2995 2996 2997
  cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
  if (threads2 && threads2->type == cJSON_Number) {
    g_Dbs.threadCountByCreateTbl = threads2->valueint;
  } else if (!threads2) {
    g_Dbs.threadCountByCreateTbl = 1;
  } else {
2998
    printf("ERROR: failed to read json, threads2 not found\n");
2999 3000 3001
    goto PARSE_OVER;
  } 

3002 3003 3004 3005 3006 3007 3008 3009 3010
  cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
  if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
    g_args.insert_interval = gInsertInterval->valueint;
  } else if (!gInsertInterval) {
    g_args.insert_interval = 0;
  } else {
    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", __func__, __LINE__);
    goto PARSE_OVER;
  }
3011

3012 3013 3014 3015 3016 3017 3018 3019 3020
  cJSON* rowsPerTbl = cJSON_GetObjectItem(root, "rows_per_tbl");
  if (rowsPerTbl && rowsPerTbl->type == cJSON_Number) {
    g_args.rows_per_tbl = rowsPerTbl->valueint;
  } else if (!rowsPerTbl) {
    g_args.rows_per_tbl = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
  } else {
    errorPrint("%s() LN%d, failed to read json, rows_per_tbl input mistake\n", __func__, __LINE__);
    goto PARSE_OVER;
  }      
3021

3022 3023 3024 3025 3026 3027 3028 3029 3030
  cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
  if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
    g_args.max_sql_len = maxSqlLen->valueint;
  } else if (!maxSqlLen) {
    g_args.max_sql_len = TSDB_PAYLOAD_SIZE;
  } else {
    errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", __func__, __LINE__);
    goto PARSE_OVER;
  }
3031
 
3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042

  cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
  if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
    g_args.num_of_RPR = numRecPerReq->valueint;
  } else if (!numRecPerReq) {
    g_args.num_of_RPR = 100;
  } else {
    errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", __func__, __LINE__);
    goto PARSE_OVER;
  }

3043
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3044 3045 3046
  if (answerPrompt 
          && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3047 3048 3049 3050 3051 3052 3053 3054 3055 3056
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
    } else {
      g_args.answer_yes = false;
    }
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
3057
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
3058 3059 3060 3061 3062
    goto PARSE_OVER;
  }  

  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (!dbs || dbs->type != cJSON_Array) {
3063
    printf("ERROR: failed to read json, databases not found\n");
3064 3065 3066 3067 3068
    goto PARSE_OVER;
  }

  int dbSize = cJSON_GetArraySize(dbs);
  if (dbSize > MAX_DB_COUNT) {
3069
    errorPrint(
3070 3071
            "ERROR: failed to read json, databases size overflow, max database is %d\n",
            MAX_DB_COUNT);
3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082
    goto PARSE_OVER;
  }

  g_Dbs.dbCount = dbSize;
  for (int i = 0; i < dbSize; ++i) {
    cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
    if (dbinfos == NULL) continue;

    // dbinfo 
    cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
    if (!dbinfo || dbinfo->type != cJSON_Object) {
3083
      printf("ERROR: failed to read json, dbinfo not found\n");
3084 3085 3086 3087 3088
      goto PARSE_OVER;
    }
    
    cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
    if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
3089
      printf("ERROR: failed to read json, db name not found\n");
3090 3091
      goto PARSE_OVER;
    }
3092
    tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE);
3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103

    cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
    if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
      if (0 == strncasecmp(drop->valuestring, "yes", 3)) {
        g_Dbs.db[i].drop = 1;
      } else {
        g_Dbs.db[i].drop = 0;
      }        
    } else if (!drop) {
      g_Dbs.db[i].drop = 0;
    } else {
3104
      printf("ERROR: failed to read json, drop not found\n");
3105 3106 3107 3108
      goto PARSE_OVER;
    }

    cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
3109 3110 3111 3112
    if (precision && precision->type == cJSON_String
            && precision->valuestring != NULL) {
      tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
              MAX_DB_NAME_SIZE);
3113
    } else if (!precision) {
3114
      //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
3115 3116
      memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE);
    } else {
3117
      printf("ERROR: failed to read json, precision not found\n");
3118 3119 3120 3121 3122 3123 3124 3125 3126
      goto PARSE_OVER;
    }

    cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
    if (update && update->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.update = update->valueint;
    } else if (!update) {
      g_Dbs.db[i].dbCfg.update = -1;
    } else {
3127
      printf("ERROR: failed to read json, update not found\n");
3128 3129 3130 3131 3132 3133 3134 3135 3136
      goto PARSE_OVER;
    }

    cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
    if (replica && replica->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.replica = replica->valueint;
    } else if (!replica) {
      g_Dbs.db[i].dbCfg.replica = -1;
    } else {
3137
      printf("ERROR: failed to read json, replica not found\n");
3138 3139 3140 3141 3142 3143 3144 3145 3146
      goto PARSE_OVER;
    }

    cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
    if (keep && keep->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.keep = keep->valueint;
    } else if (!keep) {
      g_Dbs.db[i].dbCfg.keep = -1;
    } else {
3147
     printf("ERROR: failed to read json, keep not found\n");
3148 3149 3150 3151 3152 3153 3154 3155 3156
     goto PARSE_OVER;
    }
    
    cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
    if (days && days->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.days = days->valueint;
    } else if (!days) {
      g_Dbs.db[i].dbCfg.days = -1;
    } else {
3157
     printf("ERROR: failed to read json, days not found\n");
3158 3159 3160 3161 3162 3163 3164 3165 3166
     goto PARSE_OVER;
    }
    
    cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
    if (cache && cache->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cache = cache->valueint;
    } else if (!cache) {
      g_Dbs.db[i].dbCfg.cache = -1;
    } else {
3167
     printf("ERROR: failed to read json, cache not found\n");
3168 3169 3170 3171 3172 3173 3174 3175 3176
     goto PARSE_OVER;
    }
        
    cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
    if (blocks && blocks->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
    } else if (!blocks) {
      g_Dbs.db[i].dbCfg.blocks = -1;
    } else {
3177
     printf("ERROR: failed to read json, block not found\n");
3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196
     goto PARSE_OVER;
    }

    //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
    //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
    //} else if (!maxtablesPerVnode) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
    //} else {
    // printf("failed to read json, maxtablesPerVnode not found");
    // goto PARSE_OVER;
    //}

    cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
    if (minRows && minRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
    } else if (!minRows) {
      g_Dbs.db[i].dbCfg.minRows = -1;
    } else {
3197
     printf("ERROR: failed to read json, minRows not found\n");
3198 3199 3200 3201 3202 3203 3204 3205 3206
     goto PARSE_OVER;
    }

    cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
    if (maxRows && maxRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
    } else if (!maxRows) {
      g_Dbs.db[i].dbCfg.maxRows = -1;
    } else {
3207
     printf("ERROR: failed to read json, maxRows not found\n");
3208 3209 3210 3211 3212 3213 3214 3215 3216
     goto PARSE_OVER;
    }

    cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
    if (comp && comp->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.comp = comp->valueint;
    } else if (!comp) {
      g_Dbs.db[i].dbCfg.comp = -1;
    } else {
3217
     printf("ERROR: failed to read json, comp not found\n");
3218 3219 3220 3221 3222 3223 3224 3225 3226
     goto PARSE_OVER;
    }

    cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
    if (walLevel && walLevel->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
    } else if (!walLevel) {
      g_Dbs.db[i].dbCfg.walLevel = -1;
    } else {
3227
     printf("ERROR: failed to read json, walLevel not found\n");
3228 3229 3230
     goto PARSE_OVER;
    }

3231 3232 3233 3234 3235 3236
    cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
    if (cacheLast && cacheLast->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
    } else if (!cacheLast) {
      g_Dbs.db[i].dbCfg.cacheLast = -1;
    } else {
3237
     printf("ERROR: failed to read json, cacheLast not found\n");
3238 3239 3240
     goto PARSE_OVER;
    }

3241 3242 3243 3244
    cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
    if (quorum && quorum->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
    } else if (!quorum) {
3245
      g_Dbs.db[i].dbCfg.quorum = 1;
3246
    } else {
3247
     printf("failed to read json, quorum input mistake");
3248 3249 3250 3251 3252 3253 3254 3255 3256
     goto PARSE_OVER;
    }

    cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
    if (fsync && fsync->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
    } else if (!fsync) {
      g_Dbs.db[i].dbCfg.fsync = -1;
    } else {
3257
     printf("ERROR: failed to read json, fsync not found\n");
3258 3259 3260 3261 3262 3263
     goto PARSE_OVER;
    }    

    // super_talbes 
    cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
    if (!stables || stables->type != cJSON_Array) {
3264
      printf("ERROR: failed to read json, super_tables not found\n");
3265 3266 3267 3268 3269
      goto PARSE_OVER;
    }    
    
    int stbSize = cJSON_GetArraySize(stables);
    if (stbSize > MAX_SUPER_TABLE_COUNT) {
3270
      errorPrint(
3271 3272
              "ERROR: failed to read json, databases size overflow, max database is %d\n",
              MAX_SUPER_TABLE_COUNT);
3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283
      goto PARSE_OVER;
    }

    g_Dbs.db[i].superTblCount = stbSize;
    for (int j = 0; j < stbSize; ++j) {
      cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
      if (stbInfo == NULL) continue;
    
      // dbinfo 
      cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
      if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) {
3284
        printf("ERROR: failed to read json, stb name not found\n");
3285 3286
        goto PARSE_OVER;
      }
3287
      tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE);
3288 3289 3290
    
      cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
      if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
3291
        printf("ERROR: failed to read json, childtable_prefix not found\n");
3292 3293
        goto PARSE_OVER;
      }
3294
      tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE);
3295 3296

      cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null
3297 3298 3299
      if (autoCreateTbl
              && autoCreateTbl->type == cJSON_String
              && autoCreateTbl->valuestring != NULL) {
3300 3301 3302 3303 3304 3305 3306 3307 3308 3309
        if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
        } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        } else {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        }
      } else if (!autoCreateTbl) {
        g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      } else {
3310
        printf("ERROR: failed to read json, auto_create_table not found\n");
3311 3312 3313 3314 3315 3316 3317
        goto PARSE_OVER;
      }
      
      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
      } else if (!batchCreateTbl) {
3318
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
3319
      } else {
3320
        printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
3321 3322 3323 3324
        goto PARSE_OVER;
      }      

      cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
3325 3326 3327
      if (childTblExists
              && childTblExists->type == cJSON_String
              && childTblExists->valuestring != NULL) {
3328 3329 3330 3331 3332 3333 3334 3335 3336 3337
        if (0 == strncasecmp(childTblExists->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
        } else if (0 == strncasecmp(childTblExists->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        } else {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        }
      } else if (!childTblExists) {
        g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
      } else {
3338
        errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n", __func__, __LINE__);
3339 3340 3341 3342 3343
        goto PARSE_OVER;
      }
      
      cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
      if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
3344
        errorPrint("%s() LN%d, failed to read json, childtable_count not found\n", __func__, __LINE__);
3345 3346 3347 3348 3349
        goto PARSE_OVER;
      }
      g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

      cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
3350 3351 3352 3353
      if (dataSource && dataSource->type == cJSON_String
              && dataSource->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
                dataSource->valuestring, MAX_DB_NAME_SIZE);
3354
      } else if (!dataSource) {
3355
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE);
3356
      } else {
3357
        errorPrint("%s() LN%d, failed to read json, data_source not found\n", __func__, __LINE__);
3358 3359 3360 3361
        goto PARSE_OVER;
      }

      cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , restful
3362 3363 3364 3365
      if (insertMode && insertMode->type == cJSON_String
              && insertMode->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode,
                insertMode->valuestring, MAX_DB_NAME_SIZE);
3366
      } else if (!insertMode) {
3367
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE);
3368
      } else {
3369
        printf("ERROR: failed to read json, insert_mode not found\n");
3370 3371 3372
        goto PARSE_OVER;
      }

3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394
      cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
      if (childTbl_limit) {
        if (childTbl_limit->type != cJSON_Number) {
            printf("ERROR: failed to read json, childtable_limit\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result
      }

      cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
      if (childTbl_offset) {
        if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) {
            printf("ERROR: failed to read json, childtable_offset\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblOffset = 0;
      }

3395 3396
      cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
      if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
3397 3398
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                ts->valuestring, MAX_DB_NAME_SIZE);
3399
      } else if (!ts) {
3400 3401
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                "now", MAX_DB_NAME_SIZE);
3402
      } else {
3403
        printf("ERROR: failed to read json, start_timestamp not found\n");
3404 3405
        goto PARSE_OVER;
      }
3406

3407 3408 3409 3410
      cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
      if (timestampStep && timestampStep->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
      } else if (!timestampStep) {
3411
        g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
3412
      } else {
3413
        printf("ERROR: failed to read json, timestamp_step not found\n");
3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425
        goto PARSE_OVER;
      }

      cJSON* sampleDataBufSize = cJSON_GetObjectItem(stbInfo, "sample_buf_size");
      if (sampleDataBufSize && sampleDataBufSize->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].sampleDataBufSize = sampleDataBufSize->valueint;
        if (g_Dbs.db[i].superTbls[j].sampleDataBufSize < 1024*1024) {
          g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024;
        }
      } else if (!sampleDataBufSize) {
        g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024;
      } else {
3426
        printf("ERROR: failed to read json, sample_buf_size not found\n");
3427 3428
        goto PARSE_OVER;
      }      
3429

3430 3431
      cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
      if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) {
3432 3433
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
                sampleFormat->valuestring, MAX_DB_NAME_SIZE);
3434
      } else if (!sampleFormat) {
3435
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE);
3436
      } else {
3437
        printf("ERROR: failed to read json, sample_format not found\n");
3438 3439
        goto PARSE_OVER;
      }      
3440

3441 3442
      cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
      if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) {
3443 3444
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
                sampleFile->valuestring, MAX_FILE_NAME_LEN);
3445 3446 3447
      } else if (!sampleFile) {
        memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
      } else {
3448
        printf("ERROR: failed to read json, sample_file not found\n");
3449 3450
        goto PARSE_OVER;
      }          
3451

3452 3453
      cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
      if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) {
3454 3455
        tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
                tagsFile->valuestring, MAX_FILE_NAME_LEN);
3456 3457 3458 3459 3460 3461 3462 3463 3464
        if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
          g_Dbs.db[i].superTbls[j].tagSource = 0;
        } else {
          g_Dbs.db[i].superTbls[j].tagSource = 1;
        }
      } else if (!tagsFile) {
        memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
        g_Dbs.db[i].superTbls[j].tagSource = 0;
      } else {
3465
        printf("ERROR: failed to read json, tags_file not found\n");
3466 3467
        goto PARSE_OVER;
      }
3468

3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480
      cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
      if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
        int32_t len = maxSqlLen->valueint;
        if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
          len = TSDB_MAX_ALLOWED_SQL_LEN;
        } else if (len < TSDB_MAX_SQL_LEN) {
          len = TSDB_MAX_SQL_LEN;
        }       
        g_Dbs.db[i].superTbls[j].maxSqlLen = len;
      } else if (!maxSqlLen) {
        g_Dbs.db[i].superTbls[j].maxSqlLen = TSDB_MAX_SQL_LEN;
      } else {
3481
        printf("ERROR: failed to read json, maxSqlLen not found\n");
3482 3483 3484
        goto PARSE_OVER;
      }      

3485 3486 3487 3488 3489
      cJSON *multiThreadWriteOneTbl =
          cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
      if (multiThreadWriteOneTbl
              && multiThreadWriteOneTbl->type == cJSON_String
              && multiThreadWriteOneTbl->valuestring != NULL) {
3490 3491 3492 3493 3494 3495 3496 3497
        if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
        } else {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
        }        
      } else if (!multiThreadWriteOneTbl) {
        g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
      } else {
3498
        printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
3499 3500
        goto PARSE_OVER;
      }
3501

3502 3503 3504 3505
      cJSON* rowsPerTbl = cJSON_GetObjectItem(stbInfo, "rows_per_tbl");
      if (rowsPerTbl && rowsPerTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].rowsPerTbl = rowsPerTbl->valueint;
      } else if (!rowsPerTbl) {
3506
        g_Dbs.db[i].superTbls[j].rowsPerTbl = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3507
      } else {
3508
        errorPrint("%s() LN%d, failed to read json, rowsPerTbl input mistake\n", __func__, __LINE__);
3509 3510 3511 3512 3513 3514 3515 3516 3517
        goto PARSE_OVER;
      }      

      cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
      if (disorderRatio && disorderRatio->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
      } else if (!disorderRatio) {
        g_Dbs.db[i].superTbls[j].disorderRatio = 0;
      } else {
3518
        printf("ERROR: failed to read json, disorderRatio not found\n");
3519 3520 3521 3522 3523 3524 3525 3526 3527
        goto PARSE_OVER;
      }      

      cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
      if (disorderRange && disorderRange->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
      } else if (!disorderRange) {
        g_Dbs.db[i].superTbls[j].disorderRange = 1000;
      } else {
3528
        printf("ERROR: failed to read json, disorderRange not found\n");
3529 3530
        goto PARSE_OVER;
      }
3531

3532 3533 3534 3535 3536 3537
      cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
      if (insertRows && insertRows->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
      } else if (!insertRows) {
        g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
      } else {
3538
        errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", __func__, __LINE__);
3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549
        goto PARSE_OVER;
      }

      cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
      if (insertInterval && insertInterval->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
      } else if (!insertInterval) {
        debugPrint("%s() LN%d: stable insert interval be overrided by global %d.\n",
                __func__, __LINE__, g_args.insert_interval);
        g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
      } else {
3550
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", __func__, __LINE__);
3551 3552 3553
        goto PARSE_OVER;
      }

3554
/* CBD      if (NO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable
3555
              || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
3556 3557
        continue;
      }
3558
      */
3559

3560 3561
      int retVal = getColumnAndTagTypeFromInsertJsonFile(
              stbInfo, &g_Dbs.db[i].superTbls[j]);
3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581
      if (false == retVal) {
        goto PARSE_OVER;
      }      
    }    
  }

  ret = true;

PARSE_OVER:
  //free(content);
  //cJSON_Delete(root);
  //fclose(fp);
  return ret;
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3582
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3583 3584 3585 3586
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3587
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_DB_NAME_SIZE);
3588
  } else if (!host) {
3589
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE);
3590
  } else {
3591
    printf("ERROR: failed to read json, host not found\n");
3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3604
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_DB_NAME_SIZE);   
3605
  } else if (!user) {
3606
    tstrncpy(g_queryInfo.user, "root", MAX_DB_NAME_SIZE); ;
3607 3608 3609 3610
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3611
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_DB_NAME_SIZE);
3612
  } else if (!password) {
3613
    tstrncpy(g_queryInfo.password, "taosdata", MAX_DB_NAME_SIZE);;
3614
  }
H
hzcheng 已提交
3615

3616
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3617 3618
  if (answerPrompt && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3619 3620 3621 3622
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
3623
    } else {
3624
      g_args.answer_yes = false;
H
hzcheng 已提交
3625
    }
3626 3627 3628
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
3629
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
3630 3631 3632 3633 3634
    goto PARSE_OVER;
  }  

  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
3635
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE);
3636
  } else if (!dbs) {
3637
    printf("ERROR: failed to read json, databases not found\n");
3638
    goto PARSE_OVER;
H
hzcheng 已提交
3639
  }
3640 3641 3642

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
3643
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
3644
  } else if (!queryMode) {
3645
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
3646
  } else {
3647
    printf("ERROR: failed to read json, query_mode not found\n");
3648
    goto PARSE_OVER;
H
hzcheng 已提交
3649
  }
3650 3651 3652 3653 3654 3655 3656
  
  // super_table_query 
  cJSON *superQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!superQuery) {
    g_queryInfo.superQueryInfo.concurrent = 0;
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
3657
    printf("ERROR: failed to read json, super_table_query not found\n");
3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680
    goto PARSE_OVER;
  } else {  
    cJSON* rate = cJSON_GetObjectItem(superQuery, "query_interval");
    if (rate && rate->type == cJSON_Number) {
      g_queryInfo.superQueryInfo.rate = rate->valueint;
    } else if (!rate) {
      g_queryInfo.superQueryInfo.rate = 0;
    }
  
    cJSON* concurrent = cJSON_GetObjectItem(superQuery, "concurrent");
    if (concurrent && concurrent->type == cJSON_Number) {
      g_queryInfo.superQueryInfo.concurrent = concurrent->valueint;
    } else if (!concurrent) {
      g_queryInfo.superQueryInfo.concurrent = 1;
    }
  
    cJSON* mode = cJSON_GetObjectItem(superQuery, "mode");
    if (mode && mode->type == cJSON_String && mode->valuestring != NULL) {
      if (0 == strcmp("sync", mode->valuestring)) {      
        g_queryInfo.superQueryInfo.subscribeMode = 0;
      } else if (0 == strcmp("async", mode->valuestring)) {      
        g_queryInfo.superQueryInfo.subscribeMode = 1;
      } else {
3681
        printf("ERROR: failed to read json, subscribe mod error\n");
3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703
        goto PARSE_OVER;
      }
    } else {
      g_queryInfo.superQueryInfo.subscribeMode = 0;
    }
    
    cJSON* interval = cJSON_GetObjectItem(superQuery, "interval");
    if (interval && interval->type == cJSON_Number) {
      g_queryInfo.superQueryInfo.subscribeInterval = interval->valueint;
    } else if (!interval) {    
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
    }
  
    cJSON* restart = cJSON_GetObjectItem(superQuery, "restart");
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
      if (0 == strcmp("yes", restart->valuestring)) {      
        g_queryInfo.superQueryInfo.subscribeRestart = 1;
      } else if (0 == strcmp("no", restart->valuestring)) {      
        g_queryInfo.superQueryInfo.subscribeRestart = 0;
      } else {
3704
        printf("ERROR: failed to read json, subscribe restart error\n");
3705 3706 3707 3708 3709 3710 3711
        goto PARSE_OVER;
      }
    } else {
      g_queryInfo.superQueryInfo.subscribeRestart = 1;
    }
  
    cJSON* keepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
3712 3713 3714
    if (keepProgress
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
3715 3716 3717 3718 3719
      if (0 == strcmp("yes", keepProgress->valuestring)) {      
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
      } else if (0 == strcmp("no", keepProgress->valuestring)) {      
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
      } else {
3720
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731
        goto PARSE_OVER;
      }
    } else {
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
    }

    // sqls   
    cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls");
    if (!superSqls) {
      g_queryInfo.superQueryInfo.sqlCount = 0;
    } else if (superSqls->type != cJSON_Array) {
3732
      printf("ERROR: failed to read json, super sqls not found\n");
3733 3734 3735 3736
      goto PARSE_OVER;
    } else {  
      int superSqlSize = cJSON_GetArraySize(superSqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
3737
        printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT);
3738 3739
        goto PARSE_OVER;
      }
3740

3741 3742 3743 3744 3745 3746 3747
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
      for (int j = 0; j < superSqlSize; ++j) {
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
        if (sql == NULL) continue;
      
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
3748
          printf("ERROR: failed to read json, sql not found\n");
3749 3750
          goto PARSE_OVER;
        }
3751
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
H
hzcheng 已提交
3752

3753 3754
        cJSON *result = cJSON_GetObjectItem(sql, "result");
        if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) {
3755
          tstrncpy(g_queryInfo.superQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN);
3756 3757 3758
        } else if (NULL == result) {
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
        } else {
3759
          printf("ERROR: failed to read json, super query result file not found\n");
3760 3761 3762 3763 3764
          goto PARSE_OVER;
        } 
      }
    }
  }
3765

3766 3767 3768 3769 3770 3771
  // sub_table_query 
  cJSON *subQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!subQuery) {
    g_queryInfo.subQueryInfo.threadCnt = 0;
    g_queryInfo.subQueryInfo.sqlCount = 0;
  } else if (subQuery->type != cJSON_Object) {
3772
    printf("ERROR: failed to read json, sub_table_query not found\n");
3773 3774
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
3775
  } else {
3776 3777 3778 3779 3780 3781
    cJSON* subrate = cJSON_GetObjectItem(subQuery, "query_interval");
    if (subrate && subrate->type == cJSON_Number) {
      g_queryInfo.subQueryInfo.rate = subrate->valueint;
    } else if (!subrate) {
      g_queryInfo.subQueryInfo.rate = 0;
    }
3782

3783 3784 3785 3786 3787 3788
    cJSON* threads = cJSON_GetObjectItem(subQuery, "threads");
    if (threads && threads->type == cJSON_Number) {
      g_queryInfo.subQueryInfo.threadCnt = threads->valueint;
    } else if (!threads) {
      g_queryInfo.subQueryInfo.threadCnt = 1;
    }
3789

3790 3791 3792 3793 3794 3795
    //cJSON* subTblCnt = cJSON_GetObjectItem(subQuery, "childtable_count");
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
    //  g_queryInfo.subQueryInfo.childTblCount = subTblCnt->valueint;
    //} else if (!subTblCnt) {
    //  g_queryInfo.subQueryInfo.childTblCount = 0;
    //}
3796

3797 3798
    cJSON* stblname = cJSON_GetObjectItem(subQuery, "stblname");
    if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) {
3799
      tstrncpy(g_queryInfo.subQueryInfo.sTblName, stblname->valuestring, MAX_TB_NAME_SIZE);
3800
    } else {
3801
      printf("ERROR: failed to read json, super table name not found\n");
3802 3803
      goto PARSE_OVER;
    }
3804

3805 3806 3807 3808 3809 3810 3811
    cJSON* submode = cJSON_GetObjectItem(subQuery, "mode");
    if (submode && submode->type == cJSON_String && submode->valuestring != NULL) {
      if (0 == strcmp("sync", submode->valuestring)) {      
        g_queryInfo.subQueryInfo.subscribeMode = 0;
      } else if (0 == strcmp("async", submode->valuestring)) {      
        g_queryInfo.subQueryInfo.subscribeMode = 1;
      } else {
3812
        printf("ERROR: failed to read json, subscribe mod error\n");
3813 3814 3815 3816 3817
        goto PARSE_OVER;
      }
    } else {
      g_queryInfo.subQueryInfo.subscribeMode = 0;
    }
3818

3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834
    cJSON* subinterval = cJSON_GetObjectItem(subQuery, "interval");
    if (subinterval && subinterval->type == cJSON_Number) {
      g_queryInfo.subQueryInfo.subscribeInterval = subinterval->valueint;
    } else if (!subinterval) {    
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
      g_queryInfo.subQueryInfo.subscribeInterval = 10000;
    }
  
    cJSON* subrestart = cJSON_GetObjectItem(subQuery, "restart");
    if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) {
      if (0 == strcmp("yes", subrestart->valuestring)) {      
        g_queryInfo.subQueryInfo.subscribeRestart = 1;
      } else if (0 == strcmp("no", subrestart->valuestring)) {      
        g_queryInfo.subQueryInfo.subscribeRestart = 0;
      } else {
3835
        printf("ERROR: failed to read json, subscribe restart error\n");
3836 3837 3838 3839 3840 3841 3842
        goto PARSE_OVER;
      }
    } else {
      g_queryInfo.subQueryInfo.subscribeRestart = 1;
    }
  
    cJSON* subkeepProgress = cJSON_GetObjectItem(subQuery, "keepProgress");
3843 3844 3845
    if (subkeepProgress &&
            subkeepProgress->type == cJSON_String
            && subkeepProgress->valuestring != NULL) {
3846 3847 3848 3849 3850
      if (0 == strcmp("yes", subkeepProgress->valuestring)) {      
        g_queryInfo.subQueryInfo.subscribeKeepProgress = 1;
      } else if (0 == strcmp("no", subkeepProgress->valuestring)) {      
        g_queryInfo.subQueryInfo.subscribeKeepProgress = 0;
      } else {
3851
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862
        goto PARSE_OVER;
      }
    } else {
      g_queryInfo.subQueryInfo.subscribeKeepProgress = 0;
    }  

    // sqls     
    cJSON* subsqls = cJSON_GetObjectItem(subQuery, "sqls");
    if (!subsqls) {
      g_queryInfo.subQueryInfo.sqlCount = 0;
    } else if (subsqls->type != cJSON_Array) {
3863
      printf("ERROR: failed to read json, super sqls not found\n");
3864 3865 3866 3867
      goto PARSE_OVER;
    } else {  
      int superSqlSize = cJSON_GetArraySize(subsqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
3868
        printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT);
3869 3870 3871 3872 3873 3874 3875 3876 3877 3878
        goto PARSE_OVER;
      }
    
      g_queryInfo.subQueryInfo.sqlCount = superSqlSize;
      for (int j = 0; j < superSqlSize; ++j) {      
        cJSON* sql = cJSON_GetArrayItem(subsqls, j);
        if (sql == NULL) continue;
        
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
3879
          printf("ERROR: failed to read json, sql not found\n");
3880 3881
          goto PARSE_OVER;
        }
3882
        tstrncpy(g_queryInfo.subQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
3883 3884 3885

        cJSON *result = cJSON_GetObjectItem(sql, "result");
        if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){
3886
          tstrncpy(g_queryInfo.subQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN);
3887 3888 3889
        } else if (NULL == result) {
          memset(g_queryInfo.subQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
        }  else {
3890
          printf("ERROR: failed to read json, sub query result file not found\n");
3891 3892 3893 3894
          goto PARSE_OVER;
        } 
      }
    }
H
hzcheng 已提交
3895 3896
  }

3897
  ret = true;
H
hzcheng 已提交
3898

3899 3900 3901 3902 3903 3904
PARSE_OVER:
  //free(content);
  //cJSON_Delete(root);
  //fclose(fp);
  return ret;
}
H
hzcheng 已提交
3905

3906
static bool getInfoFromJsonFile(char* file) {
3907
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
3908

3909 3910 3911 3912 3913
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
3914

3915 3916 3917 3918 3919 3920 3921 3922 3923
  bool  ret = false;
  int   maxLen = 64000;
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
3924
  }
H
Hui Li 已提交
3925

3926 3927 3928
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
3929
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
3930 3931
    goto PARSE_OVER;
  }
H
Hui Li 已提交
3932

3933 3934 3935
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
3936
      g_args.test_mode = INSERT_TEST;
3937
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
3938
      g_args.test_mode = QUERY_TEST;
3939
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
3940
      g_args.test_mode = SUBSCRIBE_TEST;
3941
    } else {
3942
      printf("ERROR: failed to read json, filetype not support\n");
3943 3944 3945
      goto PARSE_OVER;
    }
  } else if (!filetype) {
3946
    g_args.test_mode = INSERT_TEST;
3947
  } else {
3948
    printf("ERROR: failed to read json, filetype not found\n");
3949 3950
    goto PARSE_OVER;
  }
H
hzcheng 已提交
3951

3952
  if (INSERT_TEST == g_args.test_mode) {
3953
    ret = getMetaFromInsertJsonFile(root);
3954
  } else if (QUERY_TEST == g_args.test_mode) {
3955
    ret = getMetaFromQueryJsonFile(root);
3956
  } else if (SUBSCRIBE_TEST == g_args.test_mode) {
3957 3958
    ret = getMetaFromQueryJsonFile(root);
  } else {
3959
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n", __func__, __LINE__);
3960
    goto PARSE_OVER;
3961
  } 
3962 3963 3964 3965

PARSE_OVER:
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
3966
  fclose(fp);
3967 3968
  return ret;
}
H
hzcheng 已提交
3969

3970
static void prepareSampleData() {
3971 3972 3973 3974
  for (int i = 0; i < g_Dbs.dbCount; i++) {    
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
        (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]);
S
Shuaiqiang Chang 已提交
3975
      }
3976 3977 3978 3979
    }
  }
}

3980
static void postFreeResource() {
3981 3982 3983
  tmfclose(g_fpOfInsertResult);
  for (int i = 0; i < g_Dbs.dbCount; i++) {    
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
3984 3985 3986
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
3987
      }
3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4000
    }
4001 4002
  }
}
S
Shuaiqiang Chang 已提交
4003

4004 4005
static int getRowDataFromSample(char*  dataBuf, int maxLen, int64_t timestamp,
      SSuperTable* superTblInfo, int* sampleUsePos) {
4006
  if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
4007
/*    int ret = readSampleFromCsvFileToMem(superTblInfo);
4008
    if (0 != ret) {
4009
      tmfree(superTblInfo->sampleDataBuf);
4010
      superTblInfo->sampleDataBuf = NULL;
4011 4012
      return -1;
    }
4013
*/
4014 4015
    *sampleUsePos = 0;
  }
S
Shuaiqiang Chang 已提交
4016

4017
  int    dataLen = 0;
4018

4019 4020 4021 4022
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "(%" PRId64 ", ", timestamp);
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
4023
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
S
Shuaiqiang Chang 已提交
4024

4025
  (*sampleUsePos)++;
4026
 
4027 4028
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4029

4030
static int generateRowData(char*  dataBuf, int maxLen, int64_t timestamp, SSuperTable* stbInfo) {
4031 4032 4033
  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp);
  for (int i = 0; i < stbInfo->columnCount; i++) {    
4034 4035
    if ((0 == strncasecmp(stbInfo->columns[i].dataType, "binary", 6))
            || (0 == strncasecmp(stbInfo->columns[i].dataType, "nchar", 5))) {
4036
      if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
4037
        errorPrint( "binary or nchar length overflow, max size:%u\n",
4038
                (uint32_t)TSDB_MAX_BINARY_LEN);
4039 4040
        return (-1);
      }
4041

4042 4043
      char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
      if (NULL == buf) {
4044
        errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
4045 4046 4047 4048 4049
        return (-1);
      }
      rand_string(buf, stbInfo->columns[i].dataLen);
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "\'%s\', ", buf);
      tmfree(buf);
4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "int", 3)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
              "%d, ", rand_int());
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "bigint", 6)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
              "%"PRId64", ", rand_bigint());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "float", 5)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
              "%f, ", rand_float());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "double", 6)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
              "%f, ", rand_double());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "smallint", 8)) {
4068 4069 4070 4071 4072 4073 4074 4075
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_smallint());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType, "tinyint", 7)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_tinyint());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bool", 4)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_bool());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType, "timestamp", 9)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint());
    }  else {
4076
      errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType);
4077 4078
      return (-1);
    }
S
Shuaiqiang Chang 已提交
4079
  }
4080

4081 4082
  dataLen -= 2;
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
4083

4084 4085
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4086

4087
static int32_t generateData(char *res, char **data_type,
4088
        int num_of_cols, int64_t timestamp, int lenOfBinary) {
4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119
  memset(res, 0, MAX_DATA_SIZE);
  char *pstr = res;
  pstr += sprintf(pstr, "(%" PRId64, timestamp);
  int c = 0;

  for (; c < MAX_NUM_DATATYPE; c++) {
    if (data_type[c] == NULL) {
      break;
    }
  }

  if (0 == c) {
    perror("data type error!");
    exit(-1);
  }

  for (int i = 0; i < num_of_cols; i++) {
    if (strcasecmp(data_type[i % c], "tinyint") == 0) {
      pstr += sprintf(pstr, ", %d", rand_tinyint() );
    } else if (strcasecmp(data_type[i % c], "smallint") == 0) {
      pstr += sprintf(pstr, ", %d", rand_smallint());
    } else if (strcasecmp(data_type[i % c], "int") == 0) {
      pstr += sprintf(pstr, ", %d", rand_int()); 
    } else if (strcasecmp(data_type[i % c], "bigint") == 0) {
      pstr += sprintf(pstr, ", %" PRId64, rand_bigint());
    } else if (strcasecmp(data_type[i % c], "float") == 0) {
      pstr += sprintf(pstr, ", %10.4f", rand_float());
    } else if (strcasecmp(data_type[i % c], "double") == 0) {
      double t = rand_double();
      pstr += sprintf(pstr, ", %20.8f", t);
    } else if (strcasecmp(data_type[i % c], "bool") == 0) {
4120
      bool b = taosRandom() & 1;
4121 4122
      pstr += sprintf(pstr, ", %s", b ? "true" : "false");
    } else if (strcasecmp(data_type[i % c], "binary") == 0) {
4123 4124
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4125 4126 4127
      pstr += sprintf(pstr, ", \"%s\"", s);
      free(s);
    }else if (strcasecmp(data_type[i % c], "nchar") == 0) {
4128 4129
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144
      pstr += sprintf(pstr, ", \"%s\"", s);
      free(s);
    }

    if (pstr - res > MAX_DATA_SIZE) {
      perror("column length too long, abort");
      exit(-1);
    }
  }

  pstr += sprintf(pstr, ")");

  return (int32_t)(pstr - res);
}

4145
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
4146 4147
  char* sampleDataBuf = NULL;

4148
  sampleDataBuf = calloc(
4149
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
4150
  if (sampleDataBuf == NULL) {
4151 4152
      errorPrint("%s() LN%d, Failed to calloc %d Bytes, reason:%s\n", 
              __func__, __LINE__,
4153 4154
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 
              strerror(errno));
4155
      return -1;
4156
  }
4157

4158 4159 4160 4161
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
4162
      errorPrint("%s() LN%d, read sample from csv file failed.\n", __func__, __LINE__);
4163
      tmfree(sampleDataBuf);
4164
      superTblInfo->sampleDataBuf = NULL;
4165
      return -1;
H
Hui Li 已提交
4166
  }
4167

4168 4169 4170
  return 0;
}

4171
static int execInsert(threadInfo *pThreadInfo, char *buffer, int k)
4172 4173
{
  int affectedRows;
4174
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4175

4176 4177
  verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
            __func__, __LINE__, buffer);
4178
  if (superTblInfo) {
4179
    if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) {
4180
      affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE);
4181
    } else {
4182
      if (0 != postProceSql(g_Dbs.host, g_Dbs.port, buffer)) {
4183
        affectedRows = -1;
4184
        printf("========restful return fail, threadID[%d]\n", pThreadInfo->threadID);
4185 4186 4187
      } else {
        affectedRows = k;
      }
4188
    }
4189
  } else {
4190
    affectedRows = queryDbExec(pThreadInfo->taos, buffer, 1);
4191 4192 4193 4194 4195
  }

  return affectedRows;
}

4196
static void getTableName(char *pTblName, threadInfo* pThreadInfo, int tableSeq)
4197 4198
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4199 4200 4201 4202 4203 4204
  if (superTblInfo) {
    if ((superTblInfo->childTblOffset >= 0)
            && (superTblInfo->childTblLimit > 0)) {
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
    } else {
4205

4206 4207 4208
        verbosePrint("[%d] %s() LN%d: from=%d count=%d seq=%d\n",
                pThreadInfo->threadID, __func__, __LINE__,
                pThreadInfo->start_table_from,
4209 4210 4211 4212 4213 4214
                pThreadInfo->ntables, tableSeq);
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
    }
  } else {
    snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%d",
4215
        g_args.tb_prefix, tableSeq);
4216 4217 4218 4219
  }
}

static int generateDataTail(char *tableName, int32_t tableSeq,
4220
        threadInfo* pThreadInfo, SSuperTable* superTblInfo,
4221 4222 4223
        int batch, char* buffer, int64_t insertRows,
        int64_t startFrom, uint64_t startTime, int *pSamplePos, int *dataLen) {
  int len = 0;
4224 4225
  int ncols_per_record = 1; // count first col ts

4226
  if (superTblInfo == NULL) {
4227 4228 4229 4230 4231
    int datatypeSeq = 0;
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
4232 4233
  }

4234
  verbosePrint("%s() LN%d batch=%d\n", __func__, __LINE__, batch);
4235

4236 4237
  int k = 0;
  for (k = 0; k < batch;) {
4238
    if (superTblInfo) {
4239
        int retLen = 0;
4240

4241 4242 4243
        if (0 == strncasecmp(superTblInfo->dataSource,
                    "sample", strlen("sample"))) {
          retLen = getRowDataFromSample(
4244
                    buffer + len, 
4245
                    superTblInfo->maxSqlLen - len, 
4246
                    startTime + superTblInfo->timeStampStep * k,
4247
                    superTblInfo, 
4248
                    pSamplePos);
4249 4250 4251 4252
       } else if (0 == strncasecmp(superTblInfo->dataSource,
                   "rand", strlen("rand"))) {
          int rand_num = rand_tinyint() % 100;
          if (0 != superTblInfo->disorderRatio 
4253
                    && rand_num < superTblInfo->disorderRatio) {
4254 4255 4256
            int64_t d = startTime
                + superTblInfo->timeStampStep * k
                - taosRandom() % superTblInfo->disorderRange;
4257
            retLen = generateRowData(
4258
                      buffer + len, 
4259 4260
                      superTblInfo->maxSqlLen - len,
                      d, 
4261
                      superTblInfo);
4262 4263
          } else {
            retLen = generateRowData(
4264
                      buffer + len, 
4265
                      superTblInfo->maxSqlLen - len, 
4266
                      startTime + superTblInfo->timeStampStep * k,
4267
                      superTblInfo);
4268 4269
          }
       }
4270

4271 4272 4273
       if (retLen < 0) {
         return -1;
       }
4274

4275
       len += retLen;
4276

4277 4278 4279 4280
       if (len >= (superTblInfo->maxSqlLen - 256)) {    // reserve for overwrite
         k++;
         break;
       }
4281
    } else {
4282
      int rand_num = taosRandom() % 100;
4283 4284 4285 4286
          char data[MAX_DATA_SIZE];
          char **data_type = g_args.datatype;
          int lenOfBinary = g_args.len_of_binary;

4287
      if ((g_args.disorderRatio != 0)
4288 4289
                && (rand_num < g_args.disorderRange)) {
             
4290
        int64_t d = startTime - taosRandom() % 1000000 + rand_num;
4291
        len = generateData(data, data_type,
4292
                  ncols_per_record, d, lenOfBinary);
4293
      } else {
4294
            len = generateData(data, data_type,
4295 4296 4297 4298
                  ncols_per_record,
                  startTime + DEFAULT_TIMESTAMP_STEP * startFrom,
                  lenOfBinary);
      }
4299

4300 4301
      buffer += sprintf(buffer, " %s", data);
      if (strlen(buffer) >= (g_args.max_sql_len - 256)) { // too long
4302 4303
          k++;
          break;
4304 4305 4306
      }
    }

4307 4308
    verbosePrint("%s() LN%d len=%d k=%d \nbuffer=%s\n",
            __func__, __LINE__, len, k, buffer);
4309 4310 4311

    k++;
    startFrom ++;
4312

4313
    if (startFrom >= insertRows) {
4314
      break;
4315
    }
4316
  }
4317

4318 4319 4320
  *dataLen = len;
  return k;
}
4321

4322 4323
static int generateSQLHead(char *tableName, int32_t tableSeq,
        threadInfo* pThreadInfo, SSuperTable* superTblInfo, char *buffer)
4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336
{
  int len;
  if (superTblInfo) {
    if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
            tagsValBuf = generateTagVaulesForStb(superTblInfo);
      } else {
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
      }
      if (NULL == tagsValBuf) {
4337
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n", __func__, __LINE__);
4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354
        return -1;
      }

      len = snprintf(buffer,
                  superTblInfo->maxSqlLen,
                  "insert into %s.%s using %s.%s tags %s values",
                  pThreadInfo->db_name,
                  tableName,
                  pThreadInfo->db_name,
                  superTblInfo->sTblName,
                  tagsValBuf);
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
      len = snprintf(buffer,
                  superTblInfo->maxSqlLen,
                  "insert into %s.%s values",
                  pThreadInfo->db_name,
4355
                  tableName);
4356 4357
    } else {
      len = snprintf(buffer,
4358
                  superTblInfo->maxSqlLen,
4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397
                  "insert into %s.%s values",
                  pThreadInfo->db_name,
                  tableName);
    }
  } else {
      len = snprintf(buffer,
                  g_args.max_sql_len,
                  "insert into %s.%s values",
                  pThreadInfo->db_name,
                  tableName);
  }

  return len;
}

static int generateDataBuffer(char *pTblName,
        int32_t tableSeq,
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
        int64_t startFrom, int64_t startTime, int *pSamplePos)
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  int ncols_per_record = 1; // count first col ts

  if (superTblInfo == NULL) {
    int datatypeSeq = 0;
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
  }

  assert(buffer != NULL);

  memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len);

  char *pstr = buffer;

4398 4399
  int headLen = generateSQLHead(pTblName, tableSeq, pThreadInfo, superTblInfo,
          buffer);
4400 4401 4402 4403 4404 4405 4406
  pstr += headLen;

  int k;
  int dataLen;
  k = generateDataTail(pTblName, tableSeq, pThreadInfo, superTblInfo,
          g_args.num_of_RPR, pstr, insertRows, startFrom, startTime,
          pSamplePos, &dataLen);
4407 4408
  return k;
}
4409

4410
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
4411 4412
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
         pThreadInfo->threadID, __func__, __LINE__);
4413 4414 4415 4416 4417

  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1);
  if (NULL == buffer) {
4418
    errorPrint( "Failed to alloc %d Bytes, reason:%s\n",
4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452
              superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len,
              strerror(errno));
    return NULL;
  }
 
  int insertMode;
  char tableName[TSDB_TABLE_NAME_LEN];

  int rowsPerTbl = superTblInfo?superTblInfo->rowsPerTbl:g_args.rows_per_tbl;

  if (rowsPerTbl > 0) {
    insertMode = INTERLACE_INSERT_MODE;
  } else {
    insertMode = PROGRESSIVE_INSERT_MODE;
  }

  // rows per table need be less than insert batch
  if (rowsPerTbl > g_args.num_of_RPR)
      rowsPerTbl = g_args.num_of_RPR;

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

  int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
  int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
  uint64_t st = 0;
  uint64_t et = 0xffffffff;

  int64_t lastPrintTime = taosGetTimestampMs();
  int64_t startTs = taosGetTimestampUs();
  int64_t endTs;

  int tableSeq = pThreadInfo->start_table_from;

4453 4454
  debugPrint("[%d] %s() LN%d: start_table_from=%d ntables=%d insertRows=%"PRId64"\n",
          pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from,
4455 4456 4457 4458 4459
          pThreadInfo->ntables, insertRows);

  int64_t startTime = pThreadInfo->start_time;

  int batchPerTblTimes;
4460
  int batchPerTbl;
4461

4462 4463 4464 4465 4466 4467
  assert(pThreadInfo->ntables > 0);

  if (rowsPerTbl > g_args.num_of_RPR)
        rowsPerTbl = g_args.num_of_RPR;

  batchPerTbl = rowsPerTbl;
4468
  if ((rowsPerTbl > 0) && (pThreadInfo->ntables > 1)) {
4469 4470
    batchPerTblTimes =
        (g_args.num_of_RPR / (rowsPerTbl * pThreadInfo->ntables)) + 1;
4471 4472 4473 4474
  } else {
    batchPerTblTimes = 1;
  }

4475
  int generatedRecPerTbl = 0;
4476 4477
  bool flagSleep = true;
  int sleepTimeTotal = 0;
4478
  int timeShift = 0;
4479
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
4480
    if ((flagSleep) && (insert_interval)) {
4481
        st = taosGetTimestampUs();
4482
        flagSleep = false;
4483 4484 4485
    }
    // generate data
    memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len);
4486

4487
    char *pstr = buffer;
4488
    int recOfBatch = 0;
4489

4490 4491
    for (int i = 0; i < batchPerTblTimes; i ++) {
      getTableName(tableName, pThreadInfo, tableSeq);
4492

4493 4494
      int headLen;
      if (i == 0) {
4495 4496
        headLen = generateSQLHead(tableName, tableSeq, pThreadInfo,
                superTblInfo, pstr);
4497
      } else {
4498
        headLen = snprintf(pstr, TSDB_TABLE_NAME_LEN, "%s.%s values",
4499 4500
                  pThreadInfo->db_name,
                  tableName);
4501
      }
4502

4503
      // generate data buffer
4504 4505
      verbosePrint("[%d] %s() LN%d i=%d buffer:\n%s\n",
                pThreadInfo->threadID, __func__, __LINE__, i, buffer);
4506

4507 4508
      pstr += headLen;
      int dataLen = 0;
4509

4510
      verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n",
4511 4512 4513 4514 4515
                pThreadInfo->threadID, __func__, __LINE__,
                i, batchPerTblTimes, batchPerTbl);
      generateDataTail(
        tableName, tableSeq, pThreadInfo, superTblInfo,
        batchPerTbl, pstr, insertRows, 0,
4516
        startTime + timeShift + sleepTimeTotal,
4517
        &(pThreadInfo->samplePos), &dataLen);
4518
      pstr += dataLen;
4519 4520
      recOfBatch += batchPerTbl;
      pThreadInfo->totalInsertRows += batchPerTbl;
4521
      verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n",
4522 4523
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
4524

4525
      timeShift ++;
4526 4527 4528 4529 4530
      tableSeq ++;
      if (insertMode == INTERLACE_INSERT_MODE) {
          if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
            // turn to first table
            tableSeq = pThreadInfo->start_table_from;
4531 4532 4533 4534 4535 4536 4537
            generatedRecPerTbl += batchPerTbl;
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;

            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
4538
          }
4539 4540
      }

4541
      int remainRows = insertRows - generatedRecPerTbl;
4542
      if ((remainRows > 0) && (batchPerTbl > remainRows))
4543 4544
        batchPerTbl = remainRows;

4545
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%d insertRows=%"PRId64"\n",
4546 4547 4548 4549
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
4550 4551 4552
        break;
    }

4553
    verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRId64"\n",
4554 4555 4556 4557 4558
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
           pThreadInfo->threadID, __func__, __LINE__, buffer);

4559 4560
    startTs = taosGetTimestampUs();

4561
    int affectedRows = execInsert(pThreadInfo, buffer, recOfBatch);
4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572

    endTs = taosGetTimestampUs();
    int64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %10.6fms\n",
            __func__, __LINE__, delay/1000.0);

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

4573 4574
    verbosePrint("[%d] %s() LN%d affectedRows=%d\n", pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
4575 4576
    if ((affectedRows < 0) || (recOfBatch != affectedRows)) {
        errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %d\n%s\n",
4577
                pThreadInfo->threadID, __func__, __LINE__,
4578
                recOfBatch, affectedRows, buffer);
4579
        goto free_and_statistics_interlace;
4580
    }
4581

4582
    pThreadInfo->totalAffectedRows += affectedRows;
4583

4584 4585 4586
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
      printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
4587 4588 4589
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
4590 4591
      lastPrintTime = currentPrintTime;
    }
4592

4593 4594
    if ((insert_interval) && flagSleep) {
      et = taosGetTimestampUs();
4595

4596 4597
      if (insert_interval > ((et - st)/1000) ) {
        int sleepTime = insert_interval - (et -st)/1000;
4598 4599
        performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleepTime);
4600 4601 4602
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
4603
    }
4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615
  }

free_and_statistics_interlace:
  tmfree(buffer);

  printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", 
          pThreadInfo->threadID, 
          pThreadInfo->totalInsertRows, 
          pThreadInfo->totalAffectedRows);
  return NULL;
}

4616 4617 4618 4619 4620 4621 4622 4623
// sync insertion
/*
   1 thread: 100 tables * 2000  rows/s
   1 thread: 10  tables * 20000 rows/s
   6 thread: 300 tables * 2000  rows/s

   2 taosinsertdata , 1 thread:  10  tables * 20000 rows/s
*/
4624
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
4625
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
4626

4627
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4628

4629 4630
  char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1);
  if (NULL == buffer) {
4631
    errorPrint( "Failed to alloc %d Bytes, reason:%s\n",
4632 4633 4634 4635
              superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len,
              strerror(errno));
    return NULL;
  }
4636 4637 4638 4639 4640
 
  int64_t lastPrintTime = taosGetTimestampMs();
  int64_t startTs = taosGetTimestampUs();
  int64_t endTs;

4641
  int timeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
4642
  int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
4643 4644 4645
  uint64_t st = 0;
  uint64_t et = 0xffffffff;

4646 4647
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
4648

4649
  pThreadInfo->samplePos = 0;
4650

4651
  for (uint32_t tableSeq = pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to;
4652
        tableSeq ++) {
4653
    int64_t start_time = pThreadInfo->start_time;
4654 4655 4656 4657 4658 4659 4660

    int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
    verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows);

    for (int64_t i = 0; i < insertRows;) {
      if (insert_interval) {
            st = taosGetTimestampUs();
4661
      }
4662

4663 4664 4665 4666 4667 4668
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
      verbosePrint("%s() LN%d: tid=%d seq=%d tableName=%s\n",
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);

4669 4670 4671 4672
      int generated = generateDataBuffer(
              tableName, tableSeq, pThreadInfo, buffer, insertRows,
            i, start_time + pThreadInfo->totalInsertRows * timeStampStep,
            &(pThreadInfo->samplePos));
4673 4674 4675 4676 4677
      if (generated > 0)
        i += generated;
      else
        goto free_and_statistics_2;

4678
      pThreadInfo->totalInsertRows += generated;
4679 4680 4681 4682

      startTs = taosGetTimestampUs();

      int affectedRows = execInsert(pThreadInfo, buffer, generated);
4683 4684 4685

      endTs = taosGetTimestampUs();
      int64_t delay = endTs - startTs;
4686 4687 4688
      performancePrint("%s() LN%d, insert execution time is %10.6fms\n",
              __func__, __LINE__, delay/1000.0);

4689 4690 4691 4692
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
4693

4694 4695 4696 4697 4698
      if (affectedRows < 0)
        goto free_and_statistics_2;

      pThreadInfo->totalAffectedRows += affectedRows;

4699 4700
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
4701
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
4702 4703 4704
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
4705 4706 4707
        lastPrintTime = currentPrintTime;
      }

4708
      if (i >= insertRows)
4709
        break;
4710 4711 4712 4713 4714 4715

      if (insert_interval) {
        et = taosGetTimestampUs();

        if (insert_interval > ((et - st)/1000) ) {
            int sleep_time = insert_interval - (et -st)/1000;
4716 4717
            performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleep_time);
4718 4719
            taosMsleep(sleep_time); // ms
        }
4720 4721 4722
      }
    }   // num_of_DPT

4723
    if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo &&
4724 4725
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
4726 4727
          printf("%s() LN%d samplePos=%d\n",
                  __func__, __LINE__, pThreadInfo->samplePos);
4728
    }
4729
  } // tableSeq
4730

4731
free_and_statistics_2:
4732 4733
  tmfree(buffer);

4734
  printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", 
4735 4736 4737
          pThreadInfo->threadID, 
          pThreadInfo->totalInsertRows, 
          pThreadInfo->totalAffectedRows);
4738
  return NULL;
H
Hui Li 已提交
4739 4740
}

4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756
static void* syncWrite(void *sarg) {

  threadInfo *winfo = (threadInfo *)sarg; 
  SSuperTable* superTblInfo = winfo->superTblInfo;

  int rowsPerTbl = superTblInfo?superTblInfo->rowsPerTbl:g_args.rows_per_tbl;

  if (rowsPerTbl > 0) {
    // interlace mode
    return syncWriteInterlace(winfo);
  } else {
    // progressive mode
    return syncWriteProgressive(winfo);
  }
}

4757
static void callBack(void *param, TAOS_RES *res, int code) {
4758
  threadInfo* winfo = (threadInfo*)param; 
4759
  SSuperTable* superTblInfo = winfo->superTblInfo;
4760

4761 4762 4763 4764 4765
  int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
  if (insert_interval) {
    winfo->et = taosGetTimestampUs();
    if (((winfo->et - winfo->st)/1000) < insert_interval) {
      taosMsleep(insert_interval - (winfo->et - winfo->st)/1000); // ms
4766
    }
H
Hui Li 已提交
4767 4768
  }
  
4769 4770 4771
  char *buffer = calloc(1, winfo->superTblInfo->maxSqlLen);
  char *data   = calloc(1, MAX_DATA_SIZE);
  char *pstr = buffer;
4772 4773
  pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix,
          winfo->start_table_from);
4774
//  if (winfo->counter >= winfo->superTblInfo->insertRows) {
4775
  if (winfo->counter >= g_args.num_of_RPR) {
4776
    winfo->start_table_from++;
4777 4778
    winfo->counter = 0;
  }
4779
  if (winfo->start_table_from > winfo->end_table_to) {
4780 4781 4782 4783 4784 4785
    tsem_post(&winfo->lock_sem);
    free(buffer);
    free(data);
    taos_free_result(res);
    return;
  }
H
Hui Li 已提交
4786
  
4787
  for (int i = 0; i < g_args.num_of_RPR; i++) {
4788
    int rand_num = taosRandom() % 100;
4789 4790
    if (0 != winfo->superTblInfo->disorderRatio && rand_num < winfo->superTblInfo->disorderRatio)
    {
4791
      int64_t d = winfo->lastTs - taosRandom() % 1000000 + rand_num;
4792 4793 4794
      //generateData(data, datatype, ncols_per_record, d, len_of_binary);
      (void)generateRowData(data, MAX_DATA_SIZE, d, winfo->superTblInfo);
    } else {
4795
      //generateData(data, datatype, ncols_per_record, start_time += 1000, len_of_binary);
4796
      (void)generateRowData(data, MAX_DATA_SIZE, winfo->lastTs += 1000, winfo->superTblInfo);
H
Hui Li 已提交
4797
    }
4798 4799
    pstr += sprintf(pstr, "%s", data);
    winfo->counter++;
H
Hui Li 已提交
4800

4801 4802
    if (winfo->counter >= winfo->superTblInfo->insertRows) {
      break;
H
Hui Li 已提交
4803 4804
    }
  }
4805

4806 4807
  if (insert_interval) {
    winfo->st = taosGetTimestampUs();
4808 4809 4810 4811
  }
  taos_query_a(winfo->taos, buffer, callBack, winfo);
  free(buffer);
  free(data);
H
Hui Li 已提交
4812

4813
  taos_free_result(res);
H
Hui Li 已提交
4814 4815
}

4816
static void *asyncWrite(void *sarg) {
4817
  threadInfo *winfo = (threadInfo *)sarg;
4818
  SSuperTable* superTblInfo = winfo->superTblInfo;
4819 4820 4821 4822 4823

  winfo->st = 0;
  winfo->et = 0;
  winfo->lastTs = winfo->start_time;
  
4824 4825 4826
  int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
  if (insert_interval) {
    winfo->st = taosGetTimestampUs();
H
Hui Li 已提交
4827
  }
4828 4829 4830
  taos_query_a(winfo->taos, "show databases", callBack, winfo);

  tsem_wait(&(winfo->lock_sem));
H
Hui Li 已提交
4831 4832 4833 4834

  return NULL;
}

4835 4836
static void startMultiThreadInsertData(int threads, char* db_name,
        char* precision,SSuperTable* superTblInfo) {
4837 4838

    pthread_t *pids = malloc(threads * sizeof(pthread_t));
4839 4840
    assert(pids != NULL);

4841
    threadInfo *infos = malloc(threads * sizeof(threadInfo));
4842 4843
    assert(infos != NULL);

4844 4845
    memset(pids, 0, threads * sizeof(pthread_t));
    memset(infos, 0, threads * sizeof(threadInfo));
H
Hui Li 已提交
4846

4847
    int ntables = 0;
4848 4849 4850 4851 4852
    if (superTblInfo) {

        if ((superTblInfo->childTblOffset >= 0) 
            && (superTblInfo->childTblLimit > 0)) {

4853
            ntables = superTblInfo->childTblLimit;
4854
        } else {
4855
            ntables = superTblInfo->childTblCount;
4856 4857
        }
    } else {
4858
        ntables = g_args.num_of_tables;
4859
    }
H
Hui Li 已提交
4860

4861 4862 4863 4864 4865 4866 4867 4868 4869 4870
    int a = ntables / threads;
    if (a < 1) {
        threads = ntables;
        a = 1;
    }

    int b = 0;
    if (threads != 0) {
        b = ntables % threads;
    }
4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887

  //TAOS* taos;
  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
  //  if (NULL == taos) {
  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
  //    exit(-1);
  //  }
  //}

  int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
  if (0 != precision[0]) {
    if (0 == strncasecmp(precision, "ms", 2)) {
      timePrec = TSDB_TIME_PRECISION_MILLI;
    }  else if (0 == strncasecmp(precision, "us", 2)) {
      timePrec = TSDB_TIME_PRECISION_MICRO;
    }  else {
4888
      errorPrint( "No support precision: %s\n", precision);
4889 4890 4891 4892
      exit(-1);
    }
  }

4893 4894 4895 4896
  int64_t start_time; 
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
        start_time = taosGetTimestamp(timePrec);
4897
    } else {
4898 4899 4900 4901 4902
      if (TSDB_CODE_SUCCESS != taosParseTime(
        superTblInfo->startTimestamp,
        &start_time,
        strlen(superTblInfo->startTimestamp),
        timePrec, 0)) {
4903
          ERROR_EXIT("failed to parse time!\n");
4904
      }
4905
    }
4906 4907
  } else {
     start_time = 1500000000000;
4908 4909 4910
  }

  double start = getCurrentTime();
4911
 
4912
  int startFrom;
4913

4914
  if ((superTblInfo) && (superTblInfo->childTblOffset >= 0))
4915
      startFrom = superTblInfo->childTblOffset;
4916
  else
4917
      startFrom = 0;
4918

4919 4920 4921
  // read sample data from file first
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, 
              "sample", strlen("sample")))) {
4922
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
4923
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n", __func__, __LINE__);
4924 4925 4926 4927
      exit(-1);
    }
  }

4928 4929 4930 4931
  // read sample data from file first
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, 
              "sample", strlen("sample")))) {
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
4932
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n", __func__, __LINE__);
4933 4934 4935 4936 4937 4938 4939 4940
      exit(-1);
    }
  }

  TAOS* taos = taos_connect(
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
  if (NULL == taos) {
4941 4942
    errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
                __func__, __LINE__, taos_errstr(NULL));
4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960
    exit(-1);
  }

  if (superTblInfo) {

    int limit, offset;
    if (superTblInfo && (superTblInfo->childTblOffset >= 0)
            && (superTblInfo->childTblLimit > 0)) {
        limit = superTblInfo->childTblLimit;
        offset = superTblInfo->childTblOffset;
    } else {
        limit = superTblInfo->childTblCount;
        offset = 0;
    }

    superTblInfo->childTblName = (char*)calloc(1,
        limit * TSDB_TABLE_NAME_LEN);
    if (superTblInfo->childTblName == NULL) {
4961
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975
      taos_close(taos);
      exit(-1);
    }

    int childTblCount;
    getChildNameOfSuperTableWithLimitAndOffset(
        taos,
        db_name, superTblInfo->sTblName,
        &superTblInfo->childTblName, &childTblCount,
        limit,
        offset);
  }
  taos_close(taos);

H
Hui Li 已提交
4976
  for (int i = 0; i < threads; i++) {
4977
    threadInfo *t_info = infos + i;
H
Hui Li 已提交
4978 4979
    t_info->threadID = i;
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
4980 4981 4982
    t_info->superTblInfo = superTblInfo;

    t_info->start_time = start_time;
4983
    t_info->minDelay = INT16_MAX;
4984

4985 4986
    if ((NULL == superTblInfo) ||
            (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) {
4987
      //t_info->taos = taos;
4988
      t_info->taos = taos_connect(
4989
              g_Dbs.host, g_Dbs.user,
4990
              g_Dbs.password, db_name, g_Dbs.port);
4991
      if (NULL == t_info->taos) {
4992
        errorPrint( "connect to server fail from insert sub thread, reason: %s\n",
4993
                taos_errstr(NULL));
4994 4995 4996 4997 4998 4999
        exit(-1);
      }
    } else {
      t_info->taos = NULL;
    }

5000 5001
    if ((NULL == superTblInfo)
            || (0 == superTblInfo->multiThreadWriteOneTbl)) {
5002 5003 5004 5005
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
5006
    } else {
5007 5008
      t_info->start_table_from = 0;
      t_info->ntables = superTblInfo->childTblCount;
5009 5010 5011 5012 5013
      t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint();
    }

    tsem_init(&(t_info->lock_sem), 0, 0);
    if (SYNC == g_Dbs.queryMode) {
5014
      pthread_create(pids + i, NULL, syncWrite, t_info);
5015
    } else {
5016 5017
      pthread_create(pids + i, NULL, asyncWrite, t_info);
    }
H
Hui Li 已提交
5018
  }
5019

H
Hui Li 已提交
5020 5021 5022 5023
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

5024 5025 5026
  int64_t totalDelay = 0;
  int64_t maxDelay = 0;
  int64_t minDelay = INT16_MAX;
5027
  int64_t cntDelay = 1;
5028 5029
  double  avgDelay = 0;

H
Hui Li 已提交
5030
  for (int i = 0; i < threads; i++) {
5031 5032
    threadInfo *t_info = infos + i;

S
TD-1057  
Shengliang Guan 已提交
5033
    tsem_destroy(&(t_info->lock_sem));
5034 5035
    taos_close(t_info->taos);

5036 5037 5038 5039
    debugPrint("%s() LN%d, [%d] totalInsert=%"PRId64" totalAffected=%"PRId64"\n",
            __func__, __LINE__,
            t_info->threadID, t_info->totalInsertRows,
            t_info->totalAffectedRows);
5040 5041
    if (superTblInfo) {
        superTblInfo->totalAffectedRows += t_info->totalAffectedRows;
5042
        superTblInfo->totalInsertRows += t_info->totalInsertRows;
5043 5044 5045
    } else {
        g_args.totalAffectedRows += t_info->totalAffectedRows;
        g_args.totalInsertRows += t_info->totalInsertRows;
5046
    }
5047 5048 5049 5050

    totalDelay  += t_info->totalDelay;
    cntDelay   += t_info->cntDelay;
    if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay;
5051
    if (t_info->minDelay < minDelay) minDelay = t_info->minDelay;
H
Hui Li 已提交
5052
  }
5053
  cntDelay -= 1;
H
Hui Li 已提交
5054

5055
  if (cntDelay == 0)    cntDelay = 1;
5056 5057
  avgDelay = (double)totalDelay / cntDelay;

5058
  double end = getCurrentTime();
5059
  double t = end - start;
5060 5061

  if (superTblInfo) {
5062
    printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n",
5063
          t, superTblInfo->totalInsertRows,
5064 5065
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5066
          superTblInfo->totalInsertRows / t);
5067
    fprintf(g_fpOfInsertResult,
5068
          "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n",
5069
          t, superTblInfo->totalInsertRows,
5070 5071
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5072
          superTblInfo->totalInsertRows/ t);
5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084
  } else {
    printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s %2.f records/second\n\n",
          t, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          g_args.totalInsertRows / t);
    fprintf(g_fpOfInsertResult,
          "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s %2.f records/second\n\n",
          t, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          g_args.totalInsertRows / t);
5085
  }
5086 5087 5088 5089 5090

  printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n",
          avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0);
  fprintf(g_fpOfInsertResult, "insert delay, avg:%10.6fms, max: %10.6fms, min: %10.6fms\n\n",
          avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0);
5091 5092 5093
  
  //taos_close(taos);

H
Hui Li 已提交
5094
  free(pids);
5095
  free(infos);
H
Hui Li 已提交
5096 5097
}

5098
static void *readTable(void *sarg) {
5099 5100
#if 1  
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
5101 5102
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
5103
  uint64_t sTime = rinfo->start_time;
H
hzcheng 已提交
5104 5105
  char *tb_prefix = rinfo->tb_prefix;
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
5106
  if (NULL == fp) {
5107
    errorPrint( "fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
H
Hui Li 已提交
5108 5109
    return NULL;
  }
5110

5111
    int num_of_DPT;
5112
/*  if (rinfo->superTblInfo) {
5113 5114
    num_of_DPT = rinfo->superTblInfo->insertRows; //  nrecords_per_table;
  } else {
5115
  */
5116
      num_of_DPT = g_args.num_of_DPT;
5117
//  }
5118

5119
  int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
H
hzcheng 已提交
5120
  int totalData = num_of_DPT * num_of_tables;
5121
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
  printf("%d records:\n", totalData);
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

  for (int j = 0; j < n; j++) {
    double totalT = 0;
    int count = 0;
    for (int i = 0; i < num_of_tables; i++) {
L
lihui 已提交
5134
      sprintf(command, "select %s from %s%d where ts>= %" PRId64, aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
5135 5136

      double t = getCurrentTime();
S
Shuaiqiang Chang 已提交
5137 5138
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
5139

S
Shuaiqiang Chang 已提交
5140
      if (code != 0) {
5141
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
5142
        taos_free_result(pSql);
H
hzcheng 已提交
5143
        taos_close(taos);
5144
        fclose(fp);
5145
        return NULL;
H
hzcheng 已提交
5146 5147
      }

S
Shuaiqiang Chang 已提交
5148
      while (taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
5149 5150 5151 5152 5153 5154
        count++;
      }

      t = getCurrentTime() - t;
      totalT += t;

S
Shuaiqiang Chang 已提交
5155
      taos_free_result(pSql);
H
hzcheng 已提交
5156 5157 5158
    }

    fprintf(fp, "|%10s  |   %10d   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
5159
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
5160 5161 5162 5163 5164
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT);
  }
  fprintf(fp, "\n");
  fclose(fp);
5165
#endif
H
hzcheng 已提交
5166 5167 5168
  return NULL;
}

5169
static void *readMetric(void *sarg) {
5170 5171
#if 1  
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
5172 5173 5174
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
5175 5176 5177 5178
  if (NULL == fp) {
    printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
    return NULL;
  }
5179 5180

  int num_of_DPT = rinfo->superTblInfo->insertRows;
5181
  int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
H
hzcheng 已提交
5182
  int totalData = num_of_DPT * num_of_tables;
5183
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
5184 5185 5186 5187 5188 5189 5190 5191 5192

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
  printf("%d records:\n", totalData);
  fprintf(fp, "Querying On %d records:\n", totalData);

  for (int j = 0; j < n; j++) {
sangshuduo's avatar
sangshuduo 已提交
5193
    char condition[BUFFER_SIZE - 30] = "\0";
B
Bomin Zhang 已提交
5194
    char tempS[64] = "\0";
H
hzcheng 已提交
5195 5196 5197 5198 5199

    int m = 10 < num_of_tables ? 10 : num_of_tables;

    for (int i = 1; i <= m; i++) {
      if (i == 1) {
5200
        sprintf(tempS, "t1 = %d", i);
H
hzcheng 已提交
5201
      } else {
5202
        sprintf(tempS, " or t1 = %d ", i);
H
hzcheng 已提交
5203 5204 5205
      }
      strcat(condition, tempS);

L
liu0x54 已提交
5206
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
5207 5208 5209 5210 5211 5212

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

      double t = getCurrentTime();

S
Shuaiqiang Chang 已提交
5213 5214 5215 5216
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
5217
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
5218
        taos_free_result(pSql);
H
hzcheng 已提交
5219
        taos_close(taos);
5220
        fclose(fp);
5221
        return NULL;
H
hzcheng 已提交
5222 5223
      }
      int count = 0;
S
Shuaiqiang Chang 已提交
5224
      while (taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
5225 5226 5227 5228
        count++;
      }
      t = getCurrentTime() - t;

5229 5230
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
              num_of_tables * num_of_DPT / t, t * 1000);
H
hzcheng 已提交
5231 5232
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t);

S
Shuaiqiang Chang 已提交
5233
      taos_free_result(pSql);
H
hzcheng 已提交
5234 5235 5236 5237
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
5238
#endif
H
hzcheng 已提交
5239 5240 5241
  return NULL;
}

H
Hui Li 已提交
5242

5243
static int insertTestProcess() {
5244

5245 5246 5247
  setupForAnsiEscape();
  int ret = printfInsertMeta();
  resetAfterAnsiEscape();
5248

5249 5250 5251
  if (ret == -1)
    exit(EXIT_FAILURE);

5252
  debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
5253 5254
  g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
  if (NULL == g_fpOfInsertResult) {
5255
    errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
5256 5257
    return -1;
  }
5258

5259 5260
  printfInsertMetaToFile(g_fpOfInsertResult);

5261 5262 5263 5264
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void)getchar();
  }
5265
 
5266 5267 5268
  init_rand_data();

  // create database and super tables
5269
  if(createDatabases() != 0) {
5270 5271 5272
    fclose(g_fpOfInsertResult);
    return -1;
  }
5273 5274

  // pretreatement
5275
  prepareSampleData();
5276

5277 5278 5279 5280 5281 5282 5283
  double start;
  double end;

  // create child tables
  start = getCurrentTime();
  createChildTables();
  end = getCurrentTime();
5284

5285
  if (g_totalChildTables > 0) {
5286
    printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n",
5287
            end - start, g_totalChildTables, g_Dbs.threadCount);
5288 5289
    fprintf(g_fpOfInsertResult,
            "Spent %.4f seconds to create %d tables with %d thread(s)\n\n",
5290
            end - start, g_totalChildTables, g_Dbs.threadCount);
5291
  }
5292

5293
  taosMsleep(1000);
5294 5295
  // create sub threads for inserting data
  //start = getCurrentTime();
5296 5297 5298 5299 5300 5301
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    if (g_Dbs.db[i].superTblCount > 0) {
      for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
        SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];
        if (0 == g_Dbs.db[i].superTbls[j].insertRows) {
          continue;
5302
        }
5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314
        startMultiThreadInsertData(
          g_Dbs.threadCount, 
          g_Dbs.db[i].dbName, 
          g_Dbs.db[i].dbCfg.precision, 
          superTblInfo);
        }
    } else {
        startMultiThreadInsertData(
          g_Dbs.threadCount, 
          g_Dbs.db[i].dbName, 
          g_Dbs.db[i].dbCfg.precision, 
          NULL);
H
Hui Li 已提交
5315
    }
5316
  }
5317
  //end = getCurrentTime();
5318

5319
  //int64_t    totalInsertRows = 0;
5320 5321 5322
  //int64_t    totalAffectedRows = 0;
  //for (int i = 0; i < g_Dbs.dbCount; i++) {    
  //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
5323
  //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
5324 5325
  //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
  //}
5326
  //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
5327
  postFreeResource();
5328

5329 5330 5331
  return 0;
}

5332
static void *superQueryProcess(void *sarg) {
5333 5334 5335 5336 5337 5338 5339 5340 5341
  threadInfo *winfo = (threadInfo *)sarg; 

  //char sqlStr[MAX_TB_NAME_SIZE*2];
  //sprintf(sqlStr, "use %s", g_queryInfo.dbName);
  //queryDB(winfo->taos, sqlStr);
  
  int64_t st = 0;
  int64_t et = 0;
  while (1) {
5342
    if (g_queryInfo.superQueryInfo.rate && (et - st) < (int64_t)g_queryInfo.superQueryInfo.rate*1000) {
5343
      taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms
5344
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5345 5346
    }

5347
    st = taosGetTimestampUs();
5348 5349 5350 5351 5352 5353 5354 5355 5356
    for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
      if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {          
        int64_t t1 = taosGetTimestampUs();
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
        if (g_queryInfo.superQueryInfo.result[i][0] != 0) {
          sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], winfo->threadID);
        }
        selectAndGetResult(winfo->taos, g_queryInfo.superQueryInfo.sql[i], tmpFile); 
        int64_t t2 = taosGetTimestampUs();          
5357 5358
        printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %f s\n", 
                taosGetSelfPthreadId(), (t2 - t1)/1000000.0);
5359 5360
      } else {
        int64_t t1 = taosGetTimestampUs();
5361 5362
        int retCode = postProceSql(g_queryInfo.host, 
                g_queryInfo.port, g_queryInfo.superQueryInfo.sql[i]);
5363
        int64_t t2 = taosGetTimestampUs();          
5364 5365
        printf("=[restful] thread[%"PRId64"] complete one sql, Spent %f s\n", 
                taosGetSelfPthreadId(), (t2 - t1)/1000000.0);
5366 5367
        
        if (0 != retCode) {
5368
          printf("====restful return fail, threadID[%d]\n", winfo->threadID);
5369 5370 5371 5372
          return NULL;
        }
      }   
    }
5373
    et = taosGetTimestampUs();
5374 5375
    printf("==thread[%"PRId64"] complete all sqls to specify tables once queries duration:%.6fs\n\n", 
            taosGetSelfPthreadId(), (double)(et - st)/1000.0);
H
Hui Li 已提交
5376
  }
5377 5378
  return NULL;
}
H
Hui Li 已提交
5379

5380
static void replaceSubTblName(char* inSql, char* outSql, int tblIndex) {
5381 5382
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
5383 5384 5385
  sprintf(subTblName, "%s.%s", 
          g_queryInfo.dbName, 
          g_queryInfo.subQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
5386 5387 5388 5389 5390 5391

  //printf("inSql: %s\n", inSql);
  
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
    return; 
H
Hui Li 已提交
5392 5393
  }
  
5394
  tstrncpy(outSql, inSql, pos - inSql + 1);
5395 5396 5397 5398 5399
  //printf("1: %s\n", outSql);
  strcat(outSql, subTblName);  
  //printf("2: %s\n", outSql);  
  strcat(outSql, pos+strlen(sourceString));  
  //printf("3: %s\n", outSql); 
H
Hui Li 已提交
5400 5401
}

5402
static void *subQueryProcess(void *sarg) {
5403 5404 5405
  char sqlstr[1024];
  threadInfo *winfo = (threadInfo *)sarg; 
  int64_t st = 0;
5406
  int64_t et = (int64_t)g_queryInfo.subQueryInfo.rate*1000;
5407
  while (1) {
5408 5409
    if (g_queryInfo.subQueryInfo.rate
            && (et - st) < (int64_t)g_queryInfo.subQueryInfo.rate*1000) {
5410
      taosMsleep(g_queryInfo.subQueryInfo.rate*1000 - (et - st)); // ms
5411
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5412
    }
H
Hui Li 已提交
5413

5414
    st = taosGetTimestampUs();
5415
    for (int i = winfo->start_table_from; i <= winfo->end_table_to; i++) {
5416
      for (int j = 0; j < g_queryInfo.subQueryInfo.sqlCount; j++) {
5417
        memset(sqlstr,0,sizeof(sqlstr));
5418
        replaceSubTblName(g_queryInfo.subQueryInfo.sql[j], sqlstr, i);
5419 5420
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
        if (g_queryInfo.subQueryInfo.result[i][0] != 0) {
5421 5422 5423
          sprintf(tmpFile, "%s-%d", 
                  g_queryInfo.subQueryInfo.result[i], 
                  winfo->threadID);
5424 5425 5426
        }
        selectAndGetResult(winfo->taos, sqlstr, tmpFile); 
      }
H
Hui Li 已提交
5427
    }
5428
    et = taosGetTimestampUs();
5429 5430
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%d - %d] once queries duration:%.4fs\n\n", 
            taosGetSelfPthreadId(), 
5431 5432
            winfo->start_table_from, 
            winfo->end_table_to, 
5433
            (double)(et - st)/1000000.0);
5434 5435 5436 5437
  }
  return NULL;
}

5438
static int queryTestProcess() {
5439
  TAOS * taos = NULL;  
5440 5441 5442 5443 5444
  taos = taos_connect(g_queryInfo.host, 
          g_queryInfo.user, 
          g_queryInfo.password, 
          NULL, 
          g_queryInfo.port);
5445
  if (taos == NULL) {
5446
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
5447 5448 5449 5450
    exit(-1);
  }

  if (0 != g_queryInfo.subQueryInfo.sqlCount) {
5451 5452 5453 5454
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
            g_queryInfo.subQueryInfo.sTblName,
            &g_queryInfo.subQueryInfo.childTblName,
5455
            &g_queryInfo.subQueryInfo.childTblCount);
5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469
  }  
  
  printfQueryMeta();
  
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void)getchar();
  }
  
  printfQuerySystemInfo(taos);
  
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
5470 5471
  if (g_queryInfo.superQueryInfo.sqlCount > 0
          && g_queryInfo.superQueryInfo.concurrent > 0) {
5472

5473
    pids  = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t));
5474 5475 5476 5477
    if (NULL == pids) {
        taos_close(taos);
        ERROR_EXIT("memory allocation failed\n");
    }
5478
    infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo));
5479
    if (NULL == infos) {
5480
      taos_close(taos);
5481 5482
      free(pids);
      ERROR_EXIT("memory allocation failed for create threads\n");
5483
    }
5484 5485

    for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) {
5486
      threadInfo *t_info = infos + i;
5487
      t_info->threadID = i;
5488

5489 5490
      if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {
        t_info->taos = taos;
5491

5492 5493
        char sqlStr[MAX_TB_NAME_SIZE*2];
        sprintf(sqlStr, "use %s", g_queryInfo.dbName);
5494
        verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
5495
        if (0 != queryDbExec(t_info->taos, sqlStr, NO_INSERT_TYPE)) {
5496 5497
            free(infos);
            free(pids);
5498 5499 5500 5501
            errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
            return -1;
        }
5502 5503 5504
      } else {
        t_info->taos = NULL;
      }
5505

5506 5507
      pthread_create(pids + i, NULL, superQueryProcess, t_info);
    }
5508 5509 5510
  }else {
    g_queryInfo.superQueryInfo.concurrent = 0;
  }
5511

5512 5513 5514
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
5515 5516
  if ((g_queryInfo.subQueryInfo.sqlCount > 0)
          && (g_queryInfo.subQueryInfo.threadCnt > 0)) {
5517
    pidsOfSub  = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t));
5518 5519 5520 5521 5522 5523 5524 5525
    if (NULL == pidsOfSub) {
      taos_close(taos);
      free(infos);
      free(pids);

      ERROR_EXIT("memory allocation failed for create threads\n");
    }

5526
    infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo));
5527
    if (NULL == infosOfSub) {
5528
      taos_close(taos);
5529 5530 5531 5532
      free(pidsOfSub);
      free(infos);
      free(pids);
      ERROR_EXIT("memory allocation failed for create threads\n");
5533
    }
5534

5535 5536 5537 5538 5539 5540 5541 5542
    int ntables = g_queryInfo.subQueryInfo.childTblCount;
    int threads = g_queryInfo.subQueryInfo.threadCnt;
  
    int a = ntables / threads;
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
5543
 
5544 5545 5546 5547
    int b = 0;
    if (threads != 0) {
      b = ntables % threads;
    }
5548

5549
    int startFrom = 0;
5550 5551 5552 5553
    for (int i = 0; i < threads; i++) {  
      threadInfo *t_info = infosOfSub + i;
      t_info->threadID = i;
      
5554 5555 5556 5557
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
5558 5559 5560 5561 5562
      t_info->taos = taos;
      pthread_create(pidsOfSub + i, NULL, subQueryProcess, t_info);
    }

    g_queryInfo.subQueryInfo.threadCnt = threads;
5563
  } else {
5564 5565
    g_queryInfo.subQueryInfo.threadCnt = 0;
  }  
5566
 
5567 5568
  for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) {
    pthread_join(pids[i], NULL);
5569
  }
S
Shuaiqiang Chang 已提交
5570

5571
  tmfree((char*)pids);
5572 5573
  tmfree((char*)infos);
 
5574 5575
  for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) {
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
5576
  }
H
Hui Li 已提交
5577

5578
  tmfree((char*)pidsOfSub);
5579 5580
  tmfree((char*)infosOfSub);
 
5581 5582 5583 5584
  taos_close(taos);
  return 0;
}

5585
static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
5586
  if (res == NULL || taos_errno(res) != 0) {
5587 5588
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
5589 5590
    return;
  }
5591

5592 5593
  getResult(res, (char*)param);
  taos_free_result(res);
H
hzcheng 已提交
5594 5595
}

5596
static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultFileName) {
5597
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
5598

5599
  if (g_queryInfo.superQueryInfo.subscribeMode) {
5600 5601 5602
    tsub = taos_subscribe(taos,
            g_queryInfo.superQueryInfo.subscribeRestart,
            topic, sql, subscribe_callback, (void*)resultFileName,
5603
            g_queryInfo.superQueryInfo.subscribeInterval);
5604
  } else {
5605 5606
    tsub = taos_subscribe(taos,
            g_queryInfo.superQueryInfo.subscribeRestart,
5607
            topic, sql, NULL, NULL, 0);
5608
  }
5609

5610 5611 5612
  if (tsub == NULL) {
    printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
    return NULL;
5613
  }
5614

5615 5616
  return tsub;
}
H
hzcheng 已提交
5617

5618
static void *subSubscribeProcess(void *sarg) {
5619
  threadInfo *winfo = (threadInfo *)sarg;
5620
  char subSqlstr[1024];
H
hzcheng 已提交
5621

5622 5623
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
5624
  debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
5625 5626 5627
  if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)){
    return NULL;
  }
5628
 
5629 5630 5631 5632 5633
  //int64_t st = 0;
  //int64_t et = 0;
  do {
    //if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) {
    //  taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms
5634
    //  //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652
    //}

    //st = taosGetTimestampMs();
    char topic[32] = {0};
    for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) {
      sprintf(topic, "taosdemo-subscribe-%d", i);
      memset(subSqlstr,0,sizeof(subSqlstr));
      replaceSubTblName(g_queryInfo.subQueryInfo.sql[i], subSqlstr, i);
      char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
      if (g_queryInfo.subQueryInfo.result[i][0] != 0) {
        sprintf(tmpFile, "%s-%d", g_queryInfo.subQueryInfo.result[i], winfo->threadID);
      }
      g_queryInfo.subQueryInfo.tsub[i] = subscribeImpl(winfo->taos, subSqlstr, topic, tmpFile); 
      if (NULL == g_queryInfo.subQueryInfo.tsub[i]) {
        return NULL;
      }
    }
    //et = taosGetTimestampMs();
5653
    //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0);
5654 5655 5656 5657 5658 5659 5660 5661 5662
  } while (0);

  // start loop to consume result
  TAOS_RES* res = NULL;
  while (1) {
    for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) {
      if (1 == g_queryInfo.subQueryInfo.subscribeMode) {
        continue;
      }
5663

5664 5665 5666 5667
      res = taos_consume(g_queryInfo.subQueryInfo.tsub[i]);
      if (res) {
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
        if (g_queryInfo.subQueryInfo.result[i][0] != 0) {
5668 5669
          sprintf(tmpFile, "%s-%d",
                  g_queryInfo.subQueryInfo.result[i],
5670
                  winfo->threadID);
5671 5672
        }
        getResult(res, tmpFile);
H
hzcheng 已提交
5673 5674 5675
      }
    }
  }
5676
  taos_free_result(res);
5677

5678
  for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) {
5679
    taos_unsubscribe(g_queryInfo.subQueryInfo.tsub[i],
5680
            g_queryInfo.subQueryInfo.subscribeKeepProgress);
5681
  }
H
hzcheng 已提交
5682 5683 5684
  return NULL;
}

5685
static void *superSubscribeProcess(void *sarg) {
5686
  threadInfo *winfo = (threadInfo *)sarg;
H
hzcheng 已提交
5687

5688 5689
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
5690
  debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
5691 5692 5693
  if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)) {
    return NULL;
  }
5694
 
5695 5696 5697 5698 5699
  //int64_t st = 0;
  //int64_t et = 0;
  do {
    //if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) {
    //  taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms
5700
    //  //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5701 5702 5703 5704 5705 5706 5707 5708
    //}

    //st = taosGetTimestampMs();
    char topic[32] = {0};
    for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
      sprintf(topic, "taosdemo-subscribe-%d", i);
      char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
      if (g_queryInfo.subQueryInfo.result[i][0] != 0) {
5709
        sprintf(tmpFile, "%s-%d",
5710
                g_queryInfo.superQueryInfo.result[i], winfo->threadID);
5711
      }
5712 5713 5714 5715
      g_queryInfo.superQueryInfo.tsub[i] =
          subscribeImpl(winfo->taos,
                  g_queryInfo.superQueryInfo.sql[i],
                  topic, tmpFile);
5716 5717 5718 5719 5720
      if (NULL == g_queryInfo.superQueryInfo.tsub[i]) {
        return NULL;
      }
    }
    //et = taosGetTimestampMs();
5721
    //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0);
5722 5723 5724 5725 5726 5727 5728 5729 5730
  } while (0);

  // start loop to consume result
  TAOS_RES* res = NULL;
  while (1) {
    for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
      if (1 == g_queryInfo.superQueryInfo.subscribeMode) {
        continue;
      }
5731

5732 5733 5734 5735
      res = taos_consume(g_queryInfo.superQueryInfo.tsub[i]);
      if (res) {
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
        if (g_queryInfo.superQueryInfo.result[i][0] != 0) {
5736
          sprintf(tmpFile, "%s-%d",
5737
                  g_queryInfo.superQueryInfo.result[i], winfo->threadID);
5738 5739 5740 5741 5742 5743
        }
        getResult(res, tmpFile);
      }
    }
  }
  taos_free_result(res);
5744

5745
  for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
5746
    taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i],
5747
            g_queryInfo.superQueryInfo.subscribeKeepProgress);
5748
  }
H
hzcheng 已提交
5749 5750 5751
  return NULL;
}

5752
static int subscribeTestProcess() {
5753
  printfQueryMeta();
H
hzcheng 已提交
5754

5755 5756 5757
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void)getchar();
Y
yihaoDeng 已提交
5758
  }
5759 5760

  TAOS * taos = NULL;  
5761 5762 5763 5764 5765
  taos = taos_connect(g_queryInfo.host, 
          g_queryInfo.user, 
          g_queryInfo.password, 
          g_queryInfo.dbName, 
          g_queryInfo.port);
5766
  if (taos == NULL) {
5767
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
5768 5769 5770 5771
    exit(-1);
  }

  if (0 != g_queryInfo.subQueryInfo.sqlCount) {
5772
    getAllChildNameOfSuperTable(taos, 
5773 5774 5775 5776
            g_queryInfo.dbName, 
            g_queryInfo.subQueryInfo.sTblName, 
            &g_queryInfo.subQueryInfo.childTblName, 
            &g_queryInfo.subQueryInfo.childTblCount);
5777 5778 5779 5780 5781
  }

  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from super table
5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792
  if ((g_queryInfo.superQueryInfo.sqlCount <= 0) ||
          (g_queryInfo.superQueryInfo.concurrent <= 0)) {
    errorPrint("%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n",
              __func__, __LINE__, g_queryInfo.superQueryInfo.sqlCount,
              g_queryInfo.superQueryInfo.concurrent);
    exit(-1);
  }

  pids  = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t));
  infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo));
  if ((NULL == pids) || (NULL == infos)) {
5793
      printf("malloc failed for create threads\n");
5794 5795
      taos_close(taos);
      exit(-1);
5796
  }
5797

5798
  for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) {
5799 5800 5801 5802
      threadInfo *t_info = infos + i;
      t_info->threadID = i;
      t_info->taos = taos;
      pthread_create(pids + i, NULL, superSubscribeProcess, t_info);
H
hzcheng 已提交
5803
  }
5804
 
5805 5806 5807
  //==== create sub threads for query from sub table  
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
5808
  if ((g_queryInfo.subQueryInfo.sqlCount > 0)
5809
          && (g_queryInfo.subQueryInfo.threadCnt > 0)) {
5810
    pidsOfSub  = malloc(g_queryInfo.subQueryInfo.threadCnt *
5811 5812 5813
            sizeof(pthread_t));
    infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * 
            sizeof(threadInfo));
5814
    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
5815
      printf("malloc failed for create threads\n");
5816 5817
      taos_close(taos);
      exit(-1);
S
Shuaiqiang Chang 已提交
5818
    }
5819

5820 5821
    int ntables = g_queryInfo.subQueryInfo.childTblCount;
    int threads = g_queryInfo.subQueryInfo.threadCnt;
5822
 
5823 5824 5825 5826 5827
    int a = ntables / threads;
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
5828
 
5829 5830 5831 5832
    int b = 0;
    if (threads != 0) {
      b = ntables % threads;
    }
5833
 
5834
    int startFrom = 0;
5835
    for (int i = 0; i < threads; i++) {
5836 5837
      threadInfo *t_info = infosOfSub + i;
      t_info->threadID = i;
5838

5839 5840 5841 5842
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
5843 5844
      t_info->taos = taos;
      pthread_create(pidsOfSub + i, NULL, subSubscribeProcess, t_info);
H
hzcheng 已提交
5845
    }
5846
    g_queryInfo.subQueryInfo.threadCnt = threads;
H
hzcheng 已提交
5847
  }
5848
 
5849 5850 5851
  for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) {
    pthread_join(pids[i], NULL);
  }  
H
hzcheng 已提交
5852

5853
  tmfree((char*)pids);
5854
  tmfree((char*)infos);
H
hzcheng 已提交
5855

5856 5857
  for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) {
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
5858 5859
  }

5860
  tmfree((char*)pidsOfSub);
5861
  tmfree((char*)infosOfSub);
5862 5863
  taos_close(taos);
  return 0;
H
hzcheng 已提交
5864 5865
}

5866
static void initOfInsertMeta() {
5867 5868
  memset(&g_Dbs, 0, sizeof(SDbs));
   
5869 5870 5871 5872 5873 5874 5875
  // set default values
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE);
  g_Dbs.port = 6030;
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE);
  g_Dbs.threadCount = 2;
  g_Dbs.use_metric = true;
L
Liu Tao 已提交
5876 5877
}

5878
static void initOfQueryMeta() {
5879 5880
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
   
5881 5882 5883 5884 5885
  // set default values
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE);
  g_queryInfo.port = 6030;
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
5886 5887
}

5888
static void setParaFromArg(){
5889 5890 5891
  if (g_args.host) {
    strcpy(g_Dbs.host, g_args.host);
  } else {
5892
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE);
5893
  }
L
Liu Tao 已提交
5894

5895 5896 5897 5898 5899 5900 5901
  if (g_args.user) {
    strcpy(g_Dbs.user, g_args.user);
  } 

  if (g_args.password) {
    strcpy(g_Dbs.password, g_args.password);
  } 
L
Liu Tao 已提交
5902
  
5903 5904 5905
  if (g_args.port) {
    g_Dbs.port = g_args.port;
  } 
L
Liu Tao 已提交
5906

5907 5908 5909
  g_Dbs.threadCount = g_args.num_of_threads;
  g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;

5910 5911
  g_Dbs.dbCount = 1;
  g_Dbs.db[0].drop = 1;
5912

5913
  tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE);
5914
  g_Dbs.db[0].dbCfg.replica = g_args.replica;
5915
  tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
5916

5917
  tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
5918 5919 5920 5921 5922

  g_Dbs.use_metric = g_args.use_metric;
  g_Dbs.insert_only = g_args.insert_only;

  g_Dbs.do_aggreFunc = true;
L
Liu Tao 已提交
5923

5924 5925 5926 5927
  char dataString[STRING_LEN];
  char **data_type = g_args.datatype;
  
  memset(dataString, 0, STRING_LEN);
L
Liu Tao 已提交
5928

5929 5930 5931
  if (strcasecmp(data_type[0], "BINARY") == 0 
          || strcasecmp(data_type[0], "BOOL") == 0 
          || strcasecmp(data_type[0], "NCHAR") == 0 ) {
5932
    g_Dbs.do_aggreFunc = false;
L
Liu Tao 已提交
5933
  }
H
hzcheng 已提交
5934

5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953
  if (g_args.use_metric) {
    g_Dbs.db[0].superTblCount = 1;
    tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE);
    g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
    g_Dbs.threadCount = g_args.num_of_threads;
    g_Dbs.threadCountByCreateTbl = 1;
    g_Dbs.queryMode = g_args.mode;
  
    g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
    g_Dbs.db[0].superTbls[0].superTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
    g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
    tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, 
            g_args.tb_prefix, MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, 
            "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
5954
    g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
5955
  
5956
    g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
5957
    g_Dbs.db[0].superTbls[0].maxSqlLen = TSDB_PAYLOAD_SIZE;
5958

5959 5960 5961 5962 5963 5964 5965 5966 5967
    g_Dbs.db[0].superTbls[0].columnCount = 0;
    for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
      if (data_type[i] == NULL) {
        break;
      }
  
      tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, 
              data_type[i], MAX_TB_NAME_SIZE);
      g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;    
5968 5969
      g_Dbs.db[0].superTbls[0].columnCount++;
    }
5970 5971 5972 5973 5974 5975 5976 5977 5978 5979
  
    if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
      g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
    } else {
      for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) {
        tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE);
        g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;    
        g_Dbs.db[0].superTbls[0].columnCount++;
      }
    }
5980

5981
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE);
5982 5983
    g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;    
  
5984
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE);
5985 5986 5987
    g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;    
    g_Dbs.db[0].superTbls[0].tagCount = 2;  
  } else {
5988
    g_Dbs.threadCountByCreateTbl = 1;
5989
    g_Dbs.db[0].superTbls[0].tagCount = 0; 
H
hzcheng 已提交
5990
  }
5991

5992 5993 5994 5995 5996 5997
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
5998

5999 6000 6001
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
6002 6003 6004
    exit(-1);
  }

6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

6029
static void querySqlFile(TAOS* taos, char* sqlFile)
6030 6031 6032 6033 6034 6035
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
6036

6037 6038 6039 6040 6041 6042 6043
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

  double t = getCurrentTime();
6044

6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057
  while ((read_len = tgetline(&line, &line_len, fp)) != -1) {
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
6058
    }
6059 6060

    memcpy(cmd + cmd_len, line, read_len);
6061
    verbosePrint("%s() LN%d cmd: %s\n", __func__, __LINE__, cmd);
6062 6063 6064 6065 6066 6067 6068
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE)) {
        printf("queryDbExec %s failed!\n", cmd);
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
6069 6070
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
6071 6072
  }

6073 6074
  t = getCurrentTime() - t;
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
6075

6076 6077 6078 6079
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
6080 6081
}

6082
static void testMetaFile() {
6083
    if (INSERT_TEST == g_args.test_mode) {
6084
      if (g_Dbs.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
6085
      insertTestProcess();
6086
    } else if (QUERY_TEST == g_args.test_mode) {
6087 6088
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
6089
      queryTestProcess();
6090
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
6091 6092
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
6093
      subscribeTestProcess();
6094 6095 6096
    }  else {
      ;
    }
6097
}
6098

6099
static void queryResult() {
6100 6101 6102
    // select
    if (false == g_Dbs.insert_only) {
      // query data
6103

6104 6105 6106
      pthread_t read_id;
      threadInfo *rInfo = malloc(sizeof(threadInfo));
      rInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
6107
      rInfo->start_table_from = 0;
6108

6109
      //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
6110
      if (g_args.use_metric) {
6111 6112
        rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
        rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
6113 6114 6115 6116
        rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
        strcpy(rInfo->tb_prefix, 
              g_Dbs.db[0].superTbls[0].childTblPrefix);
      } else {
6117 6118
        rInfo->ntables = g_args.num_of_tables;
        rInfo->end_table_to = g_args.num_of_tables -1;
6119 6120 6121
        strcpy(rInfo->tb_prefix, g_args.tb_prefix);
      }

6122 6123 6124 6125 6126 6127
      rInfo->taos = taos_connect(
              g_Dbs.host, 
              g_Dbs.user, 
              g_Dbs.password, 
              g_Dbs.db[0].dbName, 
              g_Dbs.port);
6128
      if (rInfo->taos == NULL) {
6129
        errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
6130 6131 6132 6133
        free(rInfo);
        exit(-1);
      }

6134 6135 6136 6137 6138 6139 6140 6141 6142 6143
      strcpy(rInfo->fp, g_Dbs.resultFile);

      if (!g_Dbs.use_metric) {
        pthread_create(&read_id, NULL, readTable, rInfo);
      } else {
        pthread_create(&read_id, NULL, readMetric, rInfo);
      }
      pthread_join(read_id, NULL);
      taos_close(rInfo->taos);
      free(rInfo);
H
hzcheng 已提交
6144
    }
6145 6146
}

6147 6148
static void testCmdLine() {

6149
    g_args.test_mode = INSERT_TEST;
6150 6151 6152 6153 6154 6155 6156 6157
    insertTestProcess();

    if (g_Dbs.insert_only)
      return;
    else
        queryResult();
}

6158 6159 6160
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

6161
  debugPrint("meta file: %s\n", g_args.metaFile);
6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189

  if (g_args.metaFile) {
    initOfInsertMeta();
    initOfQueryMeta();    

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
          g_Dbs.host, 
          g_Dbs.user, 
          g_Dbs.password, 
          g_Dbs.db[0].dbName, 
          g_Dbs.port);
      querySqlFile(qtaos, g_args.sqlFile);  
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
H
hzcheng 已提交
6190
  }
6191 6192

  return 0;
H
hzcheng 已提交
6193
}
6194