taosdemo.c 208.7 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

S
slguan 已提交
21
#define _GNU_SOURCE
22 23
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
#else  
  #include <regex.h>
  #include <stdio.h>
#endif  
H
hzcheng 已提交
44

45
#include <assert.h>
46
#include <stdlib.h>
47 48
#include "cJSON.h"

49
#include "os.h"
H
hzcheng 已提交
50
#include "taos.h"
51
#include "taoserror.h"
H
Hui Li 已提交
52
#include "tutil.h"
S
slguan 已提交
53

54 55 56
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
57 58
extern char configDir[];

59 60 61 62
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

63 64 65 66 67 68
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92

#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
#define MAX_DB_NAME_SIZE   64
#define MAX_TB_NAME_SIZE   64
#define MAX_DATA_SIZE      16000
#define MAX_NUM_DATATYPE   10
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
#define MAX_FILE_NAME_LEN  256

#define   MAX_SAMPLES_ONCE_FROM_FILE   10000
#define   MAX_NUM_DATATYPE 10

#define   MAX_DB_COUNT           8
#define   MAX_SUPER_TABLE_COUNT  8
#define   MAX_COLUMN_COUNT       1024
#define   MAX_TAG_COUNT          128

#define   MAX_QUERY_SQL_COUNT    10
#define   MAX_QUERY_SQL_LENGTH   256

#define   MAX_DATABASE_COUNT     256
93
#define INPUT_BUF_LEN   256
94

95
#define DEFAULT_TIMESTAMP_STEP  1
96

97 98 99 100 101 102 103 104
typedef enum CREATE_SUB_TALBE_MOD_EN {
  PRE_CREATE_SUBTBL,
  AUTO_CREATE_SUBTBL,
  NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
  
typedef enum TALBE_EXISTS_EN {
  TBL_NO_EXISTS,
105
  TBL_ALREADY_EXISTS,
106 107 108 109 110 111 112 113
  TBL_EXISTS_BUTT
} TALBE_EXISTS_EN;

enum MODE {
  SYNC, 
  ASYNC,
  MODE_BUT
};
114

115 116 117 118 119 120
typedef enum enum_INSERT_MODE {
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
} INSERT_MODE;

121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
enum QUERY_TYPE {
  NO_INSERT_TYPE,
  INSERT_TYPE, 
  QUERY_TYPE_BUT
} ;  

enum _show_db_index {
  TSDB_SHOW_DB_NAME_INDEX,
  TSDB_SHOW_DB_CREATED_TIME_INDEX,
  TSDB_SHOW_DB_NTABLES_INDEX,
  TSDB_SHOW_DB_VGROUPS_INDEX,
  TSDB_SHOW_DB_REPLICA_INDEX,
  TSDB_SHOW_DB_QUORUM_INDEX,  
  TSDB_SHOW_DB_DAYS_INDEX,
  TSDB_SHOW_DB_KEEP_INDEX,
  TSDB_SHOW_DB_CACHE_INDEX,
  TSDB_SHOW_DB_BLOCKS_INDEX,
  TSDB_SHOW_DB_MINROWS_INDEX,
  TSDB_SHOW_DB_MAXROWS_INDEX,
  TSDB_SHOW_DB_WALLEVEL_INDEX,
  TSDB_SHOW_DB_FSYNC_INDEX,
  TSDB_SHOW_DB_COMP_INDEX,
  TSDB_SHOW_DB_CACHELAST_INDEX,
  TSDB_SHOW_DB_PRECISION_INDEX,  
  TSDB_SHOW_DB_UPDATE_INDEX,
  TSDB_SHOW_DB_STATUS_INDEX,
  TSDB_MAX_SHOW_DB
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
  TSDB_SHOW_STABLES_NAME_INDEX,
  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
  TSDB_SHOW_STABLES_COLUMNS_INDEX,
  TSDB_SHOW_STABLES_METRIC_INDEX,  
  TSDB_SHOW_STABLES_UID_INDEX,  
  TSDB_SHOW_STABLES_TID_INDEX,
  TSDB_SHOW_STABLES_VGID_INDEX,  
  TSDB_MAX_SHOW_STABLES
160 161
};

162 163 164 165 166 167 168 169 170 171 172 173 174 175
enum _describe_table_index {
  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
  TSDB_MAX_DESCRIBE_METRIC
};

typedef struct {
  char field[TSDB_COL_NAME_LEN + 1];
  char type[16];
  int length;
  char note[128];
} SColDes;
H
hzcheng 已提交
176 177

/* Used by main to communicate with parse_opt. */
178 179
typedef struct SArguments_S {
  char *   metaFile;
180
  int      test_mode;
181 182 183 184 185 186 187 188 189 190
  char *   host;
  uint16_t port;
  char *   user;
  char *   password;
  char *   database;
  int      replica;
  char *   tb_prefix;
  char *   sqlFile;
  bool     use_metric;
  bool     insert_only;
191
  bool     answer_yes;
192
  bool     debug_print;
193
  bool     verbose_print;
194
  bool     performance_print;
195 196 197 198 199 200
  char *   output_file;
  int      mode;
  char *   datatype[MAX_NUM_DATATYPE + 1];
  int      len_of_binary;
  int      num_of_CPR;
  int      num_of_threads;
201
  int      insert_interval;
202
  int      interlace_rows;
203
  int      num_of_RPR;
204
  int      max_sql_len;
205 206 207
  int      num_of_tables;
  int      num_of_DPT;
  int      abort;
208 209
  int      disorderRatio;
  int      disorderRange;
210 211
  int      method_of_delete;
  char **  arg_list;
212 213
  int64_t  totalInsertRows;
  int64_t  totalAffectedRows;
214 215 216 217 218 219 220 221 222 223
} SArguments;

typedef struct SColumn_S {
  char  field[TSDB_COL_NAME_LEN + 1];
  char  dataType[MAX_TB_NAME_SIZE];
  int   dataLen;
  char  note[128];  
} StrColumn;

typedef struct SSuperTable_S {
224
  char         sTblName[MAX_TB_NAME_SIZE+1];
225 226 227 228 229 230
  int          childTblCount;
  bool         superTblExists;    // 0: no, 1: yes
  bool         childTblExists;    // 0: no, 1: yes  
  int          batchCreateTableNum;  // 0: no batch,  > 0: batch table number in one sql
  int8_t       autoCreateTable;                  // 0: create sub table, 1: auto create sub table
  char         childTblPrefix[MAX_TB_NAME_SIZE];
231
  char         dataSource[MAX_TB_NAME_SIZE+1];  // rand_gen or sample
232
  char         insertMode[MAX_TB_NAME_SIZE];  // taosc, restful
233 234
  int           childTblLimit;
  int           childTblOffset;
235 236 237 238 239 240

  int          multiThreadWriteOneTbl;   // 0: no, 1: yes
  int          rowsPerTbl;               // 
  int          disorderRatio;            // 0: no disorder, >0: x%
  int          disorderRange;            // ms or us by database precision
  int          maxSqlLen;                // 
241 242

  int          insertInterval;          // insert interval, will override global insert interval
243 244 245 246
  int64_t      insertRows;               // 0: no limit
  int          timeStampStep;
  char         startTimestamp[MAX_TB_NAME_SIZE];  // 
  char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
247 248
  char         sampleFile[MAX_FILE_NAME_LEN+1];
  char         tagsFile[MAX_FILE_NAME_LEN+1];
249 250 251 252 253 254 255

  int          columnCount;
  StrColumn    columns[MAX_COLUMN_COUNT];
  int          tagCount;
  StrColumn    tags[MAX_TAG_COUNT];

  char*        childTblName;
256
  char*        colsOfCreateChildTable;
257 258 259 260 261 262 263 264 265 266 267 268 269 270
  int          lenOfOneRow;
  int          lenOfTagOfOneRow;

  char*        sampleDataBuf;
  int          sampleDataBufSize;
  //int          sampleRowCount;
  //int          sampleUsePos;

  int          tagSource;    // 0: rand, 1: tag sample
  char*        tagDataBuf;
  int          tagSampleCount;
  int          tagUsePos;

  // statistics  
271
  int64_t    totalInsertRows;
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302
  int64_t    totalAffectedRows;
} SSuperTable;

typedef struct {
  char     name[TSDB_DB_NAME_LEN + 1];
  char     create_time[32];
  int32_t  ntables;
  int32_t  vgroups;  
  int16_t  replica;
  int16_t  quorum;
  int16_t  days;  
  char     keeplist[32];
  int32_t  cache; //MB
  int32_t  blocks;
  int32_t  minrows;
  int32_t  maxrows;
  int8_t   wallevel;
  int32_t  fsync;
  int8_t   comp;
  int8_t   cachelast;
  char     precision[8];   // time resolution
  int8_t   update;
  char     status[16];
} SDbInfo;

typedef struct SDbCfg_S { 
//  int       maxtablesPerVnode;
  int       minRows; 
  int       maxRows;
  int       comp;
  int       walLevel;
303
  int       cacheLast;
304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
  int       fsync;  
  int       replica;
  int       update;
  int       keep;
  int       days;
  int       cache;
  int       blocks;
  int       quorum;
  char      precision[MAX_TB_NAME_SIZE];  
} SDbCfg;

typedef struct SDataBase_S {
  char         dbName[MAX_DB_NAME_SIZE];
  int          drop;  // 0: use exists, 1: if exists, drop then new create
  SDbCfg       dbCfg;
  int          superTblCount;
  SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
} SDataBase;

typedef struct SDbs_S {
324
  char         cfgDir[MAX_FILE_NAME_LEN+1];
325 326 327 328
  char         host[MAX_DB_NAME_SIZE];
  uint16_t     port;
  char         user[MAX_DB_NAME_SIZE];
  char         password[MAX_DB_NAME_SIZE];
329
  char         resultFile[MAX_FILE_NAME_LEN+1];
330 331 332 333 334 335 336 337 338 339 340
  bool         use_metric;
  bool         insert_only;
  bool         do_aggreFunc;
  bool         queryMode;
  
  int          threadCount;
  int          threadCountByCreateTbl;
  int          dbCount;
  SDataBase    db[MAX_DB_COUNT];

  // statistics
341
  int64_t    totalInsertRows;
342
  int64_t    totalAffectedRows;
343

344 345 346 347 348 349 350 351 352 353
} SDbs;

typedef struct SuperQueryInfo_S {
  int          rate;  // 0: unlimit  > 0   loop/s
  int          concurrent;
  int          sqlCount;
  int          subscribeMode; // 0: sync, 1: async
  int          subscribeInterval; // ms
  int          subscribeRestart;
  int          subscribeKeepProgress;
354 355
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];  
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
356 357 358 359
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
} SuperQueryInfo;

typedef struct SubQueryInfo_S {
360
  char         sTblName[MAX_TB_NAME_SIZE+1];
361 362 363 364 365 366 367 368 369
  int          rate;  // 0: unlimit  > 0   loop/s
  int          threadCnt;  
  int          subscribeMode; // 0: sync, 1: async
  int          subscribeInterval; // ms
  int          subscribeRestart;
  int          subscribeKeepProgress;
  int          childTblCount;
  char         childTblPrefix[MAX_TB_NAME_SIZE];
  int          sqlCount;
370 371
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];  
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
372 373 374 375 376 377
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
  
  char*        childTblName;
} SubQueryInfo;

typedef struct SQueryMetaInfo_S {
378
  char         cfgDir[MAX_FILE_NAME_LEN+1];
379 380 381 382
  char         host[MAX_DB_NAME_SIZE];
  uint16_t     port;
  char         user[MAX_DB_NAME_SIZE];
  char         password[MAX_DB_NAME_SIZE];
383
  char         dbName[MAX_DB_NAME_SIZE+1];
384 385 386 387 388 389 390 391 392
  char         queryMode[MAX_TB_NAME_SIZE];  // taosc, restful

  SuperQueryInfo  superQueryInfo;
  SubQueryInfo    subQueryInfo;  
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
  TAOS *taos;
  int threadID;
393
  char db_name[MAX_DB_NAME_SIZE+1];
394 395
  char fp[4096];
  char tb_prefix[MAX_TB_NAME_SIZE];
396 397 398
  int start_table_from;
  int end_table_to;
  int ntables;
399
  int data_of_rate;
400
  uint64_t start_time;  
401 402 403 404 405 406 407
  char* cols;  
  bool  use_metric;  
  SSuperTable* superTblInfo;

  // for async insert
  tsem_t lock_sem;
  int64_t  counter;  
408 409
  uint64_t  st;
  uint64_t  et;
410 411
  int64_t  lastTs;

412 413
  // sample data
  int samplePos;
414
  // statistics
415
  int64_t totalInsertRows;
416
  int64_t totalAffectedRows;
417 418 419 420 421 422 423 424

  // insert delay statistics
  int64_t cntDelay;
  int64_t totalDelay;
  int64_t avgDelay;
  int64_t maxDelay;
  int64_t minDelay;
  
425 426
} threadInfo;

427
#ifdef WINDOWS
428 429
#define _CRT_RAND_S

430
#include <windows.h>
431 432 433 434 435
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
436 437
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
438 439
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
440 441 442 443

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

444
static void setupForAnsiEscape(void) {
445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
  }    
}

466
static void resetAfterAnsiEscape(void) {
467 468 469 470 471 472 473 474
  // Reset colors
  printf("\x1b[0m");    

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
475

476
static int taosRandom()
477 478 479 480 481 482
{
    int number;
    rand_s(&number);

    return number;
}
483
#else
484
static void setupForAnsiEscape(void) {}
485

486
static void resetAfterAnsiEscape(void) {
487 488 489
  // Reset colors
  printf("\x1b[0m");
}
490

491
static int taosRandom()
492 493 494 495
{
    return random();
}

496 497
#endif

498 499 500
static int createDatabases();
static void createChildTables();
static int queryDbExec(TAOS *taos, char *command, int type);
501 502 503 504 505 506 507 508 509 510

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", 
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

511
SArguments g_args = {
512 513
                     NULL,            // metaFile
                     0,               // test_mode
514 515 516 517 518 519 520 521 522 523 524 525
                     "127.0.0.1",     // host
                     6030,            // port
                     "root",          // user
                     #ifdef _TD_POWER_ 
                     "powerdb",      // password
                     #else
                     "taosdata",      // password
                     #endif
                     "test",          // database
                     1,               // replica
                     "t",             // tb_prefix
                     NULL,            // sqlFile
526 527
                     true,            // use_metric
                     true,            // insert_only
528
                     false,           // debug_print
529
                     false,           // verbose_print
530
                     false,           // performance statistic print 
531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549
                     false,           // answer_yes;
                     "./output.txt",  // output_file
                     0,               // mode : sync or async
                     {
                     "TINYINT",           // datatype
                     "SMALLINT",
                     "INT",
                     "BIGINT",
                     "FLOAT",
                     "DOUBLE",
                     "BINARY",
                     "NCHAR",
                     "BOOL",
                     "TIMESTAMP"
                     },
                     16,              // len_of_binary
                     10,              // num_of_CPR
                     10,              // num_of_connections/thread
                     0,               // insert_interval
550
                     0,               // interlace_rows;
551
                     100,             // num_of_RPR
552
                     TSDB_PAYLOAD_SIZE,  // max_sql_len
553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569
                     10000,           // num_of_tables
                     10000,           // num_of_DPT
                     0,               // abort
                     0,               // disorderRatio
                     1000,            // disorderRange
                     1,               // method_of_delete
                     NULL             // arg_list
};



static SDbs            g_Dbs;
static int             g_totalChildTables = 0;
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

#define debugPrint(fmt, ...) \
570
    do { if (g_args.debug_print || g_args.verbose_print) \
571
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
572

573
#define verbosePrint(fmt, ...) \
574 575
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
576

577 578 579 580
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)

581 582 583 584
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)


585
///////////////////////////////////////////////////
H
hzcheng 已提交
586

587 588
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

589
static void printHelp() {
590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616
  char indent[10] = "        ";
  printf("%s%s%s%s\n", indent, "-f", indent, 
          "The meta file to the execution procedure. Default is './meta.json'.");
  printf("%s%s%s%s\n", indent, "-u", indent, 
          "The TDengine user name to use when connecting to the server. Default is 'root'.");
#ifdef _TD_POWER_
  printf("%s%s%s%s\n", indent, "-P", indent, 
          "The password to use when connecting to the server. Default is 'powerdb'.");
  printf("%s%s%s%s\n", indent, "-c", indent, 
          "Configuration directory. Default is '/etc/power/'.");
#else
  printf("%s%s%s%s\n", indent, "-P", indent, 
          "The password to use when connecting to the server. Default is 'taosdata'.");
  printf("%s%s%s%s\n", indent, "-c", indent, 
          "Configuration directory. Default is '/etc/taos/'.");
#endif  
  printf("%s%s%s%s\n", indent, "-h", indent, 
          "The host to connect to TDengine. Default is localhost.");
  printf("%s%s%s%s\n", indent, "-p", indent, 
          "The TCP/IP port number to use for the connection. Default is 0.");
  printf("%s%s%s%s\n", indent, "-d", indent, 
          "Destination database. Default is 'test'.");
  printf("%s%s%s%s\n", indent, "-a", indent, 
          "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
  printf("%s%s%s%s\n", indent, "-m", indent, 
          "Table prefix name. Default is 't'.");
  printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
617
  printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643
  printf("%s%s%s%s\n", indent, "-o", indent, 
          "Direct output to the named file. Default is './output.txt'.");
  printf("%s%s%s%s\n", indent, "-q", indent, 
          "Query mode--0: SYNC, 1: ASYNC. Default is SYNC.");
  printf("%s%s%s%s\n", indent, "-b", indent, 
          "The data_type of columns, default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP.");
  printf("%s%s%s%s\n", indent, "-w", indent, 
          "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
  printf("%s%s%s%s\n", indent, "-l", indent, 
          "The number of columns per record. Default is 10.");
  printf("%s%s%s%s\n", indent, "-T", indent, 
          "The number of threads. Default is 10.");
  printf("%s%s%s%s\n", indent, "-i", indent, 
          "The sleep time (ms) between insertion. Default is 0.");
  printf("%s%s%s%s\n", indent, "-r", indent, 
          "The number of records per request. Default is 100.");
  printf("%s%s%s%s\n", indent, "-t", indent, 
          "The number of tables. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-n", indent, 
          "The number of records per table. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
  printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
  printf("%s%s%s%s\n", indent, "-O", indent, 
          "Insert mode--0: In order, > 0: disorder ratio. Default is in order.");
  printf("%s%s%s%s\n", indent, "-R", indent, 
          "Out of order data's range, ms, default is 1000.");
644 645
  printf("%s%s%s%s\n", indent, "-g", indent, 
          "Print debug info.");
646 647 648 649 650
/*    printf("%s%s%s%s\n", indent, "-D", indent, 
          "if elete database if exists. 0: no, 1: yes, default is 1");
          */
}

651
static void parse_args(int argc, char *argv[], SArguments *arguments) {
652
  char **sptr;
653 654 655 656 657 658 659 660
  wordexp_t full_path;

  for (int i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-f") == 0) {
      arguments->metaFile = argv[++i];
    } else if (strcmp(argv[i], "-c") == 0) {
        char *configPath = argv[++i];
      if (wordexp(configPath, &full_path, 0) != 0) {
661
          errorPrint( "Invalid path %s\n", configPath);
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683
          return;
      }
      taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
      wordfree(&full_path);
    } else if (strcmp(argv[i], "-h") == 0) {
      arguments->host = argv[++i];
    } else if (strcmp(argv[i], "-p") == 0) {
      arguments->port = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-u") == 0) {
      arguments->user = argv[++i];
    } else if (strcmp(argv[i], "-P") == 0) {
      arguments->password = argv[++i];
    } else if (strcmp(argv[i], "-o") == 0) {
      arguments->output_file = argv[++i];
    } else if (strcmp(argv[i], "-s") == 0) {
      arguments->sqlFile = argv[++i];
    } else if (strcmp(argv[i], "-q") == 0) {
      arguments->mode = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-T") == 0) {
      arguments->num_of_threads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
      arguments->insert_interval = atoi(argv[++i]);
684
    } else if (strcmp(argv[i], "-B") == 0) {
685
      arguments->interlace_rows = atoi(argv[++i]);
686 687 688 689 690 691 692 693 694 695 696
    } else if (strcmp(argv[i], "-r") == 0) {
      arguments->num_of_RPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-t") == 0) {
      arguments->num_of_tables = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
      arguments->num_of_DPT = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-d") == 0) {
      arguments->database = argv[++i];
    } else if (strcmp(argv[i], "-l") == 0) {
      arguments->num_of_CPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-b") == 0) {
697
      sptr = arguments->datatype;
698 699
      ++i;
      if (strstr(argv[i], ",") == NULL) {
700
        // only one col
701 702 703 704 705 706 707 708 709 710
        if (strcasecmp(argv[i], "INT")
                && strcasecmp(argv[i], "FLOAT")
                && strcasecmp(argv[i], "TINYINT")
                && strcasecmp(argv[i], "BOOL")
                && strcasecmp(argv[i], "SMALLINT")
                && strcasecmp(argv[i], "BIGINT")
                && strcasecmp(argv[i], "DOUBLE")
                && strcasecmp(argv[i], "BINARY") 
                && strcasecmp(argv[i], "NCHAR")) {
          printHelp();
711
          ERROR_EXIT( "Invalid data_type!\n");
712
          exit(EXIT_FAILURE);
713
        }
714
        sptr[0] = argv[i];
715
      } else {
716
        // more than one col
717
        int index = 0;
718
        char *dupstr = strdup(argv[i]);
719 720 721
        char *running = dupstr;
        char *token = strsep(&running, ",");
        while (token != NULL) {
722 723 724 725 726 727 728 729 730 731
          if (strcasecmp(token, "INT")
                  && strcasecmp(token, "FLOAT")
                  && strcasecmp(token, "TINYINT")
                  && strcasecmp(token, "BOOL")
                  && strcasecmp(token, "SMALLINT")
                  && strcasecmp(token, "BIGINT")
                  && strcasecmp(token, "DOUBLE")
                  && strcasecmp(token, "BINARY")
                  && strcasecmp(token, "NCHAR")) {
            printHelp();
732
            ERROR_EXIT("Invalid data_type!\n");
733
            exit(EXIT_FAILURE);
H
hzcheng 已提交
734
          }
735 736 737
          sptr[index++] = token;
          token = strsep(&running, ",");
          if (index >= MAX_NUM_DATATYPE) break;
S
TD-1057  
Shengliang Guan 已提交
738
        }
739
        sptr[index] = NULL;
740
      }
741 742 743 744
    } else if (strcmp(argv[i], "-w") == 0) {
      arguments->len_of_binary = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-m") == 0) {
      arguments->tb_prefix = argv[++i];
745 746
    } else if (strcmp(argv[i], "-N") == 0) {
      arguments->use_metric = false;
747
    } else if (strcmp(argv[i], "-x") == 0) {
748
      arguments->insert_only = false;
749
    } else if (strcmp(argv[i], "-y") == 0) {
750
      arguments->answer_yes = true;
751 752
    } else if (strcmp(argv[i], "-g") == 0) {
      arguments->debug_print = true;
753 754
    } else if (strcmp(argv[i], "-gg") == 0) {
      arguments->verbose_print = true;
755 756
    } else if (strcmp(argv[i], "-pp") == 0) {
      arguments->performance_print = true;
757 758 759 760 761 762 763 764 765
    } else if (strcmp(argv[i], "-c") == 0) {
      strcpy(configDir, argv[++i]);
    } else if (strcmp(argv[i], "-O") == 0) {
      arguments->disorderRatio = atoi(argv[++i]);
      if (arguments->disorderRatio > 1 
              || arguments->disorderRatio < 0) {
        arguments->disorderRatio = 0;
      } else if (arguments->disorderRatio == 1) {
        arguments->disorderRange = 10;
766
      }
767 768 769 770 771 772
    } else if (strcmp(argv[i], "-R") == 0) {
      arguments->disorderRange = atoi(argv[++i]);
      if (arguments->disorderRange == 1 
              && (arguments->disorderRange > 50 
              || arguments->disorderRange <= 0)) {
        arguments->disorderRange = 10;
773
      }
774 775 776 777
    } else if (strcmp(argv[i], "-a") == 0) {
      arguments->replica = atoi(argv[++i]);
      if (arguments->replica > 3 || arguments->replica < 1) {
          arguments->replica = 1;
778
      }
779 780 781 782 783
    } else if (strcmp(argv[i], "-D") == 0) {
      arguments->method_of_delete = atoi(argv[++i]);
      if (arguments->method_of_delete < 0
              || arguments->method_of_delete > 3) {
        arguments->method_of_delete = 0;
S
TD-1057  
Shengliang Guan 已提交
784
      }
785 786 787 788 789
    } else if (strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else {
      printHelp();
790
      ERROR_EXIT("ERROR: wrong options\n");
791
      exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
792 793
    }
  }
794

795 796
  if (((arguments->debug_print) && (arguments->metaFile == NULL))
          || arguments->verbose_print) {
797
    printf("###################################################################\n");
798
    printf("# meta file:                         %s\n", arguments->metaFile);
799 800 801 802 803 804
    printf("# Server IP:                         %s:%hu\n", 
            arguments->host == NULL ? "localhost" : arguments->host,
            arguments->port );
    printf("# User:                              %s\n", arguments->user);
    printf("# Password:                          %s\n", arguments->password);
    printf("# Use metric:                        %s\n", arguments->use_metric ? "true" : "false");
805 806 807 808 809 810 811 812 813
    if (*(arguments->datatype)) {
        printf("# Specified data type:               ");
        for (int i = 0; i < MAX_NUM_DATATYPE; i++)
            if (arguments->datatype[i])
               printf("%s,", arguments->datatype[i]);
            else
                break;
        printf("\n");
    }
814
    printf("# Insertion interval:                %d\n", arguments->insert_interval);
815
    printf("# Number of records per req:         %d\n", arguments->num_of_RPR);
816
    printf("# Max SQL length:                    %d\n", arguments->max_sql_len);
817
    printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
818 819 820 821 822 823 824 825 826 827 828 829 830
    printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
    printf("# Number of Tables:                  %d\n", arguments->num_of_tables);
    printf("# Number of Data per Table:          %d\n", arguments->num_of_DPT);
    printf("# Database name:                     %s\n", arguments->database);
    printf("# Table prefix:                      %s\n", arguments->tb_prefix);
    if (arguments->disorderRatio) {
      printf("# Data order:                        %d\n", arguments->disorderRatio);
      printf("# Data out of order rate:            %d\n", arguments->disorderRange);
  
    }
    printf("# Delete method:                     %d\n", arguments->method_of_delete);
    printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
    printf("# Print debug info:                  %d\n", arguments->debug_print);
831
    printf("# Print verbose info:                %d\n", arguments->verbose_print);
832 833 834 835 836 837
    printf("###################################################################\n");
    if (!arguments->answer_yes) {
        printf("Press enter key to continue\n\n");
        (void) getchar();
    }
  }
838
}
H
hzcheng 已提交
839

840 841 842 843
static bool getInfoFromJsonFile(char* file);
//static int generateOneRowDataForStb(SSuperTable* stbInfo);
//static int getDataIntoMemForStb(SSuperTable* stbInfo);
static void init_rand_data();
844
static void tmfclose(FILE *fp) {
845 846 847 848
  if (NULL != fp) {
    fclose(fp);
  }
}
849

850
static void tmfree(char *buf) {
851 852
  if (NULL != buf) {
    free(buf);
H
Hui Li 已提交
853
  }
854
}
H
Hui Li 已提交
855

856
static int queryDbExec(TAOS *taos, char *command, int type) {
857 858 859
  int i;
  TAOS_RES *res = NULL;
  int32_t   code = -1;
860

861 862 863 864
  for (i = 0; i < 5; i++) {
    if (NULL != res) {
      taos_free_result(res);
      res = NULL;
H
hzcheng 已提交
865
    }
866

867 868 869 870 871
    res = taos_query(taos, command);
    code = taos_errno(res);
    if (0 == code) {
      break;
    }    
H
hzcheng 已提交
872 873
  }

874
  if (code != 0) {
875
    debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
876
    errorPrint( "Failed to run %s, reason: %s\n", command, taos_errstr(res));
877 878 879
    taos_free_result(res);
    //taos_close(taos);
    return -1;
880
  }
H
hzcheng 已提交
881

882 883 884 885
  if (INSERT_TYPE == type) {
    int affectedRows = taos_affected_rows(res);
    taos_free_result(res);
    return affectedRows;
H
hzcheng 已提交
886
  }
887
  
888
  taos_free_result(res);
889 890
  return 0;
}
H
hzcheng 已提交
891

892 893 894 895 896 897 898 899 900 901
static void getResult(TAOS_RES *res, char* resultFileName) {  
  TAOS_ROW    row = NULL;
  int         num_rows = 0;
  int         num_fields = taos_field_count(res);
  TAOS_FIELD *fields     = taos_fetch_fields(res);

  FILE *fp = NULL;
  if (resultFileName[0] != 0) {
    fp = fopen(resultFileName, "at");
    if (fp == NULL) {
902
      errorPrint("%s() LN%d, failed to open result file: %s, result will not save to file\n", __func__, __LINE__, resultFileName);
903 904 905 906 907
    }
  }
  
  char* databuf = (char*) calloc(1, 100*1024*1024);
  if (databuf == NULL) {
908
    errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n", __func__, __LINE__);
909 910
    if (fp)
        fclose(fp);
911 912
    return ;
  }
H
hzcheng 已提交
913

914 915
  int   totalLen = 0;
  char  temp[16000];
H
hzcheng 已提交
916

917 918 919 920 921 922
  // fetch the records row by row
  while ((row = taos_fetch_row(res))) {
    if (totalLen >= 100*1024*1024 - 32000) {
      if (fp) fprintf(fp, "%s", databuf);
      totalLen = 0;
      memset(databuf, 0, 100*1024*1024);
H
hzcheng 已提交
923
    }
924 925 926 927 928 929
    num_rows++;
    int len = taos_print_row(temp, row, fields, num_fields);
    len += sprintf(temp + len, "\n");
    //printf("query result:%s\n", temp);
    memcpy(databuf + totalLen, temp, len);
    totalLen += len;
H
hzcheng 已提交
930 931
  }

932 933 934 935
  if (fp) fprintf(fp, "%s", databuf);
  tmfclose(fp);
  free(databuf);
}
H
hzcheng 已提交
936

937 938 939 940 941 942
static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) {
  TAOS_RES *res = taos_query(taos, command);
  if (res == NULL || taos_errno(res) != 0) {
    printf("failed to sql:%s, reason:%s\n", command, taos_errstr(res));
    taos_free_result(res);
    return;
H
hzcheng 已提交
943
  }
H
Hui Li 已提交
944
  
945 946 947
  getResult(res, resultFileName);
  taos_free_result(res);
}
H
hzcheng 已提交
948

949
static double getCurrentTime() {
950 951 952 953
  struct timeval tv;
  if (gettimeofday(&tv, NULL) != 0) {
    perror("Failed to get current time in ms");
    return 0.0;
H
hzcheng 已提交
954
  }
S
Shuduo Sang 已提交
955

956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
  return tv.tv_sec + tv.tv_usec / 1E6;
}

static int32_t rand_bool(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 2;
}

static int32_t rand_tinyint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 128;
}

static int32_t rand_smallint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 32767;
}

static int32_t rand_int(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor];
}

static int64_t rand_bigint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randbigint[cursor];
  
}

static float rand_float(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randfloat[cursor];    
}

static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1003
static void rand_string(char *str, int size) {
1004 1005 1006 1007
  str[0] = 0;
  if (size > 0) {
    //--size;
    int n;
1008
    for (n = 0; n < size - 1; n++) {
1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
      int key = rand_tinyint() % (int)(sizeof(charset) - 1);
      str[n] = charset[key];
    }
    str[n] = 0;
  }
}

static double rand_double() {
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randdouble[cursor];

}

static void init_rand_data() {
  for (int i = 0; i < MAX_PREPARED_RAND; i++){
1026 1027 1028 1029
    randint[i] = (int)(taosRandom() % 65535);
    randbigint[i] = (int64_t)(taosRandom() % 2147483648);
    randfloat[i] = (float)(taosRandom() / 1000.0);
    randdouble[i] = (double)(taosRandom() / 1000000.0);
1030 1031 1032
  }
}

1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1053
static int printfInsertMeta() {
1054 1055
    SHOW_PARSE_RESULT_START();

1056 1057 1058 1059 1060 1061
  printf("host:                       \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port);
  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
  printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl);
1062 1063
  printf("insert interval:            \033[33m%d\033[0m\n", g_args.insert_interval);
  printf("number of records per req:  \033[33m%d\033[0m\n", g_args.num_of_RPR);
1064
  printf("max sql length:             \033[33m%d\033[0m\n", g_args.max_sql_len);
1065 1066

  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
1067

1068 1069
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    printf("database[\033[33m%d\033[0m]:\n", i);
1070
    printf("  database[%d] name:      \033[33m%s\033[0m\n", i, g_Dbs.db[i].dbName);
1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
    if (0 == g_Dbs.db[i].drop) {
      printf("  drop:                  \033[33mno\033[0m\n");     
    }else {
      printf("  drop:                  \033[33myes\033[0m\n"); 
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      printf("  blocks:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      printf("  cache:                 \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      printf("  days:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      printf("  keep:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      printf("  replica:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      printf("  update:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      printf("  minRows:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      printf("  maxRows:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      printf("  walLevel:              \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      printf("  fsync:                 \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      printf("  quorum:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1114 1115
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1116 1117
        printf("  precision:             \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision);
      } else {
1118 1119 1120
        printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
                g_Dbs.db[i].dbCfg.precision);
        return -1;
1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144
      }
    }

    printf("  super table count:     \033[33m%d\033[0m\n", g_Dbs.db[i].superTblCount);
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      printf("  super table[\033[33m%d\033[0m]:\n", j);
    
      printf("      stbName:           \033[33m%s\033[0m\n",  g_Dbs.db[i].superTbls[j].sTblName);   

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
      }
      
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
      }
1145 1146 1147 1148 1149 1150 1151 1152

      printf("      childTblCount:     \033[33m%d\033[0m\n",  g_Dbs.db[i].superTbls[j].childTblCount);
      printf("      childTblPrefix:    \033[33m%s\033[0m\n",  g_Dbs.db[i].superTbls[j].childTblPrefix);
      printf("      dataSource:        \033[33m%s\033[0m\n",  g_Dbs.db[i].superTbls[j].dataSource);
      printf("      insertMode:        \033[33m%s\033[0m\n",  g_Dbs.db[i].superTbls[j].insertMode);
      if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
        printf("      childTblLimit:     \033[33m%d\033[0m\n",  g_Dbs.db[i].superTbls[j].childTblLimit);
      }
1153
      if (g_Dbs.db[i].superTbls[j].childTblOffset >= 0) {
1154 1155 1156
        printf("      childTblOffset:    \033[33m%d\033[0m\n",  g_Dbs.db[i].superTbls[j].childTblOffset);
      }
      printf("      insertRows:        \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows);
1157 1158

      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1159
        printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
1160
      }else {
1161
        printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
1162
      }
1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
      printf("      rowsPerTbl:        \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].rowsPerTbl);
      printf("      disorderRange:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      printf("      disorderRatio:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
      printf("      maxSqlLen:         \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].maxSqlLen);
      printf("      timeStampStep:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].timeStampStep);
      printf("      startTimestamp:    \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
      printf("      sampleFormat:      \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      printf("      sampleFile:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      printf("      tagsFile:          \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
      printf("      columnCount:       \033[33m%d\033[0m\n        ",
              g_Dbs.db[i].superTbls[j].columnCount);
1183 1184
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1185 1186 1187 1188
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "nchar", 5))) {
1189
          printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
1190 1191
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1192
        } else {
1193 1194
          printf("column[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType);
1195 1196 1197
        }
      }
      printf("\n");
1198 1199 1200

      printf("      tagCount:            \033[33m%d\033[0m\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1201 1202
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1203 1204 1205
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) {
          printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
1206 1207
                  g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1208
        } else {
1209 1210
          printf("tag[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].tags[k].dataType);
1211
        }
1212 1213 1214 1215 1216
      }
      printf("\n");
    }
    printf("\n");
  }
1217 1218

  SHOW_PARSE_RESULT_END();
1219 1220

  return 0;
1221 1222 1223
}

static void printfInsertMetaToFile(FILE* fp) {
1224 1225

  SHOW_PARSE_RESULT_START_TO_FILE(fp);
1226

1227 1228 1229 1230 1231
  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
  fprintf(fp, "user:                       %s\n", g_Dbs.user);
  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
1232 1233 1234
  fprintf(fp, "insert interval:            %d\n", g_args.insert_interval);
  fprintf(fp, "number of records per req:  %d\n", g_args.num_of_RPR);
  fprintf(fp, "max sql length:             %d\n", g_args.max_sql_len);
1235
  fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
1236

1237 1238
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    fprintf(fp, "database[%d]:\n", i);
1239
    fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282
    if (0 == g_Dbs.db[i].drop) {
      fprintf(fp, "  drop:                  no\n");     
    }else {
      fprintf(fp, "  drop:                  yes\n"); 
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1283 1284
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317
        fprintf(fp, "  precision:             %s\n", g_Dbs.db[i].dbCfg.precision);
      } else {
        fprintf(fp, "  precision error:       %s\n", g_Dbs.db[i].dbCfg.precision);
      }
    }

    fprintf(fp, "  super table count:     %d\n", g_Dbs.db[i].superTblCount);
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      fprintf(fp, "  super table[%d]:\n", j);
    
      fprintf(fp, "      stbName:           %s\n",  g_Dbs.db[i].superTbls[j].sTblName);   

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
      } else {
        fprintf(fp, "      autoCreateTable:   %s\n",  "error");
      }
      
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "yes");
      } else {
        fprintf(fp, "      childTblExists:    %s\n",  "error");
      }
      
      fprintf(fp, "      childTblCount:     %d\n",  g_Dbs.db[i].superTbls[j].childTblCount);      
      fprintf(fp, "      childTblPrefix:    %s\n",  g_Dbs.db[i].superTbls[j].childTblPrefix);      
      fprintf(fp, "      dataSource:        %s\n",  g_Dbs.db[i].superTbls[j].dataSource);      
      fprintf(fp, "      insertMode:        %s\n",  g_Dbs.db[i].superTbls[j].insertMode);      
      fprintf(fp, "      insertRows:        %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); 
1318
      fprintf(fp, "      insert interval:   %d\n", g_Dbs.db[i].superTbls[j].insertInterval);
1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338

      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
        fprintf(fp, "      multiThreadWriteOneTbl:  no\n");     
      }else {
        fprintf(fp, "      multiThreadWriteOneTbl:  yes\n"); 
      }
      fprintf(fp, "      rowsPerTbl:        %d\n",  g_Dbs.db[i].superTbls[j].rowsPerTbl);     
      fprintf(fp, "      disorderRange:     %d\n",  g_Dbs.db[i].superTbls[j].disorderRange);     
      fprintf(fp, "      disorderRatio:     %d\n",  g_Dbs.db[i].superTbls[j].disorderRatio);
      fprintf(fp, "      maxSqlLen:         %d\n",  g_Dbs.db[i].superTbls[j].maxSqlLen);     
      
      fprintf(fp, "      timeStampStep:     %d\n",  g_Dbs.db[i].superTbls[j].timeStampStep);      
      fprintf(fp, "      startTimestamp:    %s\n",  g_Dbs.db[i].superTbls[j].startTimestamp);             
      fprintf(fp, "      sampleFormat:      %s\n",  g_Dbs.db[i].superTbls[j].sampleFormat);
      fprintf(fp, "      sampleFile:        %s\n",  g_Dbs.db[i].superTbls[j].sampleFile); 
      fprintf(fp, "      tagsFile:          %s\n",  g_Dbs.db[i].superTbls[j].tagsFile);   
    
      fprintf(fp, "      columnCount:       %d\n        ",  g_Dbs.db[i].superTbls[j].columnCount);
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1339 1340 1341 1342 1343 1344 1345 1346
        if ((0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "nchar", strlen("nchar")))) {
          fprintf(fp, "column[%d]:%s(%d) ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1347 1348 1349 1350 1351
        } else {
          fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
        }
      }
      fprintf(fp, "\n");
1352 1353 1354

      fprintf(fp, "      tagCount:            %d\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1355 1356
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1357 1358 1359 1360 1361 1362
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
          fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1363 1364
        } else {
          fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
1365
        }
1366 1367 1368 1369 1370
      }
      fprintf(fp, "\n");
    }
    fprintf(fp, "\n");
  }
1371

1372
  SHOW_PARSE_RESULT_END_TO_FILE(fp);
1373 1374 1375
}

static void printfQueryMeta() {
1376

1377
  SHOW_PARSE_RESULT_START();
1378

1379 1380
  printf("host:                    \033[33m%s:%u\033[0m\n",
          g_queryInfo.host, g_queryInfo.port);
1381 1382 1383 1384 1385 1386 1387 1388 1389 1390
  printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
  printf("password:                \033[33m%s\033[0m\n", g_queryInfo.password);
  printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

  printf("\n");
  printf("specified table query info:                   \n");  
  printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.rate);
  printf("concurrent:     \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.concurrent);
  printf("sqlCount:       \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.sqlCount); 

1391
  if (SUBSCRIBE_TEST == g_args.test_mode) {
1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407
    printf("mod:            \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeMode);
    printf("interval:       \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeInterval);
    printf("restart:        \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeRestart);
    printf("keepProgress:   \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeKeepProgress);
  }

  for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
    printf("  sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.superQueryInfo.sql[i]);
  }
  printf("\n");
  printf("super table query info:                   \n");  
  printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.rate);
  printf("threadCnt:      \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.threadCnt);
  printf("childTblCount:  \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.childTblCount);
  printf("stable name:    \033[33m%s\033[0m\n", g_queryInfo.subQueryInfo.sTblName);

1408
  if (SUBSCRIBE_TEST == g_args.test_mode) {
1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419
    printf("mod:            \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeMode);
    printf("interval:       \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeInterval);
    printf("restart:        \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeRestart);
    printf("keepProgress:   \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeKeepProgress);
  }
  
  printf("sqlCount:       \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.sqlCount);  
  for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) {
    printf("  sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.subQueryInfo.sql[i]);
  }  
  printf("\n");
1420

1421
  SHOW_PARSE_RESULT_END();
1422 1423 1424
}


1425
static char* formatTimestamp(char* buf, int64_t val, int precision) {
1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456
  time_t tt;
  if (precision == TSDB_TIME_PRECISION_MICRO) {
    tt = (time_t)(val / 1000000);
  } else {
    tt = (time_t)(val / 1000);
  }

/* comment out as it make testcases like select_with_tags.sim fail.
  but in windows, this may cause the call to localtime crash if tt < 0,
  need to find a better solution.
  if (tt < 0) {
    tt = 0;
  }
  */

#ifdef WINDOWS
  if (tt < 0) tt = 0;
#endif

  struct tm* ptm = localtime(&tt);
  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);

  if (precision == TSDB_TIME_PRECISION_MICRO) {
    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
  } else {
    sprintf(buf + pos, ".%03d", (int)(val % 1000));
  }

  return buf;
}

1457 1458 1459
static void xDumpFieldToFile(FILE* fp, const char* val,
        TAOS_FIELD* field, int32_t length, int precision) {

1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494
  if (val == NULL) {
    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
    return;
  }

  char buf[TSDB_MAX_BYTES_PER_ROW];
  switch (field->type) {
    case TSDB_DATA_TYPE_BOOL:
      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
      break;
    case TSDB_DATA_TYPE_TINYINT:
      fprintf(fp, "%d", *((int8_t *)val));
      break;
    case TSDB_DATA_TYPE_SMALLINT:
      fprintf(fp, "%d", *((int16_t *)val));
      break;
    case TSDB_DATA_TYPE_INT:
      fprintf(fp, "%d", *((int32_t *)val));
      break;
    case TSDB_DATA_TYPE_BIGINT:
      fprintf(fp, "%" PRId64, *((int64_t *)val));
      break;
    case TSDB_DATA_TYPE_FLOAT:
      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
      break;
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      memcpy(buf, val, length);
      buf[length] = 0;
      fprintf(fp, "\'%s\'", buf);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
1495
      formatTimestamp(buf, *(int64_t*)val, precision);
1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510
      fprintf(fp, "'%s'", buf);
      break;
    default:
      break;
  }
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  FILE* fp = fopen(fname, "at");
  if (fp == NULL) {
1511
    errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555
    return -1;
  }

  int num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int precision = taos_result_precision(tres);

  for (int col = 0; col < num_fields; col++) {
    if (col > 0) {
      fprintf(fp, ",");
    }
    fprintf(fp, "%s", fields[col].name);
  }
  fputc('\n', fp);
  
  int numOfRows = 0;
  do {
    int32_t* length = taos_fetch_lengths(tres);
    for (int i = 0; i < num_fields; i++) {
      if (i > 0) {
        fputc(',', fp);
      }
      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
    }
    fputc('\n', fp);

    numOfRows++;
    row = taos_fetch_row(tres);
  } while( row != NULL);

  fclose(fp);

  return numOfRows;
}

static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {  
  TAOS_RES * res;  
  TAOS_ROW row = NULL;
  int count = 0;
  
  res = taos_query(taos, "show databases;");  
  int32_t code = taos_errno(res);
  
  if (code != 0) {
1556
    errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567
    return -1;
  }

  TAOS_FIELD *fields = taos_fetch_fields(res);

  while ((row = taos_fetch_row(res)) != NULL) {
    // sys database name : 'log'
    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) continue;

    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
    if (dbInfos[count] == NULL) {
1568
      errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
1569 1570 1571
      return -1;
    }

1572 1573
    tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
            fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
1574
    formatTimestamp(dbInfos[count]->create_time,
1575 1576
            *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
            TSDB_TIME_PRECISION_MILLI);
1577 1578 1579 1580 1581 1582
    dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);  
    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);  

1583 1584
    tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
            fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);      
1585 1586 1587 1588 1589 1590 1591 1592 1593
    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
    dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));

1594 1595 1596
    tstrncpy(dbInfos[count]->precision, 
            (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
            fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);  
1597
    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
1598 1599 1600
    tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
            fields[TSDB_SHOW_DB_STATUS_INDEX].bytes); 

1601 1602
    count++;
    if (count > MAX_DATABASE_COUNT) {
1603
      errorPrint( "The database count overflow than %d\n", MAX_DATABASE_COUNT);      
1604 1605 1606 1607 1608 1609 1610 1611
      break;
    }
  }

  return count;
}

static void printfDbInfoForQueryToFile(char* filename, SDbInfo* dbInfos, int index) {
1612
  if (filename[0] == 0)
1613
      return;
1614 1615 1616

  FILE *fp = fopen(filename, "at");
  if (fp == NULL) {
1617
    errorPrint( "failed to open file: %s\n", filename);
1618
    return;
1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654
  }

  fprintf(fp, "================ database[%d] ================\n", index);
  fprintf(fp, "name: %s\n", dbInfos->name);
  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
  fprintf(fp, "ntables: %d\n", dbInfos->ntables);
  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);  
  fprintf(fp, "replica: %d\n", dbInfos->replica);
  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
  fprintf(fp, "days: %d\n", dbInfos->days);    
  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);      
  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
  fprintf(fp, "comp: %d\n", dbInfos->comp);
  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);  
  fprintf(fp, "precision: %s\n", dbInfos->precision);  
  fprintf(fp, "update: %d\n", dbInfos->update);
  fprintf(fp, "status: %s\n", dbInfos->status); 
  fprintf(fp, "\n");

  fclose(fp);
}

static void printfQuerySystemInfo(TAOS * taos) {
  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
  TAOS_RES* res;

  time_t t;
  struct tm* lt;
  time(&t);
  lt = localtime(&t);
1655 1656 1657 1658
  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
          lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
          lt->tm_sec);

1659 1660 1661 1662 1663 1664 1665 1666 1667
  // show variables
  res = taos_query(taos, "show variables;");
  //getResult(res, filename);
  xDumpResultToFile(filename, res);

  // show dnodes
  res = taos_query(taos, "show dnodes;");
  xDumpResultToFile(filename, res);
  //getResult(res, filename);
1668

1669 1670 1671 1672
  // show databases
  res = taos_query(taos, "show databases;");
  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
  if (dbInfos == NULL) {
1673
    errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
1674 1675 1676
    return;
  }
  int dbCount = getDbFromServer(taos, dbInfos);
1677 1678 1679 1680
  if (dbCount <= 0) {
      free(dbInfos);
      return;
  }
1681 1682 1683 1684

  for (int i = 0; i < dbCount; i++) {
    // printf database info 
    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
1685

1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702
    // show db.vgroups
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);
  
    // show db.stables
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);

    free(dbInfos[i]);
  }

  free(dbInfos);
  
}

1703
static int postProceSql(char* host, uint16_t port, char* sqlstr)
1704
{
1705
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
1706 1707 1708 1709 1710

    char *url = "/rest/sql";

    struct hostent *server;
    struct sockaddr_in serv_addr;
1711
    int bytes, sent, received, req_str_len, resp_len;
1712 1713 1714
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
1715

1716 1717 1718 1719
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;

    request_buf = malloc(req_buf_len);
    if (NULL == request_buf)
1720
        ERROR_EXIT("ERROR, cannot allocate memory.");
1721

1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};

    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
1733

1734 1735 1736 1737 1738 1739
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);

    char base64_buf[INPUT_BUF_LEN];
1740
#ifdef WINDOWS
1741
    WSADATA wsaData;
1742
    WSAStartup(MAKEWORD(2, 2), &wsaData);
1743 1744 1745 1746
    SOCKET sockfd;
#else
    int sockfd;
#endif
1747
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
1748
    if (sockfd < 0) {
1749
#ifdef WINDOWS
1750
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
1751
#endif
1752
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
1753
        free(request_buf);
1754
        ERROR_EXIT("ERROR opening socket");
1755
    }
1756 1757

    server = gethostbyname(host);
1758 1759
    if (server == NULL) {
        free(request_buf);
1760
        ERROR_EXIT("ERROR, no such host");
1761
    }
1762

1763
    debugPrint("h_name: %s\nh_addretype: %s\nh_length: %d\n",
1764 1765 1766 1767
            server->h_name,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

1768
    memset(&serv_addr, 0, sizeof(serv_addr));
1769
    serv_addr.sin_family = AF_INET;
1770
    serv_addr.sin_port = htons(rest_port);
1771 1772 1773
#ifdef WINDOWS
    serv_addr.sin_addr.s_addr = inet_addr(host);
#else
1774
    memcpy(&serv_addr.sin_addr.s_addr,server->h_addr,server->h_length);
1775
#endif
1776

1777 1778 1779
    int retConn = connect(sockfd,(struct sockaddr *)&serv_addr,sizeof(serv_addr));
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
1780
        free(request_buf);
1781
        ERROR_EXIT("ERROR connecting");
1782
    }
1783

1784 1785 1786
    memset(base64_buf, 0, INPUT_BUF_LEN);

    for (int n = 0, m = 0; n < userpass_buf_len;) {
1787
      uint32_t oct_a = n < userpass_buf_len ?
1788
        (unsigned char) userpass_buf[n++]:0;
1789
      uint32_t oct_b = n < userpass_buf_len ?
1790
        (unsigned char) userpass_buf[n++]:0;
1791
      uint32_t oct_c = n < userpass_buf_len ?
1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;

      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }

    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';

    debugPrint("%s() LN%d: auth string base64 encoded: %s\n", __func__, __LINE__, base64_buf);
    char *auth = base64_buf;

1807 1808 1809
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
1810 1811 1812 1813 1814
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
1815
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
1816

1817
    req_str_len = strlen(request_buf);
1818 1819
    sent = 0;
    do {
1820 1821 1822
#ifdef WINDOWS
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
#else
1823
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
1824
#endif
1825
        if (bytes < 0)
1826
            ERROR_EXIT("ERROR writing message to socket");
1827 1828 1829
        if (bytes == 0)
            break;
        sent+=bytes;
1830
    } while (sent < req_str_len);
1831

1832 1833
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
1834 1835
    received = 0;
    do {
1836 1837 1838
#ifdef WINDOWS
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
#else
1839
        bytes = read(sockfd, response_buf + received, resp_len - received);
1840
#endif
1841 1842
        if (bytes < 0) {
            free(request_buf);
1843
            ERROR_EXIT("ERROR reading response from socket");
1844
        }
1845 1846
        if (bytes == 0)
            break;
1847 1848
        received += bytes;
    } while (received < resp_len);
1849

1850 1851
    if (received == resp_len) {
        free(request_buf);
1852
        ERROR_EXIT("ERROR storing complete response from socket");
1853
    }
1854

1855
    response_buf[RESP_BUF_LEN - 1] = '\0';
1856 1857 1858
    printf("Response:\n%s\n", response_buf);

    free(request_buf);
1859 1860 1861 1862
#ifdef WINDOWS
    closesocket(sockfd);
    WSACleanup();
#else
1863
    close(sockfd);
1864
#endif
1865 1866 1867 1868

    return 0;
}

1869
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
1870 1871
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
1872
    errorPrint("%s() LN%d, calloc failed! size:%d\n", __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
1873 1874
    return NULL;
  }
1875

1876
  int    dataLen = 0;
1877 1878 1879
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);

1880 1881 1882
  return dataBuf;
}

1883
static char* generateTagVaulesForStb(SSuperTable* stbInfo) {
1884 1885 1886 1887 1888 1889 1890 1891 1892
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
    printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
    return NULL;
  }

  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
  for (int i = 0; i < stbInfo->tagCount; i++) {
1893 1894
    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
1895
      if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
1896 1897
        printf("binary or nchar length overflow, max size:%u\n",
                (uint32_t)TSDB_MAX_BINARY_LEN);
1898 1899 1900
        tmfree(dataBuf);
        return NULL;
      }
1901

1902 1903 1904 1905 1906 1907 1908
      char* buf = (char*)calloc(stbInfo->tags[i].dataLen+1, 1);
      if (NULL == buf) {
        printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
        tmfree(dataBuf);
        return NULL;
      }
      rand_string(buf, stbInfo->tags[i].dataLen);
1909 1910
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "\'%s\', ", buf);
1911
      tmfree(buf);
1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "int", strlen("int"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_int());
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bigint", strlen("bigint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "float", strlen("float"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_float());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "double", strlen("double"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_double());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "smallint", strlen("smallint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_smallint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "tinyint", strlen("tinyint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_tinyint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bool", strlen("bool"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_bool());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "timestamp", strlen("timestamp"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
1944 1945 1946 1947 1948 1949
    }  else {
      printf("No support data type: %s\n", stbInfo->tags[i].dataType);
      tmfree(dataBuf);
      return NULL;
    }
  }
1950

1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994
  dataLen -= 2;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");  
  return dataBuf;
}

static int calcRowLen(SSuperTable*  superTbls) {  
  int colIndex;
  int  lenOfOneRow = 0;
  
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
    
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) { 
      lenOfOneRow += 42;
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) { 
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
1995

1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) { 
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
    
  return 0;
}


2026 2027 2028 2029
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
        int* childTblCountOfSuperTbl, int limit, int offset) {

2030
  char command[BUFFER_SIZE] = "\0";
2031 2032
  char limitBuf[100] = "\0";

2033 2034 2035 2036
  TAOS_RES * res;  
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2037 2038 2039 2040 2041

  if (offset >= 0) {
    snprintf(limitBuf, 100, " limit %d offset %d", limit, offset);
  }

2042
  //get all child table name use cmd: select tbname from superTblName;  
2043 2044
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s", dbName, sTblName, limitBuf);

2045 2046 2047 2048 2049 2050 2051 2052 2053
  res = taos_query(taos, command);  
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
    taos_free_result(res);
    taos_close(taos);
    exit(-1);
  }

2054
  int childTblCount = (limit < 0)?10000:limit;
2055
  int count = 0;
2056
//  childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
2057 2058 2059
  char* pTblName = childTblName;
  while ((row = taos_fetch_row(res)) != NULL) {
    int32_t* len = taos_fetch_lengths(res);
2060
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2061 2062 2063
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2064 2065
      char *tmp = realloc(childTblName,
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2066 2067 2068
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2069 2070
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2071 2072
      } else {
        // exit, if allocate more memory failed
2073 2074
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
               __func__, __LINE__, dbName, sTblName);
2075 2076 2077 2078 2079 2080
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2081
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2082
  }
2083

2084 2085 2086 2087 2088 2089 2090
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2091 2092
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
        char* sTblName, char** childTblNameOfSuperTbl,
2093
        int* childTblCountOfSuperTbl) {
2094 2095 2096 2097 2098 2099

    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
            -1, -1);
}

2100 2101
static int getSuperTableFromServer(TAOS * taos, char* dbName,
        SSuperTable*  superTbls) {
2102 2103 2104 2105
  char command[BUFFER_SIZE] = "\0";
  TAOS_RES * res;  
  TAOS_ROW row = NULL;
  int count = 0;
2106

2107
  //get schema use cmd: describe superTblName;
2108
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2109
  res = taos_query(taos, command);
2110 2111 2112
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2113
    taos_free_result(res);
2114 2115 2116 2117 2118 2119 2120 2121 2122 2123
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
  while ((row = taos_fetch_row(res)) != NULL) {
    if (0 == count) {
      count++;
      continue;
2124
    }
2125 2126

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137
      tstrncpy(superTbls->tags[tagIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->tags[tagIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2138
      tagIndex++;
2139
    } else {
2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150
      tstrncpy(superTbls->columns[columnIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->columns[columnIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2162
/*
2163 2164
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
    //get all child table name use cmd: select tbname from superTblName;  
2165 2166 2167
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2168
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2169 2170
      return -1;
    }
2171
    getAllChildNameOfSuperTable(taos, dbName,
2172 2173 2174
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2175
  }
2176
  */
2177 2178 2179
  return 0;
}

2180
static int createSuperTable(TAOS * taos, char* dbName, SSuperTable*  superTbls, bool use_metric) {
2181
  char command[BUFFER_SIZE] = "\0";
2182

2183 2184 2185 2186 2187 2188 2189
  char cols[STRING_LEN] = "\0";
  int colIndex;
  int len = 0;

  int  lenOfOneRow = 0;
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2190
 
2191
    if (strcasecmp(dataType, "BINARY") == 0) {
2192 2193 2194
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "BINARY",
          superTbls->columns[colIndex].dataLen);
2195 2196
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
2197 2198 2199
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "NCHAR",
          superTbls->columns[colIndex].dataLen);
2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
      lenOfOneRow += 22;
2219
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2220 2221
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
      lenOfOneRow += 42;
2222
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
      lenOfOneRow += 21;
    } else {
      taos_close(taos);
      printf("config error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp
  //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName, g_Dbs.db[i].superTbls[j].columnCount, lenOfOneRow);

  // save for creating child table
2236 2237
  superTbls->colsOfCreateChildTable = (char*)calloc(len+20, 1);
  if (NULL == superTbls->colsOfCreateChildTable) {
2238 2239 2240 2241
    printf("Failed when calloc, size:%d", len+1);
    taos_close(taos);
    exit(-1);
  }
2242
  snprintf(superTbls->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
2243
  verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, superTbls->colsOfCreateChildTable);
2244 2245 2246 2247 2248 2249 2250 2251 2252 2253

  if (use_metric) {
    char tags[STRING_LEN] = "\0";
    int tagIndex;
    len = 0;

    int lenOfTagOfOneRow = 0;
    len += snprintf(tags + len, STRING_LEN - len, "(");
    for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
      char* dataType = superTbls->tags[tagIndex].dataType;
2254

2255
      if (strcasecmp(dataType, "BINARY") == 0) {
2256 2257
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
                "BINARY", superTbls->tags[tagIndex].dataLen);
2258 2259
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
      } else if (strcasecmp(dataType, "NCHAR") == 0) {
2260 2261
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
                "NCHAR", superTbls->tags[tagIndex].dataLen);
2262 2263
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
      } else if (strcasecmp(dataType, "INT") == 0)  {
2264 2265
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "INT");
2266 2267
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
      } else if (strcasecmp(dataType, "BIGINT") == 0)  {
2268 2269
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "BIGINT");
2270 2271
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
      } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
2272 2273
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "SMALLINT");
2274 2275
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
      } else if (strcasecmp(dataType, "TINYINT") == 0)  {
2276 2277
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "TINYINT");
2278 2279
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
      } else if (strcasecmp(dataType, "BOOL") == 0)  {
2280 2281
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "BOOL");
2282 2283
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
      } else if (strcasecmp(dataType, "FLOAT") == 0) {
2284 2285
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "FLOAT");
2286 2287
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
      } else if (strcasecmp(dataType, "DOUBLE") == 0) { 
2288 2289
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "DOUBLE");
2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
      } else {
        taos_close(taos);
        printf("config error tag type : %s\n", dataType);
        exit(-1);
      }
    }
    len -= 2;
    len += snprintf(tags + len, STRING_LEN - len, ")");

    superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2301 2302 2303 2304

    snprintf(command, BUFFER_SIZE,
            "create table if not exists %s.%s (ts timestamp%s) tags %s",
            dbName, superTbls->sTblName, cols, tags);
2305
    verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command);
2306

2307
    if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) {
2308
        errorPrint( "create supertable %s failed!\n\n",
2309
                superTbls->sTblName);
2310
        return -1;
2311
    }
2312
    debugPrint("create supertable %s success!\n\n", superTbls->sTblName);
2313 2314 2315 2316
  }
  return 0;
}

2317
static int createDatabases() {
2318 2319 2320 2321
  TAOS * taos = NULL;
  int    ret = 0;
  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
  if (taos == NULL) {
2322
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2323
    return -1;
2324 2325 2326 2327 2328 2329
  }
  char command[BUFFER_SIZE] = "\0";

  for (int i = 0; i < g_Dbs.dbCount; i++) {   
    if (g_Dbs.db[i].drop) {
      sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
2330
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);
2331 2332 2333 2334 2335
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) {
        taos_close(taos);
        return -1;
      }
    }
2336

2337
    int dataLen = 0;
2338
    dataLen += snprintf(command + dataLen, 
2339
        BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2340 2341

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
2342
      dataLen += snprintf(command + dataLen,
2343
          BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
2344 2345
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
2346
      dataLen += snprintf(command + dataLen,
2347
          BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
2348 2349
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
2350
      dataLen += snprintf(command + dataLen,
2351
          BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
2352 2353
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
2354
      dataLen += snprintf(command + dataLen,
2355
          BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
2356
    }
2357 2358
    if (g_Dbs.db[i].dbCfg.quorum > 1) {
      dataLen += snprintf(command + dataLen,
2359
          BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
2360 2361
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
2362
      dataLen += snprintf(command + dataLen,
2363
          BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
2364 2365
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
2366
      dataLen += snprintf(command + dataLen,
2367
          BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
2368 2369
    }
    //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
2370 2371
    //  dataLen += snprintf(command + dataLen,
    //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
2372 2373
    //}
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
2374
      dataLen += snprintf(command + dataLen,
2375
          BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
2376 2377
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
2378
      dataLen += snprintf(command + dataLen,
2379
          BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
2380 2381
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
2382
      dataLen += snprintf(command + dataLen,
2383
          BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
2384 2385
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
2386
      dataLen += snprintf(command + dataLen,
2387
          BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
2388
    }
2389
    if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
2390
      dataLen += snprintf(command + dataLen,
2391
          BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
2392
    }
2393
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
2394 2395
      dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
              " fsync %d", g_Dbs.db[i].dbCfg.fsync);
2396
    }
2397
    if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
2398 2399
            || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                    "us", strlen("us")))) {
2400
      dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2401
              " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
2402
    }
2403

2404
    debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
2405 2406
    if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) {
      taos_close(taos);
2407
      errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
2408 2409 2410 2411
      return -1;
    }
    printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);

2412 2413
    debugPrint("%s() %d supertbl count:%d\n",
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
2414 2415
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      // describe super table, if exists
2416 2417
      sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
              g_Dbs.db[i].superTbls[j].sTblName);
2418
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);
2419 2420
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) {
        g_Dbs.db[i].superTbls[j].superTblExists = TBL_NO_EXISTS;
2421 2422
        ret = createSuperTable(taos, g_Dbs.db[i].dbName,
                &g_Dbs.db[i].superTbls[j], g_Dbs.use_metric);
2423 2424
      } else {      
        g_Dbs.db[i].superTbls[j].superTblExists = TBL_ALREADY_EXISTS;
2425 2426 2427

        if (g_Dbs.db[i].superTbls[j].childTblExists != TBL_ALREADY_EXISTS) {
            ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
2428
                &g_Dbs.db[i].superTbls[j]);
2429
        }
2430 2431 2432
      }

      if (0 != ret) {
2433
        printf("\ncreate super table %d failed!\n\n", j);
2434 2435 2436 2437 2438 2439 2440 2441 2442 2443
        taos_close(taos);
        return -1;
      }
    }    
  }

  taos_close(taos);
  return 0;
}

2444
static void* createTable(void *sarg) 
2445 2446 2447 2448 2449 2450
{  
  threadInfo *winfo = (threadInfo *)sarg; 
  SSuperTable* superTblInfo = winfo->superTblInfo;

  int64_t  lastPrintTime = taosGetTimestampMs();

2451 2452 2453 2454 2455 2456
  int buff_len;
  if (superTblInfo)
    buff_len = superTblInfo->maxSqlLen;
  else
    buff_len = BUFFER_SIZE;

2457
  char *buffer = calloc(buff_len, 1);
2458
  if (buffer == NULL) {
2459
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
2460 2461
    exit(-1);
  }
2462 2463 2464

  int len = 0;
  int batchNum = 0;
2465 2466 2467 2468 2469 2470

  verbosePrint("%s() LN%d: Creating table from %d to %d\n", 
          __func__, __LINE__,
          winfo->start_table_from, winfo->end_table_to);

  for (int i = winfo->start_table_from; i <= winfo->end_table_to; i++) {
2471
    if (0 == g_Dbs.use_metric) {
2472
      snprintf(buffer, buff_len, 
2473 2474
              "create table if not exists %s.%s%d %s;",
              winfo->db_name,
2475 2476
              g_args.tb_prefix, i,
              winfo->cols);
2477 2478 2479
    } else {
      if (0 == len) {  
        batchNum = 0;
2480
        memset(buffer, 0, buff_len);
2481
        len += snprintf(buffer + len,
2482
                buff_len - len, "create table ");
2483
      }
2484

2485 2486 2487 2488
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
        tagsValBuf = generateTagVaulesForStb(superTblInfo);
      } else {
2489 2490 2491
        tagsValBuf = getTagValueFromTagSample(
                superTblInfo,
                i % superTblInfo->tagSampleCount);
2492 2493 2494 2495 2496 2497
      }
      if (NULL == tagsValBuf) {
        free(buffer);
        return NULL;
      }
      
2498 2499 2500 2501 2502 2503
      len += snprintf(buffer + len,
              superTblInfo->maxSqlLen - len,
              "if not exists %s.%s%d using %s.%s tags %s ",
              winfo->db_name, superTblInfo->childTblPrefix,
              i, winfo->db_name,
              superTblInfo->sTblName, tagsValBuf);
2504 2505 2506
      free(tagsValBuf);
      batchNum++;

2507 2508 2509
      if ((batchNum < superTblInfo->batchCreateTableNum)
              && ((superTblInfo->maxSqlLen - len) 
                  >= (superTblInfo->lenOfTagOfOneRow + 256))) {
2510 2511 2512 2513 2514
        continue;
      }
    }

    len = 0;
2515
    verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
2516
    if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE)){
2517
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2518 2519 2520 2521 2522 2523
      free(buffer);
      return NULL;
    }

    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
2524
      printf("thread[%d] already create %d - %d tables\n",
2525
              winfo->threadID, winfo->start_table_from, i);
2526 2527 2528 2529 2530
      lastPrintTime = currentPrintTime;
    }
  }
  
  if (0 != len) {
2531
    verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer);
2532
    if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE)) {
2533
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2534
    }
2535
  }
2536

2537 2538 2539 2540
  free(buffer);
  return NULL;
}

2541
static int startMultiThreadCreateChildTable(
2542
        char* cols, int threads, int startFrom, int ntables,
2543
        char* db_name, SSuperTable* superTblInfo) {
2544

2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  threadInfo *infos = malloc(threads * sizeof(threadInfo));

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

  int a = ntables / threads;
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

  int b = 0;
  b = ntables % threads;
2565
 
2566 2567 2568
  for (int i = 0; i < threads; i++) {
    threadInfo *t_info = infos + i;
    t_info->threadID = i;
H
Hui Li 已提交
2569
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
2570
    t_info->superTblInfo = superTblInfo;
2571
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
2572 2573 2574 2575 2576 2577
    t_info->taos = taos_connect(
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
2578
    if (t_info->taos == NULL) {
2579
      errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2580 2581
      free(pids);
      free(infos);  
2582 2583
      return -1;
    }
2584 2585 2586 2587 2588

    t_info->start_table_from = startFrom;
    t_info->ntables = i<b?a+1:a;
    t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
    startFrom = t_info->end_table_to + 1;
2589
    t_info->use_metric = true;
2590
    t_info->cols = cols;
2591
    t_info->minDelay = INT16_MAX;
2592 2593
    pthread_create(pids + i, NULL, createTable, t_info);
  }
2594
 
2595 2596 2597 2598 2599 2600 2601 2602 2603 2604
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
    threadInfo *t_info = infos + i;
    taos_close(t_info->taos);
  }

  free(pids);
2605
  free(infos);
2606 2607

  return 0;
2608 2609
}

2610 2611 2612 2613
static void createChildTables() {
    char tblColsBuf[MAX_SQL_SIZE];
    int len;

2614
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2615
    if (g_Dbs.db[i].superTblCount > 0) {
2616
        // with super table
2617 2618
      for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
        if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
2619
              || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
2620 2621 2622
          continue;
        }

2623
        verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
2624
                g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
2625 2626
        int startFrom = 0;
        g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;
2627 2628

        verbosePrint("%s() LN%d: create %d child tables from %d\n", __func__, __LINE__,
2629
              g_totalChildTables, startFrom);
2630
        startMultiThreadCreateChildTable(
2631
              g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
2632
              g_Dbs.threadCountByCreateTbl,
2633
              startFrom,
2634
              g_Dbs.db[i].superTbls[j].childTblCount,
2635
              g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
2636 2637
      }
    } else {
2638 2639
        // normal table
        len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
2640 2641 2642
        int j = 0;
        while (g_args.datatype[j]) {
            if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
2643 2644
                    || (strncasecmp(g_args.datatype[j],
                        "NCHAR", strlen("NCHAR")) == 0)) {
2645
                len = snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
2646
                        ", COL%d %s(60)", j, g_args.datatype[j]);
2647
            } else {
2648
                len = snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
2649
                        ", COL%d %s", j, g_args.datatype[j]);
2650 2651
            }
            len = strlen(tblColsBuf);
2652
            j++;
2653 2654
        }

2655
        len = snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");
2656

2657 2658
        verbosePrint("%s() LN%d: dbName: %s num of tb: %d schema: %s\n",
                __func__, __LINE__,
2659
                g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
2660
        startMultiThreadCreateChildTable(
2661
              tblColsBuf,
2662
              g_Dbs.threadCountByCreateTbl,
2663
              0,
2664
              g_args.num_of_tables,
2665 2666 2667
              g_Dbs.db[i].dbName,
              NULL);
    }
2668 2669 2670 2671 2672 2673
  }
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
2674
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
2675 2676 2677
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
2678

2679 2680
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
2681 2682
    printf("Failed to open tags file: %s, reason:%s\n",
            superTblInfo->tagsFile, strerror(errno));
2683 2684 2685 2686 2687 2688 2689
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
2690

2691 2692 2693 2694 2695 2696 2697 2698 2699
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

2700
  while ((readLen = tgetline(&line, &n, fp)) != -1) {
2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
2713 2714
      char *tmp = realloc(tagDataBuf,
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
2715 2716 2717
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
2718 2719
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

int readSampleFromJsonFileToMem(SSuperTable  * superTblInfo) {
  // TODO
  return 0;
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
2747 2748
static int readSampleFromCsvFileToMem(
        SSuperTable* superTblInfo) {
2749 2750 2751 2752
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
2753 2754 2755
 
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
2756
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
2757 2758 2759
              superTblInfo->sampleFile, strerror(errno));
      return -1;
  }
2760

2761
  assert(superTblInfo->sampleDataBuf);
2762 2763
  memset(superTblInfo->sampleDataBuf, 0,
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
2764
  while (1) {
2765
    readLen = tgetline(&line, &n, fp);
2766 2767
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
2768
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
2769
                superTblInfo->sampleFile, strerror(errno));
2770
        fclose(fp);
2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784
        return -1;
      }
      continue;
    }
  
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
2785 2786
      printf("sample row len[%d] overflow define schema len[%d], so discard this row\n",
              (int32_t)readLen, superTblInfo->lenOfOneRow);
2787 2788 2789
      continue;
    }

sangshuduo's avatar
sangshuduo 已提交
2790
    verbosePrint("readLen=%ld stb->lenOfOneRow=%d getRows=%d\n", (long)readLen,
2791
            superTblInfo->lenOfOneRow, getRows);
2792

2793 2794
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
          line, readLen);
2795 2796 2797 2798 2799 2800 2801
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

2802
  fclose(fp);
2803 2804 2805 2806
  tmfree(line);
  return 0;
}

2807 2808
static bool getColumnAndTagTypeFromInsertJsonFile(
        cJSON* stbInfo, SSuperTable* superTbls) {
2809
  bool  ret = false;
2810

2811 2812 2813
  // columns 
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
2814
    printf("ERROR: failed to read json, columns not found\n");
2815 2816 2817 2818 2819 2820
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
2821
 
2822 2823
  int columnSize = cJSON_GetArraySize(columns);
  if (columnSize > MAX_COLUMN_COUNT) {
2824 2825
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
            __func__, __LINE__, MAX_COLUMN_COUNT);
2826 2827 2828 2829 2830 2831
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
2832

2833
  //superTbls->columnCount = columnSize;
2834 2835 2836 2837 2838 2839 2840
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
2841
      count = countObj->valueint;
2842
    } else if (countObj && countObj->type != cJSON_Number) {
2843
      errorPrint("%s() LN%d, failed to read json, column count not found\n", __func__, __LINE__);
2844 2845 2846 2847 2848 2849 2850 2851 2852
      goto PARSE_OVER;
    } else {
      count = 1;
    }

    // column info 
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
    if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) {
2853
      errorPrint("%s() LN%d: failed to read json, column type not found\n", __func__, __LINE__);
2854 2855
      goto PARSE_OVER;
    }
2856 2857
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
2858

2859 2860
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
2861
      columnCase.dataLen = dataLen->valueint;
2862
    } else if (dataLen && dataLen->type != cJSON_Number) {
2863
      debugPrint("%s() LN%d: failed to read json, column len not found\n", __func__, __LINE__);
2864 2865 2866 2867
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
2868

2869
    for (int n = 0; n < count; ++n) {
2870 2871 2872
      tstrncpy(superTbls->columns[index].dataType,
              columnCase.dataType, MAX_TB_NAME_SIZE);
      superTbls->columns[index].dataLen = columnCase.dataLen;
2873 2874
      index++;
    }
2875 2876 2877
  }
  superTbls->columnCount = index;
 
2878 2879 2880 2881 2882
  count = 1;
  index = 0;
  // tags 
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
2883
    debugPrint("%s() LN%d, failed to read json, tags not found\n", __func__, __LINE__);
2884 2885 2886 2887 2888
    goto PARSE_OVER;
  }
  
  int tagSize = cJSON_GetArraySize(tags);
  if (tagSize > MAX_TAG_COUNT) {
2889
    debugPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", __func__, __LINE__, MAX_TAG_COUNT);
2890 2891
    goto PARSE_OVER;
  }
2892 2893

  //superTbls->tagCount = tagSize;
2894 2895 2896
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
2897

2898 2899 2900
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
2901
      count = countObj->valueint;
2902
    } else if (countObj && countObj->type != cJSON_Number) {
2903
      printf("ERROR: failed to read json, column count not found\n");
2904 2905 2906 2907 2908 2909 2910 2911 2912
      goto PARSE_OVER;
    } else {
      count = 1;
    }

    // column info 
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
    if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) {
2913
      printf("ERROR: failed to read json, tag type not found\n");
2914 2915
      goto PARSE_OVER;
    }
2916
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
2917

2918 2919
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
2920
      columnCase.dataLen = dataLen->valueint;
2921
    } else if (dataLen && dataLen->type != cJSON_Number) {
2922
      printf("ERROR: failed to read json, column len not found\n");
2923 2924 2925
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
2926 2927
    }

2928
    for (int n = 0; n < count; ++n) {
2929
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, MAX_TB_NAME_SIZE);
2930
      superTbls->tags[index].dataLen = columnCase.dataLen;
2931 2932
      index++;
    }
2933
  }
2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949
  superTbls->tagCount = index;

  ret = true;

PARSE_OVER:
  //free(content);
  //cJSON_Delete(root);
  //fclose(fp);
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
2950
    tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
2951 2952 2953 2954
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
2955
    tstrncpy(g_Dbs.host, host->valuestring, MAX_DB_NAME_SIZE);
2956
  } else if (!host) {
2957
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE);
2958
  } else {
2959
    printf("ERROR: failed to read json, host not found\n");
2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_Dbs.port = port->valueint;
  } else if (!port) {
    g_Dbs.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
2972
    tstrncpy(g_Dbs.user, user->valuestring, MAX_DB_NAME_SIZE);   
2973
  } else if (!user) {
2974
    tstrncpy(g_Dbs.user, "root", MAX_DB_NAME_SIZE);
2975 2976 2977 2978
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
2979
    tstrncpy(g_Dbs.password, password->valuestring, MAX_DB_NAME_SIZE);
2980
  } else if (!password) {
2981
    tstrncpy(g_Dbs.password, "taosdata", MAX_DB_NAME_SIZE);
2982 2983 2984 2985
  }

  cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
  if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
2986
    tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
2987
  } else if (!resultfile) {
2988
    tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
2989 2990 2991 2992 2993 2994 2995 2996
  }

  cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
  if (threads && threads->type == cJSON_Number) {
    g_Dbs.threadCount = threads->valueint;
  } else if (!threads) {
    g_Dbs.threadCount = 1;
  } else {
2997
    printf("ERROR: failed to read json, threads not found\n");
2998 2999
    goto PARSE_OVER;
  }  
3000
 
3001 3002 3003 3004 3005 3006
  cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
  if (threads2 && threads2->type == cJSON_Number) {
    g_Dbs.threadCountByCreateTbl = threads2->valueint;
  } else if (!threads2) {
    g_Dbs.threadCountByCreateTbl = 1;
  } else {
3007
    printf("ERROR: failed to read json, threads2 not found\n");
3008 3009 3010
    goto PARSE_OVER;
  } 

3011 3012 3013 3014 3015 3016 3017 3018 3019
  cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
  if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
    g_args.insert_interval = gInsertInterval->valueint;
  } else if (!gInsertInterval) {
    g_args.insert_interval = 0;
  } else {
    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", __func__, __LINE__);
    goto PARSE_OVER;
  }
3020

3021
  cJSON* rowsPerTbl = cJSON_GetObjectItem(root, "interlace_rows");
3022
  if (rowsPerTbl && rowsPerTbl->type == cJSON_Number) {
3023
    g_args.interlace_rows = rowsPerTbl->valueint;
3024
  } else if (!rowsPerTbl) {
3025
    g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3026
  } else {
3027
    errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", __func__, __LINE__);
3028 3029
    goto PARSE_OVER;
  }      
3030

3031 3032 3033 3034 3035 3036 3037 3038 3039
  cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
  if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
    g_args.max_sql_len = maxSqlLen->valueint;
  } else if (!maxSqlLen) {
    g_args.max_sql_len = TSDB_PAYLOAD_SIZE;
  } else {
    errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", __func__, __LINE__);
    goto PARSE_OVER;
  }
3040
 
3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051

  cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
  if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
    g_args.num_of_RPR = numRecPerReq->valueint;
  } else if (!numRecPerReq) {
    g_args.num_of_RPR = 100;
  } else {
    errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", __func__, __LINE__);
    goto PARSE_OVER;
  }

3052
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3053 3054 3055
  if (answerPrompt 
          && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3056 3057 3058 3059 3060 3061 3062 3063 3064 3065
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
    } else {
      g_args.answer_yes = false;
    }
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
3066
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
3067 3068 3069 3070 3071
    goto PARSE_OVER;
  }  

  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (!dbs || dbs->type != cJSON_Array) {
3072
    printf("ERROR: failed to read json, databases not found\n");
3073 3074 3075 3076 3077
    goto PARSE_OVER;
  }

  int dbSize = cJSON_GetArraySize(dbs);
  if (dbSize > MAX_DB_COUNT) {
3078
    errorPrint(
3079 3080
            "ERROR: failed to read json, databases size overflow, max database is %d\n",
            MAX_DB_COUNT);
3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091
    goto PARSE_OVER;
  }

  g_Dbs.dbCount = dbSize;
  for (int i = 0; i < dbSize; ++i) {
    cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
    if (dbinfos == NULL) continue;

    // dbinfo 
    cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
    if (!dbinfo || dbinfo->type != cJSON_Object) {
3092
      printf("ERROR: failed to read json, dbinfo not found\n");
3093 3094 3095 3096 3097
      goto PARSE_OVER;
    }
    
    cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
    if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
3098
      printf("ERROR: failed to read json, db name not found\n");
3099 3100
      goto PARSE_OVER;
    }
3101
    tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE);
3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112

    cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
    if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
      if (0 == strncasecmp(drop->valuestring, "yes", 3)) {
        g_Dbs.db[i].drop = 1;
      } else {
        g_Dbs.db[i].drop = 0;
      }        
    } else if (!drop) {
      g_Dbs.db[i].drop = 0;
    } else {
3113
      printf("ERROR: failed to read json, drop not found\n");
3114 3115 3116 3117
      goto PARSE_OVER;
    }

    cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
3118 3119 3120 3121
    if (precision && precision->type == cJSON_String
            && precision->valuestring != NULL) {
      tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
              MAX_DB_NAME_SIZE);
3122
    } else if (!precision) {
3123
      //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
3124 3125
      memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE);
    } else {
3126
      printf("ERROR: failed to read json, precision not found\n");
3127 3128 3129 3130 3131 3132 3133 3134 3135
      goto PARSE_OVER;
    }

    cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
    if (update && update->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.update = update->valueint;
    } else if (!update) {
      g_Dbs.db[i].dbCfg.update = -1;
    } else {
3136
      printf("ERROR: failed to read json, update not found\n");
3137 3138 3139 3140 3141 3142 3143 3144 3145
      goto PARSE_OVER;
    }

    cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
    if (replica && replica->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.replica = replica->valueint;
    } else if (!replica) {
      g_Dbs.db[i].dbCfg.replica = -1;
    } else {
3146
      printf("ERROR: failed to read json, replica not found\n");
3147 3148 3149 3150 3151 3152 3153 3154 3155
      goto PARSE_OVER;
    }

    cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
    if (keep && keep->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.keep = keep->valueint;
    } else if (!keep) {
      g_Dbs.db[i].dbCfg.keep = -1;
    } else {
3156
     printf("ERROR: failed to read json, keep not found\n");
3157 3158 3159 3160 3161 3162 3163 3164 3165
     goto PARSE_OVER;
    }
    
    cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
    if (days && days->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.days = days->valueint;
    } else if (!days) {
      g_Dbs.db[i].dbCfg.days = -1;
    } else {
3166
     printf("ERROR: failed to read json, days not found\n");
3167 3168 3169 3170 3171 3172 3173 3174 3175
     goto PARSE_OVER;
    }
    
    cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
    if (cache && cache->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cache = cache->valueint;
    } else if (!cache) {
      g_Dbs.db[i].dbCfg.cache = -1;
    } else {
3176
     printf("ERROR: failed to read json, cache not found\n");
3177 3178 3179 3180 3181 3182 3183 3184 3185
     goto PARSE_OVER;
    }
        
    cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
    if (blocks && blocks->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
    } else if (!blocks) {
      g_Dbs.db[i].dbCfg.blocks = -1;
    } else {
3186
     printf("ERROR: failed to read json, block not found\n");
3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205
     goto PARSE_OVER;
    }

    //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
    //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
    //} else if (!maxtablesPerVnode) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
    //} else {
    // printf("failed to read json, maxtablesPerVnode not found");
    // goto PARSE_OVER;
    //}

    cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
    if (minRows && minRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
    } else if (!minRows) {
      g_Dbs.db[i].dbCfg.minRows = -1;
    } else {
3206
     printf("ERROR: failed to read json, minRows not found\n");
3207 3208 3209 3210 3211 3212 3213 3214 3215
     goto PARSE_OVER;
    }

    cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
    if (maxRows && maxRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
    } else if (!maxRows) {
      g_Dbs.db[i].dbCfg.maxRows = -1;
    } else {
3216
     printf("ERROR: failed to read json, maxRows not found\n");
3217 3218 3219 3220 3221 3222 3223 3224 3225
     goto PARSE_OVER;
    }

    cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
    if (comp && comp->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.comp = comp->valueint;
    } else if (!comp) {
      g_Dbs.db[i].dbCfg.comp = -1;
    } else {
3226
     printf("ERROR: failed to read json, comp not found\n");
3227 3228 3229 3230 3231 3232 3233 3234 3235
     goto PARSE_OVER;
    }

    cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
    if (walLevel && walLevel->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
    } else if (!walLevel) {
      g_Dbs.db[i].dbCfg.walLevel = -1;
    } else {
3236
     printf("ERROR: failed to read json, walLevel not found\n");
3237 3238 3239
     goto PARSE_OVER;
    }

3240 3241 3242 3243 3244 3245
    cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
    if (cacheLast && cacheLast->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
    } else if (!cacheLast) {
      g_Dbs.db[i].dbCfg.cacheLast = -1;
    } else {
3246
     printf("ERROR: failed to read json, cacheLast not found\n");
3247 3248 3249
     goto PARSE_OVER;
    }

3250 3251 3252 3253
    cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
    if (quorum && quorum->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
    } else if (!quorum) {
3254
      g_Dbs.db[i].dbCfg.quorum = 1;
3255
    } else {
3256
     printf("failed to read json, quorum input mistake");
3257 3258 3259 3260 3261 3262 3263 3264 3265
     goto PARSE_OVER;
    }

    cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
    if (fsync && fsync->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
    } else if (!fsync) {
      g_Dbs.db[i].dbCfg.fsync = -1;
    } else {
3266
     printf("ERROR: failed to read json, fsync not found\n");
3267 3268 3269 3270 3271 3272
     goto PARSE_OVER;
    }    

    // super_talbes 
    cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
    if (!stables || stables->type != cJSON_Array) {
3273
      printf("ERROR: failed to read json, super_tables not found\n");
3274 3275 3276 3277 3278
      goto PARSE_OVER;
    }    
    
    int stbSize = cJSON_GetArraySize(stables);
    if (stbSize > MAX_SUPER_TABLE_COUNT) {
3279
      errorPrint(
3280 3281
              "ERROR: failed to read json, databases size overflow, max database is %d\n",
              MAX_SUPER_TABLE_COUNT);
3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292
      goto PARSE_OVER;
    }

    g_Dbs.db[i].superTblCount = stbSize;
    for (int j = 0; j < stbSize; ++j) {
      cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
      if (stbInfo == NULL) continue;
    
      // dbinfo 
      cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
      if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) {
3293
        printf("ERROR: failed to read json, stb name not found\n");
3294 3295
        goto PARSE_OVER;
      }
3296
      tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE);
3297 3298 3299
    
      cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
      if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
3300
        printf("ERROR: failed to read json, childtable_prefix not found\n");
3301 3302
        goto PARSE_OVER;
      }
3303
      tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE);
3304 3305

      cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null
3306 3307 3308
      if (autoCreateTbl
              && autoCreateTbl->type == cJSON_String
              && autoCreateTbl->valuestring != NULL) {
3309 3310 3311 3312 3313 3314 3315 3316 3317 3318
        if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
        } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        } else {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        }
      } else if (!autoCreateTbl) {
        g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      } else {
3319
        printf("ERROR: failed to read json, auto_create_table not found\n");
3320 3321 3322 3323 3324 3325 3326
        goto PARSE_OVER;
      }
      
      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
      } else if (!batchCreateTbl) {
3327
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
3328
      } else {
3329
        printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
3330 3331 3332 3333
        goto PARSE_OVER;
      }      

      cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
3334 3335 3336
      if (childTblExists
              && childTblExists->type == cJSON_String
              && childTblExists->valuestring != NULL) {
3337 3338 3339 3340 3341 3342 3343 3344 3345 3346
        if (0 == strncasecmp(childTblExists->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
        } else if (0 == strncasecmp(childTblExists->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        } else {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        }
      } else if (!childTblExists) {
        g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
      } else {
3347
        errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n", __func__, __LINE__);
3348 3349 3350 3351 3352
        goto PARSE_OVER;
      }
      
      cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
      if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
3353
        errorPrint("%s() LN%d, failed to read json, childtable_count not found\n", __func__, __LINE__);
3354 3355 3356 3357 3358
        goto PARSE_OVER;
      }
      g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

      cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
3359 3360 3361 3362
      if (dataSource && dataSource->type == cJSON_String
              && dataSource->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
                dataSource->valuestring, MAX_DB_NAME_SIZE);
3363
      } else if (!dataSource) {
3364
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE);
3365
      } else {
3366
        errorPrint("%s() LN%d, failed to read json, data_source not found\n", __func__, __LINE__);
3367 3368 3369 3370
        goto PARSE_OVER;
      }

      cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , restful
3371 3372 3373 3374
      if (insertMode && insertMode->type == cJSON_String
              && insertMode->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode,
                insertMode->valuestring, MAX_DB_NAME_SIZE);
3375
      } else if (!insertMode) {
3376
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE);
3377
      } else {
3378
        printf("ERROR: failed to read json, insert_mode not found\n");
3379 3380 3381
        goto PARSE_OVER;
      }

3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403
      cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
      if (childTbl_limit) {
        if (childTbl_limit->type != cJSON_Number) {
            printf("ERROR: failed to read json, childtable_limit\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result
      }

      cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
      if (childTbl_offset) {
        if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) {
            printf("ERROR: failed to read json, childtable_offset\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblOffset = 0;
      }

3404 3405
      cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
      if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
3406 3407
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                ts->valuestring, MAX_DB_NAME_SIZE);
3408
      } else if (!ts) {
3409 3410
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                "now", MAX_DB_NAME_SIZE);
3411
      } else {
3412
        printf("ERROR: failed to read json, start_timestamp not found\n");
3413 3414
        goto PARSE_OVER;
      }
3415

3416 3417 3418 3419
      cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
      if (timestampStep && timestampStep->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
      } else if (!timestampStep) {
3420
        g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
3421
      } else {
3422
        printf("ERROR: failed to read json, timestamp_step not found\n");
3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434
        goto PARSE_OVER;
      }

      cJSON* sampleDataBufSize = cJSON_GetObjectItem(stbInfo, "sample_buf_size");
      if (sampleDataBufSize && sampleDataBufSize->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].sampleDataBufSize = sampleDataBufSize->valueint;
        if (g_Dbs.db[i].superTbls[j].sampleDataBufSize < 1024*1024) {
          g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024;
        }
      } else if (!sampleDataBufSize) {
        g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024;
      } else {
3435
        printf("ERROR: failed to read json, sample_buf_size not found\n");
3436 3437
        goto PARSE_OVER;
      }      
3438

3439 3440
      cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
      if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) {
3441 3442
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
                sampleFormat->valuestring, MAX_DB_NAME_SIZE);
3443
      } else if (!sampleFormat) {
3444
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE);
3445
      } else {
3446
        printf("ERROR: failed to read json, sample_format not found\n");
3447 3448
        goto PARSE_OVER;
      }      
3449

3450 3451
      cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
      if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) {
3452 3453
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
                sampleFile->valuestring, MAX_FILE_NAME_LEN);
3454 3455 3456
      } else if (!sampleFile) {
        memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
      } else {
3457
        printf("ERROR: failed to read json, sample_file not found\n");
3458 3459
        goto PARSE_OVER;
      }          
3460

3461 3462
      cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
      if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) {
3463 3464
        tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
                tagsFile->valuestring, MAX_FILE_NAME_LEN);
3465 3466 3467 3468 3469 3470 3471 3472 3473
        if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
          g_Dbs.db[i].superTbls[j].tagSource = 0;
        } else {
          g_Dbs.db[i].superTbls[j].tagSource = 1;
        }
      } else if (!tagsFile) {
        memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
        g_Dbs.db[i].superTbls[j].tagSource = 0;
      } else {
3474
        printf("ERROR: failed to read json, tags_file not found\n");
3475 3476
        goto PARSE_OVER;
      }
3477

3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489
      cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
      if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
        int32_t len = maxSqlLen->valueint;
        if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
          len = TSDB_MAX_ALLOWED_SQL_LEN;
        } else if (len < TSDB_MAX_SQL_LEN) {
          len = TSDB_MAX_SQL_LEN;
        }       
        g_Dbs.db[i].superTbls[j].maxSqlLen = len;
      } else if (!maxSqlLen) {
        g_Dbs.db[i].superTbls[j].maxSqlLen = TSDB_MAX_SQL_LEN;
      } else {
3490
        printf("ERROR: failed to read json, maxSqlLen not found\n");
3491 3492 3493
        goto PARSE_OVER;
      }      

3494 3495 3496 3497 3498
      cJSON *multiThreadWriteOneTbl =
          cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
      if (multiThreadWriteOneTbl
              && multiThreadWriteOneTbl->type == cJSON_String
              && multiThreadWriteOneTbl->valuestring != NULL) {
3499 3500 3501 3502 3503 3504 3505 3506
        if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
        } else {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
        }        
      } else if (!multiThreadWriteOneTbl) {
        g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
      } else {
3507
        printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
3508 3509
        goto PARSE_OVER;
      }
3510

3511
      cJSON* rowsPerTbl = cJSON_GetObjectItem(stbInfo, "interlace_rows");
3512 3513 3514
      if (rowsPerTbl && rowsPerTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].rowsPerTbl = rowsPerTbl->valueint;
      } else if (!rowsPerTbl) {
3515
        g_Dbs.db[i].superTbls[j].rowsPerTbl = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3516
      } else {
3517
        errorPrint("%s() LN%d, failed to read json, rowsPerTbl input mistake\n", __func__, __LINE__);
3518 3519 3520 3521 3522 3523 3524 3525 3526
        goto PARSE_OVER;
      }      

      cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
      if (disorderRatio && disorderRatio->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
      } else if (!disorderRatio) {
        g_Dbs.db[i].superTbls[j].disorderRatio = 0;
      } else {
3527
        printf("ERROR: failed to read json, disorderRatio not found\n");
3528 3529 3530 3531 3532 3533 3534 3535 3536
        goto PARSE_OVER;
      }      

      cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
      if (disorderRange && disorderRange->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
      } else if (!disorderRange) {
        g_Dbs.db[i].superTbls[j].disorderRange = 1000;
      } else {
3537
        printf("ERROR: failed to read json, disorderRange not found\n");
3538 3539
        goto PARSE_OVER;
      }
3540

3541 3542 3543 3544 3545 3546
      cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
      if (insertRows && insertRows->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
      } else if (!insertRows) {
        g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
      } else {
3547
        errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", __func__, __LINE__);
3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558
        goto PARSE_OVER;
      }

      cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
      if (insertInterval && insertInterval->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
      } else if (!insertInterval) {
        debugPrint("%s() LN%d: stable insert interval be overrided by global %d.\n",
                __func__, __LINE__, g_args.insert_interval);
        g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
      } else {
3559
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", __func__, __LINE__);
3560 3561 3562
        goto PARSE_OVER;
      }

3563
/* CBD      if (NO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable
3564
              || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
3565 3566
        continue;
      }
3567
      */
3568

3569 3570
      int retVal = getColumnAndTagTypeFromInsertJsonFile(
              stbInfo, &g_Dbs.db[i].superTbls[j]);
3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590
      if (false == retVal) {
        goto PARSE_OVER;
      }      
    }    
  }

  ret = true;

PARSE_OVER:
  //free(content);
  //cJSON_Delete(root);
  //fclose(fp);
  return ret;
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3591
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3592 3593 3594 3595
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3596
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_DB_NAME_SIZE);
3597
  } else if (!host) {
3598
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE);
3599
  } else {
3600
    printf("ERROR: failed to read json, host not found\n");
3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3613
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_DB_NAME_SIZE);   
3614
  } else if (!user) {
3615
    tstrncpy(g_queryInfo.user, "root", MAX_DB_NAME_SIZE); ;
3616 3617 3618 3619
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3620
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_DB_NAME_SIZE);
3621
  } else if (!password) {
3622
    tstrncpy(g_queryInfo.password, "taosdata", MAX_DB_NAME_SIZE);;
3623
  }
H
hzcheng 已提交
3624

3625
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3626 3627
  if (answerPrompt && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3628 3629 3630 3631
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
3632
    } else {
3633
      g_args.answer_yes = false;
H
hzcheng 已提交
3634
    }
3635 3636 3637
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
3638
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
3639 3640 3641 3642 3643
    goto PARSE_OVER;
  }  

  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
3644
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE);
3645
  } else if (!dbs) {
3646
    printf("ERROR: failed to read json, databases not found\n");
3647
    goto PARSE_OVER;
H
hzcheng 已提交
3648
  }
3649 3650 3651

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
3652
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
3653
  } else if (!queryMode) {
3654
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
3655
  } else {
3656
    printf("ERROR: failed to read json, query_mode not found\n");
3657
    goto PARSE_OVER;
H
hzcheng 已提交
3658
  }
3659 3660 3661 3662 3663 3664 3665
  
  // super_table_query 
  cJSON *superQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!superQuery) {
    g_queryInfo.superQueryInfo.concurrent = 0;
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
3666
    printf("ERROR: failed to read json, super_table_query not found\n");
3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689
    goto PARSE_OVER;
  } else {  
    cJSON* rate = cJSON_GetObjectItem(superQuery, "query_interval");
    if (rate && rate->type == cJSON_Number) {
      g_queryInfo.superQueryInfo.rate = rate->valueint;
    } else if (!rate) {
      g_queryInfo.superQueryInfo.rate = 0;
    }
  
    cJSON* concurrent = cJSON_GetObjectItem(superQuery, "concurrent");
    if (concurrent && concurrent->type == cJSON_Number) {
      g_queryInfo.superQueryInfo.concurrent = concurrent->valueint;
    } else if (!concurrent) {
      g_queryInfo.superQueryInfo.concurrent = 1;
    }
  
    cJSON* mode = cJSON_GetObjectItem(superQuery, "mode");
    if (mode && mode->type == cJSON_String && mode->valuestring != NULL) {
      if (0 == strcmp("sync", mode->valuestring)) {      
        g_queryInfo.superQueryInfo.subscribeMode = 0;
      } else if (0 == strcmp("async", mode->valuestring)) {      
        g_queryInfo.superQueryInfo.subscribeMode = 1;
      } else {
3690
        printf("ERROR: failed to read json, subscribe mod error\n");
3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712
        goto PARSE_OVER;
      }
    } else {
      g_queryInfo.superQueryInfo.subscribeMode = 0;
    }
    
    cJSON* interval = cJSON_GetObjectItem(superQuery, "interval");
    if (interval && interval->type == cJSON_Number) {
      g_queryInfo.superQueryInfo.subscribeInterval = interval->valueint;
    } else if (!interval) {    
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
    }
  
    cJSON* restart = cJSON_GetObjectItem(superQuery, "restart");
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
      if (0 == strcmp("yes", restart->valuestring)) {      
        g_queryInfo.superQueryInfo.subscribeRestart = 1;
      } else if (0 == strcmp("no", restart->valuestring)) {      
        g_queryInfo.superQueryInfo.subscribeRestart = 0;
      } else {
3713
        printf("ERROR: failed to read json, subscribe restart error\n");
3714 3715 3716 3717 3718 3719 3720
        goto PARSE_OVER;
      }
    } else {
      g_queryInfo.superQueryInfo.subscribeRestart = 1;
    }
  
    cJSON* keepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
3721 3722 3723
    if (keepProgress
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
3724 3725 3726 3727 3728
      if (0 == strcmp("yes", keepProgress->valuestring)) {      
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
      } else if (0 == strcmp("no", keepProgress->valuestring)) {      
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
      } else {
3729
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740
        goto PARSE_OVER;
      }
    } else {
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
    }

    // sqls   
    cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls");
    if (!superSqls) {
      g_queryInfo.superQueryInfo.sqlCount = 0;
    } else if (superSqls->type != cJSON_Array) {
3741
      printf("ERROR: failed to read json, super sqls not found\n");
3742 3743 3744 3745
      goto PARSE_OVER;
    } else {  
      int superSqlSize = cJSON_GetArraySize(superSqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
3746
        printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT);
3747 3748
        goto PARSE_OVER;
      }
3749

3750 3751 3752 3753 3754 3755 3756
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
      for (int j = 0; j < superSqlSize; ++j) {
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
        if (sql == NULL) continue;
      
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
3757
          printf("ERROR: failed to read json, sql not found\n");
3758 3759
          goto PARSE_OVER;
        }
3760
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
H
hzcheng 已提交
3761

3762 3763
        cJSON *result = cJSON_GetObjectItem(sql, "result");
        if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) {
3764
          tstrncpy(g_queryInfo.superQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN);
3765 3766 3767
        } else if (NULL == result) {
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
        } else {
3768
          printf("ERROR: failed to read json, super query result file not found\n");
3769 3770 3771 3772 3773
          goto PARSE_OVER;
        } 
      }
    }
  }
3774

3775 3776 3777 3778 3779 3780
  // sub_table_query 
  cJSON *subQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!subQuery) {
    g_queryInfo.subQueryInfo.threadCnt = 0;
    g_queryInfo.subQueryInfo.sqlCount = 0;
  } else if (subQuery->type != cJSON_Object) {
3781
    printf("ERROR: failed to read json, sub_table_query not found\n");
3782 3783
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
3784
  } else {
3785 3786 3787 3788 3789 3790
    cJSON* subrate = cJSON_GetObjectItem(subQuery, "query_interval");
    if (subrate && subrate->type == cJSON_Number) {
      g_queryInfo.subQueryInfo.rate = subrate->valueint;
    } else if (!subrate) {
      g_queryInfo.subQueryInfo.rate = 0;
    }
3791

3792 3793 3794 3795 3796 3797
    cJSON* threads = cJSON_GetObjectItem(subQuery, "threads");
    if (threads && threads->type == cJSON_Number) {
      g_queryInfo.subQueryInfo.threadCnt = threads->valueint;
    } else if (!threads) {
      g_queryInfo.subQueryInfo.threadCnt = 1;
    }
3798

3799 3800 3801 3802 3803 3804
    //cJSON* subTblCnt = cJSON_GetObjectItem(subQuery, "childtable_count");
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
    //  g_queryInfo.subQueryInfo.childTblCount = subTblCnt->valueint;
    //} else if (!subTblCnt) {
    //  g_queryInfo.subQueryInfo.childTblCount = 0;
    //}
3805

3806 3807
    cJSON* stblname = cJSON_GetObjectItem(subQuery, "stblname");
    if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) {
3808
      tstrncpy(g_queryInfo.subQueryInfo.sTblName, stblname->valuestring, MAX_TB_NAME_SIZE);
3809
    } else {
3810
      printf("ERROR: failed to read json, super table name not found\n");
3811 3812
      goto PARSE_OVER;
    }
3813

3814 3815 3816 3817 3818 3819 3820
    cJSON* submode = cJSON_GetObjectItem(subQuery, "mode");
    if (submode && submode->type == cJSON_String && submode->valuestring != NULL) {
      if (0 == strcmp("sync", submode->valuestring)) {      
        g_queryInfo.subQueryInfo.subscribeMode = 0;
      } else if (0 == strcmp("async", submode->valuestring)) {      
        g_queryInfo.subQueryInfo.subscribeMode = 1;
      } else {
3821
        printf("ERROR: failed to read json, subscribe mod error\n");
3822 3823 3824 3825 3826
        goto PARSE_OVER;
      }
    } else {
      g_queryInfo.subQueryInfo.subscribeMode = 0;
    }
3827

3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843
    cJSON* subinterval = cJSON_GetObjectItem(subQuery, "interval");
    if (subinterval && subinterval->type == cJSON_Number) {
      g_queryInfo.subQueryInfo.subscribeInterval = subinterval->valueint;
    } else if (!subinterval) {    
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
      g_queryInfo.subQueryInfo.subscribeInterval = 10000;
    }
  
    cJSON* subrestart = cJSON_GetObjectItem(subQuery, "restart");
    if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) {
      if (0 == strcmp("yes", subrestart->valuestring)) {      
        g_queryInfo.subQueryInfo.subscribeRestart = 1;
      } else if (0 == strcmp("no", subrestart->valuestring)) {      
        g_queryInfo.subQueryInfo.subscribeRestart = 0;
      } else {
3844
        printf("ERROR: failed to read json, subscribe restart error\n");
3845 3846 3847 3848 3849 3850 3851
        goto PARSE_OVER;
      }
    } else {
      g_queryInfo.subQueryInfo.subscribeRestart = 1;
    }
  
    cJSON* subkeepProgress = cJSON_GetObjectItem(subQuery, "keepProgress");
3852 3853 3854
    if (subkeepProgress &&
            subkeepProgress->type == cJSON_String
            && subkeepProgress->valuestring != NULL) {
3855 3856 3857 3858 3859
      if (0 == strcmp("yes", subkeepProgress->valuestring)) {      
        g_queryInfo.subQueryInfo.subscribeKeepProgress = 1;
      } else if (0 == strcmp("no", subkeepProgress->valuestring)) {      
        g_queryInfo.subQueryInfo.subscribeKeepProgress = 0;
      } else {
3860
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871
        goto PARSE_OVER;
      }
    } else {
      g_queryInfo.subQueryInfo.subscribeKeepProgress = 0;
    }  

    // sqls     
    cJSON* subsqls = cJSON_GetObjectItem(subQuery, "sqls");
    if (!subsqls) {
      g_queryInfo.subQueryInfo.sqlCount = 0;
    } else if (subsqls->type != cJSON_Array) {
3872
      printf("ERROR: failed to read json, super sqls not found\n");
3873 3874 3875 3876
      goto PARSE_OVER;
    } else {  
      int superSqlSize = cJSON_GetArraySize(subsqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
3877
        printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT);
3878 3879 3880 3881 3882 3883 3884 3885 3886 3887
        goto PARSE_OVER;
      }
    
      g_queryInfo.subQueryInfo.sqlCount = superSqlSize;
      for (int j = 0; j < superSqlSize; ++j) {      
        cJSON* sql = cJSON_GetArrayItem(subsqls, j);
        if (sql == NULL) continue;
        
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
3888
          printf("ERROR: failed to read json, sql not found\n");
3889 3890
          goto PARSE_OVER;
        }
3891
        tstrncpy(g_queryInfo.subQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
3892 3893 3894

        cJSON *result = cJSON_GetObjectItem(sql, "result");
        if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){
3895
          tstrncpy(g_queryInfo.subQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN);
3896 3897 3898
        } else if (NULL == result) {
          memset(g_queryInfo.subQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
        }  else {
3899
          printf("ERROR: failed to read json, sub query result file not found\n");
3900 3901 3902 3903
          goto PARSE_OVER;
        } 
      }
    }
H
hzcheng 已提交
3904 3905
  }

3906
  ret = true;
H
hzcheng 已提交
3907

3908 3909 3910 3911 3912 3913
PARSE_OVER:
  //free(content);
  //cJSON_Delete(root);
  //fclose(fp);
  return ret;
}
H
hzcheng 已提交
3914

3915
static bool getInfoFromJsonFile(char* file) {
3916
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
3917

3918 3919 3920 3921 3922
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
3923

3924 3925 3926 3927 3928 3929 3930 3931 3932
  bool  ret = false;
  int   maxLen = 64000;
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
3933
  }
H
Hui Li 已提交
3934

3935 3936 3937
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
3938
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
3939 3940
    goto PARSE_OVER;
  }
H
Hui Li 已提交
3941

3942 3943 3944
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
3945
      g_args.test_mode = INSERT_TEST;
3946
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
3947
      g_args.test_mode = QUERY_TEST;
3948
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
3949
      g_args.test_mode = SUBSCRIBE_TEST;
3950
    } else {
3951
      printf("ERROR: failed to read json, filetype not support\n");
3952 3953 3954
      goto PARSE_OVER;
    }
  } else if (!filetype) {
3955
    g_args.test_mode = INSERT_TEST;
3956
  } else {
3957
    printf("ERROR: failed to read json, filetype not found\n");
3958 3959
    goto PARSE_OVER;
  }
H
hzcheng 已提交
3960

3961
  if (INSERT_TEST == g_args.test_mode) {
3962
    ret = getMetaFromInsertJsonFile(root);
3963
  } else if (QUERY_TEST == g_args.test_mode) {
3964
    ret = getMetaFromQueryJsonFile(root);
3965
  } else if (SUBSCRIBE_TEST == g_args.test_mode) {
3966 3967
    ret = getMetaFromQueryJsonFile(root);
  } else {
3968
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n", __func__, __LINE__);
3969
    goto PARSE_OVER;
3970
  } 
3971 3972 3973 3974

PARSE_OVER:
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
3975
  fclose(fp);
3976 3977
  return ret;
}
H
hzcheng 已提交
3978

3979
static void prepareSampleData() {
3980 3981 3982 3983
  for (int i = 0; i < g_Dbs.dbCount; i++) {    
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
        (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]);
S
Shuaiqiang Chang 已提交
3984
      }
3985 3986 3987 3988
    }
  }
}

3989
static void postFreeResource() {
3990 3991 3992
  tmfclose(g_fpOfInsertResult);
  for (int i = 0; i < g_Dbs.dbCount; i++) {    
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
3993 3994 3995
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
3996
      }
3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4009
    }
4010 4011
  }
}
S
Shuaiqiang Chang 已提交
4012

4013 4014
static int getRowDataFromSample(char*  dataBuf, int maxLen, int64_t timestamp,
      SSuperTable* superTblInfo, int* sampleUsePos) {
4015
  if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
4016
/*    int ret = readSampleFromCsvFileToMem(superTblInfo);
4017
    if (0 != ret) {
4018
      tmfree(superTblInfo->sampleDataBuf);
4019
      superTblInfo->sampleDataBuf = NULL;
4020 4021
      return -1;
    }
4022
*/
4023 4024
    *sampleUsePos = 0;
  }
S
Shuaiqiang Chang 已提交
4025

4026
  int    dataLen = 0;
4027

4028 4029 4030 4031
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "(%" PRId64 ", ", timestamp);
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
4032
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
S
Shuaiqiang Chang 已提交
4033

4034
  (*sampleUsePos)++;
4035
 
4036 4037
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4038

4039
static int generateRowData(char*  dataBuf, int maxLen, int64_t timestamp, SSuperTable* stbInfo) {
4040 4041 4042
  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp);
  for (int i = 0; i < stbInfo->columnCount; i++) {    
4043 4044
    if ((0 == strncasecmp(stbInfo->columns[i].dataType, "binary", 6))
            || (0 == strncasecmp(stbInfo->columns[i].dataType, "nchar", 5))) {
4045
      if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
4046
        errorPrint( "binary or nchar length overflow, max size:%u\n",
4047
                (uint32_t)TSDB_MAX_BINARY_LEN);
4048 4049
        return (-1);
      }
4050

4051 4052
      char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
      if (NULL == buf) {
4053
        errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
4054 4055 4056 4057 4058
        return (-1);
      }
      rand_string(buf, stbInfo->columns[i].dataLen);
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "\'%s\', ", buf);
      tmfree(buf);
4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "int", 3)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
              "%d, ", rand_int());
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "bigint", 6)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
              "%"PRId64", ", rand_bigint());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "float", 5)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
              "%f, ", rand_float());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "double", 6)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
              "%f, ", rand_double());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "smallint", 8)) {
4077 4078 4079 4080 4081 4082 4083 4084
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_smallint());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType, "tinyint", 7)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_tinyint());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bool", 4)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_bool());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType, "timestamp", 9)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint());
    }  else {
4085
      errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType);
4086 4087
      return (-1);
    }
S
Shuaiqiang Chang 已提交
4088
  }
4089

4090 4091
  dataLen -= 2;
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
4092

4093 4094
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4095

4096
static int32_t generateData(char *res, char **data_type,
4097
        int num_of_cols, int64_t timestamp, int lenOfBinary) {
4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128
  memset(res, 0, MAX_DATA_SIZE);
  char *pstr = res;
  pstr += sprintf(pstr, "(%" PRId64, timestamp);
  int c = 0;

  for (; c < MAX_NUM_DATATYPE; c++) {
    if (data_type[c] == NULL) {
      break;
    }
  }

  if (0 == c) {
    perror("data type error!");
    exit(-1);
  }

  for (int i = 0; i < num_of_cols; i++) {
    if (strcasecmp(data_type[i % c], "tinyint") == 0) {
      pstr += sprintf(pstr, ", %d", rand_tinyint() );
    } else if (strcasecmp(data_type[i % c], "smallint") == 0) {
      pstr += sprintf(pstr, ", %d", rand_smallint());
    } else if (strcasecmp(data_type[i % c], "int") == 0) {
      pstr += sprintf(pstr, ", %d", rand_int()); 
    } else if (strcasecmp(data_type[i % c], "bigint") == 0) {
      pstr += sprintf(pstr, ", %" PRId64, rand_bigint());
    } else if (strcasecmp(data_type[i % c], "float") == 0) {
      pstr += sprintf(pstr, ", %10.4f", rand_float());
    } else if (strcasecmp(data_type[i % c], "double") == 0) {
      double t = rand_double();
      pstr += sprintf(pstr, ", %20.8f", t);
    } else if (strcasecmp(data_type[i % c], "bool") == 0) {
4129
      bool b = taosRandom() & 1;
4130 4131
      pstr += sprintf(pstr, ", %s", b ? "true" : "false");
    } else if (strcasecmp(data_type[i % c], "binary") == 0) {
4132 4133
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4134 4135 4136
      pstr += sprintf(pstr, ", \"%s\"", s);
      free(s);
    }else if (strcasecmp(data_type[i % c], "nchar") == 0) {
4137 4138
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153
      pstr += sprintf(pstr, ", \"%s\"", s);
      free(s);
    }

    if (pstr - res > MAX_DATA_SIZE) {
      perror("column length too long, abort");
      exit(-1);
    }
  }

  pstr += sprintf(pstr, ")");

  return (int32_t)(pstr - res);
}

4154
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
4155 4156
  char* sampleDataBuf = NULL;

4157
  sampleDataBuf = calloc(
4158
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
4159
  if (sampleDataBuf == NULL) {
4160 4161
      errorPrint("%s() LN%d, Failed to calloc %d Bytes, reason:%s\n", 
              __func__, __LINE__,
4162 4163
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 
              strerror(errno));
4164
      return -1;
4165
  }
4166

4167 4168 4169 4170
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
4171
      errorPrint("%s() LN%d, read sample from csv file failed.\n", __func__, __LINE__);
4172
      tmfree(sampleDataBuf);
4173
      superTblInfo->sampleDataBuf = NULL;
4174
      return -1;
H
Hui Li 已提交
4175
  }
4176

4177 4178 4179
  return 0;
}

4180
static int execInsert(threadInfo *pThreadInfo, char *buffer, int k)
4181 4182
{
  int affectedRows;
4183
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4184

4185 4186
  verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
            __func__, __LINE__, buffer);
4187
  if (superTblInfo) {
4188
    if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) {
4189
      affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE);
4190
    } else {
4191
      if (0 != postProceSql(g_Dbs.host, g_Dbs.port, buffer)) {
4192
        affectedRows = -1;
4193
        printf("========restful return fail, threadID[%d]\n", pThreadInfo->threadID);
4194 4195 4196
      } else {
        affectedRows = k;
      }
4197
    }
4198
  } else {
4199
    affectedRows = queryDbExec(pThreadInfo->taos, buffer, 1);
4200 4201 4202 4203 4204
  }

  return affectedRows;
}

4205
static void getTableName(char *pTblName, threadInfo* pThreadInfo, int tableSeq)
4206 4207
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4208 4209 4210 4211 4212 4213
  if (superTblInfo) {
    if ((superTblInfo->childTblOffset >= 0)
            && (superTblInfo->childTblLimit > 0)) {
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
    } else {
4214

4215 4216 4217
        verbosePrint("[%d] %s() LN%d: from=%d count=%d seq=%d\n",
                pThreadInfo->threadID, __func__, __LINE__,
                pThreadInfo->start_table_from,
4218 4219 4220 4221 4222 4223
                pThreadInfo->ntables, tableSeq);
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
    }
  } else {
    snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%d",
4224
        g_args.tb_prefix, tableSeq);
4225 4226 4227 4228
  }
}

static int generateDataTail(char *tableName, int32_t tableSeq,
4229
        threadInfo* pThreadInfo, SSuperTable* superTblInfo,
4230 4231 4232
        int batch, char* buffer, int64_t insertRows,
        int64_t startFrom, uint64_t startTime, int *pSamplePos, int *dataLen) {
  int len = 0;
4233 4234
  int ncols_per_record = 1; // count first col ts

4235
  if (superTblInfo == NULL) {
4236 4237 4238 4239 4240
    int datatypeSeq = 0;
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
4241 4242
  }

4243
  verbosePrint("%s() LN%d batch=%d\n", __func__, __LINE__, batch);
4244

4245 4246
  int k = 0;
  for (k = 0; k < batch;) {
4247
    if (superTblInfo) {
4248
        int retLen = 0;
4249

4250 4251 4252
        if (0 == strncasecmp(superTblInfo->dataSource,
                    "sample", strlen("sample"))) {
          retLen = getRowDataFromSample(
4253
                    buffer + len, 
4254
                    superTblInfo->maxSqlLen - len, 
4255
                    startTime + superTblInfo->timeStampStep * k,
4256
                    superTblInfo, 
4257
                    pSamplePos);
4258 4259 4260 4261
       } else if (0 == strncasecmp(superTblInfo->dataSource,
                   "rand", strlen("rand"))) {
          int rand_num = rand_tinyint() % 100;
          if (0 != superTblInfo->disorderRatio 
4262
                    && rand_num < superTblInfo->disorderRatio) {
4263 4264 4265
            int64_t d = startTime
                + superTblInfo->timeStampStep * k
                - taosRandom() % superTblInfo->disorderRange;
4266
            retLen = generateRowData(
4267
                      buffer + len, 
4268 4269
                      superTblInfo->maxSqlLen - len,
                      d, 
4270
                      superTblInfo);
4271 4272
          } else {
            retLen = generateRowData(
4273
                      buffer + len, 
4274
                      superTblInfo->maxSqlLen - len, 
4275
                      startTime + superTblInfo->timeStampStep * k,
4276
                      superTblInfo);
4277 4278
          }
       }
4279

4280 4281 4282
       if (retLen < 0) {
         return -1;
       }
4283

4284
       len += retLen;
4285

4286 4287 4288 4289
       if (len >= (superTblInfo->maxSqlLen - 256)) {    // reserve for overwrite
         k++;
         break;
       }
4290
    } else {
4291
      int rand_num = taosRandom() % 100;
4292 4293 4294 4295
          char data[MAX_DATA_SIZE];
          char **data_type = g_args.datatype;
          int lenOfBinary = g_args.len_of_binary;

4296
      if ((g_args.disorderRatio != 0)
4297 4298
                && (rand_num < g_args.disorderRange)) {
             
4299 4300
        int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k 
            - taosRandom() % 1000000 + rand_num;
4301
        len = generateData(data, data_type,
4302
                  ncols_per_record, d, lenOfBinary);
4303
      } else {
4304
            len = generateData(data, data_type,
4305
                  ncols_per_record,
4306
                  startTime + DEFAULT_TIMESTAMP_STEP * k,
4307 4308
                  lenOfBinary);
      }
4309

4310 4311
      buffer += sprintf(buffer, " %s", data);
      if (strlen(buffer) >= (g_args.max_sql_len - 256)) { // too long
4312 4313
          k++;
          break;
4314 4315 4316
      }
    }

4317 4318
    verbosePrint("%s() LN%d len=%d k=%d \nbuffer=%s\n",
            __func__, __LINE__, len, k, buffer);
4319 4320 4321

    k++;
    startFrom ++;
4322

4323
    if (startFrom >= insertRows) {
4324
      break;
4325
    }
4326
  }
4327

4328 4329 4330
  *dataLen = len;
  return k;
}
4331

4332 4333
static int generateSQLHead(char *tableName, int32_t tableSeq,
        threadInfo* pThreadInfo, SSuperTable* superTblInfo, char *buffer)
4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346
{
  int len;
  if (superTblInfo) {
    if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
            tagsValBuf = generateTagVaulesForStb(superTblInfo);
      } else {
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
      }
      if (NULL == tagsValBuf) {
4347
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n", __func__, __LINE__);
4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364
        return -1;
      }

      len = snprintf(buffer,
                  superTblInfo->maxSqlLen,
                  "insert into %s.%s using %s.%s tags %s values",
                  pThreadInfo->db_name,
                  tableName,
                  pThreadInfo->db_name,
                  superTblInfo->sTblName,
                  tagsValBuf);
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
      len = snprintf(buffer,
                  superTblInfo->maxSqlLen,
                  "insert into %s.%s values",
                  pThreadInfo->db_name,
4365
                  tableName);
4366 4367
    } else {
      len = snprintf(buffer,
4368
                  superTblInfo->maxSqlLen,
4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407
                  "insert into %s.%s values",
                  pThreadInfo->db_name,
                  tableName);
    }
  } else {
      len = snprintf(buffer,
                  g_args.max_sql_len,
                  "insert into %s.%s values",
                  pThreadInfo->db_name,
                  tableName);
  }

  return len;
}

static int generateDataBuffer(char *pTblName,
        int32_t tableSeq,
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
        int64_t startFrom, int64_t startTime, int *pSamplePos)
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  int ncols_per_record = 1; // count first col ts

  if (superTblInfo == NULL) {
    int datatypeSeq = 0;
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
  }

  assert(buffer != NULL);

  memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len);

  char *pstr = buffer;

4408 4409
  int headLen = generateSQLHead(pTblName, tableSeq, pThreadInfo, superTblInfo,
          buffer);
4410 4411 4412 4413 4414
  pstr += headLen;

  int k;
  int dataLen;
  k = generateDataTail(pTblName, tableSeq, pThreadInfo, superTblInfo,
4415 4416
          g_args.num_of_RPR, pstr, insertRows, startFrom,
          startTime,
4417
          pSamplePos, &dataLen);
4418 4419
  return k;
}
4420

4421
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
4422 4423
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
         pThreadInfo->threadID, __func__, __LINE__);
4424 4425 4426 4427 4428

  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1);
  if (NULL == buffer) {
4429
    errorPrint( "Failed to alloc %d Bytes, reason:%s\n",
4430 4431 4432 4433 4434 4435 4436 4437
              superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len,
              strerror(errno));
    return NULL;
  }
 
  int insertMode;
  char tableName[TSDB_TABLE_NAME_LEN];

4438
  int rowsPerTbl = superTblInfo?superTblInfo->rowsPerTbl:g_args.interlace_rows;
4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463

  if (rowsPerTbl > 0) {
    insertMode = INTERLACE_INSERT_MODE;
  } else {
    insertMode = PROGRESSIVE_INSERT_MODE;
  }

  // rows per table need be less than insert batch
  if (rowsPerTbl > g_args.num_of_RPR)
      rowsPerTbl = g_args.num_of_RPR;

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

  int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
  int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
  uint64_t st = 0;
  uint64_t et = 0xffffffff;

  int64_t lastPrintTime = taosGetTimestampMs();
  int64_t startTs = taosGetTimestampUs();
  int64_t endTs;

  int tableSeq = pThreadInfo->start_table_from;

4464 4465
  debugPrint("[%d] %s() LN%d: start_table_from=%d ntables=%d insertRows=%"PRId64"\n",
          pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from,
4466 4467 4468 4469 4470
          pThreadInfo->ntables, insertRows);

  int64_t startTime = pThreadInfo->start_time;

  int batchPerTblTimes;
4471
  int batchPerTbl;
4472

4473 4474 4475 4476 4477 4478
  assert(pThreadInfo->ntables > 0);

  if (rowsPerTbl > g_args.num_of_RPR)
        rowsPerTbl = g_args.num_of_RPR;

  batchPerTbl = rowsPerTbl;
4479
  if ((rowsPerTbl > 0) && (pThreadInfo->ntables > 1)) {
4480 4481
    batchPerTblTimes =
        (g_args.num_of_RPR / (rowsPerTbl * pThreadInfo->ntables)) + 1;
4482 4483 4484 4485
  } else {
    batchPerTblTimes = 1;
  }

4486
  int generatedRecPerTbl = 0;
4487 4488
  bool flagSleep = true;
  int sleepTimeTotal = 0;
4489
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
4490
    if ((flagSleep) && (insert_interval)) {
4491
        st = taosGetTimestampUs();
4492
        flagSleep = false;
4493 4494 4495
    }
    // generate data
    memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len);
4496

4497
    char *pstr = buffer;
4498
    int recOfBatch = 0;
4499

4500 4501
    for (int i = 0; i < batchPerTblTimes; i ++) {
      getTableName(tableName, pThreadInfo, tableSeq);
4502

4503 4504
      int headLen;
      if (i == 0) {
4505 4506
        headLen = generateSQLHead(tableName, tableSeq, pThreadInfo,
                superTblInfo, pstr);
4507
      } else {
4508
        headLen = snprintf(pstr, TSDB_TABLE_NAME_LEN, "%s.%s values",
4509 4510
                  pThreadInfo->db_name,
                  tableName);
4511
      }
4512

4513
      // generate data buffer
4514 4515
      verbosePrint("[%d] %s() LN%d i=%d buffer:\n%s\n",
                pThreadInfo->threadID, __func__, __LINE__, i, buffer);
4516

4517 4518
      pstr += headLen;
      int dataLen = 0;
4519

4520
      verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n",
4521 4522 4523 4524 4525
                pThreadInfo->threadID, __func__, __LINE__,
                i, batchPerTblTimes, batchPerTbl);
      generateDataTail(
        tableName, tableSeq, pThreadInfo, superTblInfo,
        batchPerTbl, pstr, insertRows, 0,
4526
        startTime,
4527
        &(pThreadInfo->samplePos), &dataLen);
4528

4529
      pstr += dataLen;
4530
      recOfBatch += batchPerTbl;
4531
      startTime += batchPerTbl * superTblInfo->timeStampStep;
4532
      pThreadInfo->totalInsertRows += batchPerTbl;
4533

4534
      verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n",
4535 4536
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
4537

4538 4539 4540 4541
      tableSeq ++;
      if (insertMode == INTERLACE_INSERT_MODE) {
          if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
            // turn to first table
4542
            startTime += batchPerTbl * superTblInfo->timeStampStep;
4543
            tableSeq = pThreadInfo->start_table_from;
4544 4545 4546 4547 4548 4549 4550
            generatedRecPerTbl += batchPerTbl;
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;

            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
4551
          }
4552 4553
      }

4554
      int remainRows = insertRows - generatedRecPerTbl;
4555
      if ((remainRows > 0) && (batchPerTbl > remainRows))
4556 4557
        batchPerTbl = remainRows;

4558
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%d insertRows=%"PRId64"\n",
4559 4560 4561 4562
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
4563 4564 4565
        break;
    }

4566
    verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRId64"\n",
4567 4568 4569 4570 4571
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
           pThreadInfo->threadID, __func__, __LINE__, buffer);

4572 4573
    startTs = taosGetTimestampUs();

4574
    int affectedRows = execInsert(pThreadInfo, buffer, recOfBatch);
4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585

    endTs = taosGetTimestampUs();
    int64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %10.6fms\n",
            __func__, __LINE__, delay/1000.0);

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

4586 4587
    verbosePrint("[%d] %s() LN%d affectedRows=%d\n", pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
4588 4589
    if ((affectedRows < 0) || (recOfBatch != affectedRows)) {
        errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %d\n%s\n",
4590
                pThreadInfo->threadID, __func__, __LINE__,
4591
                recOfBatch, affectedRows, buffer);
4592
        goto free_and_statistics_interlace;
4593
    }
4594

4595
    pThreadInfo->totalAffectedRows += affectedRows;
4596

4597 4598 4599
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
      printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
4600 4601 4602
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
4603 4604
      lastPrintTime = currentPrintTime;
    }
4605

4606 4607
    if ((insert_interval) && flagSleep) {
      et = taosGetTimestampUs();
4608

4609 4610
      if (insert_interval > ((et - st)/1000) ) {
        int sleepTime = insert_interval - (et -st)/1000;
4611 4612
        performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleepTime);
4613 4614 4615
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
4616
    }
4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628
  }

free_and_statistics_interlace:
  tmfree(buffer);

  printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", 
          pThreadInfo->threadID, 
          pThreadInfo->totalInsertRows, 
          pThreadInfo->totalAffectedRows);
  return NULL;
}

4629 4630 4631 4632 4633 4634 4635 4636
// sync insertion
/*
   1 thread: 100 tables * 2000  rows/s
   1 thread: 10  tables * 20000 rows/s
   6 thread: 300 tables * 2000  rows/s

   2 taosinsertdata , 1 thread:  10  tables * 20000 rows/s
*/
4637
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
4638
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
4639

4640
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4641

4642 4643
  char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1);
  if (NULL == buffer) {
4644
    errorPrint( "Failed to alloc %d Bytes, reason:%s\n",
4645 4646 4647 4648
              superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len,
              strerror(errno));
    return NULL;
  }
4649 4650 4651 4652 4653
 
  int64_t lastPrintTime = taosGetTimestampMs();
  int64_t startTs = taosGetTimestampUs();
  int64_t endTs;

4654
  int timeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
4655
  int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
4656 4657 4658
  uint64_t st = 0;
  uint64_t et = 0xffffffff;

4659 4660
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
4661

4662
  pThreadInfo->samplePos = 0;
4663

4664
  for (uint32_t tableSeq = pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to;
4665
        tableSeq ++) {
4666
    int64_t start_time = pThreadInfo->start_time;
4667 4668 4669 4670 4671 4672 4673

    int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
    verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows);

    for (int64_t i = 0; i < insertRows;) {
      if (insert_interval) {
            st = taosGetTimestampUs();
4674
      }
4675

4676 4677 4678 4679 4680 4681
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
      verbosePrint("%s() LN%d: tid=%d seq=%d tableName=%s\n",
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);

4682 4683
      int generated = generateDataBuffer(
              tableName, tableSeq, pThreadInfo, buffer, insertRows,
4684
            i, start_time,
4685
            &(pThreadInfo->samplePos));
4686 4687 4688 4689 4690
      if (generated > 0)
        i += generated;
      else
        goto free_and_statistics_2;

4691
      start_time +=  generated * timeStampStep;
4692
      pThreadInfo->totalInsertRows += generated;
4693 4694 4695 4696

      startTs = taosGetTimestampUs();

      int affectedRows = execInsert(pThreadInfo, buffer, generated);
4697 4698 4699

      endTs = taosGetTimestampUs();
      int64_t delay = endTs - startTs;
4700 4701 4702
      performancePrint("%s() LN%d, insert execution time is %10.6fms\n",
              __func__, __LINE__, delay/1000.0);

4703 4704 4705 4706
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
4707

4708 4709 4710 4711 4712
      if (affectedRows < 0)
        goto free_and_statistics_2;

      pThreadInfo->totalAffectedRows += affectedRows;

4713 4714
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
4715
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
4716 4717 4718
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
4719 4720 4721
        lastPrintTime = currentPrintTime;
      }

4722
      if (i >= insertRows)
4723
        break;
4724 4725 4726 4727 4728 4729

      if (insert_interval) {
        et = taosGetTimestampUs();

        if (insert_interval > ((et - st)/1000) ) {
            int sleep_time = insert_interval - (et -st)/1000;
4730 4731
            performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleep_time);
4732 4733
            taosMsleep(sleep_time); // ms
        }
4734 4735 4736
      }
    }   // num_of_DPT

4737
    if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo &&
4738 4739
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
4740 4741
          printf("%s() LN%d samplePos=%d\n",
                  __func__, __LINE__, pThreadInfo->samplePos);
4742
    }
4743
  } // tableSeq
4744

4745
free_and_statistics_2:
4746 4747
  tmfree(buffer);

4748
  printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", 
4749 4750 4751
          pThreadInfo->threadID, 
          pThreadInfo->totalInsertRows, 
          pThreadInfo->totalAffectedRows);
4752
  return NULL;
H
Hui Li 已提交
4753 4754
}

4755 4756 4757 4758 4759
static void* syncWrite(void *sarg) {

  threadInfo *winfo = (threadInfo *)sarg; 
  SSuperTable* superTblInfo = winfo->superTblInfo;

4760
  int rowsPerTbl = superTblInfo?superTblInfo->rowsPerTbl:g_args.interlace_rows;
4761 4762 4763 4764 4765 4766 4767 4768 4769 4770

  if (rowsPerTbl > 0) {
    // interlace mode
    return syncWriteInterlace(winfo);
  } else {
    // progressive mode
    return syncWriteProgressive(winfo);
  }
}

4771
static void callBack(void *param, TAOS_RES *res, int code) {
4772
  threadInfo* winfo = (threadInfo*)param; 
4773
  SSuperTable* superTblInfo = winfo->superTblInfo;
4774

4775 4776 4777 4778 4779
  int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
  if (insert_interval) {
    winfo->et = taosGetTimestampUs();
    if (((winfo->et - winfo->st)/1000) < insert_interval) {
      taosMsleep(insert_interval - (winfo->et - winfo->st)/1000); // ms
4780
    }
H
Hui Li 已提交
4781 4782
  }
  
4783 4784 4785
  char *buffer = calloc(1, winfo->superTblInfo->maxSqlLen);
  char *data   = calloc(1, MAX_DATA_SIZE);
  char *pstr = buffer;
4786 4787
  pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix,
          winfo->start_table_from);
4788
//  if (winfo->counter >= winfo->superTblInfo->insertRows) {
4789
  if (winfo->counter >= g_args.num_of_RPR) {
4790
    winfo->start_table_from++;
4791 4792
    winfo->counter = 0;
  }
4793
  if (winfo->start_table_from > winfo->end_table_to) {
4794 4795 4796 4797 4798 4799
    tsem_post(&winfo->lock_sem);
    free(buffer);
    free(data);
    taos_free_result(res);
    return;
  }
H
Hui Li 已提交
4800
  
4801
  for (int i = 0; i < g_args.num_of_RPR; i++) {
4802
    int rand_num = taosRandom() % 100;
4803 4804
    if (0 != winfo->superTblInfo->disorderRatio && rand_num < winfo->superTblInfo->disorderRatio)
    {
4805
      int64_t d = winfo->lastTs - taosRandom() % 1000000 + rand_num;
4806 4807 4808
      //generateData(data, datatype, ncols_per_record, d, len_of_binary);
      (void)generateRowData(data, MAX_DATA_SIZE, d, winfo->superTblInfo);
    } else {
4809
      //generateData(data, datatype, ncols_per_record, start_time += 1000, len_of_binary);
4810
      (void)generateRowData(data, MAX_DATA_SIZE, winfo->lastTs += 1000, winfo->superTblInfo);
H
Hui Li 已提交
4811
    }
4812 4813
    pstr += sprintf(pstr, "%s", data);
    winfo->counter++;
H
Hui Li 已提交
4814

4815 4816
    if (winfo->counter >= winfo->superTblInfo->insertRows) {
      break;
H
Hui Li 已提交
4817 4818
    }
  }
4819

4820 4821
  if (insert_interval) {
    winfo->st = taosGetTimestampUs();
4822 4823 4824 4825
  }
  taos_query_a(winfo->taos, buffer, callBack, winfo);
  free(buffer);
  free(data);
H
Hui Li 已提交
4826

4827
  taos_free_result(res);
H
Hui Li 已提交
4828 4829
}

4830
static void *asyncWrite(void *sarg) {
4831
  threadInfo *winfo = (threadInfo *)sarg;
4832
  SSuperTable* superTblInfo = winfo->superTblInfo;
4833 4834 4835 4836 4837

  winfo->st = 0;
  winfo->et = 0;
  winfo->lastTs = winfo->start_time;
  
4838 4839 4840
  int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
  if (insert_interval) {
    winfo->st = taosGetTimestampUs();
H
Hui Li 已提交
4841
  }
4842 4843 4844
  taos_query_a(winfo->taos, "show databases", callBack, winfo);

  tsem_wait(&(winfo->lock_sem));
H
Hui Li 已提交
4845 4846 4847 4848

  return NULL;
}

4849 4850
static void startMultiThreadInsertData(int threads, char* db_name,
        char* precision,SSuperTable* superTblInfo) {
4851 4852

    pthread_t *pids = malloc(threads * sizeof(pthread_t));
4853 4854
    assert(pids != NULL);

4855
    threadInfo *infos = malloc(threads * sizeof(threadInfo));
4856 4857
    assert(infos != NULL);

4858 4859
    memset(pids, 0, threads * sizeof(pthread_t));
    memset(infos, 0, threads * sizeof(threadInfo));
H
Hui Li 已提交
4860

4861
    int ntables = 0;
4862 4863 4864 4865 4866
    if (superTblInfo) {

        if ((superTblInfo->childTblOffset >= 0) 
            && (superTblInfo->childTblLimit > 0)) {

4867
            ntables = superTblInfo->childTblLimit;
4868
        } else {
4869
            ntables = superTblInfo->childTblCount;
4870 4871
        }
    } else {
4872
        ntables = g_args.num_of_tables;
4873
    }
H
Hui Li 已提交
4874

4875 4876 4877 4878 4879 4880 4881 4882 4883 4884
    int a = ntables / threads;
    if (a < 1) {
        threads = ntables;
        a = 1;
    }

    int b = 0;
    if (threads != 0) {
        b = ntables % threads;
    }
4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901

  //TAOS* taos;
  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
  //  if (NULL == taos) {
  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
  //    exit(-1);
  //  }
  //}

  int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
  if (0 != precision[0]) {
    if (0 == strncasecmp(precision, "ms", 2)) {
      timePrec = TSDB_TIME_PRECISION_MILLI;
    }  else if (0 == strncasecmp(precision, "us", 2)) {
      timePrec = TSDB_TIME_PRECISION_MICRO;
    }  else {
4902
      errorPrint( "No support precision: %s\n", precision);
4903 4904 4905 4906
      exit(-1);
    }
  }

4907 4908 4909 4910
  int64_t start_time; 
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
        start_time = taosGetTimestamp(timePrec);
4911
    } else {
4912 4913 4914 4915 4916
      if (TSDB_CODE_SUCCESS != taosParseTime(
        superTblInfo->startTimestamp,
        &start_time,
        strlen(superTblInfo->startTimestamp),
        timePrec, 0)) {
4917
          ERROR_EXIT("failed to parse time!\n");
4918
      }
4919
    }
4920 4921
  } else {
     start_time = 1500000000000;
4922 4923 4924
  }

  double start = getCurrentTime();
4925
 
4926
  int startFrom;
4927

4928
  if ((superTblInfo) && (superTblInfo->childTblOffset >= 0))
4929
      startFrom = superTblInfo->childTblOffset;
4930
  else
4931
      startFrom = 0;
4932

4933 4934 4935
  // read sample data from file first
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, 
              "sample", strlen("sample")))) {
4936
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
4937
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n", __func__, __LINE__);
4938 4939 4940 4941
      exit(-1);
    }
  }

4942 4943 4944 4945
  // read sample data from file first
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, 
              "sample", strlen("sample")))) {
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
4946
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n", __func__, __LINE__);
4947 4948 4949 4950 4951 4952 4953 4954
      exit(-1);
    }
  }

  TAOS* taos = taos_connect(
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
  if (NULL == taos) {
4955 4956
    errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
                __func__, __LINE__, taos_errstr(NULL));
4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974
    exit(-1);
  }

  if (superTblInfo) {

    int limit, offset;
    if (superTblInfo && (superTblInfo->childTblOffset >= 0)
            && (superTblInfo->childTblLimit > 0)) {
        limit = superTblInfo->childTblLimit;
        offset = superTblInfo->childTblOffset;
    } else {
        limit = superTblInfo->childTblCount;
        offset = 0;
    }

    superTblInfo->childTblName = (char*)calloc(1,
        limit * TSDB_TABLE_NAME_LEN);
    if (superTblInfo->childTblName == NULL) {
4975
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989
      taos_close(taos);
      exit(-1);
    }

    int childTblCount;
    getChildNameOfSuperTableWithLimitAndOffset(
        taos,
        db_name, superTblInfo->sTblName,
        &superTblInfo->childTblName, &childTblCount,
        limit,
        offset);
  }
  taos_close(taos);

H
Hui Li 已提交
4990
  for (int i = 0; i < threads; i++) {
4991
    threadInfo *t_info = infos + i;
H
Hui Li 已提交
4992 4993
    t_info->threadID = i;
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
4994 4995 4996
    t_info->superTblInfo = superTblInfo;

    t_info->start_time = start_time;
4997
    t_info->minDelay = INT16_MAX;
4998

4999 5000
    if ((NULL == superTblInfo) ||
            (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) {
5001
      //t_info->taos = taos;
5002
      t_info->taos = taos_connect(
5003
              g_Dbs.host, g_Dbs.user,
5004
              g_Dbs.password, db_name, g_Dbs.port);
5005
      if (NULL == t_info->taos) {
5006
        errorPrint( "connect to server fail from insert sub thread, reason: %s\n",
5007
                taos_errstr(NULL));
5008 5009 5010 5011 5012 5013
        exit(-1);
      }
    } else {
      t_info->taos = NULL;
    }

5014 5015
    if ((NULL == superTblInfo)
            || (0 == superTblInfo->multiThreadWriteOneTbl)) {
5016 5017 5018 5019
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
5020
    } else {
5021 5022
      t_info->start_table_from = 0;
      t_info->ntables = superTblInfo->childTblCount;
5023 5024 5025 5026 5027
      t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint();
    }

    tsem_init(&(t_info->lock_sem), 0, 0);
    if (SYNC == g_Dbs.queryMode) {
5028
      pthread_create(pids + i, NULL, syncWrite, t_info);
5029
    } else {
5030 5031
      pthread_create(pids + i, NULL, asyncWrite, t_info);
    }
H
Hui Li 已提交
5032
  }
5033

H
Hui Li 已提交
5034 5035 5036 5037
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

5038 5039 5040
  int64_t totalDelay = 0;
  int64_t maxDelay = 0;
  int64_t minDelay = INT16_MAX;
5041
  int64_t cntDelay = 1;
5042 5043
  double  avgDelay = 0;

H
Hui Li 已提交
5044
  for (int i = 0; i < threads; i++) {
5045 5046
    threadInfo *t_info = infos + i;

S
TD-1057  
Shengliang Guan 已提交
5047
    tsem_destroy(&(t_info->lock_sem));
5048 5049
    taos_close(t_info->taos);

5050 5051 5052 5053
    debugPrint("%s() LN%d, [%d] totalInsert=%"PRId64" totalAffected=%"PRId64"\n",
            __func__, __LINE__,
            t_info->threadID, t_info->totalInsertRows,
            t_info->totalAffectedRows);
5054 5055
    if (superTblInfo) {
        superTblInfo->totalAffectedRows += t_info->totalAffectedRows;
5056
        superTblInfo->totalInsertRows += t_info->totalInsertRows;
5057 5058 5059
    } else {
        g_args.totalAffectedRows += t_info->totalAffectedRows;
        g_args.totalInsertRows += t_info->totalInsertRows;
5060
    }
5061 5062 5063 5064

    totalDelay  += t_info->totalDelay;
    cntDelay   += t_info->cntDelay;
    if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay;
5065
    if (t_info->minDelay < minDelay) minDelay = t_info->minDelay;
H
Hui Li 已提交
5066
  }
5067
  cntDelay -= 1;
H
Hui Li 已提交
5068

5069
  if (cntDelay == 0)    cntDelay = 1;
5070 5071
  avgDelay = (double)totalDelay / cntDelay;

5072
  double end = getCurrentTime();
5073
  double t = end - start;
5074 5075

  if (superTblInfo) {
5076
    printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n",
5077
          t, superTblInfo->totalInsertRows,
5078 5079
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5080
          superTblInfo->totalInsertRows / t);
5081
    fprintf(g_fpOfInsertResult,
5082
          "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n",
5083
          t, superTblInfo->totalInsertRows,
5084 5085
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5086
          superTblInfo->totalInsertRows/ t);
5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098
  } else {
    printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s %2.f records/second\n\n",
          t, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          g_args.totalInsertRows / t);
    fprintf(g_fpOfInsertResult,
          "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s %2.f records/second\n\n",
          t, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          g_args.totalInsertRows / t);
5099
  }
5100 5101 5102 5103 5104

  printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n",
          avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0);
  fprintf(g_fpOfInsertResult, "insert delay, avg:%10.6fms, max: %10.6fms, min: %10.6fms\n\n",
          avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0);
5105 5106 5107
  
  //taos_close(taos);

H
Hui Li 已提交
5108
  free(pids);
5109
  free(infos);
H
Hui Li 已提交
5110 5111
}

5112
static void *readTable(void *sarg) {
5113 5114
#if 1  
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
5115 5116
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
5117
  uint64_t sTime = rinfo->start_time;
H
hzcheng 已提交
5118 5119
  char *tb_prefix = rinfo->tb_prefix;
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
5120
  if (NULL == fp) {
5121
    errorPrint( "fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
H
Hui Li 已提交
5122 5123
    return NULL;
  }
5124

5125
    int num_of_DPT;
5126
/*  if (rinfo->superTblInfo) {
5127 5128
    num_of_DPT = rinfo->superTblInfo->insertRows; //  nrecords_per_table;
  } else {
5129
  */
5130
      num_of_DPT = g_args.num_of_DPT;
5131
//  }
5132

5133
  int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
H
hzcheng 已提交
5134
  int totalData = num_of_DPT * num_of_tables;
5135
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
  printf("%d records:\n", totalData);
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

  for (int j = 0; j < n; j++) {
    double totalT = 0;
    int count = 0;
    for (int i = 0; i < num_of_tables; i++) {
L
lihui 已提交
5148
      sprintf(command, "select %s from %s%d where ts>= %" PRId64, aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
5149 5150

      double t = getCurrentTime();
S
Shuaiqiang Chang 已提交
5151 5152
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
5153

S
Shuaiqiang Chang 已提交
5154
      if (code != 0) {
5155
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
5156
        taos_free_result(pSql);
H
hzcheng 已提交
5157
        taos_close(taos);
5158
        fclose(fp);
5159
        return NULL;
H
hzcheng 已提交
5160 5161
      }

S
Shuaiqiang Chang 已提交
5162
      while (taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
5163 5164 5165 5166 5167 5168
        count++;
      }

      t = getCurrentTime() - t;
      totalT += t;

S
Shuaiqiang Chang 已提交
5169
      taos_free_result(pSql);
H
hzcheng 已提交
5170 5171 5172
    }

    fprintf(fp, "|%10s  |   %10d   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
5173
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
5174 5175 5176 5177 5178
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT);
  }
  fprintf(fp, "\n");
  fclose(fp);
5179
#endif
H
hzcheng 已提交
5180 5181 5182
  return NULL;
}

5183
static void *readMetric(void *sarg) {
5184 5185
#if 1  
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
5186 5187 5188
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
5189 5190 5191 5192
  if (NULL == fp) {
    printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
    return NULL;
  }
5193 5194

  int num_of_DPT = rinfo->superTblInfo->insertRows;
5195
  int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
H
hzcheng 已提交
5196
  int totalData = num_of_DPT * num_of_tables;
5197
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
5198 5199 5200 5201 5202 5203 5204 5205 5206

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
  printf("%d records:\n", totalData);
  fprintf(fp, "Querying On %d records:\n", totalData);

  for (int j = 0; j < n; j++) {
sangshuduo's avatar
sangshuduo 已提交
5207
    char condition[BUFFER_SIZE - 30] = "\0";
B
Bomin Zhang 已提交
5208
    char tempS[64] = "\0";
H
hzcheng 已提交
5209 5210 5211 5212 5213

    int m = 10 < num_of_tables ? 10 : num_of_tables;

    for (int i = 1; i <= m; i++) {
      if (i == 1) {
5214
        sprintf(tempS, "t1 = %d", i);
H
hzcheng 已提交
5215
      } else {
5216
        sprintf(tempS, " or t1 = %d ", i);
H
hzcheng 已提交
5217 5218 5219
      }
      strcat(condition, tempS);

L
liu0x54 已提交
5220
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
5221 5222 5223 5224 5225 5226

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

      double t = getCurrentTime();

S
Shuaiqiang Chang 已提交
5227 5228 5229 5230
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
5231
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
5232
        taos_free_result(pSql);
H
hzcheng 已提交
5233
        taos_close(taos);
5234
        fclose(fp);
5235
        return NULL;
H
hzcheng 已提交
5236 5237
      }
      int count = 0;
S
Shuaiqiang Chang 已提交
5238
      while (taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
5239 5240 5241 5242
        count++;
      }
      t = getCurrentTime() - t;

5243 5244
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
              num_of_tables * num_of_DPT / t, t * 1000);
H
hzcheng 已提交
5245 5246
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t);

S
Shuaiqiang Chang 已提交
5247
      taos_free_result(pSql);
H
hzcheng 已提交
5248 5249 5250 5251
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
5252
#endif
H
hzcheng 已提交
5253 5254 5255
  return NULL;
}

H
Hui Li 已提交
5256

5257
static int insertTestProcess() {
5258

5259 5260 5261
  setupForAnsiEscape();
  int ret = printfInsertMeta();
  resetAfterAnsiEscape();
5262

5263 5264 5265
  if (ret == -1)
    exit(EXIT_FAILURE);

5266
  debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
5267 5268
  g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
  if (NULL == g_fpOfInsertResult) {
5269
    errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
5270 5271
    return -1;
  }
5272

5273 5274
  printfInsertMetaToFile(g_fpOfInsertResult);

5275 5276 5277 5278
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void)getchar();
  }
5279
 
5280 5281 5282
  init_rand_data();

  // create database and super tables
5283
  if(createDatabases() != 0) {
5284 5285 5286
    fclose(g_fpOfInsertResult);
    return -1;
  }
5287 5288

  // pretreatement
5289
  prepareSampleData();
5290

5291 5292 5293 5294 5295 5296 5297
  double start;
  double end;

  // create child tables
  start = getCurrentTime();
  createChildTables();
  end = getCurrentTime();
5298

5299
  if (g_totalChildTables > 0) {
5300
    printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n",
5301
            end - start, g_totalChildTables, g_Dbs.threadCount);
5302 5303
    fprintf(g_fpOfInsertResult,
            "Spent %.4f seconds to create %d tables with %d thread(s)\n\n",
5304
            end - start, g_totalChildTables, g_Dbs.threadCount);
5305
  }
5306

5307
  taosMsleep(1000);
5308 5309
  // create sub threads for inserting data
  //start = getCurrentTime();
5310 5311 5312 5313 5314 5315
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    if (g_Dbs.db[i].superTblCount > 0) {
      for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
        SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];
        if (0 == g_Dbs.db[i].superTbls[j].insertRows) {
          continue;
5316
        }
5317
        startMultiThreadInsertData(
5318 5319 5320
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
5321 5322 5323 5324
          superTblInfo);
        }
    } else {
        startMultiThreadInsertData(
5325 5326 5327
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
5328
          NULL);
H
Hui Li 已提交
5329
    }
5330
  }
5331
  //end = getCurrentTime();
5332

5333
  //int64_t    totalInsertRows = 0;
5334 5335 5336
  //int64_t    totalAffectedRows = 0;
  //for (int i = 0; i < g_Dbs.dbCount; i++) {    
  //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
5337
  //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
5338 5339
  //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
  //}
5340
  //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
5341
  postFreeResource();
5342

5343 5344 5345
  return 0;
}

5346
static void *superQueryProcess(void *sarg) {
5347
  threadInfo *winfo = (threadInfo *)sarg;
5348 5349 5350 5351 5352 5353 5354 5355

  //char sqlStr[MAX_TB_NAME_SIZE*2];
  //sprintf(sqlStr, "use %s", g_queryInfo.dbName);
  //queryDB(winfo->taos, sqlStr);
  
  int64_t st = 0;
  int64_t et = 0;
  while (1) {
5356 5357
    if (g_queryInfo.superQueryInfo.rate && (et - st) <
            (int64_t)g_queryInfo.superQueryInfo.rate*1000) {
5358
      taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms
5359
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5360 5361
    }

5362
    st = taosGetTimestampUs();
5363
    for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
5364
      if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {
5365 5366 5367
        int64_t t1 = taosGetTimestampUs();
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
        if (g_queryInfo.superQueryInfo.result[i][0] != 0) {
5368 5369
          sprintf(tmpFile, "%s-%d",
                  g_queryInfo.superQueryInfo.result[i], winfo->threadID);
5370
        }
5371 5372 5373
        selectAndGetResult(winfo->taos, g_queryInfo.superQueryInfo.sql[i], tmpFile);
        int64_t t2 = taosGetTimestampUs();
        printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %f s\n",
5374
                taosGetSelfPthreadId(), (t2 - t1)/1000000.0);
5375 5376
      } else {
        int64_t t1 = taosGetTimestampUs();
5377
        int retCode = postProceSql(g_queryInfo.host,
5378
                g_queryInfo.port, g_queryInfo.superQueryInfo.sql[i]);
5379 5380
        int64_t t2 = taosGetTimestampUs();
        printf("=[restful] thread[%"PRId64"] complete one sql, Spent %f s\n",
5381
                taosGetSelfPthreadId(), (t2 - t1)/1000000.0);
5382

5383
        if (0 != retCode) {
5384
          printf("====restful return fail, threadID[%d]\n", winfo->threadID);
5385 5386
          return NULL;
        }
5387
      }
5388
    }
5389
    et = taosGetTimestampUs();
5390
    printf("==thread[%"PRId64"] complete all sqls to specify tables once queries duration:%.6fs\n\n",
5391
            taosGetSelfPthreadId(), (double)(et - st)/1000.0);
H
Hui Li 已提交
5392
  }
5393 5394
  return NULL;
}
H
Hui Li 已提交
5395

5396
static void replaceSubTblName(char* inSql, char* outSql, int tblIndex) {
5397 5398
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
5399 5400
  sprintf(subTblName, "%s.%s",
          g_queryInfo.dbName,
5401
          g_queryInfo.subQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
5402 5403 5404 5405 5406

  //printf("inSql: %s\n", inSql);
  
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
5407
    return;
H
Hui Li 已提交
5408 5409
  }
  
5410
  tstrncpy(outSql, inSql, pos - inSql + 1);
5411
  //printf("1: %s\n", outSql);
5412 5413 5414 5415
  strcat(outSql, subTblName);
  //printf("2: %s\n", outSql);
  strcat(outSql, pos+strlen(sourceString));
  //printf("3: %s\n", outSql);
H
Hui Li 已提交
5416 5417
}

5418
static void *subQueryProcess(void *sarg) {
5419
  char sqlstr[1024];
5420
  threadInfo *winfo = (threadInfo *)sarg;
5421
  int64_t st = 0;
5422
  int64_t et = (int64_t)g_queryInfo.subQueryInfo.rate*1000;
5423
  while (1) {
5424 5425
    if (g_queryInfo.subQueryInfo.rate
            && (et - st) < (int64_t)g_queryInfo.subQueryInfo.rate*1000) {
5426
      taosMsleep(g_queryInfo.subQueryInfo.rate*1000 - (et - st)); // ms
5427
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5428
    }
H
Hui Li 已提交
5429

5430
    st = taosGetTimestampUs();
5431
    for (int i = winfo->start_table_from; i <= winfo->end_table_to; i++) {
5432
      for (int j = 0; j < g_queryInfo.subQueryInfo.sqlCount; j++) {
5433
        memset(sqlstr,0,sizeof(sqlstr));
5434
        replaceSubTblName(g_queryInfo.subQueryInfo.sql[j], sqlstr, i);
5435 5436
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
        if (g_queryInfo.subQueryInfo.result[i][0] != 0) {
5437 5438
          sprintf(tmpFile, "%s-%d",
                  g_queryInfo.subQueryInfo.result[i],
5439
                  winfo->threadID);
5440
        }
5441
        selectAndGetResult(winfo->taos, sqlstr, tmpFile);
5442
      }
H
Hui Li 已提交
5443
    }
5444
    et = taosGetTimestampUs();
5445
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%d - %d] once queries duration:%.4fs\n\n", 
5446 5447 5448
            taosGetSelfPthreadId(),
            winfo->start_table_from,
            winfo->end_table_to,
5449
            (double)(et - st)/1000000.0);
5450 5451 5452 5453
  }
  return NULL;
}

5454
static int queryTestProcess() {
5455 5456 5457 5458 5459
  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
5460
          g_queryInfo.port);
5461
  if (taos == NULL) {
5462
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
5463 5464 5465 5466
    exit(-1);
  }

  if (0 != g_queryInfo.subQueryInfo.sqlCount) {
5467 5468 5469 5470
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
            g_queryInfo.subQueryInfo.sTblName,
            &g_queryInfo.subQueryInfo.childTblName,
5471
            &g_queryInfo.subQueryInfo.childTblCount);
5472
  }
5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485
  
  printfQueryMeta();
  
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void)getchar();
  }
  
  printfQuerySystemInfo(taos);
  
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
5486 5487
  if (g_queryInfo.superQueryInfo.sqlCount > 0
          && g_queryInfo.superQueryInfo.concurrent > 0) {
5488

5489
    pids  = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t));
5490 5491 5492 5493
    if (NULL == pids) {
        taos_close(taos);
        ERROR_EXIT("memory allocation failed\n");
    }
5494
    infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo));
5495
    if (NULL == infos) {
5496
      taos_close(taos);
5497 5498
      free(pids);
      ERROR_EXIT("memory allocation failed for create threads\n");
5499
    }
5500 5501

    for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) {
5502
      threadInfo *t_info = infos + i;
5503
      t_info->threadID = i;
5504

5505 5506
      if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {
        t_info->taos = taos;
5507

5508 5509
        char sqlStr[MAX_TB_NAME_SIZE*2];
        sprintf(sqlStr, "use %s", g_queryInfo.dbName);
5510
        verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
5511
        if (0 != queryDbExec(t_info->taos, sqlStr, NO_INSERT_TYPE)) {
5512 5513
            free(infos);
            free(pids);
5514 5515 5516 5517
            errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
            return -1;
        }
5518 5519 5520
      } else {
        t_info->taos = NULL;
      }
5521

5522 5523
      pthread_create(pids + i, NULL, superQueryProcess, t_info);
    }
5524 5525 5526
  }else {
    g_queryInfo.superQueryInfo.concurrent = 0;
  }
5527

5528 5529 5530
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
5531 5532
  if ((g_queryInfo.subQueryInfo.sqlCount > 0)
          && (g_queryInfo.subQueryInfo.threadCnt > 0)) {
5533
    pidsOfSub  = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t));
5534 5535 5536 5537 5538 5539 5540 5541
    if (NULL == pidsOfSub) {
      taos_close(taos);
      free(infos);
      free(pids);

      ERROR_EXIT("memory allocation failed for create threads\n");
    }

5542
    infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo));
5543
    if (NULL == infosOfSub) {
5544
      taos_close(taos);
5545 5546 5547 5548
      free(pidsOfSub);
      free(infos);
      free(pids);
      ERROR_EXIT("memory allocation failed for create threads\n");
5549
    }
5550

5551 5552 5553 5554 5555 5556 5557 5558
    int ntables = g_queryInfo.subQueryInfo.childTblCount;
    int threads = g_queryInfo.subQueryInfo.threadCnt;
  
    int a = ntables / threads;
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
5559
 
5560 5561 5562 5563
    int b = 0;
    if (threads != 0) {
      b = ntables % threads;
    }
5564

5565
    int startFrom = 0;
5566 5567 5568
    for (int i = 0; i < threads; i++) {  
      threadInfo *t_info = infosOfSub + i;
      t_info->threadID = i;
5569

5570 5571 5572 5573
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
5574 5575 5576 5577 5578
      t_info->taos = taos;
      pthread_create(pidsOfSub + i, NULL, subQueryProcess, t_info);
    }

    g_queryInfo.subQueryInfo.threadCnt = threads;
5579
  } else {
5580
    g_queryInfo.subQueryInfo.threadCnt = 0;
5581
  }
5582
 
5583 5584
  for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) {
    pthread_join(pids[i], NULL);
5585
  }
S
Shuaiqiang Chang 已提交
5586

5587
  tmfree((char*)pids);
5588 5589
  tmfree((char*)infos);
 
5590 5591
  for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) {
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
5592
  }
H
Hui Li 已提交
5593

5594
  tmfree((char*)pidsOfSub);
5595 5596
  tmfree((char*)infosOfSub);
 
5597 5598 5599 5600
  taos_close(taos);
  return 0;
}

5601
static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
5602
  if (res == NULL || taos_errno(res) != 0) {
5603 5604
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
5605 5606
    return;
  }
5607

5608 5609
  getResult(res, (char*)param);
  taos_free_result(res);
H
hzcheng 已提交
5610 5611
}

5612
static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultFileName) {
5613
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
5614

5615
  if (g_queryInfo.superQueryInfo.subscribeMode) {
5616 5617 5618
    tsub = taos_subscribe(taos,
            g_queryInfo.superQueryInfo.subscribeRestart,
            topic, sql, subscribe_callback, (void*)resultFileName,
5619
            g_queryInfo.superQueryInfo.subscribeInterval);
5620
  } else {
5621 5622
    tsub = taos_subscribe(taos,
            g_queryInfo.superQueryInfo.subscribeRestart,
5623
            topic, sql, NULL, NULL, 0);
5624
  }
5625

5626 5627 5628
  if (tsub == NULL) {
    printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
    return NULL;
5629
  }
5630

5631 5632
  return tsub;
}
H
hzcheng 已提交
5633

5634
static void *subSubscribeProcess(void *sarg) {
5635
  threadInfo *winfo = (threadInfo *)sarg;
5636
  char subSqlstr[1024];
H
hzcheng 已提交
5637

5638 5639
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
5640
  debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
5641 5642 5643
  if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)){
    return NULL;
  }
5644
 
5645 5646 5647 5648 5649
  //int64_t st = 0;
  //int64_t et = 0;
  do {
    //if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) {
    //  taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms
5650
    //  //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668
    //}

    //st = taosGetTimestampMs();
    char topic[32] = {0};
    for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) {
      sprintf(topic, "taosdemo-subscribe-%d", i);
      memset(subSqlstr,0,sizeof(subSqlstr));
      replaceSubTblName(g_queryInfo.subQueryInfo.sql[i], subSqlstr, i);
      char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
      if (g_queryInfo.subQueryInfo.result[i][0] != 0) {
        sprintf(tmpFile, "%s-%d", g_queryInfo.subQueryInfo.result[i], winfo->threadID);
      }
      g_queryInfo.subQueryInfo.tsub[i] = subscribeImpl(winfo->taos, subSqlstr, topic, tmpFile); 
      if (NULL == g_queryInfo.subQueryInfo.tsub[i]) {
        return NULL;
      }
    }
    //et = taosGetTimestampMs();
5669
    //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0);
5670 5671 5672 5673 5674 5675 5676 5677 5678
  } while (0);

  // start loop to consume result
  TAOS_RES* res = NULL;
  while (1) {
    for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) {
      if (1 == g_queryInfo.subQueryInfo.subscribeMode) {
        continue;
      }
5679

5680 5681 5682 5683
      res = taos_consume(g_queryInfo.subQueryInfo.tsub[i]);
      if (res) {
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
        if (g_queryInfo.subQueryInfo.result[i][0] != 0) {
5684 5685
          sprintf(tmpFile, "%s-%d",
                  g_queryInfo.subQueryInfo.result[i],
5686
                  winfo->threadID);
5687 5688
        }
        getResult(res, tmpFile);
H
hzcheng 已提交
5689 5690 5691
      }
    }
  }
5692
  taos_free_result(res);
5693

5694
  for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) {
5695
    taos_unsubscribe(g_queryInfo.subQueryInfo.tsub[i],
5696
            g_queryInfo.subQueryInfo.subscribeKeepProgress);
5697
  }
H
hzcheng 已提交
5698 5699 5700
  return NULL;
}

5701
static void *superSubscribeProcess(void *sarg) {
5702
  threadInfo *winfo = (threadInfo *)sarg;
H
hzcheng 已提交
5703

5704 5705
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
5706
  debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
5707 5708 5709
  if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)) {
    return NULL;
  }
5710
 
5711 5712 5713 5714 5715
  //int64_t st = 0;
  //int64_t et = 0;
  do {
    //if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) {
    //  taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms
5716
    //  //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5717 5718 5719 5720 5721 5722 5723 5724
    //}

    //st = taosGetTimestampMs();
    char topic[32] = {0};
    for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
      sprintf(topic, "taosdemo-subscribe-%d", i);
      char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
      if (g_queryInfo.subQueryInfo.result[i][0] != 0) {
5725
        sprintf(tmpFile, "%s-%d",
5726
                g_queryInfo.superQueryInfo.result[i], winfo->threadID);
5727
      }
5728 5729 5730 5731
      g_queryInfo.superQueryInfo.tsub[i] =
          subscribeImpl(winfo->taos,
                  g_queryInfo.superQueryInfo.sql[i],
                  topic, tmpFile);
5732 5733 5734 5735 5736
      if (NULL == g_queryInfo.superQueryInfo.tsub[i]) {
        return NULL;
      }
    }
    //et = taosGetTimestampMs();
5737
    //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0);
5738 5739 5740 5741 5742 5743 5744 5745 5746
  } while (0);

  // start loop to consume result
  TAOS_RES* res = NULL;
  while (1) {
    for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
      if (1 == g_queryInfo.superQueryInfo.subscribeMode) {
        continue;
      }
5747

5748 5749 5750 5751
      res = taos_consume(g_queryInfo.superQueryInfo.tsub[i]);
      if (res) {
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
        if (g_queryInfo.superQueryInfo.result[i][0] != 0) {
5752
          sprintf(tmpFile, "%s-%d",
5753
                  g_queryInfo.superQueryInfo.result[i], winfo->threadID);
5754 5755 5756 5757 5758 5759
        }
        getResult(res, tmpFile);
      }
    }
  }
  taos_free_result(res);
5760

5761
  for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
5762
    taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i],
5763
            g_queryInfo.superQueryInfo.subscribeKeepProgress);
5764
  }
H
hzcheng 已提交
5765 5766 5767
  return NULL;
}

5768
static int subscribeTestProcess() {
5769
  printfQueryMeta();
H
hzcheng 已提交
5770

5771 5772 5773
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void)getchar();
Y
yihaoDeng 已提交
5774
  }
5775 5776

  TAOS * taos = NULL;  
5777 5778 5779 5780 5781
  taos = taos_connect(g_queryInfo.host, 
          g_queryInfo.user, 
          g_queryInfo.password, 
          g_queryInfo.dbName, 
          g_queryInfo.port);
5782
  if (taos == NULL) {
5783
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
5784 5785 5786 5787
    exit(-1);
  }

  if (0 != g_queryInfo.subQueryInfo.sqlCount) {
5788
    getAllChildNameOfSuperTable(taos, 
5789 5790 5791 5792
            g_queryInfo.dbName, 
            g_queryInfo.subQueryInfo.sTblName, 
            &g_queryInfo.subQueryInfo.childTblName, 
            &g_queryInfo.subQueryInfo.childTblCount);
5793 5794 5795 5796 5797
  }

  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from super table
5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808
  if ((g_queryInfo.superQueryInfo.sqlCount <= 0) ||
          (g_queryInfo.superQueryInfo.concurrent <= 0)) {
    errorPrint("%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n",
              __func__, __LINE__, g_queryInfo.superQueryInfo.sqlCount,
              g_queryInfo.superQueryInfo.concurrent);
    exit(-1);
  }

  pids  = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t));
  infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo));
  if ((NULL == pids) || (NULL == infos)) {
5809
      printf("malloc failed for create threads\n");
5810 5811
      taos_close(taos);
      exit(-1);
5812
  }
5813

5814
  for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) {
5815 5816 5817 5818
      threadInfo *t_info = infos + i;
      t_info->threadID = i;
      t_info->taos = taos;
      pthread_create(pids + i, NULL, superSubscribeProcess, t_info);
H
hzcheng 已提交
5819
  }
5820
 
5821 5822 5823
  //==== create sub threads for query from sub table  
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
5824
  if ((g_queryInfo.subQueryInfo.sqlCount > 0)
5825
          && (g_queryInfo.subQueryInfo.threadCnt > 0)) {
5826
    pidsOfSub  = malloc(g_queryInfo.subQueryInfo.threadCnt *
5827 5828 5829
            sizeof(pthread_t));
    infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * 
            sizeof(threadInfo));
5830
    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
5831
      printf("malloc failed for create threads\n");
5832 5833
      taos_close(taos);
      exit(-1);
S
Shuaiqiang Chang 已提交
5834
    }
5835

5836 5837
    int ntables = g_queryInfo.subQueryInfo.childTblCount;
    int threads = g_queryInfo.subQueryInfo.threadCnt;
5838
 
5839 5840 5841 5842 5843
    int a = ntables / threads;
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
5844
 
5845 5846 5847 5848
    int b = 0;
    if (threads != 0) {
      b = ntables % threads;
    }
5849
 
5850
    int startFrom = 0;
5851
    for (int i = 0; i < threads; i++) {
5852 5853
      threadInfo *t_info = infosOfSub + i;
      t_info->threadID = i;
5854

5855 5856 5857 5858
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
5859 5860
      t_info->taos = taos;
      pthread_create(pidsOfSub + i, NULL, subSubscribeProcess, t_info);
H
hzcheng 已提交
5861
    }
5862
    g_queryInfo.subQueryInfo.threadCnt = threads;
H
hzcheng 已提交
5863
  }
5864
 
5865 5866 5867
  for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) {
    pthread_join(pids[i], NULL);
  }  
H
hzcheng 已提交
5868

5869
  tmfree((char*)pids);
5870
  tmfree((char*)infos);
H
hzcheng 已提交
5871

5872 5873
  for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) {
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
5874 5875
  }

5876
  tmfree((char*)pidsOfSub);
5877
  tmfree((char*)infosOfSub);
5878 5879
  taos_close(taos);
  return 0;
H
hzcheng 已提交
5880 5881
}

5882
static void initOfInsertMeta() {
5883 5884
  memset(&g_Dbs, 0, sizeof(SDbs));
   
5885 5886 5887 5888 5889 5890 5891
  // set default values
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE);
  g_Dbs.port = 6030;
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE);
  g_Dbs.threadCount = 2;
  g_Dbs.use_metric = true;
L
Liu Tao 已提交
5892 5893
}

5894
static void initOfQueryMeta() {
5895 5896
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
   
5897 5898 5899 5900 5901
  // set default values
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE);
  g_queryInfo.port = 6030;
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
5902 5903
}

5904
static void setParaFromArg(){
5905 5906 5907
  if (g_args.host) {
    strcpy(g_Dbs.host, g_args.host);
  } else {
5908
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE);
5909
  }
L
Liu Tao 已提交
5910

5911 5912 5913 5914 5915 5916 5917
  if (g_args.user) {
    strcpy(g_Dbs.user, g_args.user);
  } 

  if (g_args.password) {
    strcpy(g_Dbs.password, g_args.password);
  } 
L
Liu Tao 已提交
5918
  
5919 5920 5921
  if (g_args.port) {
    g_Dbs.port = g_args.port;
  } 
L
Liu Tao 已提交
5922

5923 5924 5925
  g_Dbs.threadCount = g_args.num_of_threads;
  g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;

5926 5927
  g_Dbs.dbCount = 1;
  g_Dbs.db[0].drop = 1;
5928

5929
  tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE);
5930
  g_Dbs.db[0].dbCfg.replica = g_args.replica;
5931
  tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
5932

5933
  tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
5934 5935 5936 5937 5938

  g_Dbs.use_metric = g_args.use_metric;
  g_Dbs.insert_only = g_args.insert_only;

  g_Dbs.do_aggreFunc = true;
L
Liu Tao 已提交
5939

5940 5941 5942 5943
  char dataString[STRING_LEN];
  char **data_type = g_args.datatype;
  
  memset(dataString, 0, STRING_LEN);
L
Liu Tao 已提交
5944

5945 5946 5947
  if (strcasecmp(data_type[0], "BINARY") == 0 
          || strcasecmp(data_type[0], "BOOL") == 0 
          || strcasecmp(data_type[0], "NCHAR") == 0 ) {
5948
    g_Dbs.do_aggreFunc = false;
L
Liu Tao 已提交
5949
  }
H
hzcheng 已提交
5950

5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969
  if (g_args.use_metric) {
    g_Dbs.db[0].superTblCount = 1;
    tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE);
    g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
    g_Dbs.threadCount = g_args.num_of_threads;
    g_Dbs.threadCountByCreateTbl = 1;
    g_Dbs.queryMode = g_args.mode;
  
    g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
    g_Dbs.db[0].superTbls[0].superTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
    g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
    tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, 
            g_args.tb_prefix, MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, 
            "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
5970
    g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
5971
  
5972
    g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
5973
    g_Dbs.db[0].superTbls[0].maxSqlLen = TSDB_PAYLOAD_SIZE;
5974

5975 5976 5977 5978 5979 5980 5981 5982 5983
    g_Dbs.db[0].superTbls[0].columnCount = 0;
    for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
      if (data_type[i] == NULL) {
        break;
      }
  
      tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, 
              data_type[i], MAX_TB_NAME_SIZE);
      g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;    
5984 5985
      g_Dbs.db[0].superTbls[0].columnCount++;
    }
5986 5987 5988 5989 5990 5991 5992 5993 5994 5995
  
    if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
      g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
    } else {
      for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) {
        tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE);
        g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;    
        g_Dbs.db[0].superTbls[0].columnCount++;
      }
    }
5996

5997
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE);
5998 5999
    g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;    
  
6000
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE);
6001 6002 6003
    g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;    
    g_Dbs.db[0].superTbls[0].tagCount = 2;  
  } else {
6004
    g_Dbs.threadCountByCreateTbl = 1;
6005
    g_Dbs.db[0].superTbls[0].tagCount = 0; 
H
hzcheng 已提交
6006
  }
6007

6008 6009 6010 6011 6012 6013
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
6014

6015 6016 6017
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
6018 6019 6020
    exit(-1);
  }

6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

6045
static void querySqlFile(TAOS* taos, char* sqlFile)
6046 6047 6048 6049 6050 6051
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
6052

6053 6054 6055 6056 6057 6058 6059
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

  double t = getCurrentTime();
6060

6061 6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073
  while ((read_len = tgetline(&line, &line_len, fp)) != -1) {
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
6074
    }
6075 6076

    memcpy(cmd + cmd_len, line, read_len);
6077
    verbosePrint("%s() LN%d cmd: %s\n", __func__, __LINE__, cmd);
6078 6079 6080 6081 6082 6083 6084
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE)) {
        printf("queryDbExec %s failed!\n", cmd);
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
6085 6086
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
6087 6088
  }

6089 6090
  t = getCurrentTime() - t;
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
6091

6092 6093 6094 6095
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
6096 6097
}

6098
static void testMetaFile() {
6099
    if (INSERT_TEST == g_args.test_mode) {
6100
      if (g_Dbs.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
6101
      insertTestProcess();
6102
    } else if (QUERY_TEST == g_args.test_mode) {
6103 6104
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
6105
      queryTestProcess();
6106
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
6107 6108
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
6109
      subscribeTestProcess();
6110 6111 6112
    }  else {
      ;
    }
6113
}
6114

6115
static void queryResult() {
6116 6117 6118
    // select
    if (false == g_Dbs.insert_only) {
      // query data
6119

6120 6121 6122
      pthread_t read_id;
      threadInfo *rInfo = malloc(sizeof(threadInfo));
      rInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
6123
      rInfo->start_table_from = 0;
6124

6125
      //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
6126
      if (g_args.use_metric) {
6127 6128
        rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
        rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
6129 6130 6131 6132
        rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
        strcpy(rInfo->tb_prefix, 
              g_Dbs.db[0].superTbls[0].childTblPrefix);
      } else {
6133 6134
        rInfo->ntables = g_args.num_of_tables;
        rInfo->end_table_to = g_args.num_of_tables -1;
6135 6136 6137
        strcpy(rInfo->tb_prefix, g_args.tb_prefix);
      }

6138 6139 6140 6141 6142 6143
      rInfo->taos = taos_connect(
              g_Dbs.host, 
              g_Dbs.user, 
              g_Dbs.password, 
              g_Dbs.db[0].dbName, 
              g_Dbs.port);
6144
      if (rInfo->taos == NULL) {
6145
        errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
6146 6147 6148 6149
        free(rInfo);
        exit(-1);
      }

6150 6151 6152 6153 6154 6155 6156 6157 6158 6159
      strcpy(rInfo->fp, g_Dbs.resultFile);

      if (!g_Dbs.use_metric) {
        pthread_create(&read_id, NULL, readTable, rInfo);
      } else {
        pthread_create(&read_id, NULL, readMetric, rInfo);
      }
      pthread_join(read_id, NULL);
      taos_close(rInfo->taos);
      free(rInfo);
H
hzcheng 已提交
6160
    }
6161 6162
}

6163 6164
static void testCmdLine() {

6165 6166
  g_args.test_mode = INSERT_TEST;
  insertTestProcess();
6167

6168 6169 6170 6171
  if (g_Dbs.insert_only)
    return;
  else
    queryResult();
6172 6173
}

6174 6175 6176
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

6177
  debugPrint("meta file: %s\n", g_args.metaFile);
6178 6179 6180

  if (g_args.metaFile) {
    initOfInsertMeta();
6181
    initOfQueryMeta();
6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
6195 6196 6197 6198
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
6199
          g_Dbs.port);
6200
      querySqlFile(qtaos, g_args.sqlFile);
6201 6202 6203 6204 6205
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
H
hzcheng 已提交
6206
  }
6207 6208

  return 0;
H
hzcheng 已提交
6209
}
6210