taosdemo.c 216.6 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

S
slguan 已提交
21
#define _GNU_SOURCE
22 23
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
40
#else
S
TD-1057  
Shengliang Guan 已提交
41 42
  #include <regex.h>
  #include <stdio.h>
43
#endif
H
hzcheng 已提交
44

45
#include <assert.h>
46
#include <stdlib.h>
47 48
#include "cJSON.h"

49
#include "os.h"
H
hzcheng 已提交
50
#include "taos.h"
51
#include "taoserror.h"
H
Hui Li 已提交
52
#include "tutil.h"
S
slguan 已提交
53

54 55 56
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
57 58
extern char configDir[];

59 60 61 62
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

63 64 65 66 67 68
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};
69 70 71

#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
72 73
#define MAX_USERNAME_SIZE  64
#define MAX_PASSWORD_SIZE  64
74
#define MAX_DB_NAME_SIZE   64
75
#define MAX_HOSTNAME_SIZE  64
76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91
#define MAX_TB_NAME_SIZE   64
#define MAX_DATA_SIZE      16000
#define MAX_NUM_DATATYPE   10
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
#define MAX_FILE_NAME_LEN  256

#define   MAX_SAMPLES_ONCE_FROM_FILE   10000
#define   MAX_NUM_DATATYPE 10

#define   MAX_DB_COUNT           8
#define   MAX_SUPER_TABLE_COUNT  8
#define   MAX_COLUMN_COUNT       1024
#define   MAX_TAG_COUNT          128

92
#define   MAX_QUERY_SQL_COUNT    100
93 94 95
#define   MAX_QUERY_SQL_LENGTH   256

#define   MAX_DATABASE_COUNT     256
96
#define INPUT_BUF_LEN   256
97

98
#define DEFAULT_TIMESTAMP_STEP  1
99

100

101 102 103 104 105
typedef enum CREATE_SUB_TALBE_MOD_EN {
  PRE_CREATE_SUBTBL,
  AUTO_CREATE_SUBTBL,
  NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
106

107 108
typedef enum TALBE_EXISTS_EN {
  TBL_NO_EXISTS,
109
  TBL_ALREADY_EXISTS,
110 111 112 113
  TBL_EXISTS_BUTT
} TALBE_EXISTS_EN;

enum MODE {
114
  SYNC,
115 116 117
  ASYNC,
  MODE_BUT
};
118

119 120 121 122 123 124
typedef enum enum_INSERT_MODE {
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
} INSERT_MODE;

125
typedef enum enumQUERY_TYPE {
126 127 128
  NO_INSERT_TYPE,
  INSERT_TYPE, 
  QUERY_TYPE_BUT
129
} QUERY_TYPE;
130 131 132 133 134 135 136

enum _show_db_index {
  TSDB_SHOW_DB_NAME_INDEX,
  TSDB_SHOW_DB_CREATED_TIME_INDEX,
  TSDB_SHOW_DB_NTABLES_INDEX,
  TSDB_SHOW_DB_VGROUPS_INDEX,
  TSDB_SHOW_DB_REPLICA_INDEX,
137
  TSDB_SHOW_DB_QUORUM_INDEX,
138 139 140 141 142 143 144 145 146 147
  TSDB_SHOW_DB_DAYS_INDEX,
  TSDB_SHOW_DB_KEEP_INDEX,
  TSDB_SHOW_DB_CACHE_INDEX,
  TSDB_SHOW_DB_BLOCKS_INDEX,
  TSDB_SHOW_DB_MINROWS_INDEX,
  TSDB_SHOW_DB_MAXROWS_INDEX,
  TSDB_SHOW_DB_WALLEVEL_INDEX,
  TSDB_SHOW_DB_FSYNC_INDEX,
  TSDB_SHOW_DB_COMP_INDEX,
  TSDB_SHOW_DB_CACHELAST_INDEX,
148
  TSDB_SHOW_DB_PRECISION_INDEX,
149 150 151 152 153 154 155 156 157 158
  TSDB_SHOW_DB_UPDATE_INDEX,
  TSDB_SHOW_DB_STATUS_INDEX,
  TSDB_MAX_SHOW_DB
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
  TSDB_SHOW_STABLES_NAME_INDEX,
  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
  TSDB_SHOW_STABLES_COLUMNS_INDEX,
159 160
  TSDB_SHOW_STABLES_METRIC_INDEX,
  TSDB_SHOW_STABLES_UID_INDEX,
161
  TSDB_SHOW_STABLES_TID_INDEX,
162
  TSDB_SHOW_STABLES_VGID_INDEX,
163
  TSDB_MAX_SHOW_STABLES
164 165
};

166 167 168 169 170 171 172 173 174 175 176 177 178 179
enum _describe_table_index {
  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
  TSDB_MAX_DESCRIBE_METRIC
};

typedef struct {
  char field[TSDB_COL_NAME_LEN + 1];
  char type[16];
  int length;
  char note[128];
} SColDes;
H
hzcheng 已提交
180 181

/* Used by main to communicate with parse_opt. */
182 183
typedef struct SArguments_S {
  char *   metaFile;
184
  int      test_mode;
185 186 187 188 189 190 191 192 193
  char *   host;
  uint16_t port;
  char *   user;
  char *   password;
  char *   database;
  int      replica;
  char *   tb_prefix;
  char *   sqlFile;
  bool     use_metric;
194
  bool     drop_database;
195
  bool     insert_only;
196
  bool     answer_yes;
197
  bool     debug_print;
198
  bool     verbose_print;
199
  bool     performance_print;
200 201 202 203 204 205
  char *   output_file;
  int      mode;
  char *   datatype[MAX_NUM_DATATYPE + 1];
  int      len_of_binary;
  int      num_of_CPR;
  int      num_of_threads;
206
  int      insert_interval;
207
  int      query_times;
208
  int      interlace_rows;
209
  int      num_of_RPR;
210
  int      max_sql_len;
211 212 213
  int      num_of_tables;
  int      num_of_DPT;
  int      abort;
214 215
  int      disorderRatio;
  int      disorderRange;
216 217
  int      method_of_delete;
  char **  arg_list;
218 219
  int64_t  totalInsertRows;
  int64_t  totalAffectedRows;
220 221 222 223 224 225
} SArguments;

typedef struct SColumn_S {
  char  field[TSDB_COL_NAME_LEN + 1];
  char  dataType[MAX_TB_NAME_SIZE];
  int   dataLen;
226
  char  note[128];
227 228 229
} StrColumn;

typedef struct SSuperTable_S {
230
  char         sTblName[MAX_TB_NAME_SIZE+1];
231 232 233 234 235
  int          childTblCount;
  bool         childTblExists;    // 0: no, 1: yes  
  int          batchCreateTableNum;  // 0: no batch,  > 0: batch table number in one sql
  int8_t       autoCreateTable;                  // 0: create sub table, 1: auto create sub table
  char         childTblPrefix[MAX_TB_NAME_SIZE];
236
  char         dataSource[MAX_TB_NAME_SIZE+1];  // rand_gen or sample
237
  char         insertMode[MAX_TB_NAME_SIZE];  // taosc, restful
238 239
  int           childTblLimit;
  int           childTblOffset;
240 241

  int          multiThreadWriteOneTbl;   // 0: no, 1: yes
242
  int          interlaceRows;            // 
243 244 245
  int          disorderRatio;            // 0: no disorder, >0: x%
  int          disorderRange;            // ms or us by database precision
  int          maxSqlLen;                // 
246 247

  int          insertInterval;          // insert interval, will override global insert interval
248 249 250 251
  int64_t      insertRows;               // 0: no limit
  int          timeStampStep;
  char         startTimestamp[MAX_TB_NAME_SIZE];  // 
  char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
252 253
  char         sampleFile[MAX_FILE_NAME_LEN+1];
  char         tagsFile[MAX_FILE_NAME_LEN+1];
254 255 256 257 258 259 260

  int          columnCount;
  StrColumn    columns[MAX_COLUMN_COUNT];
  int          tagCount;
  StrColumn    tags[MAX_TAG_COUNT];

  char*        childTblName;
261
  char*        colsOfCreateChildTable;
262 263 264 265 266 267 268 269 270 271 272 273 274
  int          lenOfOneRow;
  int          lenOfTagOfOneRow;

  char*        sampleDataBuf;
  int          sampleDataBufSize;
  //int          sampleRowCount;
  //int          sampleUsePos;

  int          tagSource;    // 0: rand, 1: tag sample
  char*        tagDataBuf;
  int          tagSampleCount;
  int          tagUsePos;

275
  // statistics
276
  int64_t    totalInsertRows;
277 278 279 280 281 282 283
  int64_t    totalAffectedRows;
} SSuperTable;

typedef struct {
  char     name[TSDB_DB_NAME_LEN + 1];
  char     create_time[32];
  int32_t  ntables;
284
  int32_t  vgroups;
285 286
  int16_t  replica;
  int16_t  quorum;
287
  int16_t  days;
288 289 290 291 292 293 294 295 296 297 298 299 300 301
  char     keeplist[32];
  int32_t  cache; //MB
  int32_t  blocks;
  int32_t  minrows;
  int32_t  maxrows;
  int8_t   wallevel;
  int32_t  fsync;
  int8_t   comp;
  int8_t   cachelast;
  char     precision[8];   // time resolution
  int8_t   update;
  char     status[16];
} SDbInfo;

302
typedef struct SDbCfg_S {
303
//  int       maxtablesPerVnode;
304
  int       minRows;
305 306 307
  int       maxRows;
  int       comp;
  int       walLevel;
308
  int       cacheLast;
309
  int       fsync;
310 311 312 313 314 315 316
  int       replica;
  int       update;
  int       keep;
  int       days;
  int       cache;
  int       blocks;
  int       quorum;
317
  char      precision[MAX_TB_NAME_SIZE];
318 319 320 321
} SDbCfg;

typedef struct SDataBase_S {
  char         dbName[MAX_DB_NAME_SIZE];
322
  bool         drop;  // 0: use exists, 1: if exists, drop then new create
323 324 325 326 327 328
  SDbCfg       dbCfg;
  int          superTblCount;
  SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
} SDataBase;

typedef struct SDbs_S {
329
  char         cfgDir[MAX_FILE_NAME_LEN+1];
330
  char         host[MAX_HOSTNAME_SIZE];
331
  uint16_t     port;
332 333
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
334
  char         resultFile[MAX_FILE_NAME_LEN+1];
335 336 337 338
  bool         use_metric;
  bool         insert_only;
  bool         do_aggreFunc;
  bool         queryMode;
339

340 341 342 343 344 345
  int          threadCount;
  int          threadCountByCreateTbl;
  int          dbCount;
  SDataBase    db[MAX_DB_COUNT];

  // statistics
346
  int64_t    totalInsertRows;
347
  int64_t    totalAffectedRows;
348

349 350
} SDbs;

351
typedef struct SpecifiedQueryInfo_S {
352 353 354 355 356
  int          rate;  // 0: unlimit  > 0   loop/s
  int          concurrent;
  int          sqlCount;
  int          subscribeMode; // 0: sync, 1: async
  int          subscribeInterval; // ms
357
  int          queryTimes;
358 359
  int          subscribeRestart;
  int          subscribeKeepProgress;
360
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
361
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
362
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
363
} SpecifiedQueryInfo;
364

365
typedef struct SuperQueryInfo_S {
366
  char         sTblName[MAX_TB_NAME_SIZE+1];
367
  int          rate;  // 0: unlimit  > 0   loop/s
368
  int          threadCnt;
369 370 371 372
  int          subscribeMode; // 0: sync, 1: async
  int          subscribeInterval; // ms
  int          subscribeRestart;
  int          subscribeKeepProgress;
373
  int          queryTimes;
374 375 376
  int          childTblCount;
  char         childTblPrefix[MAX_TB_NAME_SIZE];
  int          sqlCount;
377
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
378
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
379
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
380

381
  char*        childTblName;
382
} SuperQueryInfo;
383 384

typedef struct SQueryMetaInfo_S {
385
  char         cfgDir[MAX_FILE_NAME_LEN+1];
386
  char         host[MAX_HOSTNAME_SIZE];
387
  uint16_t     port;
388 389
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
390
  char         dbName[MAX_DB_NAME_SIZE+1];
391 392
  char         queryMode[MAX_TB_NAME_SIZE];  // taosc, restful

393 394
  SpecifiedQueryInfo  specifiedQueryInfo;
  SuperQueryInfo    superQueryInfo;
395 396 397 398 399
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
  TAOS *taos;
  int threadID;
400
  char db_name[MAX_DB_NAME_SIZE+1];
401
  uint32_t time_precision;
402 403
  char fp[4096];
  char tb_prefix[MAX_TB_NAME_SIZE];
404 405 406
  int start_table_from;
  int end_table_to;
  int ntables;
407
  int data_of_rate;
408 409 410
  uint64_t start_time;
  char* cols;
  bool  use_metric;
411 412 413 414
  SSuperTable* superTblInfo;

  // for async insert
  tsem_t lock_sem;
415
  int64_t  counter;
416 417
  uint64_t  st;
  uint64_t  et;
418 419
  int64_t  lastTs;

420 421
  // sample data
  int samplePos;
422
  // statistics
423
  int64_t totalInsertRows;
424
  int64_t totalAffectedRows;
425 426 427 428 429 430 431

  // insert delay statistics
  int64_t cntDelay;
  int64_t totalDelay;
  int64_t avgDelay;
  int64_t maxDelay;
  int64_t minDelay;
432

433 434
} threadInfo;

435
#ifdef WINDOWS
436 437
#define _CRT_RAND_S

438
#include <windows.h>
439 440 441 442 443
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
444 445
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
446 447
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
448 449 450 451

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

452
static void setupForAnsiEscape(void) {
453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
471
  }
472 473
}

474
static void resetAfterAnsiEscape(void) {
475
  // Reset colors
476
  printf("\x1b[0m");
477 478 479 480 481 482

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
483

484
static int taosRandom()
485 486 487 488 489 490
{
    int number;
    rand_s(&number);

    return number;
}
491
#else
492
static void setupForAnsiEscape(void) {}
493

494
static void resetAfterAnsiEscape(void) {
495 496 497
  // Reset colors
  printf("\x1b[0m");
}
498

499 500
#include <time.h>

501
static int taosRandom()
502
{
503 504
  srand(time(NULL));
  return rand();
505 506
}

507 508
#endif

509 510
static int createDatabases();
static void createChildTables();
511
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet);
512 513 514 515 516 517 518

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
519
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)",
520 521
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

522
SArguments g_args = {
523 524
                     NULL,            // metaFile
                     0,               // test_mode
525 526 527
                     "127.0.0.1",     // host
                     6030,            // port
                     "root",          // user
528
                     #ifdef _TD_POWER_
529 530 531 532 533 534 535 536
                     "powerdb",      // password
                     #else
                     "taosdata",      // password
                     #endif
                     "test",          // database
                     1,               // replica
                     "t",             // tb_prefix
                     NULL,            // sqlFile
537
                     true,            // use_metric
538
                     true,            // drop_database
539
                     true,            // insert_only
540
                     false,           // debug_print
541
                     false,           // verbose_print
542
                     false,           // performance statistic print 
543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561
                     false,           // answer_yes;
                     "./output.txt",  // output_file
                     0,               // mode : sync or async
                     {
                     "TINYINT",           // datatype
                     "SMALLINT",
                     "INT",
                     "BIGINT",
                     "FLOAT",
                     "DOUBLE",
                     "BINARY",
                     "NCHAR",
                     "BOOL",
                     "TIMESTAMP"
                     },
                     16,              // len_of_binary
                     10,              // num_of_CPR
                     10,              // num_of_connections/thread
                     0,               // insert_interval
562
                     1,               // query_times
563
                     0,               // interlace_rows;
564
                     100,             // num_of_RPR
565
                     TSDB_PAYLOAD_SIZE,  // max_sql_len
566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582
                     10000,           // num_of_tables
                     10000,           // num_of_DPT
                     0,               // abort
                     0,               // disorderRatio
                     1000,            // disorderRange
                     1,               // method_of_delete
                     NULL             // arg_list
};



static SDbs            g_Dbs;
static int             g_totalChildTables = 0;
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

#define debugPrint(fmt, ...) \
583
    do { if (g_args.debug_print || g_args.verbose_print) \
584
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
585

586
#define verbosePrint(fmt, ...) \
587 588
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
589

590 591 592 593
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)

594 595 596 597
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)


598
///////////////////////////////////////////////////
H
hzcheng 已提交
599

600 601
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

602 603 604 605 606 607 608 609
#ifndef TAOSDEMO_COMMIT_SHA1
#define TAOSDEMO_COMMIT_SHA1 "unknown"
#endif

#ifndef TD_VERNUMBER
#define TD_VERNUMBER    "unknown"
#endif

610
#ifndef TAOSDEMO_STATUS
611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627
#define TAOSDEMO_STATUS "unknown"
#endif

static void printVersion() {
    char tdengine_ver[] = TD_VERNUMBER;
    char taosdemo_ver[] = TAOSDEMO_COMMIT_SHA1;
    char taosdemo_status[] = TAOSDEMO_STATUS;

    if (strlen(taosdemo_status) == 0) {
        printf("taosdemo verison %s-%s\n",
                tdengine_ver, taosdemo_ver);
    } else {
        printf("taosdemo verison %s-%s, status:%s\n",
                tdengine_ver, taosdemo_ver, taosdemo_status);
    }
}

628
static void printHelp() {
629
  char indent[10] = "        ";
630
  printf("%s%s%s%s\n", indent, "-f", indent,
631
          "The meta file to the execution procedure. Default is './meta.json'.");
632
  printf("%s%s%s%s\n", indent, "-u", indent,
633 634
          "The TDengine user name to use when connecting to the server. Default is 'root'.");
#ifdef _TD_POWER_
635
  printf("%s%s%s%s\n", indent, "-P", indent,
636
          "The password to use when connecting to the server. Default is 'powerdb'.");
637
  printf("%s%s%s%s\n", indent, "-c", indent,
638 639
          "Configuration directory. Default is '/etc/power/'.");
#else
640
  printf("%s%s%s%s\n", indent, "-P", indent,
641
          "The password to use when connecting to the server. Default is 'taosdata'.");
642
  printf("%s%s%s%s\n", indent, "-c", indent,
643 644
          "Configuration directory. Default is '/etc/taos/'.");
#endif  
645
  printf("%s%s%s%s\n", indent, "-h", indent,
646
          "The host to connect to TDengine. Default is localhost.");
647
  printf("%s%s%s%s\n", indent, "-p", indent,
648
          "The TCP/IP port number to use for the connection. Default is 0.");
649
  printf("%s%s%s%s\n", indent, "-d", indent,
650
          "Destination database. Default is 'test'.");
651
  printf("%s%s%s%s\n", indent, "-a", indent,
652
          "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
653
  printf("%s%s%s%s\n", indent, "-m", indent,
654 655
          "Table prefix name. Default is 't'.");
  printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
656
  printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
657
  printf("%s%s%s%s\n", indent, "-o", indent,
658
          "Direct output to the named file. Default is './output.txt'.");
659
  printf("%s%s%s%s\n", indent, "-q", indent,
660
          "Query mode--0: SYNC, 1: ASYNC. Default is SYNC.");
661
  printf("%s%s%s%s\n", indent, "-b", indent,
662
          "The data_type of columns, default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP.");
663
  printf("%s%s%s%s\n", indent, "-w", indent,
664
          "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
665
  printf("%s%s%s%s\n", indent, "-l", indent,
666
          "The number of columns per record. Default is 10.");
667
  printf("%s%s%s%s\n", indent, "-T", indent,
668
          "The number of threads. Default is 10.");
669
  printf("%s%s%s%s\n", indent, "-i", indent,
670
          "The sleep time (ms) between insertion. Default is 0.");
671
  printf("%s%s%s%s\n", indent, "-r", indent,
672
          "The number of records per request. Default is 100.");
673
  printf("%s%s%s%s\n", indent, "-t", indent,
674
          "The number of tables. Default is 10000.");
675
  printf("%s%s%s%s\n", indent, "-n", indent,
676 677 678
          "The number of records per table. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
  printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
679
  printf("%s%s%s%s\n", indent, "-O", indent,
680
          "Insert mode--0: In order, > 0: disorder ratio. Default is in order.");
681
  printf("%s%s%s%s\n", indent, "-R", indent,
682
          "Out of order data's range, ms, default is 1000.");
683
  printf("%s%s%s%s\n", indent, "-g", indent,
684
          "Print debug info.");
685
  printf("%s%s%s%s\n", indent, "-V, --version", indent,
686
          "Print version info.");
687 688 689 690 691
/*    printf("%s%s%s%s\n", indent, "-D", indent, 
          "if elete database if exists. 0: no, 1: yes, default is 1");
          */
}

692
static void parse_args(int argc, char *argv[], SArguments *arguments) {
693
  char **sptr;
694 695 696 697 698 699 700 701
  wordexp_t full_path;

  for (int i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-f") == 0) {
      arguments->metaFile = argv[++i];
    } else if (strcmp(argv[i], "-c") == 0) {
        char *configPath = argv[++i];
      if (wordexp(configPath, &full_path, 0) != 0) {
702
          errorPrint( "Invalid path %s\n", configPath);
703 704 705 706
          return;
      }
      taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
      wordfree(&full_path);
707

708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725
    } else if (strcmp(argv[i], "-h") == 0) {
      arguments->host = argv[++i];
    } else if (strcmp(argv[i], "-p") == 0) {
      arguments->port = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-u") == 0) {
      arguments->user = argv[++i];
    } else if (strcmp(argv[i], "-P") == 0) {
      arguments->password = argv[++i];
    } else if (strcmp(argv[i], "-o") == 0) {
      arguments->output_file = argv[++i];
    } else if (strcmp(argv[i], "-s") == 0) {
      arguments->sqlFile = argv[++i];
    } else if (strcmp(argv[i], "-q") == 0) {
      arguments->mode = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-T") == 0) {
      arguments->num_of_threads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
      arguments->insert_interval = atoi(argv[++i]);
726 727
    } else if (strcmp(argv[i], "-qt") == 0) {
      arguments->query_times = atoi(argv[++i]);
728
    } else if (strcmp(argv[i], "-B") == 0) {
729
      arguments->interlace_rows = atoi(argv[++i]);
730 731 732 733 734 735 736 737 738 739 740
    } else if (strcmp(argv[i], "-r") == 0) {
      arguments->num_of_RPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-t") == 0) {
      arguments->num_of_tables = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
      arguments->num_of_DPT = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-d") == 0) {
      arguments->database = argv[++i];
    } else if (strcmp(argv[i], "-l") == 0) {
      arguments->num_of_CPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-b") == 0) {
741
      sptr = arguments->datatype;
742 743
      ++i;
      if (strstr(argv[i], ",") == NULL) {
744
        // only one col
745 746 747 748 749 750 751 752 753 754
        if (strcasecmp(argv[i], "INT")
                && strcasecmp(argv[i], "FLOAT")
                && strcasecmp(argv[i], "TINYINT")
                && strcasecmp(argv[i], "BOOL")
                && strcasecmp(argv[i], "SMALLINT")
                && strcasecmp(argv[i], "BIGINT")
                && strcasecmp(argv[i], "DOUBLE")
                && strcasecmp(argv[i], "BINARY") 
                && strcasecmp(argv[i], "NCHAR")) {
          printHelp();
755
          ERROR_EXIT( "Invalid data_type!\n");
756
          exit(EXIT_FAILURE);
757
        }
758
        sptr[0] = argv[i];
759
      } else {
760
        // more than one col
761
        int index = 0;
762
        char *dupstr = strdup(argv[i]);
763 764 765
        char *running = dupstr;
        char *token = strsep(&running, ",");
        while (token != NULL) {
766 767 768 769 770 771 772 773 774 775
          if (strcasecmp(token, "INT")
                  && strcasecmp(token, "FLOAT")
                  && strcasecmp(token, "TINYINT")
                  && strcasecmp(token, "BOOL")
                  && strcasecmp(token, "SMALLINT")
                  && strcasecmp(token, "BIGINT")
                  && strcasecmp(token, "DOUBLE")
                  && strcasecmp(token, "BINARY")
                  && strcasecmp(token, "NCHAR")) {
            printHelp();
776
            ERROR_EXIT("Invalid data_type!\n");
777
            exit(EXIT_FAILURE);
H
hzcheng 已提交
778
          }
779 780 781
          sptr[index++] = token;
          token = strsep(&running, ",");
          if (index >= MAX_NUM_DATATYPE) break;
S
TD-1057  
Shengliang Guan 已提交
782
        }
783
        sptr[index] = NULL;
784
      }
785 786 787 788
    } else if (strcmp(argv[i], "-w") == 0) {
      arguments->len_of_binary = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-m") == 0) {
      arguments->tb_prefix = argv[++i];
789 790
    } else if (strcmp(argv[i], "-N") == 0) {
      arguments->use_metric = false;
791
    } else if (strcmp(argv[i], "-x") == 0) {
792
      arguments->insert_only = false;
793
    } else if (strcmp(argv[i], "-y") == 0) {
794
      arguments->answer_yes = true;
795 796
    } else if (strcmp(argv[i], "-g") == 0) {
      arguments->debug_print = true;
797 798
    } else if (strcmp(argv[i], "-gg") == 0) {
      arguments->verbose_print = true;
799 800
    } else if (strcmp(argv[i], "-pp") == 0) {
      arguments->performance_print = true;
801 802 803 804
    } else if (strcmp(argv[i], "-c") == 0) {
      strcpy(configDir, argv[++i]);
    } else if (strcmp(argv[i], "-O") == 0) {
      arguments->disorderRatio = atoi(argv[++i]);
805
      if (arguments->disorderRatio > 1
806 807 808 809
              || arguments->disorderRatio < 0) {
        arguments->disorderRatio = 0;
      } else if (arguments->disorderRatio == 1) {
        arguments->disorderRange = 10;
810
      }
811 812
    } else if (strcmp(argv[i], "-R") == 0) {
      arguments->disorderRange = atoi(argv[++i]);
813 814
      if (arguments->disorderRange == 1
              && (arguments->disorderRange > 50
815 816
              || arguments->disorderRange <= 0)) {
        arguments->disorderRange = 10;
817
      }
818 819 820 821
    } else if (strcmp(argv[i], "-a") == 0) {
      arguments->replica = atoi(argv[++i]);
      if (arguments->replica > 3 || arguments->replica < 1) {
          arguments->replica = 1;
822
      }
823 824 825 826 827
    } else if (strcmp(argv[i], "-D") == 0) {
      arguments->method_of_delete = atoi(argv[++i]);
      if (arguments->method_of_delete < 0
              || arguments->method_of_delete > 3) {
        arguments->method_of_delete = 0;
S
TD-1057  
Shengliang Guan 已提交
828
      }
829 830 831 832
    } else if ((strcmp(argv[i], "--version") == 0) ||
        (strcmp(argv[i], "-V") == 0)){
      printVersion();
      exit(0);
833 834 835 836 837
    } else if (strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else {
      printHelp();
838
      ERROR_EXIT("ERROR: wrong options\n");
839
      exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
840 841
    }
  }
842

843 844
  if (((arguments->debug_print) && (arguments->metaFile == NULL))
          || arguments->verbose_print) {
845
    printf("###################################################################\n");
846
    printf("# meta file:                         %s\n", arguments->metaFile);
847
    printf("# Server IP:                         %s:%hu\n",
848 849 850 851 852
            arguments->host == NULL ? "localhost" : arguments->host,
            arguments->port );
    printf("# User:                              %s\n", arguments->user);
    printf("# Password:                          %s\n", arguments->password);
    printf("# Use metric:                        %s\n", arguments->use_metric ? "true" : "false");
853 854 855 856 857 858 859 860 861
    if (*(arguments->datatype)) {
        printf("# Specified data type:               ");
        for (int i = 0; i < MAX_NUM_DATATYPE; i++)
            if (arguments->datatype[i])
               printf("%s,", arguments->datatype[i]);
            else
                break;
        printf("\n");
    }
862
    printf("# Insertion interval:                %d\n", arguments->insert_interval);
863
    printf("# Number of records per req:         %d\n", arguments->num_of_RPR);
864
    printf("# Max SQL length:                    %d\n", arguments->max_sql_len);
865
    printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
866 867 868 869 870 871 872 873
    printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
    printf("# Number of Tables:                  %d\n", arguments->num_of_tables);
    printf("# Number of Data per Table:          %d\n", arguments->num_of_DPT);
    printf("# Database name:                     %s\n", arguments->database);
    printf("# Table prefix:                      %s\n", arguments->tb_prefix);
    if (arguments->disorderRatio) {
      printf("# Data order:                        %d\n", arguments->disorderRatio);
      printf("# Data out of order rate:            %d\n", arguments->disorderRange);
874

875 876 877 878
    }
    printf("# Delete method:                     %d\n", arguments->method_of_delete);
    printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
    printf("# Print debug info:                  %d\n", arguments->debug_print);
879
    printf("# Print verbose info:                %d\n", arguments->verbose_print);
880 881 882 883 884 885
    printf("###################################################################\n");
    if (!arguments->answer_yes) {
        printf("Press enter key to continue\n\n");
        (void) getchar();
    }
  }
886
}
H
hzcheng 已提交
887

888 889 890 891
static bool getInfoFromJsonFile(char* file);
//static int generateOneRowDataForStb(SSuperTable* stbInfo);
//static int getDataIntoMemForStb(SSuperTable* stbInfo);
static void init_rand_data();
892
static void tmfclose(FILE *fp) {
893 894 895 896
  if (NULL != fp) {
    fclose(fp);
  }
}
897

898
static void tmfree(char *buf) {
899 900
  if (NULL != buf) {
    free(buf);
H
Hui Li 已提交
901
  }
902
}
H
Hui Li 已提交
903

904
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
905 906 907
  int i;
  TAOS_RES *res = NULL;
  int32_t   code = -1;
908

909 910 911 912
  for (i = 0; i < 5; i++) {
    if (NULL != res) {
      taos_free_result(res);
      res = NULL;
H
hzcheng 已提交
913
    }
914

915 916 917 918
    res = taos_query(taos, command);
    code = taos_errno(res);
    if (0 == code) {
      break;
919
    }
H
hzcheng 已提交
920 921
  }

922
  if (code != 0) {
923 924 925 926
    if (!quiet) {
      debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
      errorPrint("Failed to run %s, reason: %s\n", command, taos_errstr(res));
    }
927 928 929
    taos_free_result(res);
    //taos_close(taos);
    return -1;
930
  }
H
hzcheng 已提交
931

932 933 934 935
  if (INSERT_TYPE == type) {
    int affectedRows = taos_affected_rows(res);
    taos_free_result(res);
    return affectedRows;
H
hzcheng 已提交
936
  }
937

938
  taos_free_result(res);
939 940
  return 0;
}
H
hzcheng 已提交
941

942
static void getResult(TAOS_RES *res, char* resultFileName) {
943 944 945 946 947 948 949 950 951
  TAOS_ROW    row = NULL;
  int         num_rows = 0;
  int         num_fields = taos_field_count(res);
  TAOS_FIELD *fields     = taos_fetch_fields(res);

  FILE *fp = NULL;
  if (resultFileName[0] != 0) {
    fp = fopen(resultFileName, "at");
    if (fp == NULL) {
952 953
      errorPrint("%s() LN%d, failed to open result file: %s, result will not save to file\n",
              __func__, __LINE__, resultFileName);
954 955
    }
  }
956

957 958
  char* databuf = (char*) calloc(1, 100*1024*1024);
  if (databuf == NULL) {
959 960
    errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n",
            __func__, __LINE__);
961 962
    if (fp)
        fclose(fp);
963 964
    return ;
  }
H
hzcheng 已提交
965

966 967
  int   totalLen = 0;
  char  temp[16000];
H
hzcheng 已提交
968

969 970 971 972 973 974
  // fetch the records row by row
  while ((row = taos_fetch_row(res))) {
    if (totalLen >= 100*1024*1024 - 32000) {
      if (fp) fprintf(fp, "%s", databuf);
      totalLen = 0;
      memset(databuf, 0, 100*1024*1024);
H
hzcheng 已提交
975
    }
976 977 978 979 980 981
    num_rows++;
    int len = taos_print_row(temp, row, fields, num_fields);
    len += sprintf(temp + len, "\n");
    //printf("query result:%s\n", temp);
    memcpy(databuf + totalLen, temp, len);
    totalLen += len;
H
hzcheng 已提交
982 983
  }

984 985 986 987
  if (fp) fprintf(fp, "%s", databuf);
  tmfclose(fp);
  free(databuf);
}
H
hzcheng 已提交
988

989 990 991 992 993 994
static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) {
  TAOS_RES *res = taos_query(taos, command);
  if (res == NULL || taos_errno(res) != 0) {
    printf("failed to sql:%s, reason:%s\n", command, taos_errstr(res));
    taos_free_result(res);
    return;
H
hzcheng 已提交
995
  }
996

997 998 999
  getResult(res, resultFileName);
  taos_free_result(res);
}
H
hzcheng 已提交
1000

1001
static double getCurrentTime() {
1002 1003 1004 1005
  struct timeval tv;
  if (gettimeofday(&tv, NULL) != 0) {
    perror("Failed to get current time in ms");
    return 0.0;
H
hzcheng 已提交
1006
  }
S
Shuduo Sang 已提交
1007

1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049
  return tv.tv_sec + tv.tv_usec / 1E6;
}

static int32_t rand_bool(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 2;
}

static int32_t rand_tinyint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 128;
}

static int32_t rand_smallint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 32767;
}

static int32_t rand_int(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor];
}

static int64_t rand_bigint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randbigint[cursor];
}

static float rand_float(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
1050
  return randfloat[cursor];
1051 1052 1053
}

static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1054
static void rand_string(char *str, int size) {
1055 1056 1057 1058
  str[0] = 0;
  if (size > 0) {
    //--size;
    int n;
1059
    for (n = 0; n < size - 1; n++) {
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
      int key = rand_tinyint() % (int)(sizeof(charset) - 1);
      str[n] = charset[key];
    }
    str[n] = 0;
  }
}

static double rand_double() {
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randdouble[cursor];

}

static void init_rand_data() {
  for (int i = 0; i < MAX_PREPARED_RAND; i++){
1077 1078 1079 1080
    randint[i] = (int)(taosRandom() % 65535);
    randbigint[i] = (int64_t)(taosRandom() % 2147483648);
    randfloat[i] = (float)(taosRandom() / 1000.0);
    randdouble[i] = (double)(taosRandom() / 1000000.0);
1081 1082 1083
  }
}

1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1104
static int printfInsertMeta() {
1105 1106
    SHOW_PARSE_RESULT_START();

1107 1108 1109 1110 1111 1112
  printf("host:                       \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port);
  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
  printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl);
1113
  printf("top insert interval:        \033[33m%d\033[0m\n", g_args.insert_interval);
1114
  printf("number of records per req:  \033[33m%d\033[0m\n", g_args.num_of_RPR);
1115
  printf("max sql length:             \033[33m%d\033[0m\n", g_args.max_sql_len);
1116 1117

  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
1118

1119 1120
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    printf("database[\033[33m%d\033[0m]:\n", i);
1121
    printf("  database[%d] name:      \033[33m%s\033[0m\n", i, g_Dbs.db[i].dbName);
1122
    if (0 == g_Dbs.db[i].drop) {
1123 1124 1125
      printf("  drop:                  \033[33mno\033[0m\n");
    } else {
      printf("  drop:                  \033[33myes\033[0m\n");
1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      printf("  blocks:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      printf("  cache:                 \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      printf("  days:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      printf("  keep:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      printf("  replica:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      printf("  update:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      printf("  minRows:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      printf("  maxRows:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      printf("  walLevel:              \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      printf("  fsync:                 \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      printf("  quorum:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1165 1166
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1167 1168
        printf("  precision:             \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision);
      } else {
1169 1170 1171
        printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
                g_Dbs.db[i].dbCfg.precision);
        return -1;
1172 1173 1174 1175 1176 1177
      }
    }

    printf("  super table count:     \033[33m%d\033[0m\n", g_Dbs.db[i].superTblCount);
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      printf("  super table[\033[33m%d\033[0m]:\n", j);
1178 1179

      printf("      stbName:           \033[33m%s\033[0m\n",  g_Dbs.db[i].superTbls[j].sTblName);
1180 1181 1182 1183 1184 1185 1186 1187

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
      }
1188

1189 1190 1191 1192 1193 1194 1195
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
      }
1196 1197 1198 1199 1200 1201 1202 1203

      printf("      childTblCount:     \033[33m%d\033[0m\n",  g_Dbs.db[i].superTbls[j].childTblCount);
      printf("      childTblPrefix:    \033[33m%s\033[0m\n",  g_Dbs.db[i].superTbls[j].childTblPrefix);
      printf("      dataSource:        \033[33m%s\033[0m\n",  g_Dbs.db[i].superTbls[j].dataSource);
      printf("      insertMode:        \033[33m%s\033[0m\n",  g_Dbs.db[i].superTbls[j].insertMode);
      if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
        printf("      childTblLimit:     \033[33m%d\033[0m\n",  g_Dbs.db[i].superTbls[j].childTblLimit);
      }
1204
      if (g_Dbs.db[i].superTbls[j].childTblOffset >= 0) {
1205 1206 1207
        printf("      childTblOffset:    \033[33m%d\033[0m\n",  g_Dbs.db[i].superTbls[j].childTblOffset);
      }
      printf("      insertRows:        \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows);
1208 1209

      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1210
        printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
1211
      }else {
1212
        printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
1213
      }
1214 1215
      printf("      interlaceRows:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].interlaceRows);
1216 1217 1218 1219 1220 1221

      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
        printf("      stable insert interval:   \033[33m%d\033[0m\n",
            g_Dbs.db[i].superTbls[j].insertInterval);
      }

1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
      printf("      disorderRange:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      printf("      disorderRatio:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
      printf("      maxSqlLen:         \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].maxSqlLen);
      printf("      timeStampStep:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].timeStampStep);
      printf("      startTimestamp:    \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
      printf("      sampleFormat:      \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      printf("      sampleFile:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      printf("      tagsFile:          \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
      printf("      columnCount:       \033[33m%d\033[0m\n        ",
              g_Dbs.db[i].superTbls[j].columnCount);
1240 1241
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1242 1243 1244 1245
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "nchar", 5))) {
1246
          printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
1247 1248
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1249
        } else {
1250 1251
          printf("column[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType);
1252 1253 1254
        }
      }
      printf("\n");
1255 1256 1257

      printf("      tagCount:            \033[33m%d\033[0m\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1258 1259
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1260 1261 1262
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) {
          printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
1263 1264
                  g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1265
        } else {
1266 1267
          printf("tag[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].tags[k].dataType);
1268
        }
1269 1270 1271 1272 1273
      }
      printf("\n");
    }
    printf("\n");
  }
1274 1275

  SHOW_PARSE_RESULT_END();
1276 1277

  return 0;
1278 1279 1280
}

static void printfInsertMetaToFile(FILE* fp) {
1281 1282

  SHOW_PARSE_RESULT_START_TO_FILE(fp);
1283

1284 1285 1286 1287 1288
  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
  fprintf(fp, "user:                       %s\n", g_Dbs.user);
  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
1289 1290
  fprintf(fp, "number of records per req:  %d\n", g_args.num_of_RPR);
  fprintf(fp, "max sql length:             %d\n", g_args.max_sql_len);
1291
  fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
1292

1293 1294
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    fprintf(fp, "database[%d]:\n", i);
1295
    fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
1296
    if (0 == g_Dbs.db[i].drop) {
1297
      fprintf(fp, "  drop:                  no\n");
1298
    }else {
1299
      fprintf(fp, "  drop:                  yes\n");
1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1339 1340
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1341 1342 1343 1344 1345 1346 1347 1348 1349
        fprintf(fp, "  precision:             %s\n", g_Dbs.db[i].dbCfg.precision);
      } else {
        fprintf(fp, "  precision error:       %s\n", g_Dbs.db[i].dbCfg.precision);
      }
    }

    fprintf(fp, "  super table count:     %d\n", g_Dbs.db[i].superTblCount);
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      fprintf(fp, "  super table[%d]:\n", j);
1350 1351

      fprintf(fp, "      stbName:           %s\n",  g_Dbs.db[i].superTbls[j].sTblName);
1352 1353 1354 1355 1356 1357 1358 1359

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
      } else {
        fprintf(fp, "      autoCreateTable:   %s\n",  "error");
      }
1360

1361 1362 1363 1364 1365 1366 1367
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "yes");
      } else {
        fprintf(fp, "      childTblExists:    %s\n",  "error");
      }
1368

1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380
      fprintf(fp, "      childTblCount:     %d\n",
              g_Dbs.db[i].superTbls[j].childTblCount);
      fprintf(fp, "      childTblPrefix:    %s\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      fprintf(fp, "      dataSource:        %s\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      fprintf(fp, "      insertMode:        %s\n",
              g_Dbs.db[i].superTbls[j].insertMode);
      fprintf(fp, "      insertRows:        %"PRId64"\n",
              g_Dbs.db[i].superTbls[j].insertRows);
      fprintf(fp, "      interlace rows:    %d\n",
              g_Dbs.db[i].superTbls[j].interlaceRows);
1381
      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1382 1383
        fprintf(fp, "      stable insert interval:   %d\n",
                g_Dbs.db[i].superTbls[j].insertInterval);
1384
      }
1385 1386

      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1387
        fprintf(fp, "      multiThreadWriteOneTbl:  no\n");
1388
      }else {
1389
        fprintf(fp, "      multiThreadWriteOneTbl:  yes\n");
1390
      }
1391 1392
      fprintf(fp, "      interlaceRows:     %d\n",
              g_Dbs.db[i].superTbls[j].interlaceRows);
1393
      fprintf(fp, "      disorderRange:     %d\n",  g_Dbs.db[i].superTbls[j].disorderRange);
1394
      fprintf(fp, "      disorderRatio:     %d\n",  g_Dbs.db[i].superTbls[j].disorderRatio);
1395 1396 1397 1398
      fprintf(fp, "      maxSqlLen:         %d\n",  g_Dbs.db[i].superTbls[j].maxSqlLen);

      fprintf(fp, "      timeStampStep:     %d\n",  g_Dbs.db[i].superTbls[j].timeStampStep);
      fprintf(fp, "      startTimestamp:    %s\n",  g_Dbs.db[i].superTbls[j].startTimestamp);
1399
      fprintf(fp, "      sampleFormat:      %s\n",  g_Dbs.db[i].superTbls[j].sampleFormat);
1400 1401 1402
      fprintf(fp, "      sampleFile:        %s\n",  g_Dbs.db[i].superTbls[j].sampleFile);
      fprintf(fp, "      tagsFile:          %s\n",  g_Dbs.db[i].superTbls[j].tagsFile);

1403 1404 1405
      fprintf(fp, "      columnCount:       %d\n        ",  g_Dbs.db[i].superTbls[j].columnCount);
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1406 1407 1408 1409 1410 1411 1412 1413
        if ((0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "nchar", strlen("nchar")))) {
          fprintf(fp, "column[%d]:%s(%d) ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1414 1415 1416 1417 1418
        } else {
          fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
        }
      }
      fprintf(fp, "\n");
1419 1420 1421

      fprintf(fp, "      tagCount:            %d\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1422 1423
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1424 1425 1426 1427 1428 1429
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
          fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1430 1431
        } else {
          fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
1432
        }
1433 1434 1435 1436 1437
      }
      fprintf(fp, "\n");
    }
    fprintf(fp, "\n");
  }
1438

1439
  SHOW_PARSE_RESULT_END_TO_FILE(fp);
1440 1441 1442
}

static void printfQueryMeta() {
1443

1444
  SHOW_PARSE_RESULT_START();
1445

1446 1447
  printf("host:                    \033[33m%s:%u\033[0m\n",
          g_queryInfo.host, g_queryInfo.port);
1448 1449 1450 1451
  printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
  printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

  printf("\n");
1452
  printf("specified table query info:                   \n");
1453
  printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.rate);
1454
  printf("top query times:\033[33m%d\033[0m\n", g_args.query_times);
1455 1456
  printf("concurrent:     \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.concurrent);
  printf("sqlCount:       \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.sqlCount);
1457 1458
  printf("specified tbl query times:\n");
  printf("                \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.queryTimes);
1459

1460
  if (SUBSCRIBE_TEST == g_args.test_mode) {
1461 1462 1463 1464
    printf("mod:            \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeMode);
    printf("interval:       \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeInterval);
    printf("restart:        \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeRestart);
    printf("keepProgress:   \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
1465 1466
  }

1467 1468
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
    printf("  sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.specifiedQueryInfo.sql[i]);
1469 1470
  }
  printf("\n");
1471
  printf("super table query info:                   \n");
1472 1473 1474 1475
  printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.rate);
  printf("threadCnt:      \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.threadCnt);
  printf("childTblCount:  \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.childTblCount);
  printf("stable name:    \033[33m%s\033[0m\n", g_queryInfo.superQueryInfo.sTblName);
1476
  printf("stb query times:\033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.queryTimes);
1477

1478
  if (SUBSCRIBE_TEST == g_args.test_mode) {
1479 1480 1481 1482
    printf("mod:            \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeMode);
    printf("interval:       \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeInterval);
    printf("restart:        \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeRestart);
    printf("keepProgress:   \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeKeepProgress);
1483
  }
1484

1485 1486 1487
  printf("sqlCount:       \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.sqlCount);
  for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
    printf("  sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.superQueryInfo.sql[i]);
1488
  }
1489
  printf("\n");
1490

1491
  SHOW_PARSE_RESULT_END();
1492 1493
}

1494
static char* formatTimestamp(char* buf, int64_t val, int precision) {
1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525
  time_t tt;
  if (precision == TSDB_TIME_PRECISION_MICRO) {
    tt = (time_t)(val / 1000000);
  } else {
    tt = (time_t)(val / 1000);
  }

/* comment out as it make testcases like select_with_tags.sim fail.
  but in windows, this may cause the call to localtime crash if tt < 0,
  need to find a better solution.
  if (tt < 0) {
    tt = 0;
  }
  */

#ifdef WINDOWS
  if (tt < 0) tt = 0;
#endif

  struct tm* ptm = localtime(&tt);
  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);

  if (precision == TSDB_TIME_PRECISION_MICRO) {
    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
  } else {
    sprintf(buf + pos, ".%03d", (int)(val % 1000));
  }

  return buf;
}

1526 1527 1528
static void xDumpFieldToFile(FILE* fp, const char* val,
        TAOS_FIELD* field, int32_t length, int precision) {

1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563
  if (val == NULL) {
    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
    return;
  }

  char buf[TSDB_MAX_BYTES_PER_ROW];
  switch (field->type) {
    case TSDB_DATA_TYPE_BOOL:
      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
      break;
    case TSDB_DATA_TYPE_TINYINT:
      fprintf(fp, "%d", *((int8_t *)val));
      break;
    case TSDB_DATA_TYPE_SMALLINT:
      fprintf(fp, "%d", *((int16_t *)val));
      break;
    case TSDB_DATA_TYPE_INT:
      fprintf(fp, "%d", *((int32_t *)val));
      break;
    case TSDB_DATA_TYPE_BIGINT:
      fprintf(fp, "%" PRId64, *((int64_t *)val));
      break;
    case TSDB_DATA_TYPE_FLOAT:
      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
      break;
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      memcpy(buf, val, length);
      buf[length] = 0;
      fprintf(fp, "\'%s\'", buf);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
1564
      formatTimestamp(buf, *(int64_t*)val, precision);
1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579
      fprintf(fp, "'%s'", buf);
      break;
    default:
      break;
  }
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  FILE* fp = fopen(fname, "at");
  if (fp == NULL) {
1580
    errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594
    return -1;
  }

  int num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int precision = taos_result_precision(tres);

  for (int col = 0; col < num_fields; col++) {
    if (col > 0) {
      fprintf(fp, ",");
    }
    fprintf(fp, "%s", fields[col].name);
  }
  fputc('\n', fp);
1595

1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615
  int numOfRows = 0;
  do {
    int32_t* length = taos_fetch_lengths(tres);
    for (int i = 0; i < num_fields; i++) {
      if (i > 0) {
        fputc(',', fp);
      }
      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
    }
    fputc('\n', fp);

    numOfRows++;
    row = taos_fetch_row(tres);
  } while( row != NULL);

  fclose(fp);

  return numOfRows;
}

1616 1617
static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
  TAOS_RES * res;
1618 1619
  TAOS_ROW row = NULL;
  int count = 0;
1620 1621

  res = taos_query(taos, "show databases;");
1622
  int32_t code = taos_errno(res);
1623

1624
  if (code != 0) {
1625
    errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636
    return -1;
  }

  TAOS_FIELD *fields = taos_fetch_fields(res);

  while ((row = taos_fetch_row(res)) != NULL) {
    // sys database name : 'log'
    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) continue;

    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
    if (dbInfos[count] == NULL) {
1637
      errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
1638 1639 1640
      return -1;
    }

1641 1642
    tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
            fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
1643
    formatTimestamp(dbInfos[count]->create_time,
1644 1645
            *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
            TSDB_TIME_PRECISION_MILLI);
1646
    dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
1647
    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
1648 1649
    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
1650
    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
1651

1652
    tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
1653
            fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
1654 1655 1656 1657 1658 1659 1660 1661 1662
    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
    dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));

1663
    tstrncpy(dbInfos[count]->precision,
1664
            (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
1665
            fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
1666
    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
1667
    tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
1668
            fields[TSDB_SHOW_DB_STATUS_INDEX].bytes);
1669

1670 1671
    count++;
    if (count > MAX_DATABASE_COUNT) {
1672
      errorPrint( "The database count overflow than %d\n", MAX_DATABASE_COUNT);
1673 1674 1675 1676 1677 1678 1679 1680
      break;
    }
  }

  return count;
}

static void printfDbInfoForQueryToFile(char* filename, SDbInfo* dbInfos, int index) {
1681
  if (filename[0] == 0)
1682
      return;
1683 1684 1685

  FILE *fp = fopen(filename, "at");
  if (fp == NULL) {
1686
    errorPrint( "failed to open file: %s\n", filename);
1687
    return;
1688 1689 1690 1691 1692 1693
  }

  fprintf(fp, "================ database[%d] ================\n", index);
  fprintf(fp, "name: %s\n", dbInfos->name);
  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
  fprintf(fp, "ntables: %d\n", dbInfos->ntables);
1694
  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);
1695 1696
  fprintf(fp, "replica: %d\n", dbInfos->replica);
  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
1697 1698
  fprintf(fp, "days: %d\n", dbInfos->days);
  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);
1699 1700 1701 1702 1703 1704 1705
  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
  fprintf(fp, "comp: %d\n", dbInfos->comp);
1706 1707
  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);
  fprintf(fp, "precision: %s\n", dbInfos->precision);
1708
  fprintf(fp, "update: %d\n", dbInfos->update);
1709
  fprintf(fp, "status: %s\n", dbInfos->status);
1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723
  fprintf(fp, "\n");

  fclose(fp);
}

static void printfQuerySystemInfo(TAOS * taos) {
  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
  TAOS_RES* res;

  time_t t;
  struct tm* lt;
  time(&t);
  lt = localtime(&t);
1724 1725 1726 1727
  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
          lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
          lt->tm_sec);

1728 1729 1730 1731 1732 1733 1734 1735 1736
  // show variables
  res = taos_query(taos, "show variables;");
  //getResult(res, filename);
  xDumpResultToFile(filename, res);

  // show dnodes
  res = taos_query(taos, "show dnodes;");
  xDumpResultToFile(filename, res);
  //getResult(res, filename);
1737

1738 1739 1740 1741
  // show databases
  res = taos_query(taos, "show databases;");
  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
  if (dbInfos == NULL) {
1742
    errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
1743 1744 1745
    return;
  }
  int dbCount = getDbFromServer(taos, dbInfos);
1746 1747 1748 1749
  if (dbCount <= 0) {
      free(dbInfos);
      return;
  }
1750 1751 1752 1753

  for (int i = 0; i < dbCount; i++) {
    // printf database info 
    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
1754

1755 1756 1757 1758
    // show db.vgroups
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);
1759

1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770
    // show db.stables
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);

    free(dbInfos[i]);
  }

  free(dbInfos);
}

1771
static int postProceSql(char* host, uint16_t port, char* sqlstr)
1772
{
1773
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
1774 1775 1776 1777 1778

    char *url = "/rest/sql";

    struct hostent *server;
    struct sockaddr_in serv_addr;
1779
    int bytes, sent, received, req_str_len, resp_len;
1780 1781 1782
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
1783

1784 1785 1786 1787
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;

    request_buf = malloc(req_buf_len);
    if (NULL == request_buf)
1788
        ERROR_EXIT("ERROR, cannot allocate memory.");
1789

1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};

    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
1801

1802 1803 1804 1805 1806 1807
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);

    char base64_buf[INPUT_BUF_LEN];
1808
#ifdef WINDOWS
1809
    WSADATA wsaData;
1810
    WSAStartup(MAKEWORD(2, 2), &wsaData);
1811 1812 1813 1814
    SOCKET sockfd;
#else
    int sockfd;
#endif
1815
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
1816
    if (sockfd < 0) {
1817
#ifdef WINDOWS
1818
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
1819
#endif
1820
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
1821
        free(request_buf);
1822
        ERROR_EXIT("ERROR opening socket");
1823
    }
1824 1825

    server = gethostbyname(host);
1826 1827
    if (server == NULL) {
        free(request_buf);
1828
        ERROR_EXIT("ERROR, no such host");
1829
    }
1830

1831
    debugPrint("h_name: %s\nh_addretype: %s\nh_length: %d\n",
1832 1833 1834 1835
            server->h_name,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

1836
    memset(&serv_addr, 0, sizeof(serv_addr));
1837
    serv_addr.sin_family = AF_INET;
1838
    serv_addr.sin_port = htons(rest_port);
1839 1840 1841
#ifdef WINDOWS
    serv_addr.sin_addr.s_addr = inet_addr(host);
#else
1842
    memcpy(&serv_addr.sin_addr.s_addr,server->h_addr,server->h_length);
1843
#endif
1844

1845 1846 1847
    int retConn = connect(sockfd,(struct sockaddr *)&serv_addr,sizeof(serv_addr));
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
1848
        free(request_buf);
1849
        ERROR_EXIT("ERROR connecting");
1850
    }
1851

1852 1853 1854
    memset(base64_buf, 0, INPUT_BUF_LEN);

    for (int n = 0, m = 0; n < userpass_buf_len;) {
1855
      uint32_t oct_a = n < userpass_buf_len ?
1856
        (unsigned char) userpass_buf[n++]:0;
1857
      uint32_t oct_b = n < userpass_buf_len ?
1858
        (unsigned char) userpass_buf[n++]:0;
1859
      uint32_t oct_c = n < userpass_buf_len ?
1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;

      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }

    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';

    debugPrint("%s() LN%d: auth string base64 encoded: %s\n", __func__, __LINE__, base64_buf);
    char *auth = base64_buf;

1875 1876 1877
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
1878 1879 1880 1881 1882
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
1883
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
1884

1885
    req_str_len = strlen(request_buf);
1886 1887
    sent = 0;
    do {
1888 1889 1890
#ifdef WINDOWS
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
#else
1891
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
1892
#endif
1893
        if (bytes < 0)
1894
            ERROR_EXIT("ERROR writing message to socket");
1895 1896 1897
        if (bytes == 0)
            break;
        sent+=bytes;
1898
    } while (sent < req_str_len);
1899

1900 1901
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
1902 1903
    received = 0;
    do {
1904 1905 1906
#ifdef WINDOWS
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
#else
1907
        bytes = read(sockfd, response_buf + received, resp_len - received);
1908
#endif
1909 1910
        if (bytes < 0) {
            free(request_buf);
1911
            ERROR_EXIT("ERROR reading response from socket");
1912
        }
1913 1914
        if (bytes == 0)
            break;
1915 1916
        received += bytes;
    } while (received < resp_len);
1917

1918 1919
    if (received == resp_len) {
        free(request_buf);
1920
        ERROR_EXIT("ERROR storing complete response from socket");
1921
    }
1922

1923
    response_buf[RESP_BUF_LEN - 1] = '\0';
1924 1925 1926
    printf("Response:\n%s\n", response_buf);

    free(request_buf);
1927 1928 1929 1930
#ifdef WINDOWS
    closesocket(sockfd);
    WSACleanup();
#else
1931
    close(sockfd);
1932
#endif
1933 1934 1935 1936

    return 0;
}

1937
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
1938 1939
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
1940
    errorPrint("%s() LN%d, calloc failed! size:%d\n", __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
1941 1942
    return NULL;
  }
1943

1944
  int    dataLen = 0;
1945 1946 1947
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);

1948 1949 1950
  return dataBuf;
}

1951
static char* generateTagVaulesForStb(SSuperTable* stbInfo) {
1952 1953 1954 1955 1956 1957 1958 1959 1960
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
    printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
    return NULL;
  }

  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
  for (int i = 0; i < stbInfo->tagCount; i++) {
1961 1962
    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
1963
      if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
1964 1965
        printf("binary or nchar length overflow, max size:%u\n",
                (uint32_t)TSDB_MAX_BINARY_LEN);
1966 1967 1968
        tmfree(dataBuf);
        return NULL;
      }
1969

1970 1971 1972 1973 1974 1975 1976
      char* buf = (char*)calloc(stbInfo->tags[i].dataLen+1, 1);
      if (NULL == buf) {
        printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
        tmfree(dataBuf);
        return NULL;
      }
      rand_string(buf, stbInfo->tags[i].dataLen);
1977 1978
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "\'%s\', ", buf);
1979
      tmfree(buf);
1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "int", strlen("int"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_int());
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bigint", strlen("bigint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "float", strlen("float"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_float());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "double", strlen("double"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_double());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "smallint", strlen("smallint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_smallint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "tinyint", strlen("tinyint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_tinyint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bool", strlen("bool"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_bool());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "timestamp", strlen("timestamp"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
2012 2013 2014 2015 2016 2017
    }  else {
      printf("No support data type: %s\n", stbInfo->tags[i].dataType);
      tmfree(dataBuf);
      return NULL;
    }
  }
2018

2019
  dataLen -= 2;
2020
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");
2021 2022 2023
  return dataBuf;
}

2024
static int calcRowLen(SSuperTable*  superTbls) {
2025 2026
  int colIndex;
  int  lenOfOneRow = 0;
2027

2028 2029
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2030

2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
2047
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2048
      lenOfOneRow += 42;
2049
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
2063

2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) { 
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2089

2090 2091 2092 2093
  return 0;
}


2094 2095 2096 2097
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
        int* childTblCountOfSuperTbl, int limit, int offset) {

2098
  char command[BUFFER_SIZE] = "\0";
2099 2100
  char limitBuf[100] = "\0";

2101
  TAOS_RES * res;
2102 2103 2104
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2105 2106 2107 2108 2109

  if (offset >= 0) {
    snprintf(limitBuf, 100, " limit %d offset %d", limit, offset);
  }

2110
  //get all child table name use cmd: select tbname from superTblName;
2111 2112
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s", dbName, sTblName, limitBuf);

2113
  res = taos_query(taos, command);
2114 2115 2116 2117
  int32_t code = taos_errno(res);
  if (code != 0) {
    taos_free_result(res);
    taos_close(taos);
2118 2119
    errorPrint("%s() LN%d, failed to run command %s\n",
           __func__, __LINE__, command);
2120 2121 2122
    exit(-1);
  }

2123
  int childTblCount = (limit < 0)?10000:limit;
2124
  int count = 0;
2125 2126 2127 2128 2129 2130 2131 2132 2133 2134
  if (childTblName == NULL) {
    childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (NULL ==  childTblName) {
    taos_free_result(res);
        taos_close(taos);
        errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__);
        exit(-1);
    }
  }

2135 2136 2137
  char* pTblName = childTblName;
  while ((row = taos_fetch_row(res)) != NULL) {
    int32_t* len = taos_fetch_lengths(res);
2138
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2139 2140 2141
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2142 2143
      char *tmp = realloc(childTblName,
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2144 2145 2146
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2147 2148
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2149 2150
      } else {
        // exit, if allocate more memory failed
2151 2152
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
               __func__, __LINE__, dbName, sTblName);
2153 2154 2155 2156 2157 2158
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2159
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2160
  }
2161

2162 2163 2164 2165 2166 2167 2168
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2169 2170
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
        char* sTblName, char** childTblNameOfSuperTbl,
2171
        int* childTblCountOfSuperTbl) {
2172 2173 2174 2175 2176 2177

    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
            -1, -1);
}

2178 2179
static int getSuperTableFromServer(TAOS * taos, char* dbName,
        SSuperTable*  superTbls) {
2180

2181
  char command[BUFFER_SIZE] = "\0";
2182
  TAOS_RES * res;
2183 2184
  TAOS_ROW row = NULL;
  int count = 0;
2185

2186
  //get schema use cmd: describe superTblName;
2187
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2188
  res = taos_query(taos, command);
2189 2190 2191
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2192
    taos_free_result(res);
2193 2194 2195 2196 2197 2198 2199 2200 2201 2202
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
  while ((row = taos_fetch_row(res)) != NULL) {
    if (0 == count) {
      count++;
      continue;
2203
    }
2204 2205

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216
      tstrncpy(superTbls->tags[tagIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->tags[tagIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2217
      tagIndex++;
2218
    } else {
2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229
      tstrncpy(superTbls->columns[columnIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->columns[columnIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2241
/*
2242 2243
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
    //get all child table name use cmd: select tbname from superTblName;  
2244 2245 2246
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2247
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2248 2249
      return -1;
    }
2250
    getAllChildNameOfSuperTable(taos, dbName,
2251 2252 2253
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2254
  }
2255
  */
2256 2257 2258
  return 0;
}

2259
static int createSuperTable(TAOS * taos, char* dbName, SSuperTable*  superTbls, bool use_metric) {
2260
  char command[BUFFER_SIZE] = "\0";
2261

2262 2263 2264 2265 2266 2267 2268
  char cols[STRING_LEN] = "\0";
  int colIndex;
  int len = 0;

  int  lenOfOneRow = 0;
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2269
 
2270
    if (strcasecmp(dataType, "BINARY") == 0) {
2271 2272 2273
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "BINARY",
          superTbls->columns[colIndex].dataLen);
2274 2275
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
2276 2277 2278
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "NCHAR",
          superTbls->columns[colIndex].dataLen);
2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
      lenOfOneRow += 22;
2298
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2299 2300
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
      lenOfOneRow += 42;
2301
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
      lenOfOneRow += 21;
    } else {
      taos_close(taos);
      printf("config error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp
  //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName, g_Dbs.db[i].superTbls[j].columnCount, lenOfOneRow);

  // save for creating child table
2315 2316
  superTbls->colsOfCreateChildTable = (char*)calloc(len+20, 1);
  if (NULL == superTbls->colsOfCreateChildTable) {
2317 2318 2319 2320
    printf("Failed when calloc, size:%d", len+1);
    taos_close(taos);
    exit(-1);
  }
2321
  snprintf(superTbls->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
2322
  verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, superTbls->colsOfCreateChildTable);
2323 2324 2325 2326 2327 2328 2329 2330 2331 2332

  if (use_metric) {
    char tags[STRING_LEN] = "\0";
    int tagIndex;
    len = 0;

    int lenOfTagOfOneRow = 0;
    len += snprintf(tags + len, STRING_LEN - len, "(");
    for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
      char* dataType = superTbls->tags[tagIndex].dataType;
2333

2334
      if (strcasecmp(dataType, "BINARY") == 0) {
2335 2336
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
                "BINARY", superTbls->tags[tagIndex].dataLen);
2337 2338
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
      } else if (strcasecmp(dataType, "NCHAR") == 0) {
2339 2340
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
                "NCHAR", superTbls->tags[tagIndex].dataLen);
2341 2342
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
      } else if (strcasecmp(dataType, "INT") == 0)  {
2343 2344
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "INT");
2345 2346
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
      } else if (strcasecmp(dataType, "BIGINT") == 0)  {
2347 2348
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "BIGINT");
2349 2350
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
      } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
2351 2352
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "SMALLINT");
2353 2354
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
      } else if (strcasecmp(dataType, "TINYINT") == 0)  {
2355 2356
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "TINYINT");
2357 2358
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
      } else if (strcasecmp(dataType, "BOOL") == 0)  {
2359 2360
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "BOOL");
2361 2362
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
      } else if (strcasecmp(dataType, "FLOAT") == 0) {
2363 2364
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "FLOAT");
2365 2366
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
      } else if (strcasecmp(dataType, "DOUBLE") == 0) { 
2367 2368
        len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
                "DOUBLE");
2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379
        lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
      } else {
        taos_close(taos);
        printf("config error tag type : %s\n", dataType);
        exit(-1);
      }
    }
    len -= 2;
    len += snprintf(tags + len, STRING_LEN - len, ")");

    superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2380 2381 2382 2383

    snprintf(command, BUFFER_SIZE,
            "create table if not exists %s.%s (ts timestamp%s) tags %s",
            dbName, superTbls->sTblName, cols, tags);
2384
    verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command);
2385

2386
    if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2387
        errorPrint( "create supertable %s failed!\n\n",
2388
                superTbls->sTblName);
2389
        return -1;
2390
    }
2391
    debugPrint("create supertable %s success!\n\n", superTbls->sTblName);
2392 2393 2394 2395
  }
  return 0;
}

2396
static int createDatabases() {
2397 2398 2399 2400
  TAOS * taos = NULL;
  int    ret = 0;
  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
  if (taos == NULL) {
2401
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2402
    return -1;
2403 2404 2405
  }
  char command[BUFFER_SIZE] = "\0";

2406
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2407 2408
    if (g_Dbs.db[i].drop) {
      sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
2409
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);
2410
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2411 2412 2413 2414
        taos_close(taos);
        return -1;
      }
    }
2415

2416
    int dataLen = 0;
2417
    dataLen += snprintf(command + dataLen,
2418
        BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2419 2420

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
2421
      dataLen += snprintf(command + dataLen,
2422
          BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
2423 2424
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
2425
      dataLen += snprintf(command + dataLen,
2426
          BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
2427 2428
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
2429
      dataLen += snprintf(command + dataLen,
2430
          BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
2431 2432
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
2433
      dataLen += snprintf(command + dataLen,
2434
          BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
2435
    }
2436 2437
    if (g_Dbs.db[i].dbCfg.quorum > 1) {
      dataLen += snprintf(command + dataLen,
2438
          BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
2439 2440
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
2441
      dataLen += snprintf(command + dataLen,
2442
          BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
2443 2444
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
2445
      dataLen += snprintf(command + dataLen,
2446
          BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
2447 2448
    }
    //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
2449 2450
    //  dataLen += snprintf(command + dataLen,
    //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
2451 2452
    //}
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
2453
      dataLen += snprintf(command + dataLen,
2454
          BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
2455 2456
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
2457
      dataLen += snprintf(command + dataLen,
2458
          BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
2459 2460
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
2461
      dataLen += snprintf(command + dataLen,
2462
          BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
2463 2464
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
2465
      dataLen += snprintf(command + dataLen,
2466
          BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
2467
    }
2468
    if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
2469
      dataLen += snprintf(command + dataLen,
2470
          BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
2471
    }
2472
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
2473 2474
      dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
              " fsync %d", g_Dbs.db[i].dbCfg.fsync);
2475
    }
2476
    if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
2477 2478
            || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                    "us", strlen("us")))) {
2479
      dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2480
              " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
2481
    }
2482

2483
    debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
2484
    if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2485
      taos_close(taos);
2486
      errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
2487 2488 2489 2490
      return -1;
    }
    printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);

2491 2492
    debugPrint("%s() %d supertbl count:%d\n",
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
2493
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
2494 2495 2496 2497 2498 2499 2500
      sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
              g_Dbs.db[i].superTbls[j].sTblName);
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);

      ret = queryDbExec(taos, command, NO_INSERT_TYPE, true);

      if ((ret != 0) || (g_Dbs.db[i].drop)) {
2501 2502
        ret = createSuperTable(taos, g_Dbs.db[i].dbName,
                &g_Dbs.db[i].superTbls[j], g_Dbs.use_metric);
2503

2504 2505 2506 2507 2508 2509 2510
        if (0 != ret) {
          errorPrint("\ncreate super table %d failed!\n\n", j);
          taos_close(taos);
          return -1;
        }
      }

2511
      ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
2512
                &g_Dbs.db[i].superTbls[j]);
2513 2514 2515 2516 2517 2518
      if (0 != ret) {
        errorPrint("\nget super table %s.%s info failed!\n\n",
                g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName);
        taos_close(taos);
        return -1;
      }
2519
    }
2520 2521 2522 2523 2524 2525
  }

  taos_close(taos);
  return 0;
}

2526 2527 2528
static void* createTable(void *sarg)
{
  threadInfo *winfo = (threadInfo *)sarg;
2529 2530 2531 2532
  SSuperTable* superTblInfo = winfo->superTblInfo;

  int64_t  lastPrintTime = taosGetTimestampMs();

2533 2534 2535 2536 2537 2538
  int buff_len;
  if (superTblInfo)
    buff_len = superTblInfo->maxSqlLen;
  else
    buff_len = BUFFER_SIZE;

2539
  char *buffer = calloc(buff_len, 1);
2540
  if (buffer == NULL) {
2541
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
2542 2543
    exit(-1);
  }
2544 2545 2546

  int len = 0;
  int batchNum = 0;
2547

2548
  verbosePrint("%s() LN%d: Creating table from %d to %d\n",
2549 2550 2551 2552
          __func__, __LINE__,
          winfo->start_table_from, winfo->end_table_to);

  for (int i = winfo->start_table_from; i <= winfo->end_table_to; i++) {
2553
    if (0 == g_Dbs.use_metric) {
2554
      snprintf(buffer, buff_len,
2555 2556
              "create table if not exists %s.%s%d %s;",
              winfo->db_name,
2557 2558
              g_args.tb_prefix, i,
              winfo->cols);
2559
    } else {
2560 2561 2562
      if (superTblInfo == NULL) {
        errorPrint("%s() LN%d, use metric, but super table info is NULL\n",
                  __func__, __LINE__);
2563
        free(buffer);
2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599
        exit(-1);
      } else {
        if (0 == len) {
          batchNum = 0;
          memset(buffer, 0, buff_len);
          len += snprintf(buffer + len,
                  buff_len - len, "create table ");
        }
  
        char* tagsValBuf = NULL;
        if (0 == superTblInfo->tagSource) {
          tagsValBuf = generateTagVaulesForStb(superTblInfo);
        } else {
          tagsValBuf = getTagValueFromTagSample(
                  superTblInfo,
                  i % superTblInfo->tagSampleCount);
        }
        if (NULL == tagsValBuf) {
          free(buffer);
          return NULL;
        }
  
        len += snprintf(buffer + len,
                superTblInfo->maxSqlLen - len,
                "if not exists %s.%s%d using %s.%s tags %s ",
                winfo->db_name, superTblInfo->childTblPrefix,
                i, winfo->db_name,
                superTblInfo->sTblName, tagsValBuf);
        free(tagsValBuf);
        batchNum++;
  
        if ((batchNum < superTblInfo->batchCreateTableNum)
                && ((superTblInfo->maxSqlLen - len)
                    >= (superTblInfo->lenOfTagOfOneRow + 256))) {
          continue;
        }
2600 2601 2602 2603
      }
    }

    len = 0;
2604
    verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
2605
    if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE, false)){
2606
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2607
      free(buffer);
2608 2609 2610 2611 2612
      return NULL;
    }

    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
2613
      printf("thread[%d] already create %d - %d tables\n",
2614
              winfo->threadID, winfo->start_table_from, i);
2615 2616 2617 2618 2619
      lastPrintTime = currentPrintTime;
    }
  }
  
  if (0 != len) {
2620
    verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer);
2621
    if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE, false)) {
2622
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2623
    }
2624
  }
2625

2626 2627 2628 2629
  free(buffer);
  return NULL;
}

2630
static int startMultiThreadCreateChildTable(
2631
        char* cols, int threads, int startFrom, int ntables,
2632
        char* db_name, SSuperTable* superTblInfo) {
2633

2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  threadInfo *infos = malloc(threads * sizeof(threadInfo));

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

  int a = ntables / threads;
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

  int b = 0;
  b = ntables % threads;
2654

2655 2656 2657
  for (int i = 0; i < threads; i++) {
    threadInfo *t_info = infos + i;
    t_info->threadID = i;
H
Hui Li 已提交
2658
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
2659
    t_info->superTblInfo = superTblInfo;
2660
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
2661 2662 2663 2664 2665 2666
    t_info->taos = taos_connect(
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
2667
    if (t_info->taos == NULL) {
2668
      errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2669
      free(pids);
2670
      free(infos);
2671 2672
      return -1;
    }
2673 2674 2675 2676 2677

    t_info->start_table_from = startFrom;
    t_info->ntables = i<b?a+1:a;
    t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
    startFrom = t_info->end_table_to + 1;
2678
    t_info->use_metric = true;
2679
    t_info->cols = cols;
2680
    t_info->minDelay = INT16_MAX;
2681 2682
    pthread_create(pids + i, NULL, createTable, t_info);
  }
2683
 
2684 2685 2686 2687 2688 2689 2690 2691 2692 2693
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
    threadInfo *t_info = infos + i;
    taos_close(t_info->taos);
  }

  free(pids);
2694
  free(infos);
2695 2696

  return 0;
2697 2698
}

2699 2700 2701 2702
static void createChildTables() {
    char tblColsBuf[MAX_SQL_SIZE];
    int len;

2703
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2704
    if (g_Dbs.db[i].superTblCount > 0) {
2705
        // with super table
2706 2707
      for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
        if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
2708
              || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
2709 2710 2711
          continue;
        }

2712
        verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
2713
                g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
2714 2715
        int startFrom = 0;
        g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;
2716

2717 2718
        verbosePrint("%s() LN%d: create %d child tables from %d\n",
                __func__, __LINE__, g_totalChildTables, startFrom);
2719
        startMultiThreadCreateChildTable(
2720
              g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
2721
              g_Dbs.threadCountByCreateTbl,
2722
              startFrom,
2723
              g_Dbs.db[i].superTbls[j].childTblCount,
2724
              g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
2725 2726
      }
    } else {
2727 2728
        // normal table
        len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
2729 2730 2731
        int j = 0;
        while (g_args.datatype[j]) {
            if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
2732 2733
                    || (strncasecmp(g_args.datatype[j],
                        "NCHAR", strlen("NCHAR")) == 0)) {
2734
                snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
2735
                        ", COL%d %s(60)", j, g_args.datatype[j]);
2736
            } else {
2737
                snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
2738
                        ", COL%d %s", j, g_args.datatype[j]);
2739 2740
            }
            len = strlen(tblColsBuf);
2741
            j++;
2742 2743
        }

2744
        len = snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");
2745

2746 2747
        verbosePrint("%s() LN%d: dbName: %s num of tb: %d schema: %s\n",
                __func__, __LINE__,
2748
                g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
2749
        startMultiThreadCreateChildTable(
2750
              tblColsBuf,
2751
              g_Dbs.threadCountByCreateTbl,
2752
              0,
2753
              g_args.num_of_tables,
2754 2755 2756
              g_Dbs.db[i].dbName,
              NULL);
    }
2757 2758 2759 2760 2761 2762
  }
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
2763
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
2764 2765 2766
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
2767

2768 2769
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
2770 2771
    printf("Failed to open tags file: %s, reason:%s\n",
            superTblInfo->tagsFile, strerror(errno));
2772 2773 2774 2775 2776 2777 2778
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
2779

2780 2781 2782 2783 2784 2785 2786 2787 2788
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

2789
  while ((readLen = tgetline(&line, &n, fp)) != -1) {
2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
2802 2803
      char *tmp = realloc(tagDataBuf,
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
2804 2805 2806
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
2807 2808
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

int readSampleFromJsonFileToMem(SSuperTable  * superTblInfo) {
  // TODO
  return 0;
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
2836 2837
static int readSampleFromCsvFileToMem(
        SSuperTable* superTblInfo) {
2838 2839 2840 2841
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
2842

2843 2844
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
2845
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
2846 2847 2848
              superTblInfo->sampleFile, strerror(errno));
      return -1;
  }
2849

2850
  assert(superTblInfo->sampleDataBuf);
2851 2852
  memset(superTblInfo->sampleDataBuf, 0,
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
2853
  while (1) {
2854
    readLen = tgetline(&line, &n, fp);
2855 2856
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
2857
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
2858
                superTblInfo->sampleFile, strerror(errno));
2859
        fclose(fp);
2860 2861 2862 2863
        return -1;
      }
      continue;
    }
2864

2865 2866 2867 2868 2869 2870 2871 2872 2873
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
2874 2875
      printf("sample row len[%d] overflow define schema len[%d], so discard this row\n",
              (int32_t)readLen, superTblInfo->lenOfOneRow);
2876 2877 2878
      continue;
    }

sangshuduo's avatar
sangshuduo 已提交
2879
    verbosePrint("readLen=%ld stb->lenOfOneRow=%d getRows=%d\n", (long)readLen,
2880
            superTblInfo->lenOfOneRow, getRows);
2881

2882 2883
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
          line, readLen);
2884 2885 2886 2887 2888 2889 2890
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

2891
  fclose(fp);
2892 2893 2894 2895
  tmfree(line);
  return 0;
}

2896 2897
static bool getColumnAndTagTypeFromInsertJsonFile(
        cJSON* stbInfo, SSuperTable* superTbls) {
2898
  bool  ret = false;
2899

2900 2901 2902
  // columns 
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
2903
    printf("ERROR: failed to read json, columns not found\n");
2904 2905 2906 2907 2908 2909
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
2910
 
2911 2912
  int columnSize = cJSON_GetArraySize(columns);
  if (columnSize > MAX_COLUMN_COUNT) {
2913 2914
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
            __func__, __LINE__, MAX_COLUMN_COUNT);
2915 2916 2917 2918 2919 2920
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
2921

2922
  //superTbls->columnCount = columnSize;
2923 2924 2925 2926 2927 2928 2929
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
2930
      count = countObj->valueint;
2931
    } else if (countObj && countObj->type != cJSON_Number) {
2932
      errorPrint("%s() LN%d, failed to read json, column count not found\n", __func__, __LINE__);
2933 2934 2935 2936 2937 2938 2939 2940 2941
      goto PARSE_OVER;
    } else {
      count = 1;
    }

    // column info 
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
    if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) {
2942
      errorPrint("%s() LN%d: failed to read json, column type not found\n", __func__, __LINE__);
2943 2944
      goto PARSE_OVER;
    }
2945 2946
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
2947

2948 2949
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
2950
      columnCase.dataLen = dataLen->valueint;
2951
    } else if (dataLen && dataLen->type != cJSON_Number) {
2952
      debugPrint("%s() LN%d: failed to read json, column len not found\n", __func__, __LINE__);
2953 2954 2955 2956
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
2957

2958
    for (int n = 0; n < count; ++n) {
2959 2960 2961
      tstrncpy(superTbls->columns[index].dataType,
              columnCase.dataType, MAX_TB_NAME_SIZE);
      superTbls->columns[index].dataLen = columnCase.dataLen;
2962 2963
      index++;
    }
2964 2965 2966
  }
  superTbls->columnCount = index;
 
2967 2968 2969 2970 2971
  count = 1;
  index = 0;
  // tags 
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
2972
    debugPrint("%s() LN%d, failed to read json, tags not found\n", __func__, __LINE__);
2973 2974
    goto PARSE_OVER;
  }
2975

2976 2977
  int tagSize = cJSON_GetArraySize(tags);
  if (tagSize > MAX_TAG_COUNT) {
2978
    debugPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", __func__, __LINE__, MAX_TAG_COUNT);
2979 2980
    goto PARSE_OVER;
  }
2981 2982

  //superTbls->tagCount = tagSize;
2983 2984 2985
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
2986

2987 2988 2989
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
2990
      count = countObj->valueint;
2991
    } else if (countObj && countObj->type != cJSON_Number) {
2992
      printf("ERROR: failed to read json, column count not found\n");
2993 2994 2995 2996 2997 2998 2999 3000 3001
      goto PARSE_OVER;
    } else {
      count = 1;
    }

    // column info 
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
    if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) {
3002
      printf("ERROR: failed to read json, tag type not found\n");
3003 3004
      goto PARSE_OVER;
    }
3005
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3006

3007 3008
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3009
      columnCase.dataLen = dataLen->valueint;
3010
    } else if (dataLen && dataLen->type != cJSON_Number) {
3011
      printf("ERROR: failed to read json, column len not found\n");
3012 3013 3014
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
3015 3016
    }

3017
    for (int n = 0; n < count; ++n) {
3018
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, MAX_TB_NAME_SIZE);
3019
      superTbls->tags[index].dataLen = columnCase.dataLen;
3020 3021
      index++;
    }
3022
  }
3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038
  superTbls->tagCount = index;

  ret = true;

PARSE_OVER:
  //free(content);
  //cJSON_Delete(root);
  //fclose(fp);
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3039
    tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3040 3041 3042 3043
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3044
    tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE);
3045
  } else if (!host) {
3046
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3047
  } else {
3048
    printf("ERROR: failed to read json, host not found\n");
3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_Dbs.port = port->valueint;
  } else if (!port) {
    g_Dbs.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3061
    tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE);
3062
  } else if (!user) {
3063
    tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE);
3064 3065 3066 3067
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3068
    tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE);
3069
  } else if (!password) {
3070
    tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE);
3071 3072 3073 3074
  }

  cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
  if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
3075
    tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
3076
  } else if (!resultfile) {
3077
    tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
3078 3079 3080 3081 3082 3083 3084 3085
  }

  cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
  if (threads && threads->type == cJSON_Number) {
    g_Dbs.threadCount = threads->valueint;
  } else if (!threads) {
    g_Dbs.threadCount = 1;
  } else {
3086
    printf("ERROR: failed to read json, threads not found\n");
3087
    goto PARSE_OVER;
3088 3089
  }

3090 3091 3092 3093
  cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
  if (threads2 && threads2->type == cJSON_Number) {
    g_Dbs.threadCountByCreateTbl = threads2->valueint;
  } else if (!threads2) {
3094
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
3095
  } else {
3096 3097
    errorPrint("%s() LN%d, failed to read json, threads2 not found\n",
            __func__, __LINE__);
3098
    goto PARSE_OVER;
3099
  }
3100

3101 3102 3103 3104 3105 3106 3107 3108 3109
  cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
  if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
    g_args.insert_interval = gInsertInterval->valueint;
  } else if (!gInsertInterval) {
    g_args.insert_interval = 0;
  } else {
    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", __func__, __LINE__);
    goto PARSE_OVER;
  }
3110

3111 3112 3113
  cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows");
  if (interlaceRows && interlaceRows->type == cJSON_Number) {
    g_args.interlace_rows = interlaceRows->valueint;
3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124

    // rows per table need be less than insert batch
    if (g_args.interlace_rows > g_args.num_of_RPR) {
      printf("NOTICE: interlace rows value %d > num_of_records_per_request %d\n\n",
              g_args.interlace_rows, g_args.num_of_RPR);
      printf("        interlace rows value will be set to num_of_records_per_request %d\n\n",
              g_args.num_of_RPR);
      printf("        press Enter key to continue or Ctrl+C to stop.");
      (void)getchar();
      g_args.interlace_rows = g_args.num_of_RPR;
    }
3125
  } else if (!interlaceRows) {
3126
    g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3127
  } else {
3128
    errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", __func__, __LINE__);
3129
    goto PARSE_OVER;
3130
  }
3131

3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151
  cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
  if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
    g_args.max_sql_len = maxSqlLen->valueint;
  } else if (!maxSqlLen) {
    g_args.max_sql_len = TSDB_PAYLOAD_SIZE;
  } else {
    errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", __func__, __LINE__);
    goto PARSE_OVER;
  }

  cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
  if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
    g_args.num_of_RPR = numRecPerReq->valueint;
  } else if (!numRecPerReq) {
    g_args.num_of_RPR = 100;
  } else {
    errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", __func__, __LINE__);
    goto PARSE_OVER;
  }

3152
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3153
  if (answerPrompt
3154 3155
          && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3156 3157 3158 3159 3160 3161 3162 3163 3164 3165
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
    } else {
      g_args.answer_yes = false;
    }
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
3166
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
3167
    goto PARSE_OVER;
3168
  }
3169 3170 3171

  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (!dbs || dbs->type != cJSON_Array) {
3172
    printf("ERROR: failed to read json, databases not found\n");
3173 3174 3175 3176 3177
    goto PARSE_OVER;
  }

  int dbSize = cJSON_GetArraySize(dbs);
  if (dbSize > MAX_DB_COUNT) {
3178
    errorPrint(
3179 3180
            "ERROR: failed to read json, databases size overflow, max database is %d\n",
            MAX_DB_COUNT);
3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191
    goto PARSE_OVER;
  }

  g_Dbs.dbCount = dbSize;
  for (int i = 0; i < dbSize; ++i) {
    cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
    if (dbinfos == NULL) continue;

    // dbinfo 
    cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
    if (!dbinfo || dbinfo->type != cJSON_Object) {
3192
      printf("ERROR: failed to read json, dbinfo not found\n");
3193 3194
      goto PARSE_OVER;
    }
3195

3196 3197
    cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
    if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
3198
      printf("ERROR: failed to read json, db name not found\n");
3199 3200
      goto PARSE_OVER;
    }
3201
    tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE);
3202 3203 3204

    cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
    if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
3205 3206
      if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) {
        g_Dbs.db[i].drop = true;
3207
      } else {
3208
        g_Dbs.db[i].drop = false;
3209
      }
3210
    } else if (!drop) {
3211
      g_Dbs.db[i].drop = g_args.drop_database;
3212
    } else {
3213 3214
      errorPrint("%s() LN%d, failed to read json, drop input mistake\n",
              __func__, __LINE__);
3215 3216 3217 3218
      goto PARSE_OVER;
    }

    cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
3219 3220 3221 3222
    if (precision && precision->type == cJSON_String
            && precision->valuestring != NULL) {
      tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
              MAX_DB_NAME_SIZE);
3223
    } else if (!precision) {
3224
      //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
3225 3226
      memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE);
    } else {
3227
      printf("ERROR: failed to read json, precision not found\n");
3228 3229 3230 3231 3232 3233 3234 3235 3236
      goto PARSE_OVER;
    }

    cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
    if (update && update->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.update = update->valueint;
    } else if (!update) {
      g_Dbs.db[i].dbCfg.update = -1;
    } else {
3237
      printf("ERROR: failed to read json, update not found\n");
3238 3239 3240 3241 3242 3243 3244 3245 3246
      goto PARSE_OVER;
    }

    cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
    if (replica && replica->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.replica = replica->valueint;
    } else if (!replica) {
      g_Dbs.db[i].dbCfg.replica = -1;
    } else {
3247
      printf("ERROR: failed to read json, replica not found\n");
3248 3249 3250 3251 3252 3253 3254 3255 3256
      goto PARSE_OVER;
    }

    cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
    if (keep && keep->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.keep = keep->valueint;
    } else if (!keep) {
      g_Dbs.db[i].dbCfg.keep = -1;
    } else {
3257
     printf("ERROR: failed to read json, keep not found\n");
3258 3259
     goto PARSE_OVER;
    }
3260

3261 3262 3263 3264 3265 3266
    cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
    if (days && days->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.days = days->valueint;
    } else if (!days) {
      g_Dbs.db[i].dbCfg.days = -1;
    } else {
3267
     printf("ERROR: failed to read json, days not found\n");
3268 3269
     goto PARSE_OVER;
    }
3270

3271 3272 3273 3274 3275 3276
    cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
    if (cache && cache->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cache = cache->valueint;
    } else if (!cache) {
      g_Dbs.db[i].dbCfg.cache = -1;
    } else {
3277
     printf("ERROR: failed to read json, cache not found\n");
3278 3279
     goto PARSE_OVER;
    }
3280

3281 3282 3283 3284 3285 3286
    cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
    if (blocks && blocks->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
    } else if (!blocks) {
      g_Dbs.db[i].dbCfg.blocks = -1;
    } else {
3287
     printf("ERROR: failed to read json, block not found\n");
3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306
     goto PARSE_OVER;
    }

    //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
    //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
    //} else if (!maxtablesPerVnode) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
    //} else {
    // printf("failed to read json, maxtablesPerVnode not found");
    // goto PARSE_OVER;
    //}

    cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
    if (minRows && minRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
    } else if (!minRows) {
      g_Dbs.db[i].dbCfg.minRows = -1;
    } else {
3307
     printf("ERROR: failed to read json, minRows not found\n");
3308 3309 3310 3311 3312 3313 3314 3315 3316
     goto PARSE_OVER;
    }

    cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
    if (maxRows && maxRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
    } else if (!maxRows) {
      g_Dbs.db[i].dbCfg.maxRows = -1;
    } else {
3317
     printf("ERROR: failed to read json, maxRows not found\n");
3318 3319 3320 3321 3322 3323 3324 3325 3326
     goto PARSE_OVER;
    }

    cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
    if (comp && comp->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.comp = comp->valueint;
    } else if (!comp) {
      g_Dbs.db[i].dbCfg.comp = -1;
    } else {
3327
     printf("ERROR: failed to read json, comp not found\n");
3328 3329 3330 3331 3332 3333 3334 3335 3336
     goto PARSE_OVER;
    }

    cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
    if (walLevel && walLevel->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
    } else if (!walLevel) {
      g_Dbs.db[i].dbCfg.walLevel = -1;
    } else {
3337
     printf("ERROR: failed to read json, walLevel not found\n");
3338 3339 3340
     goto PARSE_OVER;
    }

3341 3342 3343 3344 3345 3346
    cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
    if (cacheLast && cacheLast->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
    } else if (!cacheLast) {
      g_Dbs.db[i].dbCfg.cacheLast = -1;
    } else {
3347
     printf("ERROR: failed to read json, cacheLast not found\n");
3348 3349 3350
     goto PARSE_OVER;
    }

3351 3352 3353 3354
    cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
    if (quorum && quorum->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
    } else if (!quorum) {
3355
      g_Dbs.db[i].dbCfg.quorum = 1;
3356
    } else {
3357
     printf("failed to read json, quorum input mistake");
3358 3359 3360 3361 3362 3363 3364 3365 3366
     goto PARSE_OVER;
    }

    cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
    if (fsync && fsync->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
    } else if (!fsync) {
      g_Dbs.db[i].dbCfg.fsync = -1;
    } else {
3367
     printf("ERROR: failed to read json, fsync not found\n");
3368
     goto PARSE_OVER;
3369
    }
3370

3371
    // super_talbes
3372 3373
    cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
    if (!stables || stables->type != cJSON_Array) {
3374
      printf("ERROR: failed to read json, super_tables not found\n");
3375
      goto PARSE_OVER;
3376 3377
    }

3378 3379
    int stbSize = cJSON_GetArraySize(stables);
    if (stbSize > MAX_SUPER_TABLE_COUNT) {
3380
      errorPrint(
3381 3382
              "ERROR: failed to read json, databases size overflow, max database is %d\n",
              MAX_SUPER_TABLE_COUNT);
3383 3384 3385 3386 3387 3388 3389
      goto PARSE_OVER;
    }

    g_Dbs.db[i].superTblCount = stbSize;
    for (int j = 0; j < stbSize; ++j) {
      cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
      if (stbInfo == NULL) continue;
3390 3391

      // dbinfo
3392 3393
      cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
      if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) {
3394
        printf("ERROR: failed to read json, stb name not found\n");
3395 3396
        goto PARSE_OVER;
      }
3397
      tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE);
3398

3399 3400
      cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
      if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
3401
        printf("ERROR: failed to read json, childtable_prefix not found\n");
3402 3403
        goto PARSE_OVER;
      }
3404
      tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE);
3405 3406

      cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null
3407 3408 3409
      if (autoCreateTbl
              && autoCreateTbl->type == cJSON_String
              && autoCreateTbl->valuestring != NULL) {
3410 3411 3412 3413 3414 3415 3416 3417 3418 3419
        if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
        } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        } else {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        }
      } else if (!autoCreateTbl) {
        g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      } else {
3420
        printf("ERROR: failed to read json, auto_create_table not found\n");
3421 3422
        goto PARSE_OVER;
      }
3423

3424 3425 3426 3427
      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
      } else if (!batchCreateTbl) {
3428
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
3429
      } else {
3430
        printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
3431
        goto PARSE_OVER;
3432
      }
3433 3434

      cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
3435 3436 3437
      if (childTblExists
              && childTblExists->type == cJSON_String
              && childTblExists->valuestring != NULL) {
3438 3439 3440 3441 3442 3443 3444 3445 3446 3447
        if (0 == strncasecmp(childTblExists->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
        } else if (0 == strncasecmp(childTblExists->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        } else {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        }
      } else if (!childTblExists) {
        g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
      } else {
3448 3449
        errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n",
                __func__, __LINE__);
3450 3451
        goto PARSE_OVER;
      }
3452

3453 3454
      cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
      if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
3455 3456
        errorPrint("%s() LN%d, failed to read json, childtable_count not found\n",
                __func__, __LINE__);
3457 3458 3459 3460 3461
        goto PARSE_OVER;
      }
      g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

      cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
3462 3463 3464 3465
      if (dataSource && dataSource->type == cJSON_String
              && dataSource->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
                dataSource->valuestring, MAX_DB_NAME_SIZE);
3466
      } else if (!dataSource) {
3467
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE);
3468
      } else {
3469
        errorPrint("%s() LN%d, failed to read json, data_source not found\n", __func__, __LINE__);
3470 3471 3472 3473
        goto PARSE_OVER;
      }

      cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , restful
3474 3475 3476 3477
      if (insertMode && insertMode->type == cJSON_String
              && insertMode->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode,
                insertMode->valuestring, MAX_DB_NAME_SIZE);
3478
      } else if (!insertMode) {
3479
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE);
3480
      } else {
3481
        printf("ERROR: failed to read json, insert_mode not found\n");
3482 3483 3484
        goto PARSE_OVER;
      }

3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506
      cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
      if (childTbl_limit) {
        if (childTbl_limit->type != cJSON_Number) {
            printf("ERROR: failed to read json, childtable_limit\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result
      }

      cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
      if (childTbl_offset) {
        if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) {
            printf("ERROR: failed to read json, childtable_offset\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblOffset = 0;
      }

3507 3508
      cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
      if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
3509 3510
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                ts->valuestring, MAX_DB_NAME_SIZE);
3511
      } else if (!ts) {
3512 3513
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                "now", MAX_DB_NAME_SIZE);
3514
      } else {
3515
        printf("ERROR: failed to read json, start_timestamp not found\n");
3516 3517
        goto PARSE_OVER;
      }
3518

3519 3520 3521 3522
      cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
      if (timestampStep && timestampStep->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
      } else if (!timestampStep) {
3523
        g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
3524
      } else {
3525
        printf("ERROR: failed to read json, timestamp_step not found\n");
3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537
        goto PARSE_OVER;
      }

      cJSON* sampleDataBufSize = cJSON_GetObjectItem(stbInfo, "sample_buf_size");
      if (sampleDataBufSize && sampleDataBufSize->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].sampleDataBufSize = sampleDataBufSize->valueint;
        if (g_Dbs.db[i].superTbls[j].sampleDataBufSize < 1024*1024) {
          g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024;
        }
      } else if (!sampleDataBufSize) {
        g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024;
      } else {
3538
        printf("ERROR: failed to read json, sample_buf_size not found\n");
3539
        goto PARSE_OVER;
3540
      }
3541

3542
      cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
3543 3544
      if (sampleFormat && sampleFormat->type
              == cJSON_String && sampleFormat->valuestring != NULL) {
3545 3546
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
                sampleFormat->valuestring, MAX_DB_NAME_SIZE);
3547
      } else if (!sampleFormat) {
3548
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE);
3549
      } else {
3550
        printf("ERROR: failed to read json, sample_format not found\n");
3551
        goto PARSE_OVER;
3552
      }
3553

3554 3555
      cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
      if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) {
3556 3557
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
                sampleFile->valuestring, MAX_FILE_NAME_LEN);
3558 3559 3560
      } else if (!sampleFile) {
        memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
      } else {
3561
        printf("ERROR: failed to read json, sample_file not found\n");
3562
        goto PARSE_OVER;
3563
      }
3564

3565 3566
      cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
      if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) {
3567 3568
        tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
                tagsFile->valuestring, MAX_FILE_NAME_LEN);
3569 3570 3571 3572 3573 3574 3575 3576 3577
        if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
          g_Dbs.db[i].superTbls[j].tagSource = 0;
        } else {
          g_Dbs.db[i].superTbls[j].tagSource = 1;
        }
      } else if (!tagsFile) {
        memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
        g_Dbs.db[i].superTbls[j].tagSource = 0;
      } else {
3578
        printf("ERROR: failed to read json, tags_file not found\n");
3579 3580
        goto PARSE_OVER;
      }
3581

3582 3583 3584 3585 3586 3587 3588
      cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
      if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
        int32_t len = maxSqlLen->valueint;
        if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
          len = TSDB_MAX_ALLOWED_SQL_LEN;
        } else if (len < TSDB_MAX_SQL_LEN) {
          len = TSDB_MAX_SQL_LEN;
3589
        }
3590 3591 3592 3593
        g_Dbs.db[i].superTbls[j].maxSqlLen = len;
      } else if (!maxSqlLen) {
        g_Dbs.db[i].superTbls[j].maxSqlLen = TSDB_MAX_SQL_LEN;
      } else {
3594
        printf("ERROR: failed to read json, maxSqlLen not found\n");
3595
        goto PARSE_OVER;
3596
      }
3597

3598 3599 3600 3601 3602
      cJSON *multiThreadWriteOneTbl =
          cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
      if (multiThreadWriteOneTbl
              && multiThreadWriteOneTbl->type == cJSON_String
              && multiThreadWriteOneTbl->valuestring != NULL) {
3603 3604 3605 3606
        if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
        } else {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
3607
        }
3608 3609 3610
      } else if (!multiThreadWriteOneTbl) {
        g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
      } else {
3611
        printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
3612 3613
        goto PARSE_OVER;
      }
3614

3615 3616 3617
      cJSON* interlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows");
      if (interlaceRows && interlaceRows->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].interlaceRows = interlaceRows->valueint;
3618 3619 3620 3621 3622 3623 3624 3625 3626 3627
        // rows per table need be less than insert batch
        if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) {
          printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %d > num_of_records_per_request %d\n\n",
                  i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR);
          printf("        interlace rows value will be set to num_of_records_per_request %d\n\n",
                  g_args.num_of_RPR);
          printf("        press Enter key to continue or Ctrl+C to stop.");
          (void)getchar();
          g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR;
        }
3628 3629
      } else if (!interlaceRows) {
        g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3630
      } else {
3631 3632 3633
        errorPrint(
                "%s() LN%d, failed to read json, interlace rows input mistake\n",
                __func__, __LINE__);
3634
        goto PARSE_OVER;
3635
      }
3636 3637 3638 3639 3640 3641 3642

      cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
      if (disorderRatio && disorderRatio->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
      } else if (!disorderRatio) {
        g_Dbs.db[i].superTbls[j].disorderRatio = 0;
      } else {
3643
        printf("ERROR: failed to read json, disorderRatio not found\n");
3644
        goto PARSE_OVER;
3645
      }
3646 3647 3648 3649 3650 3651 3652

      cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
      if (disorderRange && disorderRange->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
      } else if (!disorderRange) {
        g_Dbs.db[i].superTbls[j].disorderRange = 1000;
      } else {
3653
        printf("ERROR: failed to read json, disorderRange not found\n");
3654 3655
        goto PARSE_OVER;
      }
3656

3657 3658 3659 3660 3661 3662
      cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
      if (insertRows && insertRows->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
      } else if (!insertRows) {
        g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
      } else {
3663 3664
        errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
3665 3666 3667 3668 3669 3670 3671
        goto PARSE_OVER;
      }

      cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
      if (insertInterval && insertInterval->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
      } else if (!insertInterval) {
3672
        verbosePrint("%s() LN%d: stable insert interval be overrided by global %d.\n",
3673 3674 3675
                __func__, __LINE__, g_args.insert_interval);
        g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
      } else {
3676 3677
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
3678 3679 3680
        goto PARSE_OVER;
      }

3681 3682
      int retVal = getColumnAndTagTypeFromInsertJsonFile(
              stbInfo, &g_Dbs.db[i].superTbls[j]);
3683 3684
      if (false == retVal) {
        goto PARSE_OVER;
3685 3686
      }
    }
3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702
  }

  ret = true;

PARSE_OVER:
  //free(content);
  //cJSON_Delete(root);
  //fclose(fp);
  return ret;
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3703
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3704 3705 3706 3707
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3708
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE);
3709
  } else if (!host) {
3710
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3711
  } else {
3712
    printf("ERROR: failed to read json, host not found\n");
3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3725
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE);   
3726
  } else if (!user) {
3727
    tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ;
3728 3729 3730 3731
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3732
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE);
3733
  } else if (!password) {
3734
    tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);;
3735
  }
H
hzcheng 已提交
3736

3737
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3738 3739
  if (answerPrompt && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3740 3741 3742 3743
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
3744
    } else {
3745
      g_args.answer_yes = false;
H
hzcheng 已提交
3746
    }
3747 3748 3749
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
3750
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
3751
    goto PARSE_OVER;
3752
  }
3753

3754 3755 3756 3757 3758 3759 3760 3761 3762 3763
  cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
  if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
    g_args.query_times = gQueryTimes->valueint;
  } else if (!gQueryTimes) {
    g_args.query_times = 1;
  } else {
    errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__);
    goto PARSE_OVER;
  }

3764 3765
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
3766
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE);
3767
  } else if (!dbs) {
3768
    printf("ERROR: failed to read json, databases not found\n");
3769
    goto PARSE_OVER;
H
hzcheng 已提交
3770
  }
3771 3772 3773

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
3774
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
3775
  } else if (!queryMode) {
3776
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
3777
  } else {
3778
    printf("ERROR: failed to read json, query_mode not found\n");
3779
    goto PARSE_OVER;
H
hzcheng 已提交
3780
  }
3781

3782
  // super_table_query 
3783 3784 3785 3786 3787
  cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!specifiedQuery) {
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
    g_queryInfo.specifiedQueryInfo.sqlCount = 0;
  } else if (specifiedQuery->type != cJSON_Object) {
3788
    printf("ERROR: failed to read json, super_table_query not found\n");
3789
    goto PARSE_OVER;
3790
  } else {
3791
    cJSON* rate = cJSON_GetObjectItem(specifiedQuery, "query_interval");
3792
    if (rate && rate->type == cJSON_Number) {
3793
      g_queryInfo.specifiedQueryInfo.rate = rate->valueint;
3794
    } else if (!rate) {
3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805
      g_queryInfo.specifiedQueryInfo.rate = 0;
    }

    cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery, "query_times");
    if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
      g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint;
    } else if (!specifiedQueryTimes) {
      g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times;
    } else {
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__);
      goto PARSE_OVER;
3806
    }
3807

3808
    cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent");
3809
    if (concurrent && concurrent->type == cJSON_Number) {
3810
      g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint;
3811
    } else if (!concurrent) {
3812
      g_queryInfo.specifiedQueryInfo.concurrent = 1;
3813
    }
3814

3815
    cJSON* mode = cJSON_GetObjectItem(specifiedQuery, "mode");
3816
    if (mode && mode->type == cJSON_String && mode->valuestring != NULL) {
3817
      if (0 == strcmp("sync", mode->valuestring)) {
3818
        g_queryInfo.specifiedQueryInfo.subscribeMode = 0;
3819
      } else if (0 == strcmp("async", mode->valuestring)) {
3820
        g_queryInfo.specifiedQueryInfo.subscribeMode = 1;
3821
      } else {
3822
        printf("ERROR: failed to read json, subscribe mod error\n");
3823 3824 3825
        goto PARSE_OVER;
      }
    } else {
3826
      g_queryInfo.specifiedQueryInfo.subscribeMode = 0;
3827
    }
3828

3829
    cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval");
3830
    if (interval && interval->type == cJSON_Number) {
3831
      g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint;
3832
    } else if (!interval) {
3833 3834
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
3835
      g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000;
3836
    }
3837

3838
    cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart");
3839
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
3840
      if (0 == strcmp("yes", restart->valuestring)) {
3841
        g_queryInfo.specifiedQueryInfo.subscribeRestart = 1;
3842
      } else if (0 == strcmp("no", restart->valuestring)) {
3843
        g_queryInfo.specifiedQueryInfo.subscribeRestart = 0;
3844
      } else {
3845
        printf("ERROR: failed to read json, subscribe restart error\n");
3846 3847 3848
        goto PARSE_OVER;
      }
    } else {
3849
      g_queryInfo.specifiedQueryInfo.subscribeRestart = 1;
3850
    }
3851

3852
    cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress");
3853 3854 3855
    if (keepProgress
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
3856
      if (0 == strcmp("yes", keepProgress->valuestring)) {
3857
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1;
3858
      } else if (0 == strcmp("no", keepProgress->valuestring)) {
3859
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
3860
      } else {
3861
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
3862 3863 3864
        goto PARSE_OVER;
      }
    } else {
3865
      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
3866 3867
    }

3868
    // sqls
3869
    cJSON* superSqls = cJSON_GetObjectItem(specifiedQuery, "sqls");
3870
    if (!superSqls) {
3871
      g_queryInfo.specifiedQueryInfo.sqlCount = 0;
3872
    } else if (superSqls->type != cJSON_Array) {
3873
      printf("ERROR: failed to read json, super sqls not found\n");
3874
      goto PARSE_OVER;
3875
    } else {
3876 3877
      int superSqlSize = cJSON_GetArraySize(superSqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
3878
        printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT);
3879 3880
        goto PARSE_OVER;
      }
3881

3882
      g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize;
3883 3884 3885
      for (int j = 0; j < superSqlSize; ++j) {
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
        if (sql == NULL) continue;
3886

3887 3888
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
3889
          printf("ERROR: failed to read json, sql not found\n");
3890 3891
          goto PARSE_OVER;
        }
3892
        tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
H
hzcheng 已提交
3893

3894 3895
        cJSON *result = cJSON_GetObjectItem(sql, "result");
        if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) {
3896
          tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN);
3897
        } else if (NULL == result) {
3898
          memset(g_queryInfo.specifiedQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
3899
        } else {
3900
          printf("ERROR: failed to read json, super query result file not found\n");
3901
          goto PARSE_OVER;
3902
        }
3903 3904 3905
      }
    }
  }
3906

3907
  // sub_table_query 
3908 3909 3910 3911 3912
  cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!superQuery) {
    g_queryInfo.superQueryInfo.threadCnt = 0;
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
3913
    printf("ERROR: failed to read json, sub_table_query not found\n");
3914 3915
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
3916
  } else {
3917
    cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval");
3918
    if (subrate && subrate->type == cJSON_Number) {
3919
      g_queryInfo.superQueryInfo.rate = subrate->valueint;
3920
    } else if (!subrate) {
3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931
      g_queryInfo.superQueryInfo.rate = 0;
    }

    cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
    if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
      g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
    } else if (!superQueryTimes) {
      g_queryInfo.superQueryInfo.queryTimes = g_args.query_times;
    } else {
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__);
      goto PARSE_OVER;
3932
    }
3933

3934
    cJSON* threads = cJSON_GetObjectItem(superQuery, "threads");
3935
    if (threads && threads->type == cJSON_Number) {
3936
      g_queryInfo.superQueryInfo.threadCnt = threads->valueint;
3937
    } else if (!threads) {
3938
      g_queryInfo.superQueryInfo.threadCnt = 1;
3939
    }
3940

3941
    //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count");
3942
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
3943
    //  g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint;
3944
    //} else if (!subTblCnt) {
3945
    //  g_queryInfo.superQueryInfo.childTblCount = 0;
3946
    //}
3947

3948
    cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname");
3949
    if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) {
3950
      tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring, MAX_TB_NAME_SIZE);
3951
    } else {
3952
      printf("ERROR: failed to read json, super table name not found\n");
3953 3954
      goto PARSE_OVER;
    }
3955

3956
    cJSON* submode = cJSON_GetObjectItem(superQuery, "mode");
3957
    if (submode && submode->type == cJSON_String && submode->valuestring != NULL) {
3958
      if (0 == strcmp("sync", submode->valuestring)) {
3959
        g_queryInfo.superQueryInfo.subscribeMode = 0;
3960
      } else if (0 == strcmp("async", submode->valuestring)) {
3961
        g_queryInfo.superQueryInfo.subscribeMode = 1;
3962
      } else {
3963
        printf("ERROR: failed to read json, subscribe mod error\n");
3964 3965 3966
        goto PARSE_OVER;
      }
    } else {
3967
      g_queryInfo.superQueryInfo.subscribeMode = 0;
3968
    }
3969

3970
    cJSON* subinterval = cJSON_GetObjectItem(superQuery, "interval");
3971
    if (subinterval && subinterval->type == cJSON_Number) {
3972
      g_queryInfo.superQueryInfo.subscribeInterval = subinterval->valueint;
3973 3974 3975
    } else if (!subinterval) {    
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
3976
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
3977
    }
3978

3979
    cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart");
3980
    if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) {
3981
      if (0 == strcmp("yes", subrestart->valuestring)) {
3982
        g_queryInfo.superQueryInfo.subscribeRestart = 1;
3983
      } else if (0 == strcmp("no", subrestart->valuestring)) {
3984
        g_queryInfo.superQueryInfo.subscribeRestart = 0;
3985
      } else {
3986
        printf("ERROR: failed to read json, subscribe restart error\n");
3987 3988 3989
        goto PARSE_OVER;
      }
    } else {
3990
      g_queryInfo.superQueryInfo.subscribeRestart = 1;
3991
    }
3992

3993
    cJSON* subkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
3994 3995 3996
    if (subkeepProgress &&
            subkeepProgress->type == cJSON_String
            && subkeepProgress->valuestring != NULL) {
3997
      if (0 == strcmp("yes", subkeepProgress->valuestring)) {
3998
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
3999
      } else if (0 == strcmp("no", subkeepProgress->valuestring)) {
4000
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4001
      } else {
4002
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4003 4004 4005
        goto PARSE_OVER;
      }
    } else {
4006
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4007
    }
4008

4009
    // sqls
4010
    cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls");
4011
    if (!subsqls) {
4012
      g_queryInfo.superQueryInfo.sqlCount = 0;
4013
    } else if (subsqls->type != cJSON_Array) {
4014
      printf("ERROR: failed to read json, super sqls not found\n");
4015
      goto PARSE_OVER;
4016
    } else {
4017 4018
      int superSqlSize = cJSON_GetArraySize(subsqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4019
        printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT);
4020 4021
        goto PARSE_OVER;
      }
4022

4023
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
4024
      for (int j = 0; j < superSqlSize; ++j) {
4025 4026
        cJSON* sql = cJSON_GetArrayItem(subsqls, j);
        if (sql == NULL) continue;
4027

4028 4029
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
4030
          printf("ERROR: failed to read json, sql not found\n");
4031 4032
          goto PARSE_OVER;
        }
4033
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
4034 4035 4036

        cJSON *result = cJSON_GetObjectItem(sql, "result");
        if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){
4037
          tstrncpy(g_queryInfo.superQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN);
4038
        } else if (NULL == result) {
4039
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4040
        }  else {
4041
          printf("ERROR: failed to read json, sub query result file not found\n");
4042
          goto PARSE_OVER;
4043
        }
4044 4045
      }
    }
H
hzcheng 已提交
4046 4047
  }

4048
  ret = true;
H
hzcheng 已提交
4049

4050 4051 4052 4053 4054 4055
PARSE_OVER:
  //free(content);
  //cJSON_Delete(root);
  //fclose(fp);
  return ret;
}
H
hzcheng 已提交
4056

4057
static bool getInfoFromJsonFile(char* file) {
4058
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
4059

4060 4061 4062 4063 4064
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
4065

4066 4067 4068 4069 4070 4071 4072 4073 4074
  bool  ret = false;
  int   maxLen = 64000;
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
4075
  }
H
Hui Li 已提交
4076

4077 4078 4079
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
4080
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
4081 4082
    goto PARSE_OVER;
  }
H
Hui Li 已提交
4083

4084 4085 4086
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
4087
      g_args.test_mode = INSERT_TEST;
4088
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
4089
      g_args.test_mode = QUERY_TEST;
4090
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
4091
      g_args.test_mode = SUBSCRIBE_TEST;
4092
    } else {
4093
      printf("ERROR: failed to read json, filetype not support\n");
4094 4095 4096
      goto PARSE_OVER;
    }
  } else if (!filetype) {
4097
    g_args.test_mode = INSERT_TEST;
4098
  } else {
4099
    printf("ERROR: failed to read json, filetype not found\n");
4100 4101
    goto PARSE_OVER;
  }
H
hzcheng 已提交
4102

4103
  if (INSERT_TEST == g_args.test_mode) {
4104
    ret = getMetaFromInsertJsonFile(root);
4105 4106
  } else if ((QUERY_TEST == g_args.test_mode)
          || (SUBSCRIBE_TEST == g_args.test_mode)) {
4107 4108
    ret = getMetaFromQueryJsonFile(root);
  } else {
4109 4110
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n",
            __func__, __LINE__);
4111
    goto PARSE_OVER;
4112
  }
4113 4114 4115 4116

PARSE_OVER:
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
4117
  fclose(fp);
4118 4119
  return ret;
}
H
hzcheng 已提交
4120

4121
static void prepareSampleData() {
4122
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4123 4124 4125
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
        (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]);
S
Shuaiqiang Chang 已提交
4126
      }
4127 4128 4129 4130
    }
  }
}

4131
static void postFreeResource() {
4132
  tmfclose(g_fpOfInsertResult);
4133
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4134
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
4135 4136 4137
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
4138
      }
4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4151
    }
4152 4153
  }
}
S
Shuaiqiang Chang 已提交
4154

4155 4156
static int getRowDataFromSample(char*  dataBuf, int maxLen, int64_t timestamp,
      SSuperTable* superTblInfo, int* sampleUsePos) {
4157
  if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
4158
/*    int ret = readSampleFromCsvFileToMem(superTblInfo);
4159
    if (0 != ret) {
4160
      tmfree(superTblInfo->sampleDataBuf);
4161
      superTblInfo->sampleDataBuf = NULL;
4162 4163
      return -1;
    }
4164
*/
4165 4166
    *sampleUsePos = 0;
  }
S
Shuaiqiang Chang 已提交
4167

4168
  int    dataLen = 0;
4169

4170 4171 4172 4173
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "(%" PRId64 ", ", timestamp);
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
4174
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
S
Shuaiqiang Chang 已提交
4175

4176
  (*sampleUsePos)++;
4177
 
4178 4179
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4180

4181
static int generateRowData(char*  dataBuf, int maxLen, int64_t timestamp, SSuperTable* stbInfo) {
4182 4183
  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp);
4184
  for (int i = 0; i < stbInfo->columnCount; i++) {
4185 4186
    if ((0 == strncasecmp(stbInfo->columns[i].dataType, "binary", 6))
            || (0 == strncasecmp(stbInfo->columns[i].dataType, "nchar", 5))) {
4187
      if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
4188
        errorPrint( "binary or nchar length overflow, max size:%u\n",
4189
                (uint32_t)TSDB_MAX_BINARY_LEN);
4190 4191
        return (-1);
      }
4192

4193 4194
      char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
      if (NULL == buf) {
4195
        errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
4196 4197 4198 4199 4200
        return (-1);
      }
      rand_string(buf, stbInfo->columns[i].dataLen);
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "\'%s\', ", buf);
      tmfree(buf);
4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "int", 3)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
              "%d, ", rand_int());
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "bigint", 6)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
              "%"PRId64", ", rand_bigint());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "float", 5)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
              "%f, ", rand_float());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "double", 6)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
              "%f, ", rand_double());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "smallint", 8)) {
4219 4220 4221 4222 4223 4224 4225 4226
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_smallint());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType, "tinyint", 7)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_tinyint());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bool", 4)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_bool());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType, "timestamp", 9)) {
      dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint());
    }  else {
4227
      errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType);
4228 4229
      return (-1);
    }
S
Shuaiqiang Chang 已提交
4230
  }
4231

4232 4233
  dataLen -= 2;
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
4234

4235 4236
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4237

4238
static int32_t generateData(char *res, char **data_type,
4239
        int num_of_cols, int64_t timestamp, int lenOfBinary) {
4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270
  memset(res, 0, MAX_DATA_SIZE);
  char *pstr = res;
  pstr += sprintf(pstr, "(%" PRId64, timestamp);
  int c = 0;

  for (; c < MAX_NUM_DATATYPE; c++) {
    if (data_type[c] == NULL) {
      break;
    }
  }

  if (0 == c) {
    perror("data type error!");
    exit(-1);
  }

  for (int i = 0; i < num_of_cols; i++) {
    if (strcasecmp(data_type[i % c], "tinyint") == 0) {
      pstr += sprintf(pstr, ", %d", rand_tinyint() );
    } else if (strcasecmp(data_type[i % c], "smallint") == 0) {
      pstr += sprintf(pstr, ", %d", rand_smallint());
    } else if (strcasecmp(data_type[i % c], "int") == 0) {
      pstr += sprintf(pstr, ", %d", rand_int()); 
    } else if (strcasecmp(data_type[i % c], "bigint") == 0) {
      pstr += sprintf(pstr, ", %" PRId64, rand_bigint());
    } else if (strcasecmp(data_type[i % c], "float") == 0) {
      pstr += sprintf(pstr, ", %10.4f", rand_float());
    } else if (strcasecmp(data_type[i % c], "double") == 0) {
      double t = rand_double();
      pstr += sprintf(pstr, ", %20.8f", t);
    } else if (strcasecmp(data_type[i % c], "bool") == 0) {
4271
      bool b = taosRandom() & 1;
4272 4273
      pstr += sprintf(pstr, ", %s", b ? "true" : "false");
    } else if (strcasecmp(data_type[i % c], "binary") == 0) {
4274 4275
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4276 4277 4278
      pstr += sprintf(pstr, ", \"%s\"", s);
      free(s);
    }else if (strcasecmp(data_type[i % c], "nchar") == 0) {
4279 4280
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295
      pstr += sprintf(pstr, ", \"%s\"", s);
      free(s);
    }

    if (pstr - res > MAX_DATA_SIZE) {
      perror("column length too long, abort");
      exit(-1);
    }
  }

  pstr += sprintf(pstr, ")");

  return (int32_t)(pstr - res);
}

4296
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
4297 4298
  char* sampleDataBuf = NULL;

4299
  sampleDataBuf = calloc(
4300
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
4301
  if (sampleDataBuf == NULL) {
4302 4303
      errorPrint("%s() LN%d, Failed to calloc %d Bytes, reason:%s\n", 
              __func__, __LINE__,
4304 4305
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 
              strerror(errno));
4306
      return -1;
4307
  }
4308

4309 4310 4311 4312
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
4313
      errorPrint("%s() LN%d, read sample from csv file failed.\n", __func__, __LINE__);
4314
      tmfree(sampleDataBuf);
4315
      superTblInfo->sampleDataBuf = NULL;
4316
      return -1;
H
Hui Li 已提交
4317
  }
4318

4319 4320 4321
  return 0;
}

4322
static int execInsert(threadInfo *pThreadInfo, char *buffer, int k)
4323 4324
{
  int affectedRows;
4325
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4326

4327 4328
  verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
            __func__, __LINE__, buffer);
4329
  if (superTblInfo) {
4330
    if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) {
4331
      affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false);
4332
    } else {
4333
      if (0 != postProceSql(g_Dbs.host, g_Dbs.port, buffer)) {
4334
        affectedRows = -1;
4335
        printf("========restful return fail, threadID[%d]\n", pThreadInfo->threadID);
4336 4337 4338
      } else {
        affectedRows = k;
      }
4339
    }
4340
  } else {
4341
    affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false);
4342 4343 4344 4345 4346
  }

  return affectedRows;
}

4347
static void getTableName(char *pTblName, threadInfo* pThreadInfo, int tableSeq)
4348 4349
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4350 4351 4352 4353 4354 4355
  if (superTblInfo) {
    if ((superTblInfo->childTblOffset >= 0)
            && (superTblInfo->childTblLimit > 0)) {
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
    } else {
4356

4357 4358 4359
        verbosePrint("[%d] %s() LN%d: from=%d count=%d seq=%d\n",
                pThreadInfo->threadID, __func__, __LINE__,
                pThreadInfo->start_table_from,
4360 4361 4362 4363 4364 4365
                pThreadInfo->ntables, tableSeq);
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
    }
  } else {
    snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%d",
4366
        g_args.tb_prefix, tableSeq);
4367 4368 4369 4370
  }
}

static int generateDataTail(char *tableName, int32_t tableSeq,
4371
        threadInfo* pThreadInfo, SSuperTable* superTblInfo,
4372 4373 4374
        int batch, char* buffer, int64_t insertRows,
        int64_t startFrom, uint64_t startTime, int *pSamplePos, int *dataLen) {
  int len = 0;
4375 4376
  int ncols_per_record = 1; // count first col ts

4377
  if (superTblInfo == NULL) {
4378 4379 4380 4381 4382
    int datatypeSeq = 0;
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
4383 4384
  }

4385
  verbosePrint("%s() LN%d batch=%d\n", __func__, __LINE__, batch);
4386

4387 4388
  int k = 0;
  for (k = 0; k < batch;) {
4389
    if (superTblInfo) {
4390
        int retLen = 0;
4391

4392 4393 4394
        if (0 == strncasecmp(superTblInfo->dataSource,
                    "sample", strlen("sample"))) {
          retLen = getRowDataFromSample(
4395 4396
                    buffer + len,
                    superTblInfo->maxSqlLen - len,
4397
                    startTime + superTblInfo->timeStampStep * k,
4398
                    superTblInfo,
4399
                    pSamplePos);
4400 4401 4402
       } else if (0 == strncasecmp(superTblInfo->dataSource,
                   "rand", strlen("rand"))) {
          int rand_num = rand_tinyint() % 100;
4403
          if (0 != superTblInfo->disorderRatio
4404
                    && rand_num < superTblInfo->disorderRatio) {
4405 4406 4407
            int64_t d = startTime
                + superTblInfo->timeStampStep * k
                - taosRandom() % superTblInfo->disorderRange;
4408
            retLen = generateRowData(
4409
                      buffer + len,
4410
                      superTblInfo->maxSqlLen - len,
4411
                      d,
4412
                      superTblInfo);
4413 4414
          } else {
            retLen = generateRowData(
4415 4416
                      buffer + len,
                      superTblInfo->maxSqlLen - len,
4417
                      startTime + superTblInfo->timeStampStep * k,
4418
                      superTblInfo);
4419 4420
          }
       }
4421

4422 4423 4424
       if (retLen < 0) {
         return -1;
       }
4425

4426
       len += retLen;
4427

4428 4429 4430 4431
       if (len >= (superTblInfo->maxSqlLen - 256)) {    // reserve for overwrite
         k++;
         break;
       }
4432
    } else {
4433
      int rand_num = taosRandom() % 100;
4434 4435 4436 4437
          char data[MAX_DATA_SIZE];
          char **data_type = g_args.datatype;
          int lenOfBinary = g_args.len_of_binary;

4438
      if ((g_args.disorderRatio != 0)
4439
                && (rand_num < g_args.disorderRange)) {
4440

4441 4442
        int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k 
            - taosRandom() % 1000000 + rand_num;
4443
        len = generateData(data, data_type,
4444
                  ncols_per_record, d, lenOfBinary);
4445
      } else {
4446
            len = generateData(data, data_type,
4447
                  ncols_per_record,
4448
                  startTime + DEFAULT_TIMESTAMP_STEP * k,
4449 4450
                  lenOfBinary);
      }
4451

4452 4453
      buffer += sprintf(buffer, " %s", data);
      if (strlen(buffer) >= (g_args.max_sql_len - 256)) { // too long
4454 4455
          k++;
          break;
4456 4457 4458
      }
    }

4459 4460
    verbosePrint("%s() LN%d len=%d k=%d \nbuffer=%s\n",
            __func__, __LINE__, len, k, buffer);
4461 4462 4463

    k++;
    startFrom ++;
4464

4465
    if (startFrom >= insertRows) {
4466
      break;
4467
    }
4468
  }
4469

4470 4471 4472
  *dataLen = len;
  return k;
}
4473

4474 4475
static int generateSQLHead(char *tableName, int32_t tableSeq,
        threadInfo* pThreadInfo, SSuperTable* superTblInfo, char *buffer)
4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488
{
  int len;
  if (superTblInfo) {
    if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
            tagsValBuf = generateTagVaulesForStb(superTblInfo);
      } else {
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
      }
      if (NULL == tagsValBuf) {
4489
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n", __func__, __LINE__);
4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506
        return -1;
      }

      len = snprintf(buffer,
                  superTblInfo->maxSqlLen,
                  "insert into %s.%s using %s.%s tags %s values",
                  pThreadInfo->db_name,
                  tableName,
                  pThreadInfo->db_name,
                  superTblInfo->sTblName,
                  tagsValBuf);
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
      len = snprintf(buffer,
                  superTblInfo->maxSqlLen,
                  "insert into %s.%s values",
                  pThreadInfo->db_name,
4507
                  tableName);
4508 4509
    } else {
      len = snprintf(buffer,
4510
                  superTblInfo->maxSqlLen,
4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525
                  "insert into %s.%s values",
                  pThreadInfo->db_name,
                  tableName);
    }
  } else {
      len = snprintf(buffer,
                  g_args.max_sql_len,
                  "insert into %s.%s values",
                  pThreadInfo->db_name,
                  tableName);
  }

  return len;
}

4526
static int generateProgressiveDataBuffer(char *pTblName,
4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549
        int32_t tableSeq,
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
        int64_t startFrom, int64_t startTime, int *pSamplePos)
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  int ncols_per_record = 1; // count first col ts

  if (superTblInfo == NULL) {
    int datatypeSeq = 0;
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
  }

  assert(buffer != NULL);

  memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len);

  char *pstr = buffer;

4550 4551
  int headLen = generateSQLHead(pTblName, tableSeq, pThreadInfo, superTblInfo,
          buffer);
4552 4553 4554 4555 4556
  pstr += headLen;

  int k;
  int dataLen;
  k = generateDataTail(pTblName, tableSeq, pThreadInfo, superTblInfo,
4557 4558
          g_args.num_of_RPR, pstr, insertRows, startFrom,
          startTime,
4559
          pSamplePos, &dataLen);
4560 4561
  return k;
}
4562

4563
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
4564 4565
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
         pThreadInfo->threadID, __func__, __LINE__);
4566 4567

  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579
  int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows;

  int insertMode;

  if (interlaceRows > 0) {
    insertMode = INTERLACE_INSERT_MODE;
  } else {
    insertMode = PROGRESSIVE_INSERT_MODE;
  }

  // TODO: prompt tbl count multple interlace rows and batch
  //
4580 4581 4582

  char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1);
  if (NULL == buffer) {
4583
    errorPrint( "Failed to alloc %d Bytes, reason:%s\n",
4584 4585 4586 4587 4588 4589
              superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len,
              strerror(errno));
    return NULL;
  }


4590
  char tableName[TSDB_TABLE_NAME_LEN];
4591 4592 4593 4594 4595

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

  int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
4596 4597
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
4598 4599 4600 4601 4602 4603 4604 4605 4606
  uint64_t st = 0;
  uint64_t et = 0xffffffff;

  int64_t lastPrintTime = taosGetTimestampMs();
  int64_t startTs = taosGetTimestampUs();
  int64_t endTs;

  int tableSeq = pThreadInfo->start_table_from;

4607 4608
  debugPrint("[%d] %s() LN%d: start_table_from=%d ntables=%d insertRows=%"PRId64"\n",
          pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from,
4609 4610 4611 4612 4613
          pThreadInfo->ntables, insertRows);

  int64_t startTime = pThreadInfo->start_time;

  int batchPerTblTimes;
4614
  int batchPerTbl;
4615

4616 4617
  assert(pThreadInfo->ntables > 0);

4618 4619
  if (interlaceRows > g_args.num_of_RPR)
        interlaceRows = g_args.num_of_RPR;
4620

4621 4622
  batchPerTbl = interlaceRows;
  if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) {
4623
    batchPerTblTimes =
4624
        (g_args.num_of_RPR / (interlaceRows * pThreadInfo->ntables)) + 1;
4625 4626 4627 4628
  } else {
    batchPerTblTimes = 1;
  }

4629
  int generatedRecPerTbl = 0;
4630 4631
  bool flagSleep = true;
  int sleepTimeTotal = 0;
4632
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
4633
    if ((flagSleep) && (insert_interval)) {
4634
        st = taosGetTimestampUs();
4635
        flagSleep = false;
4636 4637 4638
    }
    // generate data
    memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len);
4639

4640
    char *pstr = buffer;
4641
    int recOfBatch = 0;
4642

4643 4644
    for (int i = 0; i < batchPerTblTimes; i ++) {
      getTableName(tableName, pThreadInfo, tableSeq);
4645

4646 4647
      int headLen;
      if (i == 0) {
4648 4649
        headLen = generateSQLHead(tableName, tableSeq, pThreadInfo,
                superTblInfo, pstr);
4650
      } else {
4651
        headLen = snprintf(pstr, TSDB_TABLE_NAME_LEN, "%s.%s values",
4652 4653
                  pThreadInfo->db_name,
                  tableName);
4654
      }
4655

4656
      // generate data buffer
4657 4658
      verbosePrint("[%d] %s() LN%d i=%d buffer:\n%s\n",
                pThreadInfo->threadID, __func__, __LINE__, i, buffer);
4659

4660 4661
      pstr += headLen;
      int dataLen = 0;
4662

4663
      verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n",
4664 4665
                pThreadInfo->threadID, __func__, __LINE__,
                i, batchPerTblTimes, batchPerTbl);
4666 4667 4668 4669 4670 4671 4672

      if (superTblInfo) {
        if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
          startTime = taosGetTimestamp(pThreadInfo->time_precision);
        }
      } else {
          startTime = 1500000000000;
4673
      }
4674 4675 4676
      generateDataTail(
        tableName, tableSeq, pThreadInfo, superTblInfo,
        batchPerTbl, pstr, insertRows, 0,
4677
        startTime,
4678
        &(pThreadInfo->samplePos), &dataLen);
4679

4680
      pstr += dataLen;
4681
      recOfBatch += batchPerTbl;
4682
      startTime += batchPerTbl * superTblInfo->timeStampStep;
4683
      pThreadInfo->totalInsertRows += batchPerTbl;
4684
      verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n",
4685 4686
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
4687

4688 4689 4690 4691
      tableSeq ++;
      if (insertMode == INTERLACE_INSERT_MODE) {
          if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
            // turn to first table
4692
            startTime += batchPerTbl * superTblInfo->timeStampStep;
4693
            tableSeq = pThreadInfo->start_table_from;
4694 4695 4696 4697 4698 4699 4700
            generatedRecPerTbl += batchPerTbl;
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;

            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
4701
          }
4702 4703
      }

4704
      int remainRows = insertRows - generatedRecPerTbl;
4705
      if ((remainRows > 0) && (batchPerTbl > remainRows))
4706 4707
        batchPerTbl = remainRows;

4708
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%d insertRows=%"PRId64"\n",
4709 4710 4711 4712
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
4713 4714 4715
        break;
    }

4716
    verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRId64"\n",
4717 4718 4719 4720 4721
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
           pThreadInfo->threadID, __func__, __LINE__, buffer);

4722 4723
    startTs = taosGetTimestampUs();

4724
    int affectedRows = execInsert(pThreadInfo, buffer, recOfBatch);
4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735

    endTs = taosGetTimestampUs();
    int64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %10.6fms\n",
            __func__, __LINE__, delay/1000.0);

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

4736 4737
    verbosePrint("[%d] %s() LN%d affectedRows=%d\n", pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
4738 4739
    if ((affectedRows < 0) || (recOfBatch != affectedRows)) {
        errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %d\n%s\n",
4740
                pThreadInfo->threadID, __func__, __LINE__,
4741
                recOfBatch, affectedRows, buffer);
4742
        goto free_and_statistics_interlace;
4743
    }
4744

4745
    pThreadInfo->totalAffectedRows += affectedRows;
4746

4747 4748 4749
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
      printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
4750 4751 4752
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
4753 4754
      lastPrintTime = currentPrintTime;
    }
4755

4756 4757
    if ((insert_interval) && flagSleep) {
      et = taosGetTimestampUs();
4758

4759 4760
      if (insert_interval > ((et - st)/1000) ) {
        int sleepTime = insert_interval - (et -st)/1000;
4761 4762
        performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleepTime);
4763 4764 4765
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
4766
    }
4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778
  }

free_and_statistics_interlace:
  tmfree(buffer);

  printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", 
          pThreadInfo->threadID, 
          pThreadInfo->totalInsertRows, 
          pThreadInfo->totalAffectedRows);
  return NULL;
}

4779 4780 4781 4782 4783 4784 4785 4786
// sync insertion
/*
   1 thread: 100 tables * 2000  rows/s
   1 thread: 10  tables * 20000 rows/s
   6 thread: 300 tables * 2000  rows/s

   2 taosinsertdata , 1 thread:  10  tables * 20000 rows/s
*/
4787
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
4788
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
4789

4790
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4791

4792 4793
  char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1);
  if (NULL == buffer) {
4794
    errorPrint( "Failed to alloc %d Bytes, reason:%s\n",
4795 4796 4797 4798
              superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len,
              strerror(errno));
    return NULL;
  }
4799

4800 4801 4802 4803
  int64_t lastPrintTime = taosGetTimestampMs();
  int64_t startTs = taosGetTimestampUs();
  int64_t endTs;

4804 4805
  int timeStampStep =
      superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
4806
/*  int insert_interval =
4807
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
4808 4809
  uint64_t st = 0;
  uint64_t et = 0xffffffff;
4810
  */
4811

4812 4813
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
4814

4815
  pThreadInfo->samplePos = 0;
4816

4817 4818
  for (uint32_t tableSeq =
          pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to;
4819
        tableSeq ++) {
4820
    int64_t start_time = pThreadInfo->start_time;
4821 4822 4823 4824 4825

    int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
    verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows);

    for (int64_t i = 0; i < insertRows;) {
4826
        /*
4827 4828
      if (insert_interval) {
            st = taosGetTimestampUs();
4829
      }
4830
      */
4831

4832 4833 4834 4835 4836 4837
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
      verbosePrint("%s() LN%d: tid=%d seq=%d tableName=%s\n",
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);

4838
      int generated = generateProgressiveDataBuffer(
4839
              tableName, tableSeq, pThreadInfo, buffer, insertRows,
4840
            i, start_time,
4841
            &(pThreadInfo->samplePos));
4842 4843 4844 4845 4846
      if (generated > 0)
        i += generated;
      else
        goto free_and_statistics_2;

4847
      start_time +=  generated * timeStampStep;
4848
      pThreadInfo->totalInsertRows += generated;
4849 4850 4851 4852

      startTs = taosGetTimestampUs();

      int affectedRows = execInsert(pThreadInfo, buffer, generated);
4853 4854 4855

      endTs = taosGetTimestampUs();
      int64_t delay = endTs - startTs;
4856 4857 4858
      performancePrint("%s() LN%d, insert execution time is %10.6fms\n",
              __func__, __LINE__, delay/1000.0);

4859 4860 4861 4862
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
4863

4864 4865 4866 4867 4868
      if (affectedRows < 0)
        goto free_and_statistics_2;

      pThreadInfo->totalAffectedRows += affectedRows;

4869 4870
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
4871
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
4872 4873 4874
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
4875 4876 4877
        lastPrintTime = currentPrintTime;
      }

4878
      if (i >= insertRows)
4879
        break;
4880
/*
4881 4882 4883 4884 4885
      if (insert_interval) {
        et = taosGetTimestampUs();

        if (insert_interval > ((et - st)/1000) ) {
            int sleep_time = insert_interval - (et -st)/1000;
4886 4887
            performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleep_time);
4888 4889
            taosMsleep(sleep_time); // ms
        }
4890
      }
4891
      */
4892 4893
    }   // num_of_DPT

4894
    if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo &&
4895 4896
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
4897 4898
          printf("%s() LN%d samplePos=%d\n",
                  __func__, __LINE__, pThreadInfo->samplePos);
4899
    }
4900
  } // tableSeq
4901

4902
free_and_statistics_2:
4903 4904
  tmfree(buffer);

4905
  printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", 
4906 4907 4908
          pThreadInfo->threadID, 
          pThreadInfo->totalInsertRows, 
          pThreadInfo->totalAffectedRows);
4909
  return NULL;
H
Hui Li 已提交
4910 4911
}

4912 4913 4914 4915 4916
static void* syncWrite(void *sarg) {

  threadInfo *winfo = (threadInfo *)sarg; 
  SSuperTable* superTblInfo = winfo->superTblInfo;

4917
  int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows;
4918

4919
  if (interlaceRows > 0) {
4920 4921 4922 4923 4924 4925 4926 4927
    // interlace mode
    return syncWriteInterlace(winfo);
  } else {
    // progressive mode
    return syncWriteProgressive(winfo);
  }
}

4928
static void callBack(void *param, TAOS_RES *res, int code) {
4929
  threadInfo* winfo = (threadInfo*)param; 
4930
  SSuperTable* superTblInfo = winfo->superTblInfo;
4931

4932 4933
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
4934 4935 4936 4937
  if (insert_interval) {
    winfo->et = taosGetTimestampUs();
    if (((winfo->et - winfo->st)/1000) < insert_interval) {
      taosMsleep(insert_interval - (winfo->et - winfo->st)/1000); // ms
4938
    }
H
Hui Li 已提交
4939
  }
4940

4941 4942 4943
  char *buffer = calloc(1, winfo->superTblInfo->maxSqlLen);
  char *data   = calloc(1, MAX_DATA_SIZE);
  char *pstr = buffer;
4944 4945
  pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix,
          winfo->start_table_from);
4946
//  if (winfo->counter >= winfo->superTblInfo->insertRows) {
4947
  if (winfo->counter >= g_args.num_of_RPR) {
4948
    winfo->start_table_from++;
4949 4950
    winfo->counter = 0;
  }
4951
  if (winfo->start_table_from > winfo->end_table_to) {
4952 4953 4954 4955 4956 4957
    tsem_post(&winfo->lock_sem);
    free(buffer);
    free(data);
    taos_free_result(res);
    return;
  }
4958

4959
  for (int i = 0; i < g_args.num_of_RPR; i++) {
4960
    int rand_num = taosRandom() % 100;
4961 4962
    if (0 != winfo->superTblInfo->disorderRatio
            && rand_num < winfo->superTblInfo->disorderRatio) {
4963
      int64_t d = winfo->lastTs - taosRandom() % 1000000 + rand_num;
4964
      //generateData(data, datatype, ncols_per_record, d, len_of_binary);
4965
      generateRowData(data, MAX_DATA_SIZE, d, winfo->superTblInfo);
4966
    } else {
4967
      //generateData(data, datatype, ncols_per_record, start_time += 1000, len_of_binary);
4968
      generateRowData(data, MAX_DATA_SIZE, winfo->lastTs += 1000, winfo->superTblInfo);
H
Hui Li 已提交
4969
    }
4970 4971
    pstr += sprintf(pstr, "%s", data);
    winfo->counter++;
H
Hui Li 已提交
4972

4973 4974
    if (winfo->counter >= winfo->superTblInfo->insertRows) {
      break;
H
Hui Li 已提交
4975 4976
    }
  }
4977

4978 4979
  if (insert_interval) {
    winfo->st = taosGetTimestampUs();
4980 4981 4982 4983
  }
  taos_query_a(winfo->taos, buffer, callBack, winfo);
  free(buffer);
  free(data);
H
Hui Li 已提交
4984

4985
  taos_free_result(res);
H
Hui Li 已提交
4986 4987
}

4988
static void *asyncWrite(void *sarg) {
4989
  threadInfo *winfo = (threadInfo *)sarg;
4990
  SSuperTable* superTblInfo = winfo->superTblInfo;
4991 4992 4993 4994

  winfo->st = 0;
  winfo->et = 0;
  winfo->lastTs = winfo->start_time;
4995

4996
  int insert_interval =
4997
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
4998 4999
  if (insert_interval) {
    winfo->st = taosGetTimestampUs();
H
Hui Li 已提交
5000
  }
5001 5002 5003
  taos_query_a(winfo->taos, "show databases", callBack, winfo);

  tsem_wait(&(winfo->lock_sem));
H
Hui Li 已提交
5004 5005 5006 5007

  return NULL;
}

5008 5009
static void startMultiThreadInsertData(int threads, char* db_name,
        char* precision,SSuperTable* superTblInfo) {
5010 5011

    pthread_t *pids = malloc(threads * sizeof(pthread_t));
5012 5013
    assert(pids != NULL);

5014
    threadInfo *infos = malloc(threads * sizeof(threadInfo));
5015 5016
    assert(infos != NULL);

5017 5018
    memset(pids, 0, threads * sizeof(pthread_t));
    memset(infos, 0, threads * sizeof(threadInfo));
H
Hui Li 已提交
5019

5020
    int ntables = 0;
5021 5022
    if (superTblInfo) {

5023
        if ((superTblInfo->childTblOffset >= 0)
5024 5025
            && (superTblInfo->childTblLimit > 0)) {

5026
            ntables = superTblInfo->childTblLimit;
5027
        } else {
5028
            ntables = superTblInfo->childTblCount;
5029 5030
        }
    } else {
5031
        ntables = g_args.num_of_tables;
5032
    }
H
Hui Li 已提交
5033

5034 5035 5036 5037 5038 5039 5040 5041 5042 5043
    int a = ntables / threads;
    if (a < 1) {
        threads = ntables;
        a = 1;
    }

    int b = 0;
    if (threads != 0) {
        b = ntables % threads;
    }
5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060

  //TAOS* taos;
  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
  //  if (NULL == taos) {
  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
  //    exit(-1);
  //  }
  //}

  int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
  if (0 != precision[0]) {
    if (0 == strncasecmp(precision, "ms", 2)) {
      timePrec = TSDB_TIME_PRECISION_MILLI;
    }  else if (0 == strncasecmp(precision, "us", 2)) {
      timePrec = TSDB_TIME_PRECISION_MICRO;
    }  else {
5061
      errorPrint("Not support precision: %s\n", precision);
5062 5063 5064 5065
      exit(-1);
    }
  }

5066
  int64_t start_time;
5067 5068 5069
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
        start_time = taosGetTimestamp(timePrec);
5070
    } else {
5071 5072 5073 5074 5075
      if (TSDB_CODE_SUCCESS != taosParseTime(
        superTblInfo->startTimestamp,
        &start_time,
        strlen(superTblInfo->startTimestamp),
        timePrec, 0)) {
5076
          ERROR_EXIT("failed to parse time!\n");
5077
      }
5078
    }
5079 5080
  } else {
     start_time = 1500000000000;
5081 5082 5083
  }

  double start = getCurrentTime();
5084

5085
  int startFrom;
5086

5087
  if ((superTblInfo) && (superTblInfo->childTblOffset >= 0))
5088
      startFrom = superTblInfo->childTblOffset;
5089
  else
5090
      startFrom = 0;
5091

5092
  // read sample data from file first
5093
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5094
              "sample", strlen("sample")))) {
5095
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5096 5097
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
              __func__, __LINE__);
5098 5099 5100 5101
      exit(-1);
    }
  }

5102
  // read sample data from file first
5103
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5104 5105
              "sample", strlen("sample")))) {
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5106 5107
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
              __func__, __LINE__);
5108 5109 5110 5111 5112 5113 5114 5115
      exit(-1);
    }
  }

  TAOS* taos = taos_connect(
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
  if (NULL == taos) {
5116 5117
    errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
                __func__, __LINE__, taos_errstr(NULL));
5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135
    exit(-1);
  }

  if (superTblInfo) {

    int limit, offset;
    if (superTblInfo && (superTblInfo->childTblOffset >= 0)
            && (superTblInfo->childTblLimit > 0)) {
        limit = superTblInfo->childTblLimit;
        offset = superTblInfo->childTblOffset;
    } else {
        limit = superTblInfo->childTblCount;
        offset = 0;
    }

    superTblInfo->childTblName = (char*)calloc(1,
        limit * TSDB_TABLE_NAME_LEN);
    if (superTblInfo->childTblName == NULL) {
5136
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150
      taos_close(taos);
      exit(-1);
    }

    int childTblCount;
    getChildNameOfSuperTableWithLimitAndOffset(
        taos,
        db_name, superTblInfo->sTblName,
        &superTblInfo->childTblName, &childTblCount,
        limit,
        offset);
  }
  taos_close(taos);

H
Hui Li 已提交
5151
  for (int i = 0; i < threads; i++) {
5152
    threadInfo *t_info = infos + i;
H
Hui Li 已提交
5153 5154
    t_info->threadID = i;
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
5155
    t_info->time_precision = timePrec;
5156 5157 5158
    t_info->superTblInfo = superTblInfo;

    t_info->start_time = start_time;
5159
    t_info->minDelay = INT16_MAX;
5160

5161 5162
    if ((NULL == superTblInfo) ||
            (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) {
5163
      //t_info->taos = taos;
5164
      t_info->taos = taos_connect(
5165
              g_Dbs.host, g_Dbs.user,
5166
              g_Dbs.password, db_name, g_Dbs.port);
5167
      if (NULL == t_info->taos) {
5168 5169
        errorPrint(
                "connect to server fail from insert sub thread, reason: %s\n",
5170
                taos_errstr(NULL));
5171 5172 5173 5174 5175 5176
        exit(-1);
      }
    } else {
      t_info->taos = NULL;
    }

5177 5178
    if ((NULL == superTblInfo)
            || (0 == superTblInfo->multiThreadWriteOneTbl)) {
5179 5180 5181 5182
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
5183
    } else {
5184 5185
      t_info->start_table_from = 0;
      t_info->ntables = superTblInfo->childTblCount;
5186 5187 5188 5189 5190
      t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint();
    }

    tsem_init(&(t_info->lock_sem), 0, 0);
    if (SYNC == g_Dbs.queryMode) {
5191
      pthread_create(pids + i, NULL, syncWrite, t_info);
5192
    } else {
5193 5194
      pthread_create(pids + i, NULL, asyncWrite, t_info);
    }
H
Hui Li 已提交
5195
  }
5196

H
Hui Li 已提交
5197 5198 5199 5200
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

5201 5202 5203
  int64_t totalDelay = 0;
  int64_t maxDelay = 0;
  int64_t minDelay = INT16_MAX;
5204
  int64_t cntDelay = 1;
5205 5206
  double  avgDelay = 0;

H
Hui Li 已提交
5207
  for (int i = 0; i < threads; i++) {
5208 5209
    threadInfo *t_info = infos + i;

S
TD-1057  
Shengliang Guan 已提交
5210
    tsem_destroy(&(t_info->lock_sem));
5211 5212
    taos_close(t_info->taos);

5213 5214 5215 5216
    debugPrint("%s() LN%d, [%d] totalInsert=%"PRId64" totalAffected=%"PRId64"\n",
            __func__, __LINE__,
            t_info->threadID, t_info->totalInsertRows,
            t_info->totalAffectedRows);
5217 5218
    if (superTblInfo) {
        superTblInfo->totalAffectedRows += t_info->totalAffectedRows;
5219
        superTblInfo->totalInsertRows += t_info->totalInsertRows;
5220 5221 5222
    } else {
        g_args.totalAffectedRows += t_info->totalAffectedRows;
        g_args.totalInsertRows += t_info->totalInsertRows;
5223
    }
5224 5225 5226 5227

    totalDelay  += t_info->totalDelay;
    cntDelay   += t_info->cntDelay;
    if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay;
5228
    if (t_info->minDelay < minDelay) minDelay = t_info->minDelay;
H
Hui Li 已提交
5229
  }
5230
  cntDelay -= 1;
H
Hui Li 已提交
5231

5232
  if (cntDelay == 0)    cntDelay = 1;
5233 5234
  avgDelay = (double)totalDelay / cntDelay;

5235
  double end = getCurrentTime();
5236
  double t = end - start;
5237 5238

  if (superTblInfo) {
5239
    printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n",
5240
          t, superTblInfo->totalInsertRows,
5241 5242
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5243
          superTblInfo->totalInsertRows / t);
5244
    fprintf(g_fpOfInsertResult,
5245
          "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n",
5246
          t, superTblInfo->totalInsertRows,
5247 5248
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5249
          superTblInfo->totalInsertRows/ t);
5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261
  } else {
    printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s %2.f records/second\n\n",
          t, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          g_args.totalInsertRows / t);
    fprintf(g_fpOfInsertResult,
          "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s %2.f records/second\n\n",
          t, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          g_args.totalInsertRows / t);
5262
  }
5263 5264 5265 5266 5267

  printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n",
          avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0);
  fprintf(g_fpOfInsertResult, "insert delay, avg:%10.6fms, max: %10.6fms, min: %10.6fms\n\n",
          avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0);
5268

5269 5270
  //taos_close(taos);

H
Hui Li 已提交
5271
  free(pids);
5272
  free(infos);
H
Hui Li 已提交
5273 5274
}

5275
static void *readTable(void *sarg) {
5276
#if 1
5277
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
5278 5279
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
5280
  uint64_t sTime = rinfo->start_time;
H
hzcheng 已提交
5281 5282
  char *tb_prefix = rinfo->tb_prefix;
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
5283
  if (NULL == fp) {
5284
    errorPrint( "fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
H
Hui Li 已提交
5285 5286
    return NULL;
  }
5287

5288
    int num_of_DPT;
5289
/*  if (rinfo->superTblInfo) {
5290 5291
    num_of_DPT = rinfo->superTblInfo->insertRows; //  nrecords_per_table;
  } else {
5292
  */
5293
      num_of_DPT = g_args.num_of_DPT;
5294
//  }
5295

5296
  int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
H
hzcheng 已提交
5297
  int totalData = num_of_DPT * num_of_tables;
5298
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
  printf("%d records:\n", totalData);
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

  for (int j = 0; j < n; j++) {
    double totalT = 0;
    int count = 0;
    for (int i = 0; i < num_of_tables; i++) {
5311 5312
      sprintf(command, "select %s from %s%d where ts>= %" PRId64,
              aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
5313 5314

      double t = getCurrentTime();
S
Shuaiqiang Chang 已提交
5315 5316
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
5317

S
Shuaiqiang Chang 已提交
5318
      if (code != 0) {
5319
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
5320
        taos_free_result(pSql);
H
hzcheng 已提交
5321
        taos_close(taos);
5322
        fclose(fp);
5323
        return NULL;
H
hzcheng 已提交
5324 5325
      }

S
Shuaiqiang Chang 已提交
5326
      while (taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
5327 5328 5329 5330 5331 5332
        count++;
      }

      t = getCurrentTime() - t;
      totalT += t;

S
Shuaiqiang Chang 已提交
5333
      taos_free_result(pSql);
H
hzcheng 已提交
5334 5335 5336
    }

    fprintf(fp, "|%10s  |   %10d   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
5337
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
5338 5339 5340 5341 5342
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT);
  }
  fprintf(fp, "\n");
  fclose(fp);
5343
#endif
H
hzcheng 已提交
5344 5345 5346
  return NULL;
}

5347
static void *readMetric(void *sarg) {
5348 5349
#if 1  
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
5350 5351 5352
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
5353 5354 5355 5356
  if (NULL == fp) {
    printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
    return NULL;
  }
5357 5358

  int num_of_DPT = rinfo->superTblInfo->insertRows;
5359
  int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
H
hzcheng 已提交
5360
  int totalData = num_of_DPT * num_of_tables;
5361
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
5362 5363 5364 5365 5366 5367 5368 5369 5370

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
  printf("%d records:\n", totalData);
  fprintf(fp, "Querying On %d records:\n", totalData);

  for (int j = 0; j < n; j++) {
sangshuduo's avatar
sangshuduo 已提交
5371
    char condition[BUFFER_SIZE - 30] = "\0";
B
Bomin Zhang 已提交
5372
    char tempS[64] = "\0";
H
hzcheng 已提交
5373 5374 5375 5376 5377

    int m = 10 < num_of_tables ? 10 : num_of_tables;

    for (int i = 1; i <= m; i++) {
      if (i == 1) {
5378
        sprintf(tempS, "t1 = %d", i);
H
hzcheng 已提交
5379
      } else {
5380
        sprintf(tempS, " or t1 = %d ", i);
H
hzcheng 已提交
5381 5382 5383
      }
      strcat(condition, tempS);

L
liu0x54 已提交
5384
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
5385 5386 5387 5388 5389 5390

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

      double t = getCurrentTime();

S
Shuaiqiang Chang 已提交
5391 5392 5393 5394
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
5395
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
5396
        taos_free_result(pSql);
H
hzcheng 已提交
5397
        taos_close(taos);
5398
        fclose(fp);
5399
        return NULL;
H
hzcheng 已提交
5400 5401
      }
      int count = 0;
S
Shuaiqiang Chang 已提交
5402
      while (taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
5403 5404 5405 5406
        count++;
      }
      t = getCurrentTime() - t;

5407 5408
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
              num_of_tables * num_of_DPT / t, t * 1000);
H
hzcheng 已提交
5409 5410
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t);

S
Shuaiqiang Chang 已提交
5411
      taos_free_result(pSql);
H
hzcheng 已提交
5412 5413 5414 5415
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
5416
#endif
H
hzcheng 已提交
5417 5418 5419
  return NULL;
}

H
Hui Li 已提交
5420

5421
static int insertTestProcess() {
5422

5423 5424 5425
  setupForAnsiEscape();
  int ret = printfInsertMeta();
  resetAfterAnsiEscape();
5426

5427 5428 5429
  if (ret == -1)
    exit(EXIT_FAILURE);

5430
  debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
5431 5432
  g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
  if (NULL == g_fpOfInsertResult) {
5433
    errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
5434 5435
    return -1;
  }
5436

5437 5438
  printfInsertMetaToFile(g_fpOfInsertResult);

5439 5440 5441 5442
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void)getchar();
  }
5443

5444 5445 5446
  init_rand_data();

  // create database and super tables
5447
  if(createDatabases() != 0) {
5448 5449 5450
    fclose(g_fpOfInsertResult);
    return -1;
  }
5451 5452

  // pretreatement
5453
  prepareSampleData();
5454

5455 5456 5457 5458 5459 5460 5461
  double start;
  double end;

  // create child tables
  start = getCurrentTime();
  createChildTables();
  end = getCurrentTime();
5462

5463
  if (g_totalChildTables > 0) {
5464
    printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n",
5465
            end - start, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
5466 5467
    fprintf(g_fpOfInsertResult,
            "Spent %.4f seconds to create %d tables with %d thread(s)\n\n",
5468
            end - start, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
5469
  }
5470

5471
  taosMsleep(1000);
5472 5473
  // create sub threads for inserting data
  //start = getCurrentTime();
5474 5475 5476 5477 5478 5479
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    if (g_Dbs.db[i].superTblCount > 0) {
      for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
        SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];
        if (0 == g_Dbs.db[i].superTbls[j].insertRows) {
          continue;
5480
        }
5481
        startMultiThreadInsertData(
5482 5483 5484
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
5485 5486 5487 5488
          superTblInfo);
        }
    } else {
        startMultiThreadInsertData(
5489 5490 5491
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
5492
          NULL);
H
Hui Li 已提交
5493
    }
5494
  }
5495
  //end = getCurrentTime();
5496

5497
  //int64_t    totalInsertRows = 0;
5498 5499 5500
  //int64_t    totalAffectedRows = 0;
  //for (int i = 0; i < g_Dbs.dbCount; i++) {    
  //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
5501
  //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
5502 5503
  //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
  //}
5504
  //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
5505
  postFreeResource();
5506

5507 5508 5509
  return 0;
}

5510
static void *superQueryProcess(void *sarg) {
5511
  threadInfo *winfo = (threadInfo *)sarg;
5512

5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528
  if (winfo->taos == NULL) {
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
            winfo->threadID, taos_errstr(NULL));
      return NULL;
    } else {
      winfo->taos = taos;
    }
  }

5529 5530 5531
  //char sqlStr[MAX_TB_NAME_SIZE*2];
  //sprintf(sqlStr, "use %s", g_queryInfo.dbName);
  //queryDB(winfo->taos, sqlStr);
5532

5533 5534
  int64_t st = 0;
  int64_t et = 0;
5535

5536
  int queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes;
5537 5538

  while(queryTimes --) {
5539 5540 5541
    if (g_queryInfo.specifiedQueryInfo.rate && (et - st) <
            (int64_t)g_queryInfo.specifiedQueryInfo.rate*1000) {
      taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms
5542
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5543 5544
    }

5545
    st = taosGetTimestampUs();
5546
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
5547
      if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {
5548 5549
        int64_t t1 = taosGetTimestampUs();
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
5550
        if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) {
5551
          sprintf(tmpFile, "%s-%d",
5552
                  g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID);
5553
        }
5554
        selectAndGetResult(winfo->taos, g_queryInfo.specifiedQueryInfo.sql[i], tmpFile);
5555 5556
        int64_t t2 = taosGetTimestampUs();
        printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %f s\n",
5557
                taosGetSelfPthreadId(), (t2 - t1)/1000000.0);
5558 5559
      } else {
        int64_t t1 = taosGetTimestampUs();
5560
        int retCode = postProceSql(g_queryInfo.host,
5561
                g_queryInfo.port, g_queryInfo.specifiedQueryInfo.sql[i]);
5562 5563
        int64_t t2 = taosGetTimestampUs();
        printf("=[restful] thread[%"PRId64"] complete one sql, Spent %f s\n",
5564
                taosGetSelfPthreadId(), (t2 - t1)/1000000.0);
5565

5566
        if (0 != retCode) {
5567
          printf("====restful return fail, threadID[%d]\n", winfo->threadID);
5568 5569
          return NULL;
        }
5570
      }
5571
    }
5572
    et = taosGetTimestampUs();
5573
    printf("==thread[%"PRId64"] complete all sqls to specify tables once queries duration:%.6fs\n\n",
5574
            taosGetSelfPthreadId(), (double)(et - st)/1000.0);
H
Hui Li 已提交
5575
  }
5576 5577
  return NULL;
}
H
Hui Li 已提交
5578

5579
static void replaceSubTblName(char* inSql, char* outSql, int tblIndex) {
5580 5581
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
5582 5583
  sprintf(subTblName, "%s.%s",
          g_queryInfo.dbName,
5584
          g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
5585 5586

  //printf("inSql: %s\n", inSql);
5587

5588 5589
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
5590
    return;
H
Hui Li 已提交
5591
  }
5592

5593
  tstrncpy(outSql, inSql, pos - inSql + 1);
5594
  //printf("1: %s\n", outSql);
5595 5596 5597 5598
  strcat(outSql, subTblName);
  //printf("2: %s\n", outSql);
  strcat(outSql, pos+strlen(sourceString));
  //printf("3: %s\n", outSql);
H
Hui Li 已提交
5599 5600
}

5601
static void *subQueryProcess(void *sarg) {
5602
  char sqlstr[1024];
5603
  threadInfo *winfo = (threadInfo *)sarg;
5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620

  if (winfo->taos == NULL) {
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
            winfo->threadID, taos_errstr(NULL));
      return NULL;
    } else {
      winfo->taos = taos;
    }
  }

5621
  int64_t st = 0;
5622 5623 5624
  int64_t et = (int64_t)g_queryInfo.superQueryInfo.rate*1000;

  int queryTimes = g_queryInfo.superQueryInfo.queryTimes;
5625

5626
  while(queryTimes --) {
5627 5628 5629
    if (g_queryInfo.superQueryInfo.rate
            && (et - st) < (int64_t)g_queryInfo.superQueryInfo.rate*1000) {
      taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms
5630
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5631
    }
H
Hui Li 已提交
5632

5633
    st = taosGetTimestampUs();
5634
    for (int i = winfo->start_table_from; i <= winfo->end_table_to; i++) {
5635
      for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
5636
        memset(sqlstr,0,sizeof(sqlstr));
5637
        replaceSubTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i);
5638
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
5639
        if (g_queryInfo.superQueryInfo.result[j][0] != 0) {
5640
          sprintf(tmpFile, "%s-%d",
5641
                  g_queryInfo.superQueryInfo.result[j],
5642
                  winfo->threadID);
5643
        }
5644
        selectAndGetResult(winfo->taos, sqlstr, tmpFile);
5645
      }
H
Hui Li 已提交
5646
    }
5647
    et = taosGetTimestampUs();
5648
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%d - %d] once queries duration:%.4fs\n\n",
5649 5650 5651
            taosGetSelfPthreadId(),
            winfo->start_table_from,
            winfo->end_table_to,
5652
            (double)(et - st)/1000000.0);
5653
  }
5654

5655 5656 5657
  return NULL;
}

5658
static int queryTestProcess() {
5659 5660 5661 5662

  setupForAnsiEscape();
  printfQueryMeta();
  resetAfterAnsiEscape();
5663 5664 5665 5666 5667 5668

  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
5669
          g_queryInfo.port);
5670
  if (taos == NULL) {
5671 5672
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
5673 5674 5675
    exit(-1);
  }

5676
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
5677 5678
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
5679 5680 5681
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
5682
  }
5683

5684 5685 5686 5687
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void)getchar();
  }
5688

5689
  printfQuerySystemInfo(taos);
5690

5691 5692 5693
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
5694 5695
  if (g_queryInfo.specifiedQueryInfo.sqlCount > 0
          && g_queryInfo.specifiedQueryInfo.concurrent > 0) {
5696

5697
    pids  = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t));
5698 5699 5700 5701
    if (NULL == pids) {
        taos_close(taos);
        ERROR_EXIT("memory allocation failed\n");
    }
5702
    infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo));
5703
    if (NULL == infos) {
5704
      taos_close(taos);
5705 5706
      free(pids);
      ERROR_EXIT("memory allocation failed for create threads\n");
5707
    }
5708

5709
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) {
5710
      threadInfo *t_info = infos + i;
5711
      t_info->threadID = i;
5712

5713
      if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {
5714

5715 5716
        char sqlStr[MAX_TB_NAME_SIZE*2];
        sprintf(sqlStr, "use %s", g_queryInfo.dbName);
5717
        verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
5718 5719
        if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) {
            taos_close(taos);
5720 5721
            free(infos);
            free(pids);
5722 5723 5724 5725
            errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
            return -1;
        }
5726
      }
5727

5728 5729
      t_info->taos = NULL;// TODO: workaround to use separate taos connection;

5730 5731
      pthread_create(pids + i, NULL, superQueryProcess, t_info);
    }
5732
  } else {
5733
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
5734
  }
5735

5736 5737
  taos_close(taos);

5738 5739 5740
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
5741 5742 5743
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
    pidsOfSub  = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t));
5744 5745 5746 5747 5748 5749 5750
    if (NULL == pidsOfSub) {
      free(infos);
      free(pids);

      ERROR_EXIT("memory allocation failed for create threads\n");
    }

5751
    infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo));
5752 5753 5754 5755 5756
    if (NULL == infosOfSub) {
      free(pidsOfSub);
      free(infos);
      free(pids);
      ERROR_EXIT("memory allocation failed for create threads\n");
5757
    }
5758

5759 5760
    int ntables = g_queryInfo.superQueryInfo.childTblCount;
    int threads = g_queryInfo.superQueryInfo.threadCnt;
5761

5762 5763 5764 5765 5766
    int a = ntables / threads;
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
5767

5768 5769 5770 5771
    int b = 0;
    if (threads != 0) {
      b = ntables % threads;
    }
5772

5773
    int startFrom = 0;
5774
    for (int i = 0; i < threads; i++) {
5775 5776
      threadInfo *t_info = infosOfSub + i;
      t_info->threadID = i;
5777

5778 5779 5780 5781
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
5782
      t_info->taos = NULL; // TODO: workaround to use separate taos connection;
5783 5784 5785
      pthread_create(pidsOfSub + i, NULL, subQueryProcess, t_info);
    }

5786
    g_queryInfo.superQueryInfo.threadCnt = threads;
5787
  } else {
5788
    g_queryInfo.superQueryInfo.threadCnt = 0;
5789
  }
5790

5791
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) {
5792
    pthread_join(pids[i], NULL);
5793
  }
S
Shuaiqiang Chang 已提交
5794

5795
  tmfree((char*)pids);
5796
  tmfree((char*)infos);
5797

5798
  for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
5799
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
5800
  }
H
Hui Li 已提交
5801

5802
  tmfree((char*)pidsOfSub);
5803
  tmfree((char*)infosOfSub);
5804

5805
//  taos_close(taos);// TODO: workaround to use separate taos connection;
5806 5807 5808
  return 0;
}

5809
static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
5810
  if (res == NULL || taos_errno(res) != 0) {
5811 5812
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
5813 5814
    return;
  }
5815

5816 5817
  getResult(res, (char*)param);
  taos_free_result(res);
H
hzcheng 已提交
5818 5819
}

5820
static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultFileName) {
5821
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
5822

5823
  if (g_queryInfo.specifiedQueryInfo.subscribeMode) {
5824
    tsub = taos_subscribe(taos,
5825
            g_queryInfo.specifiedQueryInfo.subscribeRestart,
5826
            topic, sql, subscribe_callback, (void*)resultFileName,
5827
            g_queryInfo.specifiedQueryInfo.subscribeInterval);
5828
  } else {
5829
    tsub = taos_subscribe(taos,
5830
            g_queryInfo.specifiedQueryInfo.subscribeRestart,
5831
            topic, sql, NULL, NULL, 0);
5832
  }
5833

5834 5835 5836
  if (tsub == NULL) {
    printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
    return NULL;
5837
  }
5838

5839 5840
  return tsub;
}
H
hzcheng 已提交
5841

5842
static void *subSubscribeProcess(void *sarg) {
5843
  threadInfo *winfo = (threadInfo *)sarg;
5844
  char subSqlstr[1024];
H
hzcheng 已提交
5845

5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861
  if (winfo->taos == NULL) {
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
            winfo->threadID, taos_errstr(NULL));
      return NULL;
    } else {
      winfo->taos = taos;
    }
  }

5862 5863
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
5864
  debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
5865 5866
  if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(winfo->taos);
5867 5868
    return NULL;
  }
5869

5870 5871 5872
  //int64_t st = 0;
  //int64_t et = 0;
  do {
5873 5874
    //if (g_queryInfo.specifiedQueryInfo.rate && (et - st) < g_queryInfo.specifiedQueryInfo.rate*1000) {
    //  taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms
5875
    //  //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5876 5877 5878 5879
    //}

    //st = taosGetTimestampMs();
    char topic[32] = {0};
5880
    for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
5881 5882
      sprintf(topic, "taosdemo-subscribe-%d", i);
      memset(subSqlstr,0,sizeof(subSqlstr));
5883
      replaceSubTblName(g_queryInfo.superQueryInfo.sql[i], subSqlstr, i);
5884
      char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
5885
      if (g_queryInfo.superQueryInfo.result[i][0] != 0) {
5886
        sprintf(tmpFile, "%s-%d",
5887
                g_queryInfo.superQueryInfo.result[i], winfo->threadID);
5888
      }
5889
      g_queryInfo.superQueryInfo.tsub[i] = subscribeImpl(
5890
              winfo->taos, subSqlstr, topic, tmpFile); 
5891
      if (NULL == g_queryInfo.superQueryInfo.tsub[i]) {
5892
        taos_close(winfo->taos);
5893 5894 5895 5896
        return NULL;
      }
    }
    //et = taosGetTimestampMs();
5897
    //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0);
5898 5899 5900 5901 5902
  } while (0);

  // start loop to consume result
  TAOS_RES* res = NULL;
  while (1) {
5903 5904
    for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
      if (1 == g_queryInfo.superQueryInfo.subscribeMode) {
5905 5906
        continue;
      }
5907

5908
      res = taos_consume(g_queryInfo.superQueryInfo.tsub[i]);
5909 5910
      if (res) {
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
5911
        if (g_queryInfo.superQueryInfo.result[i][0] != 0) {
5912
          sprintf(tmpFile, "%s-%d",
5913
                  g_queryInfo.superQueryInfo.result[i],
5914
                  winfo->threadID);
5915 5916
        }
        getResult(res, tmpFile);
H
hzcheng 已提交
5917 5918 5919
      }
    }
  }
5920
  taos_free_result(res);
5921

5922 5923 5924
  for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
    taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i],
            g_queryInfo.superQueryInfo.subscribeKeepProgress);
5925
  }
5926 5927

  taos_close(winfo->taos);
H
hzcheng 已提交
5928 5929 5930
  return NULL;
}

5931
static void *superSubscribeProcess(void *sarg) {
5932
  threadInfo *winfo = (threadInfo *)sarg;
H
hzcheng 已提交
5933

5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949
  if (winfo->taos == NULL) {
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
            winfo->threadID, taos_errstr(NULL));
      return NULL;
    } else {
      winfo->taos = taos;
    }
  }

5950 5951
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
5952
  debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
5953
  if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
5954
    taos_close(winfo->taos);
5955 5956
    return NULL;
  }
5957

5958 5959 5960
  //int64_t st = 0;
  //int64_t et = 0;
  do {
5961 5962
    //if (g_queryInfo.specifiedQueryInfo.rate && (et - st) < g_queryInfo.specifiedQueryInfo.rate*1000) {
    //  taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms
5963
    //  //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5964 5965 5966 5967
    //}

    //st = taosGetTimestampMs();
    char topic[32] = {0};
5968
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
5969 5970
      sprintf(topic, "taosdemo-subscribe-%d", i);
      char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
5971
      if (g_queryInfo.superQueryInfo.result[i][0] != 0) {
5972
        sprintf(tmpFile, "%s-%d",
5973
                g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID);
5974
      }
5975
      g_queryInfo.specifiedQueryInfo.tsub[i] =
5976
          subscribeImpl(winfo->taos,
5977
                  g_queryInfo.specifiedQueryInfo.sql[i],
5978
                  topic, tmpFile);
5979
      if (NULL == g_queryInfo.specifiedQueryInfo.tsub[i]) {
5980
        taos_close(winfo->taos);
5981 5982 5983 5984
        return NULL;
      }
    }
    //et = taosGetTimestampMs();
5985
    //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0);
5986 5987 5988 5989 5990
  } while (0);

  // start loop to consume result
  TAOS_RES* res = NULL;
  while (1) {
5991 5992
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
      if (1 == g_queryInfo.specifiedQueryInfo.subscribeMode) {
5993 5994
        continue;
      }
5995

5996
      res = taos_consume(g_queryInfo.specifiedQueryInfo.tsub[i]);
5997 5998
      if (res) {
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
5999
        if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) {
6000
          sprintf(tmpFile, "%s-%d",
6001
                  g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID);
6002 6003 6004 6005 6006 6007
        }
        getResult(res, tmpFile);
      }
    }
  }
  taos_free_result(res);
6008

6009 6010 6011
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
    taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[i],
            g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
6012
  }
6013 6014

  taos_close(winfo->taos);
H
hzcheng 已提交
6015 6016 6017
  return NULL;
}

6018
static int subscribeTestProcess() {
6019
  setupForAnsiEscape();
6020
  printfQueryMeta();
6021
  resetAfterAnsiEscape();
6022

6023 6024
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
6025
    (void) getchar();
Y
yihaoDeng 已提交
6026
  }
6027

6028 6029 6030 6031 6032
  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
6033
          g_queryInfo.port);
6034
  if (taos == NULL) {
6035 6036
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6037 6038 6039
    exit(-1);
  }

6040
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6041 6042
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6043 6044 6045
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6046 6047
  }

6048 6049
  taos_close(taos); // TODO: workaround to use separate taos connection;

6050 6051 6052
  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from super table
6053 6054
  if ((g_queryInfo.specifiedQueryInfo.sqlCount <= 0) ||
          (g_queryInfo.specifiedQueryInfo.concurrent <= 0)) {
6055
    errorPrint("%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n",
6056 6057
              __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount,
              g_queryInfo.specifiedQueryInfo.concurrent);
6058 6059 6060
    exit(-1);
  }

6061 6062
  pids  = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t));
  infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo));
6063
  if ((NULL == pids) || (NULL == infos)) {
6064
      errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
6065
      exit(-1);
6066
  }
6067

6068
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) {
6069 6070
      threadInfo *t_info = infos + i;
      t_info->threadID = i;
6071
      t_info->taos = NULL;  // TODO: workaround to use separate taos connection;
6072
      pthread_create(pids + i, NULL, superSubscribeProcess, t_info);
H
hzcheng 已提交
6073
  }
6074

6075
  //==== create sub threads for query from sub table
6076 6077
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
6078 6079 6080
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
    pidsOfSub  = malloc(g_queryInfo.superQueryInfo.threadCnt *
6081
            sizeof(pthread_t));
6082
    infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * 
6083
            sizeof(threadInfo));
6084
    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
6085 6086 6087
      errorPrint("%s() LN%d, malloc failed for create threads\n",
              __func__, __LINE__);
      // taos_close(taos);
6088
      exit(-1);
S
Shuaiqiang Chang 已提交
6089
    }
6090

6091 6092
    int ntables = g_queryInfo.superQueryInfo.childTblCount;
    int threads = g_queryInfo.superQueryInfo.threadCnt;
6093

6094 6095 6096 6097 6098
    int a = ntables / threads;
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
6099

6100 6101 6102 6103
    int b = 0;
    if (threads != 0) {
      b = ntables % threads;
    }
6104

6105
    int startFrom = 0;
6106
    for (int i = 0; i < threads; i++) {
6107 6108
      threadInfo *t_info = infosOfSub + i;
      t_info->threadID = i;
6109

6110 6111 6112 6113
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
6114
      t_info->taos = NULL; // TODO: workaround to use separate taos connection;
6115
      pthread_create(pidsOfSub + i, NULL, subSubscribeProcess, t_info);
H
hzcheng 已提交
6116
    }
6117

6118
    g_queryInfo.superQueryInfo.threadCnt = threads;
6119

6120
    for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
6121 6122
      pthread_join(pidsOfSub[i], NULL);
    }
H
hzcheng 已提交
6123
  }
6124

6125
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) {
6126
    pthread_join(pids[i], NULL);
6127
  }
H
hzcheng 已提交
6128

6129
  tmfree((char*)pids);
6130
  tmfree((char*)infos);
H
hzcheng 已提交
6131

6132
  tmfree((char*)pidsOfSub);
6133
  tmfree((char*)infosOfSub);
6134
//   taos_close(taos);
6135
  return 0;
H
hzcheng 已提交
6136 6137
}

6138
static void initOfInsertMeta() {
6139
  memset(&g_Dbs, 0, sizeof(SDbs));
6140

6141
  // set default values
6142
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
6143
  g_Dbs.port = 6030;
6144 6145
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
6146
  g_Dbs.threadCount = 2;
6147 6148

  g_Dbs.use_metric = g_args.use_metric;
L
Liu Tao 已提交
6149 6150
}

6151
static void initOfQueryMeta() {
6152
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
6153

6154
  // set default values
6155
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
6156
  g_queryInfo.port = 6030;
6157 6158
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
L
Liu Tao 已提交
6159 6160
}

6161
static void setParaFromArg(){
6162
  if (g_args.host) {
6163
    tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE);
6164
  } else {
6165
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
6166
  }
L
Liu Tao 已提交
6167

6168
  if (g_args.user) {
6169
    tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE);
6170
  }
6171 6172

  if (g_args.password) {
6173
    tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE);
6174 6175
  }

6176 6177
  if (g_args.port) {
    g_Dbs.port = g_args.port;
6178
  }
L
Liu Tao 已提交
6179

6180 6181 6182
  g_Dbs.threadCount = g_args.num_of_threads;
  g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;

6183 6184
  g_Dbs.dbCount = 1;
  g_Dbs.db[0].drop = 1;
6185

6186
  tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE);
6187
  g_Dbs.db[0].dbCfg.replica = g_args.replica;
6188
  tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
6189

6190
  tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
6191 6192 6193 6194 6195

  g_Dbs.use_metric = g_args.use_metric;
  g_Dbs.insert_only = g_args.insert_only;

  g_Dbs.do_aggreFunc = true;
L
Liu Tao 已提交
6196

6197 6198
  char dataString[STRING_LEN];
  char **data_type = g_args.datatype;
6199

6200
  memset(dataString, 0, STRING_LEN);
L
Liu Tao 已提交
6201

6202 6203
  if (strcasecmp(data_type[0], "BINARY") == 0
          || strcasecmp(data_type[0], "BOOL") == 0
6204
          || strcasecmp(data_type[0], "NCHAR") == 0 ) {
6205
    g_Dbs.do_aggreFunc = false;
L
Liu Tao 已提交
6206
  }
H
hzcheng 已提交
6207

6208 6209 6210 6211 6212
  if (g_args.use_metric) {
    g_Dbs.db[0].superTblCount = 1;
    tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE);
    g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
    g_Dbs.threadCount = g_args.num_of_threads;
6213
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
6214
    g_Dbs.queryMode = g_args.mode;
6215

6216 6217 6218 6219
    g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
    g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
    g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
6220
    tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix,
6221 6222 6223
            g_args.tb_prefix, MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE);
6224
    tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp,
6225
            "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
6226
    g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
6227

6228
    g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
6229
    g_Dbs.db[0].superTbls[0].maxSqlLen = TSDB_PAYLOAD_SIZE;
6230

6231 6232 6233 6234 6235
    g_Dbs.db[0].superTbls[0].columnCount = 0;
    for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
      if (data_type[i] == NULL) {
        break;
      }
6236

6237
      tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
6238
              data_type[i], MAX_TB_NAME_SIZE);
6239
      g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;
6240 6241
      g_Dbs.db[0].superTbls[0].columnCount++;
    }
6242

6243 6244 6245 6246 6247
    if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
      g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
    } else {
      for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) {
        tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE);
6248
        g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;
6249 6250 6251
        g_Dbs.db[0].superTbls[0].columnCount++;
      }
    }
6252

6253
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE);
6254
    g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;
6255

6256
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE);
6257 6258
    g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;
    g_Dbs.db[0].superTbls[0].tagCount = 2;
6259
  } else {
6260 6261
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
    g_Dbs.db[0].superTbls[0].tagCount = 0;
H
hzcheng 已提交
6262
  }
6263 6264 6265 6266 6267 6268
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
6269

6270 6271 6272
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
6273 6274 6275
    exit(-1);
  }

6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

6300
static void querySqlFile(TAOS* taos, char* sqlFile)
6301 6302 6303 6304 6305 6306
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
6307

6308 6309 6310 6311 6312 6313 6314
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

  double t = getCurrentTime();
6315

6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327 6328
  while ((read_len = tgetline(&line, &line_len, fp)) != -1) {
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
6329
    }
6330 6331

    memcpy(cmd + cmd_len, line, read_len);
6332
    verbosePrint("%s() LN%d cmd: %s\n", __func__, __LINE__, cmd);
6333 6334 6335
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) {
        errorPrint("%s() LN%d, queryDbExec %s failed!\n",
               __func__, __LINE__, cmd);
6336 6337 6338 6339 6340
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
6341 6342
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
6343 6344
  }

6345 6346
  t = getCurrentTime() - t;
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
6347

6348 6349 6350 6351
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
6352 6353
}

6354
static void testMetaFile() {
6355
    if (INSERT_TEST == g_args.test_mode) {
6356 6357
      if (g_Dbs.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
6358

6359
      insertTestProcess();
6360

6361
    } else if (QUERY_TEST == g_args.test_mode) {
6362 6363
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
6364

6365
      queryTestProcess();
6366

6367
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
6368 6369
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
6370

6371
      subscribeTestProcess();
6372

6373 6374 6375
    }  else {
      ;
    }
6376
}
6377

6378
static void queryResult() {
6379 6380 6381
    // select
    if (false == g_Dbs.insert_only) {
      // query data
6382

6383 6384 6385
      pthread_t read_id;
      threadInfo *rInfo = malloc(sizeof(threadInfo));
      rInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
6386
      rInfo->start_table_from = 0;
6387

6388
      //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
6389
      if (g_args.use_metric) {
6390 6391
        rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
        rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
6392
        rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
6393 6394
        tstrncpy(rInfo->tb_prefix,
              g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE);
6395
      } else {
6396 6397
        rInfo->ntables = g_args.num_of_tables;
        rInfo->end_table_to = g_args.num_of_tables -1;
6398
        tstrncpy(rInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE);
6399 6400
      }

6401
      rInfo->taos = taos_connect(
6402 6403 6404 6405
              g_Dbs.host,
              g_Dbs.user,
              g_Dbs.password,
              g_Dbs.db[0].dbName,
6406
              g_Dbs.port);
6407
      if (rInfo->taos == NULL) {
6408 6409
        errorPrint( "Failed to connect to TDengine, reason:%s\n",
                taos_errstr(NULL));
6410 6411 6412 6413
        free(rInfo);
        exit(-1);
      }

6414
      tstrncpy(rInfo->fp, g_Dbs.resultFile, MAX_FILE_NAME_LEN);
6415 6416 6417 6418 6419 6420 6421 6422 6423

      if (!g_Dbs.use_metric) {
        pthread_create(&read_id, NULL, readTable, rInfo);
      } else {
        pthread_create(&read_id, NULL, readMetric, rInfo);
      }
      pthread_join(read_id, NULL);
      taos_close(rInfo->taos);
      free(rInfo);
H
hzcheng 已提交
6424
    }
6425 6426
}

6427 6428
static void testCmdLine() {

6429 6430
  g_args.test_mode = INSERT_TEST;
  insertTestProcess();
6431

6432 6433 6434 6435
  if (g_Dbs.insert_only)
    return;
  else
    queryResult();
6436 6437
}

6438 6439 6440
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

6441
  debugPrint("meta file: %s\n", g_args.metaFile);
6442 6443 6444

  if (g_args.metaFile) {
    initOfInsertMeta();
6445
    initOfQueryMeta();
6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
6459 6460 6461 6462
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
6463
          g_Dbs.port);
6464
      querySqlFile(qtaos, g_args.sqlFile);
6465 6466 6467 6468 6469
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
H
hzcheng 已提交
6470
  }
6471 6472

  return 0;
H
hzcheng 已提交
6473
}
6474