taosdemo.c 247.1 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

21
#include <stdint.h>
S
slguan 已提交
22
#define _GNU_SOURCE
23 24
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
41
#else
S
TD-1057  
Shengliang Guan 已提交
42 43
  #include <regex.h>
  #include <stdio.h>
44
#endif
H
hzcheng 已提交
45

46
#include <assert.h>
47
#include <stdlib.h>
48 49
#include "cJSON.h"

50
#include "os.h"
H
hzcheng 已提交
51
#include "taos.h"
52
#include "taoserror.h"
H
Hui Li 已提交
53
#include "tutil.h"
S
slguan 已提交
54

55 56 57
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
58 59
extern char configDir[];

60 61 62 63
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

64 65 66 67 68 69
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};
70

71 72
#define MAX_RECORDS_PER_REQ     32766

73 74
#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
75
#define COND_BUF_LEN        BUFFER_SIZE - 30
76 77
#define MAX_USERNAME_SIZE  64
#define MAX_PASSWORD_SIZE  64
78
#define MAX_DB_NAME_SIZE   64
79
#define MAX_HOSTNAME_SIZE  64
80
#define MAX_TB_NAME_SIZE   64
81
#define MAX_DATA_SIZE      (16*1024)+20     // max record len: 16*1024, timestamp string and ,('') need extra space
82 83 84 85 86 87 88 89 90 91
#define MAX_NUM_DATATYPE   10
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
#define MAX_FILE_NAME_LEN  256

#define   MAX_SAMPLES_ONCE_FROM_FILE   10000
#define   MAX_NUM_DATATYPE 10

#define   MAX_DB_COUNT           8
92
#define   MAX_SUPER_TABLE_COUNT  200
93 94 95
#define   MAX_COLUMN_COUNT       1024
#define   MAX_TAG_COUNT          128

96
#define   MAX_QUERY_SQL_COUNT    100
97
#define   MAX_QUERY_SQL_LENGTH   1024
98 99

#define   MAX_DATABASE_COUNT     256
100
#define INPUT_BUF_LEN   256
101

102
#define DEFAULT_TIMESTAMP_STEP  1
103

104

105 106 107 108 109
typedef enum CREATE_SUB_TALBE_MOD_EN {
  PRE_CREATE_SUBTBL,
  AUTO_CREATE_SUBTBL,
  NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
110

111 112
typedef enum TALBE_EXISTS_EN {
  TBL_NO_EXISTS,
113
  TBL_ALREADY_EXISTS,
114 115 116
  TBL_EXISTS_BUTT
} TALBE_EXISTS_EN;

117
enum enumSYNC_MODE {
118 119
  SYNC_MODE,
  ASYNC_MODE,
120 121
  MODE_BUT
};
122

123 124 125 126 127 128
typedef enum enumQUERY_CLASS {
    SPECIFIED_CLASS,
    STABLE_CLASS,
    CLASS_BUT
} QUERY_CLASS;

129 130 131 132 133 134
typedef enum enum_INSERT_MODE {
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
} INSERT_MODE;

135
typedef enum enumQUERY_TYPE {
136
  NO_INSERT_TYPE,
137
  INSERT_TYPE,
138
  QUERY_TYPE_BUT
139
} QUERY_TYPE;
140 141 142 143 144 145 146

enum _show_db_index {
  TSDB_SHOW_DB_NAME_INDEX,
  TSDB_SHOW_DB_CREATED_TIME_INDEX,
  TSDB_SHOW_DB_NTABLES_INDEX,
  TSDB_SHOW_DB_VGROUPS_INDEX,
  TSDB_SHOW_DB_REPLICA_INDEX,
147
  TSDB_SHOW_DB_QUORUM_INDEX,
148 149 150 151 152 153 154 155 156 157
  TSDB_SHOW_DB_DAYS_INDEX,
  TSDB_SHOW_DB_KEEP_INDEX,
  TSDB_SHOW_DB_CACHE_INDEX,
  TSDB_SHOW_DB_BLOCKS_INDEX,
  TSDB_SHOW_DB_MINROWS_INDEX,
  TSDB_SHOW_DB_MAXROWS_INDEX,
  TSDB_SHOW_DB_WALLEVEL_INDEX,
  TSDB_SHOW_DB_FSYNC_INDEX,
  TSDB_SHOW_DB_COMP_INDEX,
  TSDB_SHOW_DB_CACHELAST_INDEX,
158
  TSDB_SHOW_DB_PRECISION_INDEX,
159 160 161 162 163 164 165 166 167 168
  TSDB_SHOW_DB_UPDATE_INDEX,
  TSDB_SHOW_DB_STATUS_INDEX,
  TSDB_MAX_SHOW_DB
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
  TSDB_SHOW_STABLES_NAME_INDEX,
  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
  TSDB_SHOW_STABLES_COLUMNS_INDEX,
169 170
  TSDB_SHOW_STABLES_METRIC_INDEX,
  TSDB_SHOW_STABLES_UID_INDEX,
171
  TSDB_SHOW_STABLES_TID_INDEX,
172
  TSDB_SHOW_STABLES_VGID_INDEX,
173
  TSDB_MAX_SHOW_STABLES
174 175
};

176 177 178 179 180 181 182 183 184 185 186 187 188 189
enum _describe_table_index {
  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
  TSDB_MAX_DESCRIBE_METRIC
};

typedef struct {
  char field[TSDB_COL_NAME_LEN + 1];
  char type[16];
  int length;
  char note[128];
} SColDes;
H
hzcheng 已提交
190 191

/* Used by main to communicate with parse_opt. */
192 193
static char *g_dupstr = NULL;

194 195
typedef struct SArguments_S {
  char *   metaFile;
196
  uint32_t test_mode;
197 198 199 200 201 202 203 204 205
  char *   host;
  uint16_t port;
  char *   user;
  char *   password;
  char *   database;
  int      replica;
  char *   tb_prefix;
  char *   sqlFile;
  bool     use_metric;
206
  bool     drop_database;
207
  bool     insert_only;
208
  bool     answer_yes;
209
  bool     debug_print;
210
  bool     verbose_print;
211
  bool     performance_print;
212
  char *   output_file;
213
  bool     async_mode;
214
  char *   datatype[MAX_NUM_DATATYPE + 1];
215 216 217 218
  uint32_t len_of_binary;
  uint32_t num_of_CPR;
  uint32_t num_of_threads;
  uint64_t insert_interval;
219
  int64_t  query_times;
220 221 222
  uint64_t interlace_rows;
  uint64_t num_of_RPR;                  // num_of_records_per_req
  uint64_t max_sql_len;
223 224
  int64_t  num_of_tables;
  int64_t  num_of_DPT;
225
  int      abort;
226 227
  int      disorderRatio;               // 0: no disorder, >0: x%
  int      disorderRange;               // ms or us by database precision
228
  uint32_t method_of_delete;
229
  char **  arg_list;
230 231
  uint64_t totalInsertRows;
  uint64_t totalAffectedRows;
232 233 234
} SArguments;

typedef struct SColumn_S {
235 236 237
  char      field[TSDB_COL_NAME_LEN + 1];
  char      dataType[MAX_TB_NAME_SIZE];
  uint32_t  dataLen;
238
  char      note[128];
239 240 241
} StrColumn;

typedef struct SSuperTable_S {
242
  char         sTblName[MAX_TB_NAME_SIZE+1];
243 244 245 246
  char         dataSource[MAX_TB_NAME_SIZE+1];  // rand_gen or sample
  char         childTblPrefix[MAX_TB_NAME_SIZE];
  char         insertMode[MAX_TB_NAME_SIZE];    // taosc, rest
  uint16_t     childTblExists;
247
  int64_t      childTblCount;
248 249
  uint64_t     batchCreateTableNum;     // 0: no batch,  > 0: batch table number in one sql
  uint8_t      autoCreateTable;         // 0: create sub table, 1: auto create sub table
250
  int64_t      childTblLimit;
251
  uint64_t     childTblOffset;
252

253
//  int          multiThreadWriteOneTbl;  // 0: no, 1: yes
254
  uint64_t     interlaceRows;           //
255 256
  int          disorderRatio;           // 0: no disorder, >0: x%
  int          disorderRange;           // ms or us by database precision
257
  uint64_t     maxSqlLen;               //
258

259
  uint64_t     insertInterval;          // insert interval, will override global insert interval
260
  int64_t      insertRows;
261
  int64_t      timeStampStep;
262
  char         startTimestamp[MAX_TB_NAME_SIZE];
263
  char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
264 265
  char         sampleFile[MAX_FILE_NAME_LEN+1];
  char         tagsFile[MAX_FILE_NAME_LEN+1];
266

267
  uint32_t     columnCount;
268
  StrColumn    columns[MAX_COLUMN_COUNT];
269
  uint32_t     tagCount;
270 271 272
  StrColumn    tags[MAX_TAG_COUNT];

  char*        childTblName;
273
  char*        colsOfCreateChildTable;
274 275
  uint64_t     lenOfOneRow;
  uint64_t     lenOfTagOfOneRow;
276 277 278 279 280

  char*        sampleDataBuf;
  //int          sampleRowCount;
  //int          sampleUsePos;

281
  uint32_t     tagSource;    // 0: rand, 1: tag sample
282
  char*        tagDataBuf;
283 284
  uint32_t     tagSampleCount;
  uint32_t     tagUsePos;
285

286
  // statistics
287 288
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
289 290 291 292 293
} SSuperTable;

typedef struct {
  char     name[TSDB_DB_NAME_LEN + 1];
  char     create_time[32];
294
  int64_t  ntables;
295
  int32_t  vgroups;
296 297
  int16_t  replica;
  int16_t  quorum;
298
  int16_t  days;
299 300 301 302 303 304 305 306 307 308 309 310 311 312
  char     keeplist[32];
  int32_t  cache; //MB
  int32_t  blocks;
  int32_t  minrows;
  int32_t  maxrows;
  int8_t   wallevel;
  int32_t  fsync;
  int8_t   comp;
  int8_t   cachelast;
  char     precision[8];   // time resolution
  int8_t   update;
  char     status[16];
} SDbInfo;

313
typedef struct SDbCfg_S {
314
//  int       maxtablesPerVnode;
315 316
  uint32_t  minRows;        // 0 means default
  uint32_t  maxRows;        // 0 means default
317 318
  int       comp;
  int       walLevel;
319
  int       cacheLast;
320
  int       fsync;
321 322 323 324 325 326 327
  int       replica;
  int       update;
  int       keep;
  int       days;
  int       cache;
  int       blocks;
  int       quorum;
328
  char      precision[MAX_TB_NAME_SIZE];
329 330 331 332
} SDbCfg;

typedef struct SDataBase_S {
  char         dbName[MAX_DB_NAME_SIZE];
333
  bool         drop;  // 0: use exists, 1: if exists, drop then new create
334
  SDbCfg       dbCfg;
335
  uint64_t     superTblCount;
336 337 338 339
  SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
} SDataBase;

typedef struct SDbs_S {
340
  char         cfgDir[MAX_FILE_NAME_LEN+1];
341
  char         host[MAX_HOSTNAME_SIZE];
342 343
  struct sockaddr_in serv_addr;

344
  uint16_t     port;
345 346
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
347
  char         resultFile[MAX_FILE_NAME_LEN+1];
348 349 350
  bool         use_metric;
  bool         insert_only;
  bool         do_aggreFunc;
351
  bool         asyncMode;
352

353 354 355
  uint32_t     threadCount;
  uint32_t     threadCountByCreateTbl;
  uint32_t     dbCount;
356 357 358
  SDataBase    db[MAX_DB_COUNT];

  // statistics
359 360
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
361

362 363
} SDbs;

364
typedef struct SpecifiedQueryInfo_S {
365
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
366
  uint32_t     concurrent;
367
  int          sqlCount;
368
  uint32_t     asyncMode; // 0: sync, 1: async
369 370
  uint64_t     subscribeInterval; // ms
  uint64_t     queryTimes;
371
  bool         subscribeRestart;
372
  int          subscribeKeepProgress;
373
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
374
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
375
  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
376
  int          endAfterConsume[MAX_QUERY_SQL_COUNT];
377
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
378 379 380
  char         topic[MAX_QUERY_SQL_COUNT][32];
  int          consumed[MAX_QUERY_SQL_COUNT];
  TAOS_RES*    res[MAX_QUERY_SQL_COUNT];
381
  uint64_t     totalQueried;
382
} SpecifiedQueryInfo;
383

384
typedef struct SuperQueryInfo_S {
385
  char         sTblName[MAX_TB_NAME_SIZE+1];
386 387
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint32_t     threadCnt;
388
  uint32_t     asyncMode; // 0: sync, 1: async
389
  uint64_t     subscribeInterval; // ms
390
  bool         subscribeRestart;
391
  int          subscribeKeepProgress;
392
  uint64_t     queryTimes;
393
  int64_t      childTblCount;
394
  char         childTblPrefix[MAX_TB_NAME_SIZE];
395
  int          sqlCount;
396
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
397
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
398
  int          resubAfterConsume;
399
  int          endAfterConsume;
400
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
401

402
  char*        childTblName;
403
  uint64_t     totalQueried;
404
} SuperQueryInfo;
405 406

typedef struct SQueryMetaInfo_S {
407
  char         cfgDir[MAX_FILE_NAME_LEN+1];
408
  char         host[MAX_HOSTNAME_SIZE];
409
  uint16_t     port;
410
  struct       sockaddr_in serv_addr;
411 412
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
413
  char         dbName[MAX_DB_NAME_SIZE+1];
414
  char         queryMode[MAX_TB_NAME_SIZE];  // taosc, rest
415

416
  SpecifiedQueryInfo  specifiedQueryInfo;
417
  SuperQueryInfo      superQueryInfo;
418
  uint64_t     totalQueried;
419 420 421
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
422 423 424 425
  TAOS *    taos;
  int       threadID;
  char      db_name[MAX_DB_NAME_SIZE+1];
  uint32_t  time_precision;
426 427
  char      filePath[4096];
  FILE      *fp;
428 429 430
  char      tb_prefix[MAX_TB_NAME_SIZE];
  uint64_t  start_table_from;
  uint64_t  end_table_to;
431
  int64_t   ntables;
432 433 434 435
  uint64_t  data_of_rate;
  int64_t   start_time;
  char*     cols;
  bool      use_metric;
436 437 438
  SSuperTable* superTblInfo;

  // for async insert
439 440
  tsem_t    lock_sem;
  int64_t   counter;
441 442
  uint64_t  st;
  uint64_t  et;
443
  uint64_t  lastTs;
444

445
  // sample data
446
  int64_t   samplePos;
447
  // statistics
448 449
  uint64_t  totalInsertRows;
  uint64_t  totalAffectedRows;
450 451

  // insert delay statistics
452 453 454 455 456
  uint64_t  cntDelay;
  uint64_t  totalDelay;
  uint64_t  avgDelay;
  uint64_t  maxDelay;
  uint64_t  minDelay;
457

458
  // seq of query or subscribe
459
  uint64_t  querySeq;   // sequence number of sql command
460
  TAOS_SUB*  tsub;
461

462 463
} threadInfo;

464
#ifdef WINDOWS
465 466
#define _CRT_RAND_S

467
#include <windows.h>
468 469 470 471 472
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
473 474
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
475 476
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
477 478 479 480

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

481
static void setupForAnsiEscape(void) {
482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
500
  }
501 502
}

503
static void resetAfterAnsiEscape(void) {
504
  // Reset colors
505
  printf("\x1b[0m");
506 507 508 509 510 511

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
512

513
static int taosRandom()
514 515 516 517 518 519
{
    int number;
    rand_s(&number);

    return number;
}
520
#else   // Not windows
521
static void setupForAnsiEscape(void) {}
522

523
static void resetAfterAnsiEscape(void) {
524 525 526
  // Reset colors
  printf("\x1b[0m");
}
527

528 529
#include <time.h>

530
static int taosRandom()
531
{
532
  return rand();
533 534
}

535
#endif // ifdef Windows
536

537
static void prompt();
538
static int createDatabasesAndStables();
539
static void createChildTables();
540
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet);
541
static int postProceSql(char *host, struct sockaddr_in *pServAddr,
542
        uint16_t port, char* sqlstr, threadInfo *pThreadInfo);
543 544
static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq,
                  int disorderRatio, int disorderRange);
545 546 547 548 549 550 551

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
552
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)",
553 554
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

555
SArguments g_args = {
556 557
                     NULL,            // metaFile
                     0,               // test_mode
558 559 560
                     "127.0.0.1",     // host
                     6030,            // port
                     "root",          // user
561
                     #ifdef _TD_POWER_
562 563 564 565 566 567 568 569
                     "powerdb",      // password
                     #else
                     "taosdata",      // password
                     #endif
                     "test",          // database
                     1,               // replica
                     "t",             // tb_prefix
                     NULL,            // sqlFile
570
                     true,            // use_metric
571
                     true,            // drop_database
572
                     true,            // insert_only
573
                     false,           // debug_print
574
                     false,           // verbose_print
575
                     false,           // performance statistic print
576 577 578 579
                     false,           // answer_yes;
                     "./output.txt",  // output_file
                     0,               // mode : sync or async
                     {
580 581 582 583
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
584 585
                     },
                     16,              // len_of_binary
586
                     4,               // num_of_CPR
587 588
                     10,              // num_of_connections/thread
                     0,               // insert_interval
589
                     1,               // query_times
590
                     0,               // interlace_rows;
591
                     30000,           // num_of_RPR
592
                     (1024*1024),         // max_sql_len
593 594 595 596 597 598 599 600 601 602 603 604
                     10000,           // num_of_tables
                     10000,           // num_of_DPT
                     0,               // abort
                     0,               // disorderRatio
                     1000,            // disorderRange
                     1,               // method_of_delete
                     NULL             // arg_list
};



static SDbs            g_Dbs;
605
static int64_t         g_totalChildTables = 0;
606 607 608 609
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

#define debugPrint(fmt, ...) \
610
    do { if (g_args.debug_print || g_args.verbose_print) \
611
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
612

613
#define verbosePrint(fmt, ...) \
614 615
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
616

617 618 619 620
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)

621 622 623 624
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)


625
///////////////////////////////////////////////////
H
hzcheng 已提交
626

627 628
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

629 630 631 632 633 634 635 636
#ifndef TAOSDEMO_COMMIT_SHA1
#define TAOSDEMO_COMMIT_SHA1 "unknown"
#endif

#ifndef TD_VERNUMBER
#define TD_VERNUMBER    "unknown"
#endif

637
#ifndef TAOSDEMO_STATUS
638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654
#define TAOSDEMO_STATUS "unknown"
#endif

static void printVersion() {
    char tdengine_ver[] = TD_VERNUMBER;
    char taosdemo_ver[] = TAOSDEMO_COMMIT_SHA1;
    char taosdemo_status[] = TAOSDEMO_STATUS;

    if (strlen(taosdemo_status) == 0) {
        printf("taosdemo verison %s-%s\n",
                tdengine_ver, taosdemo_ver);
    } else {
        printf("taosdemo verison %s-%s, status:%s\n",
                tdengine_ver, taosdemo_ver, taosdemo_status);
    }
}

655
static void printHelp() {
656
  char indent[10] = "        ";
657
  printf("%s%s%s%s\n", indent, "-f", indent,
658
          "The meta file to the execution procedure. Default is './meta.json'.");
659
  printf("%s%s%s%s\n", indent, "-u", indent,
660 661
          "The TDengine user name to use when connecting to the server. Default is 'root'.");
#ifdef _TD_POWER_
662
  printf("%s%s%s%s\n", indent, "-P", indent,
663
          "The password to use when connecting to the server. Default is 'powerdb'.");
664
  printf("%s%s%s%s\n", indent, "-c", indent,
665 666
          "Configuration directory. Default is '/etc/power/'.");
#else
667
  printf("%s%s%s%s\n", indent, "-P", indent,
668
          "The password to use when connecting to the server. Default is 'taosdata'.");
669
  printf("%s%s%s%s\n", indent, "-c", indent,
670
          "Configuration directory. Default is '/etc/taos/'.");
671
#endif
672
  printf("%s%s%s%s\n", indent, "-h", indent,
673
          "The host to connect to TDengine. Default is localhost.");
674
  printf("%s%s%s%s\n", indent, "-p", indent,
675
          "The TCP/IP port number to use for the connection. Default is 0.");
676
  printf("%s%s%s%s\n", indent, "-d", indent,
677
          "Destination database. Default is 'test'.");
678
  printf("%s%s%s%s\n", indent, "-a", indent,
679
          "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
680
  printf("%s%s%s%s\n", indent, "-m", indent,
681 682
          "Table prefix name. Default is 't'.");
  printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
683
  printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
684
  printf("%s%s%s%s\n", indent, "-o", indent,
685
          "Direct output to the named file. Default is './output.txt'.");
686
  printf("%s%s%s%s\n", indent, "-q", indent,
687
          "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC.");
688
  printf("%s%s%s%s\n", indent, "-b", indent,
689
          "The data_type of columns, default: INT,INT,INT,INT.");
690
  printf("%s%s%s%s\n", indent, "-w", indent,
691
          "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
692 693 694
  printf("%s%s%s%s%d\n", indent, "-l", indent,
          "The number of columns per record. Default is 4. Max values is ",
       MAX_NUM_DATATYPE);
695
  printf("%s%s%s%s\n", indent, "-T", indent,
696
          "The number of threads. Default is 10.");
697
  printf("%s%s%s%s\n", indent, "-i", indent,
698
          "The sleep time (ms) between insertion. Default is 0.");
699
  printf("%s%s%s%s\n", indent, "-r", indent,
700
          "The number of records per request. Default is 30000.");
701
  printf("%s%s%s%s\n", indent, "-t", indent,
702
          "The number of tables. Default is 10000.");
703
  printf("%s%s%s%s\n", indent, "-n", indent,
704 705 706
          "The number of records per table. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
  printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
707
  printf("%s%s%s%s\n", indent, "-O", indent,
708
          "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order.");
709
  printf("%s%s%s%s\n", indent, "-R", indent,
710
          "Out of order data's range, ms, default is 1000.");
711
  printf("%s%s%s%s\n", indent, "-g", indent,
712
          "Print debug info.");
713
  printf("%s%s%s\n", indent, "-V, --version\t",
714
          "Print version info.");
715 716
  printf("%s%s%s%s\n", indent, "--help\t", indent,
          "Print command line arguments list info.");
717
/*    printf("%s%s%s%s\n", indent, "-D", indent,
718 719 720 721
          "if elete database if exists. 0: no, 1: yes, default is 1");
          */
}

722 723 724 725 726 727 728 729 730 731 732 733 734 735 736
static bool isStringNumber(char *input)
{
  int len = strlen(input);
  if (0 == len) {
    return false;
  }

  for (int i = 0; i < len; i++) {
    if (!isdigit(input[i]))
      return false;
  }

  return true;
}

737
static void parse_args(int argc, char *argv[], SArguments *arguments) {
738 739 740 741 742

  for (int i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-f") == 0) {
      arguments->metaFile = argv[++i];
    } else if (strcmp(argv[i], "-c") == 0) {
743 744 745 746 747
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-c need a valid path following!\n");
        exit(EXIT_FAILURE);
      }
748
      tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN);
749
    } else if (strcmp(argv[i], "-h") == 0) {
750 751 752 753 754
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-h need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
755 756
      arguments->host = argv[++i];
    } else if (strcmp(argv[i], "-p") == 0) {
757 758 759 760 761 762
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-p need a number following!\n");
        exit(EXIT_FAILURE);
      }
763 764
      arguments->port = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-u") == 0) {
765 766 767 768 769
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-u need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
770 771
      arguments->user = argv[++i];
    } else if (strcmp(argv[i], "-P") == 0) {
772 773 774 775 776
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-P need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
777 778
      arguments->password = argv[++i];
    } else if (strcmp(argv[i], "-o") == 0) {
779 780 781 782 783
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-o need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
784 785
      arguments->output_file = argv[++i];
    } else if (strcmp(argv[i], "-s") == 0) {
786 787 788 789 790
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-s need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
791 792
      arguments->sqlFile = argv[++i];
    } else if (strcmp(argv[i], "-q") == 0) {
793 794
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
795
        printHelp();
796
        errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, not-0: ASYNC. Default is SYNC.\n");
797 798
        exit(EXIT_FAILURE);
      }
799
      arguments->async_mode = atoi(argv[++i]);
800
    } else if (strcmp(argv[i], "-T") == 0) {
801 802
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
803 804 805 806
        printHelp();
        errorPrint("%s", "\n\t-T need a number following!\n");
        exit(EXIT_FAILURE);
      }
807 808
      arguments->num_of_threads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
809 810
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
811 812 813 814
        printHelp();
        errorPrint("%s", "\n\t-i need a number following!\n");
        exit(EXIT_FAILURE);
      }
815
      arguments->insert_interval = atoi(argv[++i]);
816
    } else if (strcmp(argv[i], "-qt") == 0) {
817 818
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
819
        printHelp();
820
        errorPrint("%s", "\n\t-qt need a number following!\n");
821 822
        exit(EXIT_FAILURE);
      }
823
      arguments->query_times = atoi(argv[++i]);
824
    } else if (strcmp(argv[i], "-B") == 0) {
825 826
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
827 828 829 830
        printHelp();
        errorPrint("%s", "\n\t-B need a number following!\n");
        exit(EXIT_FAILURE);
      }
831
      arguments->interlace_rows = atoi(argv[++i]);
832
    } else if (strcmp(argv[i], "-r") == 0) {
833 834
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
835 836 837 838
        printHelp();
        errorPrint("%s", "\n\t-r need a number following!\n");
        exit(EXIT_FAILURE);
      }
839 840
      arguments->num_of_RPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-t") == 0) {
841 842 843 844 845 846
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-t need a number following!\n");
        exit(EXIT_FAILURE);
      }
847 848
      arguments->num_of_tables = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
849 850 851 852 853 854
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-n need a number following!\n");
        exit(EXIT_FAILURE);
      }
855 856
      arguments->num_of_DPT = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-d") == 0) {
857 858 859 860 861
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-d need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
862 863
      arguments->database = argv[++i];
    } else if (strcmp(argv[i], "-l") == 0) {
864 865 866 867 868 869
      if (argc == i+1) {
        if (!isStringNumber(argv[i+1])) {
            printHelp();
            errorPrint("%s", "\n\t-l need a number following!\n");
            exit(EXIT_FAILURE);
        }
870
      }
871
      arguments->num_of_CPR = atoi(argv[++i]);
872 873 874 875 876 877 878 879 880 881 882

      if (arguments->num_of_CPR > MAX_NUM_DATATYPE) {
          printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE);
          prompt();
          arguments->num_of_CPR = MAX_NUM_DATATYPE;
      }

      for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) {
          arguments->datatype[col] = NULL;
      }

883
    } else if (strcmp(argv[i], "-b") == 0) {
884 885 886 887 888
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-b need valid string following!\n");
        exit(EXIT_FAILURE);
      }
889 890
      ++i;
      if (strstr(argv[i], ",") == NULL) {
891
        // only one col
892 893 894 895 896 897 898
        if (strcasecmp(argv[i], "INT")
                && strcasecmp(argv[i], "FLOAT")
                && strcasecmp(argv[i], "TINYINT")
                && strcasecmp(argv[i], "BOOL")
                && strcasecmp(argv[i], "SMALLINT")
                && strcasecmp(argv[i], "BIGINT")
                && strcasecmp(argv[i], "DOUBLE")
899
                && strcasecmp(argv[i], "BINARY")
900 901
                && strcasecmp(argv[i], "NCHAR")) {
          printHelp();
902
          errorPrint("%s", "-b: Invalid data_type!\n");
903
          exit(EXIT_FAILURE);
904
        }
905
        arguments->datatype[0] = argv[i];
906
      } else {
907
        // more than one col
908
        int index = 0;
909 910
        g_dupstr = strdup(argv[i]);
        char *running = g_dupstr;
911
        char *token = strsep(&running, ",");
912
        while(token != NULL) {
913 914 915 916 917 918 919 920 921 922
          if (strcasecmp(token, "INT")
                  && strcasecmp(token, "FLOAT")
                  && strcasecmp(token, "TINYINT")
                  && strcasecmp(token, "BOOL")
                  && strcasecmp(token, "SMALLINT")
                  && strcasecmp(token, "BIGINT")
                  && strcasecmp(token, "DOUBLE")
                  && strcasecmp(token, "BINARY")
                  && strcasecmp(token, "NCHAR")) {
            printHelp();
923
            free(g_dupstr);
924
            errorPrint("%s", "-b: Invalid data_type!\n");
925
            exit(EXIT_FAILURE);
H
hzcheng 已提交
926
          }
927
          arguments->datatype[index++] = token;
928 929
          token = strsep(&running, ",");
          if (index >= MAX_NUM_DATATYPE) break;
S
TD-1057  
Shengliang Guan 已提交
930
        }
931
        arguments->datatype[index] = NULL;
932
      }
933
    } else if (strcmp(argv[i], "-w") == 0) {
934 935 936 937 938 939
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-w need a number following!\n");
        exit(EXIT_FAILURE);
      }
940 941
      arguments->len_of_binary = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-m") == 0) {
942 943 944 945 946 947
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-m need a number following!\n");
        exit(EXIT_FAILURE);
      }
948
      arguments->tb_prefix = argv[++i];
949 950
    } else if (strcmp(argv[i], "-N") == 0) {
      arguments->use_metric = false;
951
    } else if (strcmp(argv[i], "-x") == 0) {
952
      arguments->insert_only = false;
953
    } else if (strcmp(argv[i], "-y") == 0) {
954
      arguments->answer_yes = true;
955 956
    } else if (strcmp(argv[i], "-g") == 0) {
      arguments->debug_print = true;
957 958
    } else if (strcmp(argv[i], "-gg") == 0) {
      arguments->verbose_print = true;
959 960
    } else if (strcmp(argv[i], "-pp") == 0) {
      arguments->performance_print = true;
961
    } else if (strcmp(argv[i], "-O") == 0) {
962 963 964 965 966 967
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-O need a number following!\n");
        exit(EXIT_FAILURE);
      }
968

969
      arguments->disorderRatio = atoi(argv[++i]);
970

971
      if (arguments->disorderRatio > 50) {
972
        arguments->disorderRatio = 50;
973
      }
974

975
      if (arguments->disorderRatio < 0) {
976
        arguments->disorderRatio = 0;
977
      }
978

979
    } else if (strcmp(argv[i], "-R") == 0) {
980 981 982 983 984 985
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-R need a number following!\n");
        exit(EXIT_FAILURE);
      }
986

987
      arguments->disorderRange = atoi(argv[++i]);
988 989 990
      if (arguments->disorderRange < 0)
        arguments->disorderRange = 1000;

991
    } else if (strcmp(argv[i], "-a") == 0) {
992 993 994 995 996 997
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-a need a number following!\n");
        exit(EXIT_FAILURE);
      }
998 999 1000
      arguments->replica = atoi(argv[++i]);
      if (arguments->replica > 3 || arguments->replica < 1) {
          arguments->replica = 1;
1001
      }
1002 1003
    } else if (strcmp(argv[i], "-D") == 0) {
      arguments->method_of_delete = atoi(argv[++i]);
1004 1005 1006
      if (arguments->method_of_delete > 3) {
        errorPrint("%s", "\n\t-D need a valud (0~3) number following!\n");
        exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1007
      }
1008 1009 1010 1011
    } else if ((strcmp(argv[i], "--version") == 0) ||
        (strcmp(argv[i], "-V") == 0)){
      printVersion();
      exit(0);
1012 1013 1014 1015 1016
    } else if (strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else {
      printHelp();
1017
      errorPrint("%s", "ERROR: wrong options\n");
1018
      exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1019 1020
    }
  }
1021

1022 1023
  if (((arguments->debug_print) && (arguments->metaFile == NULL))
          || arguments->verbose_print) {
1024
    printf("###################################################################\n");
1025
    printf("# meta file:                         %s\n", arguments->metaFile);
1026
    printf("# Server IP:                         %s:%hu\n",
1027 1028 1029 1030 1031
            arguments->host == NULL ? "localhost" : arguments->host,
            arguments->port );
    printf("# User:                              %s\n", arguments->user);
    printf("# Password:                          %s\n", arguments->password);
    printf("# Use metric:                        %s\n", arguments->use_metric ? "true" : "false");
1032 1033 1034 1035 1036 1037 1038 1039 1040
    if (*(arguments->datatype)) {
        printf("# Specified data type:               ");
        for (int i = 0; i < MAX_NUM_DATATYPE; i++)
            if (arguments->datatype[i])
               printf("%s,", arguments->datatype[i]);
            else
                break;
        printf("\n");
    }
1041
    printf("# Insertion interval:                %"PRIu64"\n",
1042
            arguments->insert_interval);
1043
    printf("# Number of records per req:         %"PRIu64"\n",
1044
            arguments->num_of_RPR);
1045
    printf("# Max SQL length:                    %"PRIu64"\n",
1046
            arguments->max_sql_len);
1047
    printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
1048
    printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
1049
    printf("# Number of Tables:                  %"PRId64"\n",
1050
            arguments->num_of_tables);
1051
    printf("# Number of Data per Table:          %"PRId64"\n",
1052
            arguments->num_of_DPT);
1053 1054 1055 1056 1057
    printf("# Database name:                     %s\n", arguments->database);
    printf("# Table prefix:                      %s\n", arguments->tb_prefix);
    if (arguments->disorderRatio) {
      printf("# Data order:                        %d\n", arguments->disorderRatio);
      printf("# Data out of order rate:            %d\n", arguments->disorderRange);
1058

1059 1060 1061 1062
    }
    printf("# Delete method:                     %d\n", arguments->method_of_delete);
    printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
    printf("# Print debug info:                  %d\n", arguments->debug_print);
1063
    printf("# Print verbose info:                %d\n", arguments->verbose_print);
1064
    printf("###################################################################\n");
1065 1066

    prompt();
1067
  }
1068
}
H
hzcheng 已提交
1069

1070 1071 1072 1073
static bool getInfoFromJsonFile(char* file);
//static int generateOneRowDataForStb(SSuperTable* stbInfo);
//static int getDataIntoMemForStb(SSuperTable* stbInfo);
static void init_rand_data();
1074
static void tmfclose(FILE *fp) {
1075 1076 1077 1078
  if (NULL != fp) {
    fclose(fp);
  }
}
1079

1080
static void tmfree(char *buf) {
1081 1082
  if (NULL != buf) {
    free(buf);
H
Hui Li 已提交
1083
  }
1084
}
H
Hui Li 已提交
1085

1086
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
1087 1088 1089
  int i;
  TAOS_RES *res = NULL;
  int32_t   code = -1;
1090

1091 1092 1093 1094
  for (i = 0; i < 5; i++) {
    if (NULL != res) {
      taos_free_result(res);
      res = NULL;
H
hzcheng 已提交
1095
    }
1096

1097 1098 1099 1100
    res = taos_query(taos, command);
    code = taos_errno(res);
    if (0 == code) {
      break;
1101
    }
H
hzcheng 已提交
1102 1103
  }

1104
  verbosePrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
1105
  if (code != 0) {
1106
    if (!quiet) {
1107
      errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res));
1108
    }
1109 1110 1111
    taos_free_result(res);
    //taos_close(taos);
    return -1;
1112
  }
H
hzcheng 已提交
1113

1114 1115 1116 1117
  if (INSERT_TYPE == type) {
    int affectedRows = taos_affected_rows(res);
    taos_free_result(res);
    return affectedRows;
H
hzcheng 已提交
1118
  }
1119

1120
  taos_free_result(res);
1121 1122
  return 0;
}
H
hzcheng 已提交
1123

1124
static void appendResultBufToFile(char *resultBuf, threadInfo *pThreadInfo)
1125
{
1126 1127
  pThreadInfo->fp = fopen(pThreadInfo->filePath, "at");
  if (pThreadInfo->fp == NULL) {
1128 1129
      errorPrint(
              "%s() LN%d, failed to open result file: %s, result will not save to file\n",
1130
              __func__, __LINE__, pThreadInfo->filePath);
1131 1132
      return;
  }
1133

1134 1135 1136
  fprintf(pThreadInfo->fp, "%s", resultBuf);
  tmfclose(pThreadInfo->fp);
  pThreadInfo->fp = NULL;
1137
}
1138

1139
static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) {
1140 1141 1142 1143 1144
  TAOS_ROW    row = NULL;
  int         num_rows = 0;
  int         num_fields = taos_field_count(res);
  TAOS_FIELD *fields     = taos_fetch_fields(res);

1145 1146
  char* databuf = (char*) calloc(1, 100*1024*1024);
  if (databuf == NULL) {
1147 1148
    errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n",
            __func__, __LINE__);
1149 1150
    return ;
  }
H
hzcheng 已提交
1151

1152 1153
  int   totalLen = 0;
  char  temp[16000];
H
hzcheng 已提交
1154

1155
  // fetch the records row by row
1156
  while((row = taos_fetch_row(res))) {
1157
    if ((strlen(pThreadInfo->filePath) > 0)
1158
            && (totalLen >= 100*1024*1024 - 32000)) {
1159
        appendResultBufToFile(databuf, pThreadInfo);
1160 1161
        totalLen = 0;
        memset(databuf, 0, 100*1024*1024);
H
hzcheng 已提交
1162
    }
1163 1164 1165 1166 1167 1168
    num_rows++;
    int len = taos_print_row(temp, row, fields, num_fields);
    len += sprintf(temp + len, "\n");
    //printf("query result:%s\n", temp);
    memcpy(databuf + totalLen, temp, len);
    totalLen += len;
H
hzcheng 已提交
1169 1170
  }

1171
  verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n",
1172 1173 1174
          __func__, __LINE__, databuf, pThreadInfo->filePath);
  if (strlen(pThreadInfo->filePath) > 0) {
    appendResultBufToFile(databuf, pThreadInfo);
1175
  }
1176 1177
  free(databuf);
}
H
hzcheng 已提交
1178

1179
static void selectAndGetResult(
1180
        threadInfo *pThreadInfo, char *command)
1181 1182 1183 1184 1185 1186 1187 1188 1189 1190
{
  if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) {
    TAOS_RES *res = taos_query(pThreadInfo->taos, command);
    if (res == NULL || taos_errno(res) != 0) {
        errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n",
            __func__, __LINE__, command, taos_errstr(res));
        taos_free_result(res);
        return;
    }

1191
    fetchResult(res, pThreadInfo);
1192
    taos_free_result(res);
1193

1194 1195 1196 1197
  } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
      int retCode = postProceSql(
              g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port,
              command,
1198
              pThreadInfo);
1199 1200 1201 1202 1203 1204 1205 1206
      if (0 != retCode) {
        printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID);
      }

  } else {
      errorPrint("%s() LN%d, unknown query mode: %s\n",
        __func__, __LINE__, g_queryInfo.queryMode);
  }
1207
}
H
hzcheng 已提交
1208

1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247
static int32_t rand_bool(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 2;
}

static int32_t rand_tinyint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 128;
}

static int32_t rand_smallint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 32767;
}

static int32_t rand_int(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor];
}

static int64_t rand_bigint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randbigint[cursor];
}

static float rand_float(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
1248
  return randfloat[cursor];
1249 1250
}

1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267
#if 0
static const char charNum[] = "0123456789";

static void nonrand_string(char *, int) __attribute__ ((unused));   // reserve for debugging purpose
static void nonrand_string(char *str, int size)
{
  str[0] = 0;
  if (size > 0) {
    int n;
    for (n = 0; n < size; n++) {
      str[n] = charNum[n % 10];
    }
    str[n] = 0;
  }
}
#endif

1268
static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1269

1270
static void rand_string(char *str, int size) {
1271 1272 1273 1274
  str[0] = 0;
  if (size > 0) {
    //--size;
    int n;
1275
    for (n = 0; n < size; n++) {
1276
      int key = abs(rand_tinyint()) % (int)(sizeof(charset) - 1);
1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
      str[n] = charset[key];
    }
    str[n] = 0;
  }
}

static double rand_double() {
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randdouble[cursor];

}

static void init_rand_data() {
  for (int i = 0; i < MAX_PREPARED_RAND; i++){
1293 1294 1295 1296
    randint[i] = (int)(taosRandom() % 65535);
    randbigint[i] = (int64_t)(taosRandom() % 2147483648);
    randfloat[i] = (float)(taosRandom() / 1000.0);
    randdouble[i] = (double)(taosRandom() / 1000000.0);
1297 1298 1299
  }
}

1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1320
static int printfInsertMeta() {
1321 1322
    SHOW_PARSE_RESULT_START();

1323 1324
  printf("host:                       \033[33m%s:%u\033[0m\n",
          g_Dbs.host, g_Dbs.port);
1325 1326
  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
1327
  printf("configDir:                  \033[33m%s\033[0m\n", configDir);
1328 1329
  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
1330 1331
  printf("thread num of create table: \033[33m%d\033[0m\n",
          g_Dbs.threadCountByCreateTbl);
1332
  printf("top insert interval:        \033[33m%"PRIu64"\033[0m\n",
1333
          g_args.insert_interval);
1334
  printf("number of records per req:  \033[33m%"PRIu64"\033[0m\n",
1335
          g_args.num_of_RPR);
1336
  printf("max sql length:             \033[33m%"PRIu64"\033[0m\n",
1337
          g_args.max_sql_len);
1338 1339

  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
1340

1341 1342
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    printf("database[\033[33m%d\033[0m]:\n", i);
1343 1344
    printf("  database[%d] name:      \033[33m%s\033[0m\n",
            i, g_Dbs.db[i].dbName);
1345
    if (0 == g_Dbs.db[i].drop) {
1346 1347 1348
      printf("  drop:                  \033[33mno\033[0m\n");
    } else {
      printf("  drop:                  \033[33myes\033[0m\n");
1349 1350 1351
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
1352 1353
      printf("  blocks:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.blocks);
1354 1355
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
1356 1357
      printf("  cache:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.cache);
1358 1359
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
1360 1361
      printf("  days:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.days);
1362 1363
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
1364 1365
      printf("  keep:                  \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.keep);
1366 1367
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
1368 1369
      printf("  replica:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.replica);
1370 1371
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
1372 1373
      printf("  update:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.update);
1374 1375
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
1376 1377
      printf("  minRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.minRows);
1378 1379
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
1380 1381
      printf("  maxRows:               \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.maxRows);
1382 1383 1384 1385 1386
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
1387 1388
      printf("  walLevel:              \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.walLevel);
1389 1390
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
1391 1392
      printf("  fsync:                 \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.fsync);
1393 1394
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
1395 1396
      printf("  quorum:                \033[33m%d\033[0m\n",
              g_Dbs.db[i].dbCfg.quorum);
1397 1398
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1399 1400
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1401 1402
        printf("  precision:             \033[33m%s\033[0m\n",
            g_Dbs.db[i].dbCfg.precision);
1403
      } else {
1404 1405 1406
        printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
                g_Dbs.db[i].dbCfg.precision);
        return -1;
1407 1408 1409
      }
    }

1410
    printf("  super table count:     \033[33m%"PRIu64"\033[0m\n",
1411
        g_Dbs.db[i].superTblCount);
1412 1413
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      printf("  super table[\033[33m%"PRIu64"\033[0m]:\n", j);
1414

1415 1416
      printf("      stbName:           \033[33m%s\033[0m\n",
          g_Dbs.db[i].superTbls[j].sTblName);
1417 1418 1419

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
1420 1421
      } else if (AUTO_CREATE_SUBTBL ==
              g_Dbs.db[i].superTbls[j].autoCreateTable) {
1422 1423 1424 1425
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
      }
1426

1427 1428 1429 1430 1431 1432 1433
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
      }
1434

1435
      printf("      childTblCount:     \033[33m%"PRId64"\033[0m\n",
1436 1437 1438 1439 1440 1441 1442
              g_Dbs.db[i].superTbls[j].childTblCount);
      printf("      childTblPrefix:    \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      printf("      dataSource:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      printf("      insertMode:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].insertMode);
1443
      if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
1444
        printf("      childTblLimit:     \033[33m%"PRId64"\033[0m\n",
1445
                g_Dbs.db[i].superTbls[j].childTblLimit);
1446
      }
1447
      if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) {
1448
        printf("      childTblOffset:    \033[33m%"PRIu64"\033[0m\n",
1449
                g_Dbs.db[i].superTbls[j].childTblOffset);
1450
      }
1451
      printf("      insertRows:        \033[33m%"PRId64"\033[0m\n",
1452
              g_Dbs.db[i].superTbls[j].insertRows);
1453
/*
1454
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1455
        printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
1456
      }else {
1457
        printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
1458
      }
1459
      */
1460
      printf("      interlaceRows:     \033[33m%"PRIu64"\033[0m\n",
1461
              g_Dbs.db[i].superTbls[j].interlaceRows);
1462 1463

      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1464
        printf("      stable insert interval:   \033[33m%"PRIu64"\033[0m\n",
1465 1466 1467
            g_Dbs.db[i].superTbls[j].insertInterval);
      }

1468 1469 1470 1471
      printf("      disorderRange:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      printf("      disorderRatio:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
1472
      printf("      maxSqlLen:         \033[33m%"PRIu64"\033[0m\n",
1473
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1474
      printf("      timeStampStep:     \033[33m%"PRId64"\033[0m\n",
1475 1476 1477 1478 1479 1480 1481 1482 1483
              g_Dbs.db[i].superTbls[j].timeStampStep);
      printf("      startTimestamp:    \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
      printf("      sampleFormat:      \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      printf("      sampleFile:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      printf("      tagsFile:          \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
1484
      printf("      columnCount:       \033[33m%d\033[0m\n",
1485
              g_Dbs.db[i].superTbls[j].columnCount);
1486 1487
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1488 1489 1490 1491
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "nchar", 5))) {
1492
          printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
1493 1494
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1495
        } else {
1496 1497
          printf("column[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType);
1498 1499 1500
        }
      }
      printf("\n");
1501 1502 1503

      printf("      tagCount:            \033[33m%d\033[0m\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1504 1505
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1506 1507 1508 1509
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1510
          printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
1511 1512
                  g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1513
        } else {
1514 1515
          printf("tag[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].tags[k].dataType);
1516
        }
1517 1518 1519 1520 1521
      }
      printf("\n");
    }
    printf("\n");
  }
1522 1523

  SHOW_PARSE_RESULT_END();
1524 1525

  return 0;
1526 1527 1528
}

static void printfInsertMetaToFile(FILE* fp) {
1529 1530

  SHOW_PARSE_RESULT_START_TO_FILE(fp);
1531

1532 1533
  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
  fprintf(fp, "user:                       %s\n", g_Dbs.user);
1534
  fprintf(fp, "configDir:                  %s\n", configDir);
1535 1536 1537
  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
1538 1539
  fprintf(fp, "number of records per req:  %"PRIu64"\n", g_args.num_of_RPR);
  fprintf(fp, "max sql length:             %"PRIu64"\n", g_args.max_sql_len);
1540
  fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
1541

1542 1543
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    fprintf(fp, "database[%d]:\n", i);
1544
    fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
1545
    if (0 == g_Dbs.db[i].drop) {
1546
      fprintf(fp, "  drop:                  no\n");
1547
    }else {
1548
      fprintf(fp, "  drop:                  yes\n");
1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1588 1589
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1590 1591
        fprintf(fp, "  precision:             %s\n",
                g_Dbs.db[i].dbCfg.precision);
1592
      } else {
1593 1594
        fprintf(fp, "  precision error:       %s\n",
                g_Dbs.db[i].dbCfg.precision);
1595 1596 1597
      }
    }

1598 1599
    fprintf(fp, "  super table count:     %"PRIu64"\n",
            g_Dbs.db[i].superTblCount);
1600 1601
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      fprintf(fp, "  super table[%d]:\n", j);
1602

1603 1604
      fprintf(fp, "      stbName:           %s\n",
              g_Dbs.db[i].superTbls[j].sTblName);
1605 1606 1607

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "no");
1608 1609
      } else if (AUTO_CREATE_SUBTBL
              == g_Dbs.db[i].superTbls[j].autoCreateTable) {
1610 1611 1612 1613
        fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
      } else {
        fprintf(fp, "      autoCreateTable:   %s\n",  "error");
      }
1614

1615 1616
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "no");
1617 1618
      } else if (TBL_ALREADY_EXISTS
              == g_Dbs.db[i].superTbls[j].childTblExists) {
1619 1620 1621 1622
        fprintf(fp, "      childTblExists:    %s\n",  "yes");
      } else {
        fprintf(fp, "      childTblExists:    %s\n",  "error");
      }
1623

1624
      fprintf(fp, "      childTblCount:     %"PRId64"\n",
1625 1626 1627 1628 1629 1630 1631
              g_Dbs.db[i].superTbls[j].childTblCount);
      fprintf(fp, "      childTblPrefix:    %s\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      fprintf(fp, "      dataSource:        %s\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      fprintf(fp, "      insertMode:        %s\n",
              g_Dbs.db[i].superTbls[j].insertMode);
1632
      fprintf(fp, "      insertRows:        %"PRId64"\n",
1633
              g_Dbs.db[i].superTbls[j].insertRows);
1634
      fprintf(fp, "      interlace rows:    %"PRIu64"\n",
1635
              g_Dbs.db[i].superTbls[j].interlaceRows);
1636
      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1637
        fprintf(fp, "      stable insert interval:   %"PRIu64"\n",
1638
                g_Dbs.db[i].superTbls[j].insertInterval);
1639
      }
1640
/*
1641
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1642
        fprintf(fp, "      multiThreadWriteOneTbl:  no\n");
1643
      }else {
1644
        fprintf(fp, "      multiThreadWriteOneTbl:  yes\n");
1645
      }
1646
      */
1647
      fprintf(fp, "      interlaceRows:     %"PRIu64"\n",
1648
              g_Dbs.db[i].superTbls[j].interlaceRows);
1649 1650 1651 1652
      fprintf(fp, "      disorderRange:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      fprintf(fp, "      disorderRatio:     %d\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
1653
      fprintf(fp, "      maxSqlLen:         %"PRIu64"\n",
1654
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1655

1656 1657 1658 1659
      fprintf(fp, "      timeStampStep:     %"PRId64"\n",
              g_Dbs.db[i].superTbls[j].timeStampStep);
      fprintf(fp, "      startTimestamp:    %s\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
1660 1661 1662 1663 1664 1665
      fprintf(fp, "      sampleFormat:      %s\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      fprintf(fp, "      sampleFile:        %s\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      fprintf(fp, "      tagsFile:          %s\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
1666

1667 1668
      fprintf(fp, "      columnCount:       %d\n        ",
              g_Dbs.db[i].superTbls[j].columnCount);
1669 1670
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1671 1672 1673
        if ((0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "binary", strlen("binary")))
1674 1675
                || (0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
1676 1677 1678 1679
                        "nchar", strlen("nchar")))) {
          fprintf(fp, "column[%d]:%s(%d) ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1680
        } else {
1681 1682
          fprintf(fp, "column[%d]:%s ",
                  k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
1683 1684 1685
        }
      }
      fprintf(fp, "\n");
1686 1687 1688

      fprintf(fp, "      tagCount:            %d\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1689 1690
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1691 1692 1693 1694
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1695 1696
          fprintf(fp, "tag[%d]:%s(%d) ",
                  k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
1697
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1698 1699
        } else {
          fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
1700
        }
1701 1702 1703 1704 1705
      }
      fprintf(fp, "\n");
    }
    fprintf(fp, "\n");
  }
1706

1707
  SHOW_PARSE_RESULT_END_TO_FILE(fp);
1708 1709 1710
}

static void printfQueryMeta() {
1711

1712
  SHOW_PARSE_RESULT_START();
1713

1714 1715
  printf("host:                    \033[33m%s:%u\033[0m\n",
          g_queryInfo.host, g_queryInfo.port);
1716 1717 1718 1719
  printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
  printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

  printf("\n");
1720 1721 1722

  if ((SUBSCRIBE_TEST == g_args.test_mode) || (QUERY_TEST == g_args.test_mode)) {
    printf("specified table query info:                   \n");
1723
    printf("sqlCount:       \033[33m%d\033[0m\n",
1724
      g_queryInfo.specifiedQueryInfo.sqlCount);
1725 1726 1727
    if (g_queryInfo.specifiedQueryInfo.sqlCount > 0) {
      printf("specified tbl query times:\n");
      printf("                \033[33m%"PRIu64"\033[0m\n",
1728
      g_queryInfo.specifiedQueryInfo.queryTimes);
1729 1730 1731
      printf("query interval: \033[33m%"PRIu64" ms\033[0m\n",
        g_queryInfo.specifiedQueryInfo.queryInterval);
      printf("top query times:\033[33m%"PRIu64"\033[0m\n", g_args.query_times);
1732
      printf("concurrent:     \033[33m%d\033[0m\n",
1733 1734
      g_queryInfo.specifiedQueryInfo.concurrent);
      printf("mod:            \033[33m%s\033[0m\n",
1735
        (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync");
1736
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1737
        g_queryInfo.specifiedQueryInfo.subscribeInterval);
1738
      printf("restart:        \033[33m%d\033[0m\n",
1739
        g_queryInfo.specifiedQueryInfo.subscribeRestart);
1740
      printf("keepProgress:   \033[33m%d\033[0m\n",
1741
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
1742

1743 1744
      for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        printf("  sql[%d]: \033[33m%s\033[0m\n",
1745 1746 1747 1748 1749 1750
            i, g_queryInfo.specifiedQueryInfo.sql[i]);
      }
      printf("\n");
    }

    printf("super table query info:\n");
1751
    printf("sqlCount:       \033[33m%d\033[0m\n",
1752 1753 1754 1755 1756 1757 1758
      g_queryInfo.superQueryInfo.sqlCount);

    if (g_queryInfo.superQueryInfo.sqlCount > 0) {
      printf("query interval: \033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryInterval);
      printf("threadCnt:      \033[33m%d\033[0m\n",
        g_queryInfo.superQueryInfo.threadCnt);
1759
      printf("childTblCount:  \033[33m%"PRId64"\033[0m\n",
1760 1761 1762 1763 1764 1765 1766
        g_queryInfo.superQueryInfo.childTblCount);
      printf("stable name:    \033[33m%s\033[0m\n",
        g_queryInfo.superQueryInfo.sTblName);
      printf("stb query times:\033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryTimes);

      printf("mod:            \033[33m%s\033[0m\n",
1767
        (g_queryInfo.superQueryInfo.asyncMode)?"async":"sync");
1768
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1769
        g_queryInfo.superQueryInfo.subscribeInterval);
1770
      printf("restart:        \033[33m%d\033[0m\n",
1771
        g_queryInfo.superQueryInfo.subscribeRestart);
1772
      printf("keepProgress:   \033[33m%d\033[0m\n",
1773 1774
        g_queryInfo.superQueryInfo.subscribeKeepProgress);

1775 1776 1777 1778 1779 1780
      for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
        printf("  sql[%d]: \033[33m%s\033[0m\n",
            i, g_queryInfo.superQueryInfo.sql[i]);
      }
      printf("\n");
    }
1781
  }
1782

1783
  SHOW_PARSE_RESULT_END();
1784 1785
}

1786
static char* formatTimestamp(char* buf, int64_t val, int precision) {
1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817
  time_t tt;
  if (precision == TSDB_TIME_PRECISION_MICRO) {
    tt = (time_t)(val / 1000000);
  } else {
    tt = (time_t)(val / 1000);
  }

/* comment out as it make testcases like select_with_tags.sim fail.
  but in windows, this may cause the call to localtime crash if tt < 0,
  need to find a better solution.
  if (tt < 0) {
    tt = 0;
  }
  */

#ifdef WINDOWS
  if (tt < 0) tt = 0;
#endif

  struct tm* ptm = localtime(&tt);
  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);

  if (precision == TSDB_TIME_PRECISION_MICRO) {
    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
  } else {
    sprintf(buf + pos, ".%03d", (int)(val % 1000));
  }

  return buf;
}

1818 1819 1820
static void xDumpFieldToFile(FILE* fp, const char* val,
        TAOS_FIELD* field, int32_t length, int precision) {

1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855
  if (val == NULL) {
    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
    return;
  }

  char buf[TSDB_MAX_BYTES_PER_ROW];
  switch (field->type) {
    case TSDB_DATA_TYPE_BOOL:
      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
      break;
    case TSDB_DATA_TYPE_TINYINT:
      fprintf(fp, "%d", *((int8_t *)val));
      break;
    case TSDB_DATA_TYPE_SMALLINT:
      fprintf(fp, "%d", *((int16_t *)val));
      break;
    case TSDB_DATA_TYPE_INT:
      fprintf(fp, "%d", *((int32_t *)val));
      break;
    case TSDB_DATA_TYPE_BIGINT:
      fprintf(fp, "%" PRId64, *((int64_t *)val));
      break;
    case TSDB_DATA_TYPE_FLOAT:
      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
      break;
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      memcpy(buf, val, length);
      buf[length] = 0;
      fprintf(fp, "\'%s\'", buf);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
1856
      formatTimestamp(buf, *(int64_t*)val, precision);
1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871
      fprintf(fp, "'%s'", buf);
      break;
    default:
      break;
  }
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  FILE* fp = fopen(fname, "at");
  if (fp == NULL) {
1872
    errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886
    return -1;
  }

  int num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int precision = taos_result_precision(tres);

  for (int col = 0; col < num_fields; col++) {
    if (col > 0) {
      fprintf(fp, ",");
    }
    fprintf(fp, "%s", fields[col].name);
  }
  fputc('\n', fp);
1887

1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907
  int numOfRows = 0;
  do {
    int32_t* length = taos_fetch_lengths(tres);
    for (int i = 0; i < num_fields; i++) {
      if (i > 0) {
        fputc(',', fp);
      }
      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
    }
    fputc('\n', fp);

    numOfRows++;
    row = taos_fetch_row(tres);
  } while( row != NULL);

  fclose(fp);

  return numOfRows;
}

1908 1909
static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
  TAOS_RES * res;
1910 1911
  TAOS_ROW row = NULL;
  int count = 0;
1912 1913

  res = taos_query(taos, "show databases;");
1914
  int32_t code = taos_errno(res);
1915

1916
  if (code != 0) {
1917
    errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
1918 1919 1920 1921 1922
    return -1;
  }

  TAOS_FIELD *fields = taos_fetch_fields(res);

1923
  while((row = taos_fetch_row(res)) != NULL) {
1924
    // sys database name : 'log'
1925 1926 1927 1928
    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log",
                fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) {
      continue;
    }
1929 1930 1931

    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
    if (dbInfos[count] == NULL) {
1932
      errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
1933 1934 1935
      return -1;
    }

1936 1937
    tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
            fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
1938
    formatTimestamp(dbInfos[count]->create_time,
1939 1940
            *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
            TSDB_TIME_PRECISION_MILLI);
1941
    dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
1942
    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
1943 1944
    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
1945
    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
1946

1947
    tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
1948
            fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
1949 1950 1951 1952 1953 1954 1955
    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
1956 1957
    dbInfos[count]->cachelast =
      (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));
1958

1959
    tstrncpy(dbInfos[count]->precision,
1960
            (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
1961
            fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
1962
    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
1963
    tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
1964
            fields[TSDB_SHOW_DB_STATUS_INDEX].bytes);
1965

1966 1967
    count++;
    if (count > MAX_DATABASE_COUNT) {
1968 1969
      errorPrint("%s() LN%d, The database count overflow than %d\n",
         __func__, __LINE__, MAX_DATABASE_COUNT);
1970 1971 1972 1973 1974 1975 1976
      break;
    }
  }

  return count;
}

1977 1978
static void printfDbInfoForQueryToFile(
        char* filename, SDbInfo* dbInfos, int index) {
1979

1980
  if (filename[0] == 0)
1981
      return;
1982 1983 1984

  FILE *fp = fopen(filename, "at");
  if (fp == NULL) {
1985
    errorPrint( "failed to open file: %s\n", filename);
1986
    return;
1987 1988 1989 1990 1991
  }

  fprintf(fp, "================ database[%d] ================\n", index);
  fprintf(fp, "name: %s\n", dbInfos->name);
  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
1992
  fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables);
1993
  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);
1994 1995
  fprintf(fp, "replica: %d\n", dbInfos->replica);
  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
1996 1997
  fprintf(fp, "days: %d\n", dbInfos->days);
  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);
1998 1999 2000 2001 2002 2003 2004
  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
  fprintf(fp, "comp: %d\n", dbInfos->comp);
2005 2006
  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);
  fprintf(fp, "precision: %s\n", dbInfos->precision);
2007
  fprintf(fp, "update: %d\n", dbInfos->update);
2008
  fprintf(fp, "status: %s\n", dbInfos->status);
2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022
  fprintf(fp, "\n");

  fclose(fp);
}

static void printfQuerySystemInfo(TAOS * taos) {
  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
  TAOS_RES* res;

  time_t t;
  struct tm* lt;
  time(&t);
  lt = localtime(&t);
2023 2024 2025 2026
  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
          lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
          lt->tm_sec);

2027 2028
  // show variables
  res = taos_query(taos, "show variables;");
2029
  //fetchResult(res, filename);
2030 2031 2032 2033 2034
  xDumpResultToFile(filename, res);

  // show dnodes
  res = taos_query(taos, "show dnodes;");
  xDumpResultToFile(filename, res);
2035
  //fetchResult(res, filename);
2036

2037 2038 2039 2040
  // show databases
  res = taos_query(taos, "show databases;");
  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
  if (dbInfos == NULL) {
2041
    errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
2042 2043 2044
    return;
  }
  int dbCount = getDbFromServer(taos, dbInfos);
2045 2046 2047 2048
  if (dbCount <= 0) {
      free(dbInfos);
      return;
  }
2049 2050

  for (int i = 0; i < dbCount; i++) {
2051
    // printf database info
2052
    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
2053

2054 2055 2056 2057
    // show db.vgroups
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);
2058

2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069
    // show db.stables
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);

    free(dbInfos[i]);
  }

  free(dbInfos);
}

2070
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
2071
        char* sqlstr, threadInfo *pThreadInfo)
2072
{
2073
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
2074 2075 2076

    char *url = "/rest/sql";

2077
    int bytes, sent, received, req_str_len, resp_len;
2078 2079 2080
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
2081

2082 2083 2084
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;

    request_buf = malloc(req_buf_len);
2085 2086 2087 2088
    if (NULL == request_buf) {
      errorPrint("%s", "ERROR, cannot allocate memory.\n");
      exit(EXIT_FAILURE);
    }
2089

2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};

    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
2101

2102 2103 2104 2105 2106 2107
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);

    char base64_buf[INPUT_BUF_LEN];
2108
#ifdef WINDOWS
2109
    WSADATA wsaData;
2110
    WSAStartup(MAKEWORD(2, 2), &wsaData);
2111 2112 2113 2114
    SOCKET sockfd;
#else
    int sockfd;
#endif
2115
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
2116
    if (sockfd < 0) {
2117
#ifdef WINDOWS
2118
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
2119
#endif
2120
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
2121
        free(request_buf);
2122
        ERROR_EXIT("ERROR opening socket");
2123
    }
2124

2125
    int retConn = connect(sockfd, (struct sockaddr *)pServAddr, sizeof(struct sockaddr));
2126 2127
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
2128
        free(request_buf);
2129
        ERROR_EXIT("ERROR connecting");
2130
    }
2131

2132 2133 2134
    memset(base64_buf, 0, INPUT_BUF_LEN);

    for (int n = 0, m = 0; n < userpass_buf_len;) {
2135
      uint32_t oct_a = n < userpass_buf_len ?
2136
        (unsigned char) userpass_buf[n++]:0;
2137
      uint32_t oct_b = n < userpass_buf_len ?
2138
        (unsigned char) userpass_buf[n++]:0;
2139
      uint32_t oct_c = n < userpass_buf_len ?
2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;

      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }

    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';

2152 2153
    debugPrint("%s() LN%d: auth string base64 encoded: %s\n",
            __func__, __LINE__, base64_buf);
2154 2155
    char *auth = base64_buf;

2156 2157 2158
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
2159 2160 2161 2162 2163
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
2164
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
2165

2166
    req_str_len = strlen(request_buf);
2167 2168
    sent = 0;
    do {
2169 2170 2171
#ifdef WINDOWS
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
#else
2172
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
2173
#endif
2174
        if (bytes < 0)
2175
            ERROR_EXIT("ERROR writing message to socket");
2176 2177 2178
        if (bytes == 0)
            break;
        sent+=bytes;
2179
    } while(sent < req_str_len);
2180

2181 2182
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
2183 2184
    received = 0;
    do {
2185 2186 2187
#ifdef WINDOWS
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
#else
2188
        bytes = read(sockfd, response_buf + received, resp_len - received);
2189
#endif
2190 2191
        if (bytes < 0) {
            free(request_buf);
2192
            ERROR_EXIT("ERROR reading response from socket");
2193
        }
2194 2195
        if (bytes == 0)
            break;
2196
        received += bytes;
2197
    } while(received < resp_len);
2198

2199 2200
    if (received == resp_len) {
        free(request_buf);
2201
        ERROR_EXIT("ERROR storing complete response from socket");
2202
    }
2203

2204
    response_buf[RESP_BUF_LEN - 1] = '\0';
2205 2206
    printf("Response:\n%s\n", response_buf);

2207 2208
    if (strlen(pThreadInfo->filePath) > 0) {
       appendResultBufToFile(response_buf, pThreadInfo);
2209 2210
    }

2211
    free(request_buf);
2212 2213 2214 2215
#ifdef WINDOWS
    closesocket(sockfd);
    WSACleanup();
#else
2216
    close(sockfd);
2217
#endif
2218 2219 2220 2221

    return 0;
}

2222
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
2223 2224
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
2225 2226
    errorPrint("%s() LN%d, calloc failed! size:%d\n",
        __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
2227 2228
    return NULL;
  }
2229

2230
  int    dataLen = 0;
2231 2232 2233
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);

2234 2235 2236
  return dataBuf;
}

2237
static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) {
2238 2239 2240 2241 2242 2243 2244 2245 2246
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
    printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
    return NULL;
  }

  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
  for (int i = 0; i < stbInfo->tagCount; i++) {
2247 2248
    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
2249
      if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
2250 2251
        printf("binary or nchar length overflow, max size:%u\n",
                (uint32_t)TSDB_MAX_BINARY_LEN);
2252 2253 2254
        tmfree(dataBuf);
        return NULL;
      }
2255

2256 2257
      int tagBufLen = stbInfo->tags[i].dataLen + 1;
      char* buf = (char*)calloc(tagBufLen, 1);
2258 2259 2260 2261 2262
      if (NULL == buf) {
        printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
        tmfree(dataBuf);
        return NULL;
      }
2263 2264 2265 2266 2267 2268 2269

      if (tableSeq % 2) {
        tstrncpy(buf, "beijing", tagBufLen);
      } else {
        tstrncpy(buf, "shanghai", tagBufLen);
      }
      //rand_string(buf, stbInfo->tags[i].dataLen);
2270 2271
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "\'%s\', ", buf);
2272
      tmfree(buf);
2273 2274 2275
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "int", strlen("int"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2276
              "%d, ", tableSeq);
2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bigint", strlen("bigint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "float", strlen("float"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_float());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "double", strlen("double"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_double());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "smallint", strlen("smallint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_smallint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "tinyint", strlen("tinyint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_tinyint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bool", strlen("bool"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_bool());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "timestamp", strlen("timestamp"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
2305 2306 2307 2308 2309 2310
    }  else {
      printf("No support data type: %s\n", stbInfo->tags[i].dataType);
      tmfree(dataBuf);
      return NULL;
    }
  }
2311

2312
  dataLen -= 2;
2313
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");
2314 2315 2316
  return dataBuf;
}

2317
static int calcRowLen(SSuperTable*  superTbls) {
2318 2319
  int colIndex;
  int  lenOfOneRow = 0;
2320

2321 2322
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2323

2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
2340
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2341
      lenOfOneRow += 42;
2342
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
2356

2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
2373
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2374 2375 2376 2377 2378 2379 2380 2381
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2382

2383 2384 2385 2386
  return 0;
}


2387 2388
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
2389
        int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) {
2390

2391
  char command[BUFFER_SIZE] = "\0";
2392 2393
  char limitBuf[100] = "\0";

2394
  TAOS_RES * res;
2395 2396 2397
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2398 2399

  if (offset >= 0) {
2400
    snprintf(limitBuf, 100, " limit %"PRId64" offset %"PRIu64"",
2401
            limit, offset);
2402 2403
  }

2404
  //get all child table name use cmd: select tbname from superTblName;
2405 2406
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s",
          dbName, sTblName, limitBuf);
2407

2408
  res = taos_query(taos, command);
2409 2410 2411 2412
  int32_t code = taos_errno(res);
  if (code != 0) {
    taos_free_result(res);
    taos_close(taos);
2413 2414
    errorPrint("%s() LN%d, failed to run command %s\n",
           __func__, __LINE__, command);
2415 2416 2417
    exit(-1);
  }

2418 2419
  int64_t childTblCount = (limit < 0)?10000:limit;
  int64_t count = 0;
2420 2421 2422 2423 2424 2425 2426 2427 2428 2429
  if (childTblName == NULL) {
    childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (NULL ==  childTblName) {
    taos_free_result(res);
        taos_close(taos);
        errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__);
        exit(-1);
    }
  }

2430
  char* pTblName = childTblName;
2431
  while((row = taos_fetch_row(res)) != NULL) {
2432
    int32_t* len = taos_fetch_lengths(res);
2433
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2434 2435 2436
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2437 2438
      char *tmp = realloc(childTblName,
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2439 2440 2441
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2442 2443
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2444 2445
      } else {
        // exit, if allocate more memory failed
2446 2447
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
               __func__, __LINE__, dbName, sTblName);
2448 2449 2450 2451 2452 2453
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2454
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2455
  }
2456

2457 2458 2459 2460 2461 2462 2463
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2464 2465
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
        char* sTblName, char** childTblNameOfSuperTbl,
2466
        int64_t* childTblCountOfSuperTbl) {
2467 2468 2469

    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
2470
            -1, 0);
2471 2472
}

2473 2474
static int getSuperTableFromServer(TAOS * taos, char* dbName,
        SSuperTable*  superTbls) {
2475

2476
  char command[BUFFER_SIZE] = "\0";
2477
  TAOS_RES * res;
2478 2479
  TAOS_ROW row = NULL;
  int count = 0;
2480

2481
  //get schema use cmd: describe superTblName;
2482
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2483
  res = taos_query(taos, command);
2484 2485 2486
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2487
    taos_free_result(res);
2488 2489 2490 2491 2492 2493
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
2494
  while((row = taos_fetch_row(res)) != NULL) {
2495 2496 2497
    if (0 == count) {
      count++;
      continue;
2498
    }
2499 2500

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511
      tstrncpy(superTbls->tags[tagIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->tags[tagIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2512
      tagIndex++;
2513
    } else {
2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524
      tstrncpy(superTbls->columns[columnIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->columns[columnIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2536
/*
2537
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
2538
    //get all child table name use cmd: select tbname from superTblName;
2539 2540 2541
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2542
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2543 2544
      return -1;
    }
2545
    getAllChildNameOfSuperTable(taos, dbName,
2546 2547 2548
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2549
  }
2550
  */
2551 2552 2553
  return 0;
}

2554 2555
static int createSuperTable(
        TAOS * taos, char* dbName,
2556
        SSuperTable*  superTbl) {
2557

2558
  char command[BUFFER_SIZE] = "\0";
2559

2560 2561 2562 2563 2564
  char cols[STRING_LEN] = "\0";
  int colIndex;
  int len = 0;

  int  lenOfOneRow = 0;
2565 2566 2567 2568 2569 2570 2571 2572 2573

  if (superTbl->columnCount == 0) {
    errorPrint("%s() LN%d, super table column count is %d\n",
            __func__, __LINE__, superTbl->columnCount);
    return -1;
  }

  for (colIndex = 0; colIndex < superTbl->columnCount; colIndex++) {
    char* dataType = superTbl->columns[colIndex].dataType;
2574

2575
    if (strcasecmp(dataType, "BINARY") == 0) {
2576 2577
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "BINARY",
2578 2579
          superTbl->columns[colIndex].dataLen);
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2580
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
2581 2582
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "NCHAR",
2583 2584
          superTbl->columns[colIndex].dataLen);
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
      lenOfOneRow += 22;
2603
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2604 2605
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
      lenOfOneRow += 42;
2606
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2607 2608 2609 2610
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
      lenOfOneRow += 21;
    } else {
      taos_close(taos);
2611 2612
      errorPrint("%s() LN%d, config error data type : %s\n",
         __func__, __LINE__, dataType);
2613 2614 2615 2616
      exit(-1);
    }
  }

2617 2618
  superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp
  //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbl[j].sTblName, g_Dbs.db[i].superTbl[j].columnCount, lenOfOneRow);
2619 2620

  // save for creating child table
2621 2622 2623 2624
  superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1);
  if (NULL == superTbl->colsOfCreateChildTable) {
    errorPrint("%s() LN%d, Failed when calloc, size:%d",
           __func__, __LINE__, len+1);
2625 2626 2627 2628
    taos_close(taos);
    exit(-1);
  }

2629
  snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
2630 2631
  verbosePrint("%s() LN%d: %s\n",
      __func__, __LINE__, superTbl->colsOfCreateChildTable);
2632

2633 2634 2635 2636 2637
  if (superTbl->tagCount == 0) {
    errorPrint("%s() LN%d, super table tag count is %d\n",
            __func__, __LINE__, superTbl->tagCount);
    return -1;
  }
2638

2639 2640 2641
  char tags[STRING_LEN] = "\0";
  int tagIndex;
  len = 0;
2642

2643 2644 2645 2646
  int lenOfTagOfOneRow = 0;
  len += snprintf(tags + len, STRING_LEN - len, "(");
  for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) {
    char* dataType = superTbl->tags[tagIndex].dataType;
2647

2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685
    if (strcasecmp(dataType, "BINARY") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
              "BINARY", superTbl->tags[tagIndex].dataLen);
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
              "NCHAR", superTbl->tags[tagIndex].dataLen);
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "INT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "BIGINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "SMALLINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "TINYINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "BOOL");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "FLOAT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "DOUBLE");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42;
    } else {
      taos_close(taos);
2686 2687
      errorPrint("%s() LN%d, config error tag type : %s\n",
         __func__, __LINE__, dataType);
2688
      exit(-1);
2689 2690
    }
  }
2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705

  len -= 2;
  len += snprintf(tags + len, STRING_LEN - len, ")");

  superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow;

  snprintf(command, BUFFER_SIZE,
          "create table if not exists %s.%s (ts timestamp%s) tags %s",
          dbName, superTbl->sTblName, cols, tags);
  if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
      errorPrint( "create supertable %s failed!\n\n",
              superTbl->sTblName);
      return -1;
  }
  debugPrint("create supertable %s success!\n\n", superTbl->sTblName);
2706 2707 2708
  return 0;
}

2709
static int createDatabasesAndStables() {
2710 2711 2712 2713
  TAOS * taos = NULL;
  int    ret = 0;
  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
  if (taos == NULL) {
2714
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2715
    return -1;
2716 2717 2718
  }
  char command[BUFFER_SIZE] = "\0";

2719
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2720 2721
    if (g_Dbs.db[i].drop) {
      sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
2722
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2723 2724 2725 2726
        taos_close(taos);
        return -1;
      }

2727
      int dataLen = 0;
2728
      dataLen += snprintf(command + dataLen,
2729
          BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2730

2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799
      if (g_Dbs.db[i].dbCfg.blocks > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
      }
      if (g_Dbs.db[i].dbCfg.cache > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
      }
      if (g_Dbs.db[i].dbCfg.days > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
      }
      if (g_Dbs.db[i].dbCfg.keep > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
      }
      if (g_Dbs.db[i].dbCfg.quorum > 1) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
      }
      if (g_Dbs.db[i].dbCfg.replica > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
      }
      if (g_Dbs.db[i].dbCfg.update > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
      }
      //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
      //  dataLen += snprintf(command + dataLen,
      //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
      //}
      if (g_Dbs.db[i].dbCfg.minRows > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
      }
      if (g_Dbs.db[i].dbCfg.maxRows > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
      }
      if (g_Dbs.db[i].dbCfg.comp > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
      }
      if (g_Dbs.db[i].dbCfg.walLevel > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
      }
      if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
      }
      if (g_Dbs.db[i].dbCfg.fsync > 0) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
                " fsync %d", g_Dbs.db[i].dbCfg.fsync);
      }
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                      "us", strlen("us")))) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
                " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
      }

      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
        taos_close(taos);
        errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
        return -1;
      }
      printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
2800 2801
    }

2802
    debugPrint("%s() LN%d supertbl count:%"PRIu64"\n",
2803
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
2804 2805 2806

    int validStbCount = 0;

2807
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
2808 2809 2810 2811 2812
      sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
              g_Dbs.db[i].superTbls[j].sTblName);
      ret = queryDbExec(taos, command, NO_INSERT_TYPE, true);

      if ((ret != 0) || (g_Dbs.db[i].drop)) {
2813
        ret = createSuperTable(taos, g_Dbs.db[i].dbName,
2814
                &g_Dbs.db[i].superTbls[j]);
2815

2816
        if (0 != ret) {
2817 2818
          errorPrint("create super table %d failed!\n\n", j);
          continue;
2819 2820 2821
        }
      }

2822
      ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
2823
                &g_Dbs.db[i].superTbls[j]);
2824 2825 2826
      if (0 != ret) {
        errorPrint("\nget super table %s.%s info failed!\n\n",
                g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName);
2827
        continue;
2828
      }
2829 2830

      validStbCount ++;
2831
    }
2832 2833

    g_Dbs.db[i].superTblCount = validStbCount;
2834 2835 2836 2837 2838 2839
  }

  taos_close(taos);
  return 0;
}

2840 2841
static void* createTable(void *sarg)
{
2842 2843
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
2844 2845 2846

  int64_t  lastPrintTime = taosGetTimestampMs();

2847
  int buff_len;
2848
  buff_len = BUFFER_SIZE / 8;
2849

2850
  char *buffer = calloc(buff_len, 1);
2851
  if (buffer == NULL) {
2852
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
2853 2854
    exit(-1);
  }
2855 2856 2857

  int len = 0;
  int batchNum = 0;
2858

2859
  verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n",
2860
          __func__, __LINE__,
2861
          pThreadInfo->start_table_from, pThreadInfo->end_table_to);
2862

2863
  for (uint64_t i = pThreadInfo->start_table_from;
2864
          i <= pThreadInfo->end_table_to; i++) {
2865
    if (0 == g_Dbs.use_metric) {
2866
      snprintf(buffer, buff_len,
2867
              "create table if not exists %s.%s%"PRIu64" %s;",
2868
              pThreadInfo->db_name,
2869
              g_args.tb_prefix, i,
2870
              pThreadInfo->cols);
2871
    } else {
2872 2873 2874
      if (superTblInfo == NULL) {
        errorPrint("%s() LN%d, use metric, but super table info is NULL\n",
                  __func__, __LINE__);
2875
        free(buffer);
2876 2877 2878 2879 2880 2881 2882 2883 2884 2885
        exit(-1);
      } else {
        if (0 == len) {
          batchNum = 0;
          memset(buffer, 0, buff_len);
          len += snprintf(buffer + len,
                  buff_len - len, "create table ");
        }
        char* tagsValBuf = NULL;
        if (0 == superTblInfo->tagSource) {
2886
          tagsValBuf = generateTagVaulesForStb(superTblInfo, i);
2887 2888 2889 2890 2891 2892 2893 2894 2895 2896
        } else {
          tagsValBuf = getTagValueFromTagSample(
                  superTblInfo,
                  i % superTblInfo->tagSampleCount);
        }
        if (NULL == tagsValBuf) {
          free(buffer);
          return NULL;
        }
        len += snprintf(buffer + len,
2897
                buff_len - len,
2898
                "if not exists %s.%s%"PRIu64" using %s.%s tags %s ",
2899 2900
                pThreadInfo->db_name, superTblInfo->childTblPrefix,
                i, pThreadInfo->db_name,
2901 2902 2903 2904
                superTblInfo->sTblName, tagsValBuf);
        free(tagsValBuf);
        batchNum++;
        if ((batchNum < superTblInfo->batchCreateTableNum)
2905
                && ((buff_len - len)
2906 2907 2908
                    >= (superTblInfo->lenOfTagOfOneRow + 256))) {
          continue;
        }
2909 2910 2911 2912
      }
    }

    len = 0;
2913
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)){
2914
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2915
      free(buffer);
2916 2917 2918 2919 2920
      return NULL;
    }

    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
2921
      printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n",
2922
              pThreadInfo->threadID, pThreadInfo->start_table_from, i);
2923 2924 2925
      lastPrintTime = currentPrintTime;
    }
  }
2926

2927
  if (0 != len) {
2928
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)) {
2929
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2930
    }
2931
  }
2932

2933 2934 2935 2936
  free(buffer);
  return NULL;
}

2937
static int startMultiThreadCreateChildTable(
2938
        char* cols, int threads, uint64_t startFrom, int64_t ntables,
2939
        char* db_name, SSuperTable* superTblInfo) {
2940

2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  threadInfo *infos = malloc(threads * sizeof(threadInfo));

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

2953
  int64_t a = ntables / threads;
2954 2955 2956 2957 2958
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

2959
  int64_t b = 0;
2960
  b = ntables % threads;
2961

2962
  for (int64_t i = 0; i < threads; i++) {
2963 2964 2965 2966
    threadInfo *pThreadInfo = infos + i;
    pThreadInfo->threadID = i;
    tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE);
    pThreadInfo->superTblInfo = superTblInfo;
2967
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
2968
    pThreadInfo->taos = taos_connect(
2969 2970 2971 2972 2973
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
2974
    if (pThreadInfo->taos == NULL) {
2975 2976
      errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n",
         __func__, __LINE__, taos_errstr(NULL));
2977
      free(pids);
2978
      free(infos);
2979 2980
      return -1;
    }
2981

2982 2983 2984 2985 2986 2987 2988 2989
    pThreadInfo->start_table_from = startFrom;
    pThreadInfo->ntables = i<b?a+1:a;
    pThreadInfo->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
    startFrom = pThreadInfo->end_table_to + 1;
    pThreadInfo->use_metric = true;
    pThreadInfo->cols = cols;
    pThreadInfo->minDelay = UINT64_MAX;
    pthread_create(pids + i, NULL, createTable, pThreadInfo);
2990
  }
2991

2992 2993 2994 2995 2996
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
2997 2998
    threadInfo *pThreadInfo = infos + i;
    taos_close(pThreadInfo->taos);
2999 3000 3001
  }

  free(pids);
3002
  free(infos);
3003 3004

  return 0;
3005 3006
}

3007 3008 3009 3010
static void createChildTables() {
    char tblColsBuf[MAX_SQL_SIZE];
    int len;

3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050
    for (int i = 0; i < g_Dbs.dbCount; i++) {
        if (g_Dbs.use_metric) {
            if (g_Dbs.db[i].superTblCount > 0) {
                // with super table
                for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
                    if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
                            || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
                        continue;
                    }
                    verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
                          g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
                    uint64_t startFrom = 0;
                    g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;

                    verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n",
                            __func__, __LINE__, g_totalChildTables, startFrom);

                    startMultiThreadCreateChildTable(
                            g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
                            g_Dbs.threadCountByCreateTbl,
                            startFrom,
                            g_Dbs.db[i].superTbls[j].childTblCount,
                            g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
                }
            }
        } else {
            // normal table
            len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
            for (int j = 0; j < g_args.num_of_CPR; j++) {
                if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
                        || (strncasecmp(g_args.datatype[j],
                                "NCHAR", strlen("NCHAR")) == 0)) {
                    snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                            ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary);
                } else {
                    snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                            ", COL%d %s", j, g_args.datatype[j]);
                }
                len = strlen(tblColsBuf);
            }
3051

3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063
            snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");

            verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n",
                    __func__, __LINE__,
                    g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
            startMultiThreadCreateChildTable(
                    tblColsBuf,
                    g_Dbs.threadCountByCreateTbl,
                    0,
                    g_args.num_of_tables,
                    g_Dbs.db[i].dbName,
                    NULL);
3064
        }
3065
    }
3066 3067 3068 3069 3070
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3071
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
3072 3073 3074
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
3075

3076 3077
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
3078 3079
    printf("Failed to open tags file: %s, reason:%s\n",
            superTblInfo->tagsFile, strerror(errno));
3080 3081 3082 3083 3084 3085 3086
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
3087

3088 3089 3090 3091 3092 3093 3094 3095 3096
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

3097
  while((readLen = tgetline(&line, &n, fp)) != -1) {
3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
3110 3111
      char *tmp = realloc(tagDataBuf,
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
3112 3113 3114
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
3115 3116
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

int readSampleFromJsonFileToMem(SSuperTable  * superTblInfo) {
  // TODO
  return 0;
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3144 3145
static int readSampleFromCsvFileToMem(
        SSuperTable* superTblInfo) {
3146 3147 3148 3149
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
3150

3151 3152
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
3153
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
3154 3155 3156
              superTblInfo->sampleFile, strerror(errno));
      return -1;
  }
3157

3158
  assert(superTblInfo->sampleDataBuf);
3159 3160
  memset(superTblInfo->sampleDataBuf, 0,
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
3161
  while(1) {
3162
    readLen = tgetline(&line, &n, fp);
3163 3164
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
3165
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
3166
                superTblInfo->sampleFile, strerror(errno));
3167
        fclose(fp);
3168 3169 3170 3171
        return -1;
      }
      continue;
    }
3172

3173 3174 3175 3176 3177 3178 3179 3180 3181
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
3182
      printf("sample row len[%d] overflow define schema len[%"PRIu64"], so discard this row\n",
3183
              (int32_t)readLen, superTblInfo->lenOfOneRow);
3184 3185 3186
      continue;
    }

3187 3188
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
          line, readLen);
3189 3190 3191 3192 3193 3194 3195
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

3196
  fclose(fp);
3197 3198 3199 3200
  tmfree(line);
  return 0;
}

3201 3202
static bool getColumnAndTagTypeFromInsertJsonFile(
        cJSON* stbInfo, SSuperTable* superTbls) {
3203
  bool  ret = false;
3204

3205
  // columns
3206 3207
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
3208
    printf("ERROR: failed to read json, columns not found\n");
3209 3210 3211 3212 3213 3214
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
3215

3216
  int columnSize = cJSON_GetArraySize(columns);
3217
  if ((columnSize + 1/* ts */) > MAX_COLUMN_COUNT) {
3218 3219
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
            __func__, __LINE__, MAX_COLUMN_COUNT);
3220 3221 3222 3223 3224 3225
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
3226

3227
  //superTbls->columnCount = columnSize;
3228 3229 3230 3231 3232 3233 3234
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
3235
      count = countObj->valueint;
3236
    } else if (countObj && countObj->type != cJSON_Number) {
3237 3238
      errorPrint("%s() LN%d, failed to read json, column count not found\n",
          __func__, __LINE__);
3239 3240 3241 3242 3243
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3244
    // column info
3245 3246
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
3247 3248 3249 3250
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d: failed to read json, column type not found\n",
          __func__, __LINE__);
3251 3252
      goto PARSE_OVER;
    }
3253 3254
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3255

3256 3257
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3258
      columnCase.dataLen = dataLen->valueint;
3259
    } else if (dataLen && dataLen->type != cJSON_Number) {
3260 3261
      debugPrint("%s() LN%d: failed to read json, column len not found\n",
          __func__, __LINE__);
3262 3263 3264 3265
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
3266

3267
    for (int n = 0; n < count; ++n) {
3268 3269 3270
      tstrncpy(superTbls->columns[index].dataType,
              columnCase.dataType, MAX_TB_NAME_SIZE);
      superTbls->columns[index].dataLen = columnCase.dataLen;
3271 3272
      index++;
    }
3273
  }
3274

3275 3276
  if ((index + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n",
3277 3278 3279 3280
            __func__, __LINE__, MAX_COLUMN_COUNT);
    goto PARSE_OVER;
  }

3281
  superTbls->columnCount = index;
3282

3283 3284
  count = 1;
  index = 0;
3285
  // tags
3286 3287
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
3288 3289
    errorPrint("%s() LN%d, failed to read json, tags not found\n",
        __func__, __LINE__);
3290 3291
    goto PARSE_OVER;
  }
3292

3293 3294
  int tagSize = cJSON_GetArraySize(tags);
  if (tagSize > MAX_TAG_COUNT) {
3295 3296
    errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n",
        __func__, __LINE__, MAX_TAG_COUNT);
3297 3298
    goto PARSE_OVER;
  }
3299 3300

  //superTbls->tagCount = tagSize;
3301 3302 3303
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
3304

3305 3306 3307
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
3308
      count = countObj->valueint;
3309
    } else if (countObj && countObj->type != cJSON_Number) {
3310
      printf("ERROR: failed to read json, column count not found\n");
3311 3312 3313 3314 3315
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3316
    // column info
3317 3318
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
3319 3320 3321 3322
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d, failed to read json, tag type not found\n",
          __func__, __LINE__);
3323 3324
      goto PARSE_OVER;
    }
3325
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3326

3327 3328
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3329
      columnCase.dataLen = dataLen->valueint;
3330
    } else if (dataLen && dataLen->type != cJSON_Number) {
3331 3332
      errorPrint("%s() LN%d, failed to read json, column len not found\n",
          __func__, __LINE__);
3333 3334 3335
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
3336 3337
    }

3338
    for (int n = 0; n < count; ++n) {
3339 3340
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType,
          MAX_TB_NAME_SIZE);
3341
      superTbls->tags[index].dataLen = columnCase.dataLen;
3342 3343
      index++;
    }
3344
  }
3345 3346

  if (index > MAX_TAG_COUNT) {
3347
    errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n",
3348 3349 3350 3351
        __func__, __LINE__, MAX_TAG_COUNT);
    goto PARSE_OVER;
  }

3352 3353
  superTbls->tagCount = index;

3354 3355 3356
  if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n",
        __func__, __LINE__, MAX_COLUMN_COUNT);
3357 3358
    goto PARSE_OVER;
  }
3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369
  ret = true;

PARSE_OVER:
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3370
    tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3371 3372 3373 3374
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3375
    tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE);
3376
  } else if (!host) {
3377
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3378
  } else {
3379
    printf("ERROR: failed to read json, host not found\n");
3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_Dbs.port = port->valueint;
  } else if (!port) {
    g_Dbs.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3392
    tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE);
3393
  } else if (!user) {
3394
    tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE);
3395 3396 3397 3398
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3399
    tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE);
3400
  } else if (!password) {
3401
    tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE);
3402 3403 3404 3405
  }

  cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
  if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
3406
    tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
3407
  } else if (!resultfile) {
3408
    tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
3409 3410 3411 3412 3413 3414 3415 3416
  }

  cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
  if (threads && threads->type == cJSON_Number) {
    g_Dbs.threadCount = threads->valueint;
  } else if (!threads) {
    g_Dbs.threadCount = 1;
  } else {
3417
    printf("ERROR: failed to read json, threads not found\n");
3418
    goto PARSE_OVER;
3419 3420
  }

3421 3422 3423 3424
  cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
  if (threads2 && threads2->type == cJSON_Number) {
    g_Dbs.threadCountByCreateTbl = threads2->valueint;
  } else if (!threads2) {
3425
    g_Dbs.threadCountByCreateTbl = 1;
3426
  } else {
3427 3428
    errorPrint("%s() LN%d, failed to read json, threads2 not found\n",
            __func__, __LINE__);
3429
    goto PARSE_OVER;
3430
  }
3431

3432 3433
  cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
  if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
3434 3435 3436 3437 3438
    if (gInsertInterval->valueint <0) {
      errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n",
            __func__, __LINE__);
      goto PARSE_OVER;
    }
3439 3440 3441 3442
    g_args.insert_interval = gInsertInterval->valueint;
  } else if (!gInsertInterval) {
    g_args.insert_interval = 0;
  } else {
3443 3444
    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
        __func__, __LINE__);
3445 3446
    goto PARSE_OVER;
  }
3447

3448 3449
  cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows");
  if (interlaceRows && interlaceRows->type == cJSON_Number) {
3450 3451 3452 3453 3454 3455
    if (interlaceRows->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;

    }
3456 3457
    g_args.interlace_rows = interlaceRows->valueint;
  } else if (!interlaceRows) {
3458
    g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3459
  } else {
3460 3461
    errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
        __func__, __LINE__);
3462
    goto PARSE_OVER;
3463
  }
3464

3465 3466
  cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
  if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
3467 3468 3469 3470 3471
    if (maxSqlLen->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
    }
3472 3473
    g_args.max_sql_len = maxSqlLen->valueint;
  } else if (!maxSqlLen) {
3474
    g_args.max_sql_len = (1024*1024);
3475
  } else {
3476 3477
    errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
        __func__, __LINE__);
3478 3479 3480 3481 3482
    goto PARSE_OVER;
  }

  cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
  if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
3483
    if (numRecPerReq->valueint <= 0) {
3484 3485 3486
      errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
3487
    } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) {
3488 3489 3490 3491 3492
      printf("NOTICE: number of records per request value %"PRIu64" > %d\n\n",
              numRecPerReq->valueint, MAX_RECORDS_PER_REQ);
      printf("        number of records per request value will be set to %d\n\n",
              MAX_RECORDS_PER_REQ);
      prompt();
3493
      numRecPerReq->valueint = MAX_RECORDS_PER_REQ;
3494
    }
3495 3496
    g_args.num_of_RPR = numRecPerReq->valueint;
  } else if (!numRecPerReq) {
3497
    g_args.num_of_RPR = MAX_RECORDS_PER_REQ;
3498
  } else {
3499 3500
    errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n",
        __func__, __LINE__);
3501 3502 3503
    goto PARSE_OVER;
  }

3504
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3505
  if (answerPrompt
3506 3507
          && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3508 3509 3510 3511 3512 3513 3514 3515
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
    } else {
      g_args.answer_yes = false;
    }
  } else if (!answerPrompt) {
3516
    g_args.answer_yes = true;   // default is no, mean answer_yes.
3517
  } else {
3518
    errorPrint("%s", "failed to read json, confirm_parameter_prompt input mistake\n");
3519
    goto PARSE_OVER;
3520
  }
3521

3522 3523 3524 3525 3526 3527 3528 3529 3530 3531
  // rows per table need be less than insert batch
  if (g_args.interlace_rows > g_args.num_of_RPR) {
      printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
              g_args.interlace_rows, g_args.num_of_RPR);
      printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
              g_args.num_of_RPR);
      prompt();
      g_args.interlace_rows = g_args.num_of_RPR;
  }

3532 3533
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (!dbs || dbs->type != cJSON_Array) {
3534
    printf("ERROR: failed to read json, databases not found\n");
3535 3536 3537 3538 3539
    goto PARSE_OVER;
  }

  int dbSize = cJSON_GetArraySize(dbs);
  if (dbSize > MAX_DB_COUNT) {
3540
    errorPrint(
3541 3542
            "ERROR: failed to read json, databases size overflow, max database is %d\n",
            MAX_DB_COUNT);
3543 3544 3545 3546 3547 3548 3549 3550
    goto PARSE_OVER;
  }

  g_Dbs.dbCount = dbSize;
  for (int i = 0; i < dbSize; ++i) {
    cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
    if (dbinfos == NULL) continue;

3551
    // dbinfo
3552 3553
    cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
    if (!dbinfo || dbinfo->type != cJSON_Object) {
3554
      printf("ERROR: failed to read json, dbinfo not found\n");
3555 3556
      goto PARSE_OVER;
    }
3557

3558 3559
    cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
    if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
3560
      printf("ERROR: failed to read json, db name not found\n");
3561 3562
      goto PARSE_OVER;
    }
3563
    tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE);
3564 3565 3566

    cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
    if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
3567 3568
      if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) {
        g_Dbs.db[i].drop = true;
3569
      } else {
3570
        g_Dbs.db[i].drop = false;
3571
      }
3572
    } else if (!drop) {
3573
      g_Dbs.db[i].drop = g_args.drop_database;
3574
    } else {
3575 3576
      errorPrint("%s() LN%d, failed to read json, drop input mistake\n",
              __func__, __LINE__);
3577 3578 3579 3580
      goto PARSE_OVER;
    }

    cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
3581 3582 3583 3584
    if (precision && precision->type == cJSON_String
            && precision->valuestring != NULL) {
      tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
              MAX_DB_NAME_SIZE);
3585
    } else if (!precision) {
3586
      //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
3587 3588
      memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE);
    } else {
3589
      printf("ERROR: failed to read json, precision not found\n");
3590 3591 3592 3593 3594 3595 3596 3597 3598
      goto PARSE_OVER;
    }

    cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
    if (update && update->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.update = update->valueint;
    } else if (!update) {
      g_Dbs.db[i].dbCfg.update = -1;
    } else {
3599
      printf("ERROR: failed to read json, update not found\n");
3600 3601 3602 3603 3604 3605 3606 3607 3608
      goto PARSE_OVER;
    }

    cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
    if (replica && replica->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.replica = replica->valueint;
    } else if (!replica) {
      g_Dbs.db[i].dbCfg.replica = -1;
    } else {
3609
      printf("ERROR: failed to read json, replica not found\n");
3610 3611 3612 3613 3614 3615 3616 3617 3618
      goto PARSE_OVER;
    }

    cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
    if (keep && keep->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.keep = keep->valueint;
    } else if (!keep) {
      g_Dbs.db[i].dbCfg.keep = -1;
    } else {
3619
     printf("ERROR: failed to read json, keep not found\n");
3620 3621
     goto PARSE_OVER;
    }
3622

3623 3624 3625 3626 3627 3628
    cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
    if (days && days->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.days = days->valueint;
    } else if (!days) {
      g_Dbs.db[i].dbCfg.days = -1;
    } else {
3629
     printf("ERROR: failed to read json, days not found\n");
3630 3631
     goto PARSE_OVER;
    }
3632

3633 3634 3635 3636 3637 3638
    cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
    if (cache && cache->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cache = cache->valueint;
    } else if (!cache) {
      g_Dbs.db[i].dbCfg.cache = -1;
    } else {
3639
     printf("ERROR: failed to read json, cache not found\n");
3640 3641
     goto PARSE_OVER;
    }
3642

3643 3644 3645 3646 3647 3648
    cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
    if (blocks && blocks->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
    } else if (!blocks) {
      g_Dbs.db[i].dbCfg.blocks = -1;
    } else {
3649
     printf("ERROR: failed to read json, block not found\n");
3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666
     goto PARSE_OVER;
    }

    //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
    //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
    //} else if (!maxtablesPerVnode) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
    //} else {
    // printf("failed to read json, maxtablesPerVnode not found");
    // goto PARSE_OVER;
    //}

    cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
    if (minRows && minRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
    } else if (!minRows) {
3667
      g_Dbs.db[i].dbCfg.minRows = 0;    // 0 means default
3668
    } else {
3669
     printf("ERROR: failed to read json, minRows not found\n");
3670 3671 3672 3673 3674 3675 3676
     goto PARSE_OVER;
    }

    cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
    if (maxRows && maxRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
    } else if (!maxRows) {
3677
      g_Dbs.db[i].dbCfg.maxRows = 0;    // 0 means default
3678
    } else {
3679
     printf("ERROR: failed to read json, maxRows not found\n");
3680 3681 3682 3683 3684 3685 3686 3687 3688
     goto PARSE_OVER;
    }

    cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
    if (comp && comp->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.comp = comp->valueint;
    } else if (!comp) {
      g_Dbs.db[i].dbCfg.comp = -1;
    } else {
3689
     printf("ERROR: failed to read json, comp not found\n");
3690 3691 3692 3693 3694 3695 3696 3697 3698
     goto PARSE_OVER;
    }

    cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
    if (walLevel && walLevel->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
    } else if (!walLevel) {
      g_Dbs.db[i].dbCfg.walLevel = -1;
    } else {
3699
     printf("ERROR: failed to read json, walLevel not found\n");
3700 3701 3702
     goto PARSE_OVER;
    }

3703 3704 3705 3706 3707 3708
    cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
    if (cacheLast && cacheLast->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
    } else if (!cacheLast) {
      g_Dbs.db[i].dbCfg.cacheLast = -1;
    } else {
3709
     printf("ERROR: failed to read json, cacheLast not found\n");
3710 3711 3712
     goto PARSE_OVER;
    }

3713 3714 3715 3716
    cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
    if (quorum && quorum->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
    } else if (!quorum) {
3717
      g_Dbs.db[i].dbCfg.quorum = 1;
3718
    } else {
3719
     printf("failed to read json, quorum input mistake");
3720 3721 3722 3723 3724 3725 3726 3727 3728
     goto PARSE_OVER;
    }

    cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
    if (fsync && fsync->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
    } else if (!fsync) {
      g_Dbs.db[i].dbCfg.fsync = -1;
    } else {
3729 3730 3731
      errorPrint("%s() LN%d, failed to read json, fsync input mistake\n",
              __func__, __LINE__);
      goto PARSE_OVER;
3732
    }
3733

3734
    // super_talbes
3735 3736
    cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
    if (!stables || stables->type != cJSON_Array) {
3737 3738
      errorPrint("%s() LN%d, failed to read json, super_tables not found\n",
              __func__, __LINE__);
3739
      goto PARSE_OVER;
3740 3741
    }

3742 3743
    int stbSize = cJSON_GetArraySize(stables);
    if (stbSize > MAX_SUPER_TABLE_COUNT) {
3744
      errorPrint(
3745 3746
              "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n",
              __func__, __LINE__, MAX_SUPER_TABLE_COUNT);
3747 3748 3749 3750 3751 3752 3753
      goto PARSE_OVER;
    }

    g_Dbs.db[i].superTblCount = stbSize;
    for (int j = 0; j < stbSize; ++j) {
      cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
      if (stbInfo == NULL) continue;
3754 3755

      // dbinfo
3756
      cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
3757 3758
      if (!stbName || stbName->type != cJSON_String
              || stbName->valuestring == NULL) {
3759 3760
        errorPrint("%s() LN%d, failed to read json, stb name not found\n",
                __func__, __LINE__);
3761 3762
        goto PARSE_OVER;
      }
3763 3764
      tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring,
              MAX_TB_NAME_SIZE);
3765

3766 3767
      cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
      if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
3768
        printf("ERROR: failed to read json, childtable_prefix not found\n");
3769 3770
        goto PARSE_OVER;
      }
3771 3772
      tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring,
              MAX_DB_NAME_SIZE);
3773

3774
      cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table");
3775 3776 3777
      if (autoCreateTbl
              && autoCreateTbl->type == cJSON_String
              && autoCreateTbl->valuestring != NULL) {
3778 3779 3780 3781 3782 3783 3784 3785
          if ((0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3))
                  && (TBL_ALREADY_EXISTS != g_Dbs.db[i].superTbls[j].childTblExists)) {
              g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
          } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
              g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
          } else {
              g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
          }
3786
      } else if (!autoCreateTbl) {
3787
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
3788
      } else {
3789 3790
          printf("ERROR: failed to read json, auto_create_table not found\n");
          goto PARSE_OVER;
3791
      }
3792

3793 3794 3795 3796
      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
      } else if (!batchCreateTbl) {
3797
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
3798
      } else {
3799
        printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
3800
        goto PARSE_OVER;
3801
      }
3802 3803

      cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
3804 3805 3806
      if (childTblExists
              && childTblExists->type == cJSON_String
              && childTblExists->valuestring != NULL) {
3807 3808
        if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3))
            && (g_Dbs.db[i].drop == false)) {
3809
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
3810 3811
        } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2)
              || (g_Dbs.db[i].drop == true))) {
3812 3813 3814 3815 3816 3817 3818
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        } else {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        }
      } else if (!childTblExists) {
        g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
      } else {
3819 3820
        errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n",
                __func__, __LINE__);
3821 3822
        goto PARSE_OVER;
      }
3823

3824 3825 3826 3827
      if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      }

3828 3829
      cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
      if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
3830
        errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n",
3831
                __func__, __LINE__);
3832 3833 3834 3835 3836
        goto PARSE_OVER;
      }
      g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

      cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
3837 3838 3839 3840
      if (dataSource && dataSource->type == cJSON_String
              && dataSource->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
                dataSource->valuestring, MAX_DB_NAME_SIZE);
3841
      } else if (!dataSource) {
3842
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE);
3843
      } else {
3844 3845
        errorPrint("%s() LN%d, failed to read json, data_source not found\n",
            __func__, __LINE__);
3846 3847 3848
        goto PARSE_OVER;
      }

3849
      cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest
3850 3851 3852 3853
      if (insertMode && insertMode->type == cJSON_String
              && insertMode->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode,
                insertMode->valuestring, MAX_DB_NAME_SIZE);
3854
      } else if (!insertMode) {
3855
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE);
3856
      } else {
3857
        printf("ERROR: failed to read json, insert_mode not found\n");
3858 3859 3860
        goto PARSE_OVER;
      }

3861
      cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
3862 3863
      if ((childTbl_limit) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3864 3865 3866 3867 3868 3869
        if (childTbl_limit->type != cJSON_Number) {
            printf("ERROR: failed to read json, childtable_limit\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
      } else {
3870
        g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid.
3871 3872 3873
      }

      cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
3874 3875
      if ((childTbl_offset) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3876 3877
        if ((childTbl_offset->type != cJSON_Number)
                || (0 > childTbl_offset->valueint)) {
3878 3879 3880 3881 3882 3883 3884 3885
            printf("ERROR: failed to read json, childtable_offset\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblOffset = 0;
      }

3886 3887
      cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
      if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
3888 3889
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                ts->valuestring, MAX_DB_NAME_SIZE);
3890
      } else if (!ts) {
3891 3892
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                "now", MAX_DB_NAME_SIZE);
3893
      } else {
3894
        printf("ERROR: failed to read json, start_timestamp not found\n");
3895 3896
        goto PARSE_OVER;
      }
3897

3898 3899 3900 3901
      cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
      if (timestampStep && timestampStep->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
      } else if (!timestampStep) {
3902
        g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
3903
      } else {
3904
        printf("ERROR: failed to read json, timestamp_step not found\n");
3905 3906 3907 3908
        goto PARSE_OVER;
      }

      cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
3909 3910
      if (sampleFormat && sampleFormat->type
              == cJSON_String && sampleFormat->valuestring != NULL) {
3911 3912
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
                sampleFormat->valuestring, MAX_DB_NAME_SIZE);
3913
      } else if (!sampleFormat) {
3914
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE);
3915
      } else {
3916
        printf("ERROR: failed to read json, sample_format not found\n");
3917
        goto PARSE_OVER;
3918
      }
3919

3920
      cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
3921 3922
      if (sampleFile && sampleFile->type == cJSON_String
          && sampleFile->valuestring != NULL) {
3923 3924
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
                sampleFile->valuestring, MAX_FILE_NAME_LEN);
3925 3926 3927
      } else if (!sampleFile) {
        memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
      } else {
3928
        printf("ERROR: failed to read json, sample_file not found\n");
3929
        goto PARSE_OVER;
3930
      }
3931

3932
      cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
3933 3934
      if ((tagsFile && tagsFile->type == cJSON_String)
              && (tagsFile->valuestring != NULL)) {
3935 3936
        tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
                tagsFile->valuestring, MAX_FILE_NAME_LEN);
3937 3938 3939 3940 3941 3942 3943 3944 3945
        if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
          g_Dbs.db[i].superTbls[j].tagSource = 0;
        } else {
          g_Dbs.db[i].superTbls[j].tagSource = 1;
        }
      } else if (!tagsFile) {
        memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
        g_Dbs.db[i].superTbls[j].tagSource = 0;
      } else {
3946
        printf("ERROR: failed to read json, tags_file not found\n");
3947 3948
        goto PARSE_OVER;
      }
3949

3950 3951 3952 3953 3954
      cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
      if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
        int32_t len = maxSqlLen->valueint;
        if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
          len = TSDB_MAX_ALLOWED_SQL_LEN;
3955 3956
        } else if (len < 5) {
          len = 5;
3957
        }
3958 3959
        g_Dbs.db[i].superTbls[j].maxSqlLen = len;
      } else if (!maxSqlLen) {
3960
        g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len;
3961
      } else {
3962 3963
        errorPrint("%s() LN%d, failed to read json, maxSqlLen input mistake\n",
            __func__, __LINE__);
3964
        goto PARSE_OVER;
3965
      }
3966
/*
3967 3968 3969 3970 3971
      cJSON *multiThreadWriteOneTbl =
          cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
      if (multiThreadWriteOneTbl
              && multiThreadWriteOneTbl->type == cJSON_String
              && multiThreadWriteOneTbl->valuestring != NULL) {
3972 3973 3974 3975
        if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
        } else {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
3976
        }
3977 3978 3979
      } else if (!multiThreadWriteOneTbl) {
        g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
      } else {
3980
        printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
3981 3982
        goto PARSE_OVER;
      }
3983
*/
3984 3985
      cJSON* interlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows");
      if (interlaceRows && interlaceRows->type == cJSON_Number) {
3986 3987 3988 3989 3990
        if (interlaceRows->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n",
            __func__, __LINE__);
          goto PARSE_OVER;
        }
3991
        g_Dbs.db[i].superTbls[j].interlaceRows = interlaceRows->valueint;
3992 3993
        // rows per table need be less than insert batch
        if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) {
3994
          printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
3995 3996 3997
                  i, j,
                  g_Dbs.db[i].superTbls[j].interlaceRows,
                  g_args.num_of_RPR);
3998
          printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
3999
                  g_args.num_of_RPR);
4000
          prompt();
4001 4002
          g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR;
        }
4003 4004
      } else if (!interlaceRows) {
        g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
4005
      } else {
4006 4007 4008
        errorPrint(
                "%s() LN%d, failed to read json, interlace rows input mistake\n",
                __func__, __LINE__);
4009
        goto PARSE_OVER;
4010
      }
4011 4012 4013

      cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
      if (disorderRatio && disorderRatio->type == cJSON_Number) {
4014 4015 4016 4017 4018 4019
        if (disorderRatio->valueint > 50)
          disorderRatio->valueint = 50;

        if (disorderRatio->valueint < 0)
          disorderRatio->valueint = 0;

4020 4021 4022 4023
        g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
      } else if (!disorderRatio) {
        g_Dbs.db[i].superTbls[j].disorderRatio = 0;
      } else {
4024
        printf("ERROR: failed to read json, disorderRatio not found\n");
4025
        goto PARSE_OVER;
4026
      }
4027 4028 4029 4030 4031 4032 4033

      cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
      if (disorderRange && disorderRange->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
      } else if (!disorderRange) {
        g_Dbs.db[i].superTbls[j].disorderRange = 1000;
      } else {
4034
        printf("ERROR: failed to read json, disorderRange not found\n");
4035 4036
        goto PARSE_OVER;
      }
4037

4038 4039
      cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
      if (insertRows && insertRows->type == cJSON_Number) {
4040 4041 4042 4043 4044
        if (insertRows->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4045 4046 4047 4048
        g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
      } else if (!insertRows) {
        g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
      } else {
4049 4050
        errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
4051 4052 4053 4054 4055 4056
        goto PARSE_OVER;
      }

      cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
      if (insertInterval && insertInterval->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
4057 4058 4059 4060 4061
        if (insertInterval->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4062
      } else if (!insertInterval) {
4063
        verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n",
4064 4065 4066
                __func__, __LINE__, g_args.insert_interval);
        g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
      } else {
4067 4068
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
4069 4070 4071
        goto PARSE_OVER;
      }

4072 4073
      int retVal = getColumnAndTagTypeFromInsertJsonFile(
              stbInfo, &g_Dbs.db[i].superTbls[j]);
4074 4075
      if (false == retVal) {
        goto PARSE_OVER;
4076 4077
      }
    }
4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090
  }

  ret = true;

PARSE_OVER:
  return ret;
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
4091
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
4092 4093 4094 4095
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
4096
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE);
4097
  } else if (!host) {
4098
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
4099
  } else {
4100
    printf("ERROR: failed to read json, host not found\n");
4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
4113
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE);
4114
  } else if (!user) {
4115
    tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ;
4116 4117 4118 4119
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
4120
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE);
4121
  } else if (!password) {
4122
    tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);;
4123
  }
H
hzcheng 已提交
4124

4125
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
4126 4127
  if (answerPrompt && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
4128 4129 4130 4131
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
4132
    } else {
4133
      g_args.answer_yes = false;
H
hzcheng 已提交
4134
    }
4135 4136 4137
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
4138
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
4139
    goto PARSE_OVER;
4140
  }
4141

4142 4143
  cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
  if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
4144
    if (gQueryTimes->valueint <= 0) {
4145 4146
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4147 4148
      goto PARSE_OVER;
    }
4149 4150 4151 4152
    g_args.query_times = gQueryTimes->valueint;
  } else if (!gQueryTimes) {
    g_args.query_times = 1;
  } else {
4153 4154
    errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4155 4156 4157
    goto PARSE_OVER;
  }

4158 4159
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
4160
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE);
4161
  } else if (!dbs) {
4162
    printf("ERROR: failed to read json, databases not found\n");
4163
    goto PARSE_OVER;
H
hzcheng 已提交
4164
  }
4165 4166 4167

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
4168
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
4169
  } else if (!queryMode) {
4170
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
4171
  } else {
4172
    printf("ERROR: failed to read json, query_mode not found\n");
4173
    goto PARSE_OVER;
H
hzcheng 已提交
4174
  }
4175

4176
  // specified_table_query
4177 4178
  cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!specifiedQuery) {
4179
    g_queryInfo.specifiedQueryInfo.concurrent = 1;
4180 4181
    g_queryInfo.specifiedQueryInfo.sqlCount = 0;
  } else if (specifiedQuery->type != cJSON_Object) {
4182
    printf("ERROR: failed to read json, super_table_query not found\n");
4183
    goto PARSE_OVER;
4184
  } else {
4185 4186 4187 4188 4189
    cJSON* queryInterval = cJSON_GetObjectItem(specifiedQuery, "query_interval");
    if (queryInterval && queryInterval->type == cJSON_Number) {
      g_queryInfo.specifiedQueryInfo.queryInterval = queryInterval->valueint;
    } else if (!queryInterval) {
      g_queryInfo.specifiedQueryInfo.queryInterval = 0;
4190 4191
    }

4192 4193
    cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery,
        "query_times");
4194
    if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
4195
      if (specifiedQueryTimes->valueint <= 0) {
4196 4197
        errorPrint(
                "%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4198
          __func__, __LINE__, specifiedQueryTimes->valueint);
4199 4200 4201
        goto PARSE_OVER;

      }
4202 4203 4204 4205
      g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint;
    } else if (!specifiedQueryTimes) {
      g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times;
    } else {
4206 4207
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
          __func__, __LINE__);
4208
      goto PARSE_OVER;
4209
    }
4210

4211
    cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent");
4212
    if (concurrent && concurrent->type == cJSON_Number) {
4213
      if (concurrent->valueint <= 0) {
4214
        errorPrint(
4215
                "%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n",
4216 4217
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount,
4218 4219 4220
              g_queryInfo.specifiedQueryInfo.concurrent);
        goto PARSE_OVER;
      }
4221
      g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint;
4222
    } else if (!concurrent) {
4223
      g_queryInfo.specifiedQueryInfo.concurrent = 1;
4224
    }
4225

4226 4227 4228 4229 4230 4231 4232
    cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode");
    if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String
        && specifiedAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE;
4233
      } else {
4234
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4235
            __func__, __LINE__);
4236 4237 4238
        goto PARSE_OVER;
      }
    } else {
4239
      g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
4240
    }
4241

4242
    cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval");
4243
    if (interval && interval->type == cJSON_Number) {
4244
      g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint;
4245
    } else if (!interval) {
4246 4247
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4248
      g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000;
4249
    }
4250

4251
    cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart");
4252
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
4253
      if (0 == strcmp("yes", restart->valuestring)) {
4254
        g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4255
      } else if (0 == strcmp("no", restart->valuestring)) {
4256
        g_queryInfo.specifiedQueryInfo.subscribeRestart = false;
4257
      } else {
4258
        printf("ERROR: failed to read json, subscribe restart error\n");
4259 4260 4261
        goto PARSE_OVER;
      }
    } else {
4262
      g_queryInfo.specifiedQueryInfo.subscribeRestart = true;
4263
    }
4264

4265
    cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress");
4266 4267 4268
    if (keepProgress
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
4269
      if (0 == strcmp("yes", keepProgress->valuestring)) {
4270
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1;
4271
      } else if (0 == strcmp("no", keepProgress->valuestring)) {
4272
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4273
      } else {
4274
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4275 4276 4277
        goto PARSE_OVER;
      }
    } else {
4278
      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4279 4280
    }

4281
    // sqls
4282 4283
    cJSON* specifiedSqls = cJSON_GetObjectItem(specifiedQuery, "sqls");
    if (!specifiedSqls) {
4284
      g_queryInfo.specifiedQueryInfo.sqlCount = 0;
4285
    } else if (specifiedSqls->type != cJSON_Array) {
4286 4287
      errorPrint("%s() LN%d, failed to read json, super sqls not found\n",
          __func__, __LINE__);
4288
      goto PARSE_OVER;
4289
    } else {
4290 4291 4292 4293 4294 4295 4296 4297
      int superSqlSize = cJSON_GetArraySize(specifiedSqls);
      if (superSqlSize * g_queryInfo.specifiedQueryInfo.concurrent
              > MAX_QUERY_SQL_COUNT) {
        errorPrint("%s() LN%d, failed to read json, query sql(%d) * concurrent(%d) overflow, max is %d\n",
           __func__, __LINE__,
           superSqlSize,
           g_queryInfo.specifiedQueryInfo.concurrent,
           MAX_QUERY_SQL_COUNT);
4298 4299
        goto PARSE_OVER;
      }
4300

4301
      g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize;
4302
      for (int j = 0; j < superSqlSize; ++j) {
4303
        cJSON* sql = cJSON_GetArrayItem(specifiedSqls, j);
4304
        if (sql == NULL) continue;
4305

4306 4307
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
4308
          printf("ERROR: failed to read json, sql not found\n");
4309 4310
          goto PARSE_OVER;
        }
4311 4312 4313
        tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j],
                sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);

4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324
        cJSON* endAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "endAfterConsume");
        if (endAfterConsume
                && endAfterConsume->type == cJSON_Number) {
            g_queryInfo.specifiedQueryInfo.endAfterConsume[j]
                = endAfterConsume->valueint;
        } else if (!endAfterConsume) {
            // default value is -1, which mean infinite loop
            g_queryInfo.specifiedQueryInfo.endAfterConsume[j] = -1;
        }

4325 4326 4327 4328 4329 4330 4331
        cJSON* resubAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume");
        if (resubAfterConsume
                && resubAfterConsume->type == cJSON_Number) {
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j]
                = resubAfterConsume->valueint;
        } else if (!resubAfterConsume) {
4332 4333
            // default value is -1, which mean do not resub
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = -1;
4334
        }
H
hzcheng 已提交
4335

4336
        cJSON *result = cJSON_GetObjectItem(sql, "result");
4337 4338 4339 4340
        if ((NULL != result) && (result->type == cJSON_String)
                && (result->valuestring != NULL)) {
          tstrncpy(g_queryInfo.specifiedQueryInfo.result[j],
                  result->valuestring, MAX_FILE_NAME_LEN);
4341
        } else if (NULL == result) {
4342 4343
          memset(g_queryInfo.specifiedQueryInfo.result[j],
                  0, MAX_FILE_NAME_LEN);
4344
        } else {
4345
          printf("ERROR: failed to read json, super query result file not found\n");
4346
          goto PARSE_OVER;
4347
        }
4348 4349 4350
      }
    }
  }
4351

4352
  // super_table_query
4353 4354
  cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!superQuery) {
4355
    g_queryInfo.superQueryInfo.threadCnt = 1;
4356 4357
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
4358
    printf("ERROR: failed to read json, sub_table_query not found\n");
4359 4360
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
4361
  } else {
4362
    cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval");
4363
    if (subrate && subrate->type == cJSON_Number) {
4364
      g_queryInfo.superQueryInfo.queryInterval = subrate->valueint;
4365
    } else if (!subrate) {
4366
      g_queryInfo.superQueryInfo.queryInterval = 0;
4367 4368 4369 4370
    }

    cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
    if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
4371 4372 4373
      if (superQueryTimes->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
          __func__, __LINE__, superQueryTimes->valueint);
4374 4375
        goto PARSE_OVER;
      }
4376 4377 4378 4379
      g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
    } else if (!superQueryTimes) {
      g_queryInfo.superQueryInfo.queryTimes = g_args.query_times;
    } else {
4380 4381
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
          __func__, __LINE__);
4382
      goto PARSE_OVER;
4383
    }
4384

4385
    cJSON* threads = cJSON_GetObjectItem(superQuery, "threads");
4386
    if (threads && threads->type == cJSON_Number) {
4387 4388 4389 4390 4391 4392
      if (threads->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, threads input mistake\n",
          __func__, __LINE__);
        goto PARSE_OVER;

      }
4393
      g_queryInfo.superQueryInfo.threadCnt = threads->valueint;
4394
    } else if (!threads) {
4395
      g_queryInfo.superQueryInfo.threadCnt = 1;
4396
    }
4397

4398
    //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count");
4399
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
4400
    //  g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint;
4401
    //} else if (!subTblCnt) {
4402
    //  g_queryInfo.superQueryInfo.childTblCount = 0;
4403
    //}
4404

4405
    cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname");
4406 4407 4408 4409
    if (stblname && stblname->type == cJSON_String
        && stblname->valuestring != NULL) {
      tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring,
          MAX_TB_NAME_SIZE);
4410
    } else {
4411 4412
      errorPrint("%s() LN%d, failed to read json, super table name input error\n",
          __func__, __LINE__);
4413 4414
      goto PARSE_OVER;
    }
4415

4416 4417 4418 4419 4420 4421 4422
    cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode");
    if (superAsyncMode && superAsyncMode->type == cJSON_String
        && superAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE;
4423
      } else {
4424
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4425
            __func__, __LINE__);
4426 4427 4428
        goto PARSE_OVER;
      }
    } else {
4429
      g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
4430
    }
4431

4432 4433 4434 4435 4436 4437 4438 4439 4440
    cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval");
    if (superInterval && superInterval->type == cJSON_Number) {
      if (superInterval->valueint < 0) {
        errorPrint("%s() LN%d, failed to read json, interval input mistake\n",
            __func__, __LINE__);
        goto PARSE_OVER;
      }
      g_queryInfo.superQueryInfo.subscribeInterval = superInterval->valueint;
    } else if (!superInterval) {
4441 4442
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4443
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
4444
    }
4445

4446
    cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart");
4447 4448
    if (subrestart && subrestart->type == cJSON_String
        && subrestart->valuestring != NULL) {
4449
      if (0 == strcmp("yes", subrestart->valuestring)) {
4450
        g_queryInfo.superQueryInfo.subscribeRestart = true;
4451
      } else if (0 == strcmp("no", subrestart->valuestring)) {
4452
        g_queryInfo.superQueryInfo.subscribeRestart = false;
4453
      } else {
4454
        printf("ERROR: failed to read json, subscribe restart error\n");
4455 4456 4457
        goto PARSE_OVER;
      }
    } else {
4458
      g_queryInfo.superQueryInfo.subscribeRestart = true;
4459
    }
4460

4461 4462 4463 4464 4465
    cJSON* superkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
    if (superkeepProgress &&
            superkeepProgress->type == cJSON_String
            && superkeepProgress->valuestring != NULL) {
      if (0 == strcmp("yes", superkeepProgress->valuestring)) {
4466
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
4467
      } else if (0 == strcmp("no", superkeepProgress->valuestring)) {
4468
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4469
      } else {
4470
        printf("ERROR: failed to read json, subscribe super table keepProgress error\n");
4471 4472 4473
        goto PARSE_OVER;
      }
    } else {
4474
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4475
    }
4476

4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487
    cJSON* superEndAfterConsume =
            cJSON_GetObjectItem(superQuery, "endAfterConsume");
    if (superEndAfterConsume
            && superEndAfterConsume->type == cJSON_Number) {
        g_queryInfo.superQueryInfo.endAfterConsume =
            superEndAfterConsume->valueint;
    } else if (!superEndAfterConsume) {
        // default value is -1, which mean do not resub
        g_queryInfo.superQueryInfo.endAfterConsume = -1;
    }

4488
    cJSON* superResubAfterConsume =
4489
            cJSON_GetObjectItem(superQuery, "endAfterConsume");
4490 4491
    if (superResubAfterConsume
            && superResubAfterConsume->type == cJSON_Number) {
4492
        g_queryInfo.superQueryInfo.endAfterConsume =
4493 4494
            superResubAfterConsume->valueint;
    } else if (!superResubAfterConsume) {
4495 4496
        // default value is -1, which mean do not resub
        g_queryInfo.superQueryInfo.endAfterConsume = -1;
4497 4498
    }

4499 4500 4501
    // supert table sqls
    cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls");
    if (!superSqls) {
4502
      g_queryInfo.superQueryInfo.sqlCount = 0;
4503
    } else if (superSqls->type != cJSON_Array) {
4504 4505
      errorPrint("%s() LN%d: failed to read json, super sqls not found\n",
          __func__, __LINE__);
4506
      goto PARSE_OVER;
4507
    } else {
4508
      int superSqlSize = cJSON_GetArraySize(superSqls);
4509
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4510 4511
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4512 4513
        goto PARSE_OVER;
      }
4514

4515
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
4516
      for (int j = 0; j < superSqlSize; ++j) {
4517
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
4518
        if (sql == NULL) continue;
4519

4520
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
4521 4522 4523 4524
        if (!sqlStr || sqlStr->type != cJSON_String
            || sqlStr->valuestring == NULL) {
          errorPrint("%s() LN%d, failed to read json, sql not found\n",
              __func__, __LINE__);
4525 4526
          goto PARSE_OVER;
        }
4527 4528
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring,
            MAX_QUERY_SQL_LENGTH);
4529 4530

        cJSON *result = cJSON_GetObjectItem(sql, "result");
4531 4532 4533 4534
        if (result != NULL && result->type == cJSON_String
            && result->valuestring != NULL){
          tstrncpy(g_queryInfo.superQueryInfo.result[j],
              result->valuestring, MAX_FILE_NAME_LEN);
4535
        } else if (NULL == result) {
4536
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4537
        }  else {
4538 4539
          errorPrint("%s() LN%d, failed to read json, sub query result file not found\n",
              __func__, __LINE__);
4540
          goto PARSE_OVER;
4541
        }
4542 4543
      }
    }
H
hzcheng 已提交
4544 4545
  }

4546
  ret = true;
H
hzcheng 已提交
4547

4548 4549 4550
PARSE_OVER:
  return ret;
}
H
hzcheng 已提交
4551

4552
static bool getInfoFromJsonFile(char* file) {
4553
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
4554

4555 4556 4557 4558 4559
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
4560

4561
  bool  ret = false;
4562
  int   maxLen = 6400000;
4563 4564 4565 4566 4567 4568 4569
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
4570
  }
H
Hui Li 已提交
4571

4572 4573 4574
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
4575
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
4576 4577
    goto PARSE_OVER;
  }
H
Hui Li 已提交
4578

4579 4580 4581
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
4582
      g_args.test_mode = INSERT_TEST;
4583
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
4584
      g_args.test_mode = QUERY_TEST;
4585
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
4586
      g_args.test_mode = SUBSCRIBE_TEST;
4587
    } else {
4588
      printf("ERROR: failed to read json, filetype not support\n");
4589 4590 4591
      goto PARSE_OVER;
    }
  } else if (!filetype) {
4592
    g_args.test_mode = INSERT_TEST;
4593
  } else {
4594
    printf("ERROR: failed to read json, filetype not found\n");
4595 4596
    goto PARSE_OVER;
  }
H
hzcheng 已提交
4597

4598
  if (INSERT_TEST == g_args.test_mode) {
4599
    ret = getMetaFromInsertJsonFile(root);
4600 4601
  } else if ((QUERY_TEST == g_args.test_mode)
          || (SUBSCRIBE_TEST == g_args.test_mode)) {
4602 4603
    ret = getMetaFromQueryJsonFile(root);
  } else {
4604 4605
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n",
            __func__, __LINE__);
4606
    goto PARSE_OVER;
4607
  }
4608 4609 4610 4611

PARSE_OVER:
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
4612
  fclose(fp);
4613 4614
  return ret;
}
H
hzcheng 已提交
4615

4616
static void prepareSampleData() {
4617
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4618 4619 4620
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
        (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]);
S
Shuaiqiang Chang 已提交
4621
      }
4622 4623 4624 4625
    }
  }
}

4626
static void postFreeResource() {
4627
  tmfclose(g_fpOfInsertResult);
4628
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4629
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
4630 4631 4632
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
4633
      }
4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4646
    }
4647 4648
  }
}
S
Shuaiqiang Chang 已提交
4649

4650 4651 4652
static int getRowDataFromSample(
        char* dataBuf, int64_t maxLen, int64_t timestamp,
      SSuperTable* superTblInfo, int64_t* sampleUsePos) {
4653
  if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
4654
/*    int ret = readSampleFromCsvFileToMem(superTblInfo);
4655
    if (0 != ret) {
4656
      tmfree(superTblInfo->sampleDataBuf);
4657
      superTblInfo->sampleDataBuf = NULL;
4658 4659
      return -1;
    }
4660
*/
4661 4662
    *sampleUsePos = 0;
  }
S
Shuaiqiang Chang 已提交
4663

4664
  int    dataLen = 0;
4665

4666 4667 4668 4669
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "(%" PRId64 ", ", timestamp);
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
4670
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
S
Shuaiqiang Chang 已提交
4671

4672
  (*sampleUsePos)++;
4673

4674 4675
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4676

4677 4678
static int64_t generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stbInfo) {
  int64_t   dataLen = 0;
4679
  char  *pstr = recBuf;
4680
  int64_t maxLen = MAX_DATA_SIZE;
4681

4682
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "(%" PRId64 ",", timestamp);
4683

4684
  for (int i = 0; i < stbInfo->columnCount; i++) {
4685 4686
    if ((0 == strncasecmp(stbInfo->columns[i].dataType, "BINARY", strlen("BINARY")))
            || (0 == strncasecmp(stbInfo->columns[i].dataType, "NCHAR", strlen("NCHAR")))) {
4687
      if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
4688
        errorPrint( "binary or nchar length overflow, max size:%u\n",
4689
                (uint32_t)TSDB_MAX_BINARY_LEN);
4690
        return -1;
4691
      }
4692

4693 4694
      char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
      if (NULL == buf) {
4695
        errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
4696
        return -1;
4697 4698
      }
      rand_string(buf, stbInfo->columns[i].dataLen);
4699
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf);
4700
      tmfree(buf);
4701
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4702
                "INT", 3)) {
4703
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4704
              "%d,", rand_int());
4705
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4706
                "BIGINT", 6)) {
4707
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4708
              "%"PRId64",", rand_bigint());
4709
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4710
                "FLOAT", 5)) {
4711
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4712
              "%f,", rand_float());
4713
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4714
                "DOUBLE", 6)) {
4715
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4716
              "%f,", rand_double());
4717
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4718
                "SMALLINT", 8)) {
4719
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4720
          "%d,", rand_smallint());
4721
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4722
          "TINYINT", strlen("TINYINT"))) {
4723
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4724
          "%d,", rand_tinyint());
4725
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4726
          "BOOL", strlen("BOOL"))) {
4727
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4728
          "%d,", rand_bool());
4729
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4730
          "TIMESTAMP", strlen("TIMESTAMP"))) {
4731
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4732
          "%"PRId64",", rand_bigint());
4733
    }  else {
4734
      errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType);
4735
      return -1;
4736
    }
S
Shuaiqiang Chang 已提交
4737
  }
4738

4739
  dataLen -= 1;
4740
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")");
4741

4742 4743 4744
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

  return strlen(recBuf);
4745
}
S
Shuaiqiang Chang 已提交
4746

4747
static int64_t generateData(char *recBuf, char **data_type,
4748
        int num_of_cols, int64_t timestamp, int lenOfBinary) {
4749 4750
  memset(recBuf, 0, MAX_DATA_SIZE);
  char *pstr = recBuf;
4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764
  pstr += sprintf(pstr, "(%" PRId64, timestamp);
  int c = 0;

  for (; c < MAX_NUM_DATATYPE; c++) {
    if (data_type[c] == NULL) {
      break;
    }
  }

  if (0 == c) {
    perror("data type error!");
    exit(-1);
  }

4765
  for (int i = 0; i < c; i++) {
4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776
    if (strcasecmp(data_type[i % c], "TINYINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_tinyint() );
    } else if (strcasecmp(data_type[i % c], "SMALLINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_smallint());
    } else if (strcasecmp(data_type[i % c], "INT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_int());
    } else if (strcasecmp(data_type[i % c], "BIGINT") == 0) {
      pstr += sprintf(pstr, ",%" PRId64, rand_bigint());
    } else if (strcasecmp(data_type[i % c], "FLOAT") == 0) {
      pstr += sprintf(pstr, ",%10.4f", rand_float());
    } else if (strcasecmp(data_type[i % c], "DOUBLE") == 0) {
4777
      double t = rand_double();
4778 4779
      pstr += sprintf(pstr, ",%20.8f", t);
    } else if (strcasecmp(data_type[i % c], "BOOL") == 0) {
4780
      bool b = rand_bool() & 1;
4781 4782
      pstr += sprintf(pstr, ",%s", b ? "true" : "false");
    } else if (strcasecmp(data_type[i % c], "BINARY") == 0) {
4783 4784
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4785
      pstr += sprintf(pstr, ",\"%s\"", s);
4786
      free(s);
4787
    } else if (strcasecmp(data_type[i % c], "NCHAR") == 0) {
4788 4789
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4790
      pstr += sprintf(pstr, ",\"%s\"", s);
4791 4792 4793
      free(s);
    }

4794
    if (strlen(recBuf) > MAX_DATA_SIZE) {
4795 4796 4797 4798 4799 4800 4801
      perror("column length too long, abort");
      exit(-1);
    }
  }

  pstr += sprintf(pstr, ")");

4802 4803
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

4804
  return (int32_t)strlen(recBuf);
4805 4806
}

4807
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
4808 4809
  char* sampleDataBuf = NULL;

4810
  sampleDataBuf = calloc(
4811
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
4812
  if (sampleDataBuf == NULL) {
4813
      errorPrint("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n",
4814
              __func__, __LINE__,
4815
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE,
4816
              strerror(errno));
4817
      return -1;
4818
  }
4819

4820 4821 4822 4823
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
4824 4825
      errorPrint("%s() LN%d, read sample from csv file failed.\n",
          __func__, __LINE__);
4826
      tmfree(sampleDataBuf);
4827
      superTblInfo->sampleDataBuf = NULL;
4828
      return -1;
H
Hui Li 已提交
4829
  }
4830

4831 4832 4833
  return 0;
}

4834
static int64_t execInsert(threadInfo *pThreadInfo, char *buffer, uint64_t k)
4835 4836
{
  int affectedRows;
4837
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4838

4839 4840
  verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
            __func__, __LINE__, buffer);
4841
  if (superTblInfo) {
4842
    if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) {
4843
      affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false);
4844
    } else if (0 == strncasecmp(superTblInfo->insertMode, "rest", strlen("rest"))) {
4845 4846
      if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port,
                  buffer, NULL /* not set result file */)) {
4847
        affectedRows = -1;
4848 4849
        printf("========restful return fail, threadID[%d]\n",
            pThreadInfo->threadID);
4850 4851 4852
      } else {
        affectedRows = k;
      }
4853 4854 4855 4856
    } else {
      errorPrint("%s() LN%d: unknown insert mode: %s\n",
        __func__, __LINE__, superTblInfo->insertMode);
      affectedRows = 0;
4857
    }
4858
  } else {
4859
    affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false);
4860 4861 4862 4863 4864
  }

  return affectedRows;
}

4865 4866
static void getTableName(char *pTblName,
        threadInfo* pThreadInfo, uint64_t tableSeq)
4867
{
4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886
    SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
    if (superTblInfo) {
        if (AUTO_CREATE_SUBTBL != superTblInfo->autoCreateTable) {
            if (superTblInfo->childTblLimit > 0) {
                snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
                        superTblInfo->childTblName +
                        (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
            } else {
                verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n",
                        pThreadInfo->threadID, __func__, __LINE__,
                        pThreadInfo->start_table_from,
                        pThreadInfo->ntables, tableSeq);
                snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
                        superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
            }
        } else {
            snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"",
                    superTblInfo->childTblPrefix, tableSeq);
        }
4887
    } else {
4888 4889
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"",
                g_args.tb_prefix, tableSeq);
4890 4891 4892
    }
}

4893
static int64_t generateDataTail(
4894
        SSuperTable* superTblInfo,
4895
        uint64_t batch, char* buffer, int64_t remainderBufLen, int64_t insertRows,
4896
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) {
4897 4898
  uint64_t len = 0;
  uint32_t ncols_per_record = 1; // count first col ts
4899

4900 4901
  char *pstr = buffer;

4902
  if (superTblInfo == NULL) {
4903
    uint32_t datatypeSeq = 0;
4904 4905 4906 4907
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
4908 4909
  }

4910
  verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch);
4911

4912 4913 4914 4915 4916 4917 4918 4919
  bool tsRand;
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
                  "rand", strlen("rand")))) {
        tsRand = true;
    } else {
        tsRand = false;
  }

4920
  uint64_t k = 0;
4921
  for (k = 0; k < batch;) {
4922
    char data[MAX_DATA_SIZE];
4923 4924
    memset(data, 0, MAX_DATA_SIZE);

4925
    int64_t retLen = 0;
4926

4927
    if (superTblInfo) {
4928 4929 4930 4931 4932 4933 4934 4935 4936
        if (tsRand) {
            retLen = generateRowData(
                    data,
                    startTime + getTSRandTail(
                        superTblInfo->timeStampStep, k,
                        superTblInfo->disorderRatio,
                        superTblInfo->disorderRange),
                    superTblInfo);
        } else {
4937
          retLen = getRowDataFromSample(
4938 4939
                    data,
                    remainderBufLen,
4940
                    startTime + superTblInfo->timeStampStep * k,
4941
                    superTblInfo,
4942
                    pSamplePos);
4943
        }
4944 4945
        if (retLen > remainderBufLen) {
            break;
4946
        }
4947

4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963
        pstr += snprintf(pstr , retLen + 1, "%s", data);
        k++;
        len += retLen;
        remainderBufLen -= retLen;
    } else {
        char **data_type = g_args.datatype;
        int lenOfBinary = g_args.len_of_binary;
        retLen = generateData(data, data_type,
                ncols_per_record,
                startTime + getTSRandTail(
                    DEFAULT_TIMESTAMP_STEP, k,
                    g_args.disorderRatio,
                    g_args.disorderRange),
                lenOfBinary);
        if (len > remainderBufLen)
            break;
4964

4965 4966 4967 4968
        pstr += sprintf(pstr, "%s", data);
        k++;
        len += retLen;
        remainderBufLen -= retLen;
4969 4970
    }

4971
    verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n",
4972
            __func__, __LINE__, len, k, buffer);
4973 4974

    startFrom ++;
4975

4976
    if (startFrom >= insertRows) {
4977
      break;
4978
    }
4979
  }
4980

4981 4982 4983
  *dataLen = len;
  return k;
}
4984

4985
static int generateSQLHead(char *tableName, int32_t tableSeq,
4986 4987
        threadInfo* pThreadInfo, SSuperTable* superTblInfo,
        char *buffer, int remainderBufLen)
4988 4989
{
  int len;
4990 4991 4992 4993

#define HEAD_BUFF_LEN    1024*24  // 16*1024 + (192+32)*2 + insert into ..
  char headBuf[HEAD_BUFF_LEN];

4994
  if (superTblInfo) {
4995 4996
    if ((AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable)
            && (TBL_ALREADY_EXISTS != superTblInfo->childTblExists)) {
4997 4998
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
4999
            tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq);
5000 5001 5002 5003 5004 5005
      } else {
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
      }
      if (NULL == tagsValBuf) {
5006 5007
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n",
            __func__, __LINE__);
5008 5009 5010
        return -1;
      }

5011 5012 5013
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
5014
                  "%s.%s using %s.%s tags %s values",
5015 5016 5017 5018 5019 5020 5021
                  pThreadInfo->db_name,
                  tableName,
                  pThreadInfo->db_name,
                  superTblInfo->sTblName,
                  tagsValBuf);
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
5022 5023 5024
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
5025
                  "%s.%s values",
5026
                  pThreadInfo->db_name,
5027
                  tableName);
5028
    } else {
5029 5030 5031
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
5032
                  "%s.%s values",
5033 5034 5035 5036
                  pThreadInfo->db_name,
                  tableName);
    }
  } else {
5037 5038 5039
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
5040
                  "%s.%s values",
5041 5042 5043 5044
                  pThreadInfo->db_name,
                  tableName);
  }

5045 5046 5047 5048 5049
  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

5050 5051 5052
  return len;
}

5053
static int64_t generateInterlaceDataBuffer(
5054 5055
        char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes,
        uint64_t tableSeq,
5056
        threadInfo *pThreadInfo, char *buffer,
5057
        int64_t insertRows,
5058
        int64_t startTime,
5059
        uint64_t *pRemainderBufLen)
5060
{
5061
  assert(buffer);
5062 5063 5064 5065 5066 5067 5068 5069 5070 5071
  char *pstr = buffer;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo,
            superTblInfo, pstr, *pRemainderBufLen);

  if (headLen <= 0) {
    return 0;
  }
  // generate data buffer
5072
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n",
5073 5074 5075 5076 5077
            pThreadInfo->threadID, __func__, __LINE__, i, buffer);

  pstr += headLen;
  *pRemainderBufLen -= headLen;

5078
  int64_t dataLen = 0;
5079

5080
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%"PRIu64" batchPerTbl = %"PRIu64"\n",
5081 5082 5083 5084 5085 5086 5087 5088 5089 5090
            pThreadInfo->threadID, __func__, __LINE__,
            i, batchPerTblTimes, batchPerTbl);

  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
      startTime = taosGetTimestamp(pThreadInfo->time_precision);
    }
  } else {
      startTime = 1500000000000;
  }
5091

5092
  int64_t k = generateDataTail(
5093
    superTblInfo,
5094 5095 5096 5097
    batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
    startTime,
    &(pThreadInfo->samplePos), &dataLen);

5098
  if (k == batchPerTbl) {
5099 5100 5101
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
  } else {
5102
    debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n",
5103
            __func__, __LINE__, k, batchPerTbl);
5104 5105
    pstr -= headLen;
    pstr[0] = '\0';
5106
    k = 0;
5107 5108 5109 5110 5111
  }

  return k;
}

5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127
static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq,
          int disorderRatio, int disorderRange)
{
    int64_t randTail = timeStampStep * seq;
    if (disorderRatio > 0) {
        int rand_num = taosRandom() % 100;
        if(rand_num < disorderRatio) {
            randTail = (randTail +
                    (taosRandom() % disorderRange + 1)) * (-1);
            debugPrint("rand data generated, back %"PRId64"\n", randTail);
        }
    }

    return randTail;
}

5128
static int64_t generateProgressiveDataBuffer(
5129
        char *tableName,
5130
        int64_t tableSeq,
5131 5132
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
5133
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos,
5134
        int64_t *pRemainderBufLen)
5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  int ncols_per_record = 1; // count first col ts

  if (superTblInfo == NULL) {
    int datatypeSeq = 0;
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
  }

  assert(buffer != NULL);
5149
  char *pstr = buffer;
5150

5151
  int64_t k = 0;
5152

5153
  memset(buffer, 0, *pRemainderBufLen);
5154

5155 5156
  int64_t headLen = generateSQLHead(tableName, tableSeq, pThreadInfo,
          superTblInfo,
5157
          buffer, *pRemainderBufLen);
5158 5159 5160 5161

  if (headLen <= 0) {
    return 0;
  }
5162
  pstr += headLen;
5163
  *pRemainderBufLen -= headLen;
5164

5165
  int64_t dataLen;
5166
  k = generateDataTail(superTblInfo,
5167
          g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom,
5168
          startTime,
5169
          pSamplePos, &dataLen);
5170

5171 5172
  return k;
}
5173

5174 5175 5176 5177 5178 5179 5180 5181 5182
static void printStatPerThread(threadInfo *pThreadInfo)
{
  fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n",
          pThreadInfo->threadID,
          pThreadInfo->totalInsertRows,
          pThreadInfo->totalAffectedRows,
          (double)(pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay/1000.0)));
}

5183
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
5184 5185
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
         pThreadInfo->threadID, __func__, __LINE__);
5186

5187
  int64_t insertRows;
5188
  uint64_t interlaceRows;
5189

5190
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5191

5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204
  if (superTblInfo) {
    insertRows = superTblInfo->insertRows;

    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    insertRows = g_args.num_of_DPT;
    interlaceRows = g_args.interlace_rows;
  }
5205

5206
  if (interlaceRows > insertRows)
5207
    interlaceRows = insertRows;
5208

5209 5210
  if (interlaceRows > g_args.num_of_RPR)
    interlaceRows = g_args.num_of_RPR;
5211

5212 5213 5214 5215 5216 5217 5218 5219 5220 5221
  int insertMode;

  if (interlaceRows > 0) {
    insertMode = INTERLACE_INSERT_MODE;
  } else {
    insertMode = PROGRESSIVE_INSERT_MODE;
  }

  // TODO: prompt tbl count multple interlace rows and batch
  //
5222

5223
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
5224
  char* buffer = calloc(maxSqlLen, 1);
5225
  if (NULL == buffer) {
5226
    errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5227
              __func__, __LINE__, maxSqlLen, strerror(errno));
5228 5229 5230
    return NULL;
  }

5231
  char tableName[TSDB_TABLE_NAME_LEN];
5232 5233 5234 5235

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

5236
  int64_t nTimeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
5237

5238
  uint64_t insert_interval =
5239
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5240 5241
  uint64_t st = 0;
  uint64_t et = UINT64_MAX;
5242

5243 5244 5245
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5246

5247
  uint64_t tableSeq = pThreadInfo->start_table_from;
5248

5249
  debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n",
5250
          pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from,
5251 5252 5253 5254
          pThreadInfo->ntables, insertRows);

  int64_t startTime = pThreadInfo->start_time;

5255 5256
  uint64_t batchPerTbl = interlaceRows;
  uint64_t batchPerTblTimes;
5257

5258
  if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) {
5259
    batchPerTblTimes =
5260
        g_args.num_of_RPR / interlaceRows;
5261 5262 5263 5264
  } else {
    batchPerTblTimes = 1;
  }

5265
  uint64_t generatedRecPerTbl = 0;
5266
  bool flagSleep = true;
5267
  uint64_t sleepTimeTotal = 0;
5268

5269 5270 5271
  char *strInsertInto = "insert into ";
  int nInsertBufLen = strlen(strInsertInto);

5272
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
5273
    if ((flagSleep) && (insert_interval)) {
5274
        st = taosGetTimestampMs();
5275
        flagSleep = false;
5276 5277
    }
    // generate data
5278
    memset(buffer, 0, maxSqlLen);
5279
    uint64_t remainderBufLen = maxSqlLen;
5280

5281
    char *pstr = buffer;
5282 5283 5284 5285 5286

    int len = snprintf(pstr, nInsertBufLen + 1, "%s", strInsertInto);
    pstr += len;
    remainderBufLen -= len;

5287
    uint64_t recOfBatch = 0;
5288

5289
    for (uint64_t i = 0; i < batchPerTblTimes; i ++) {
5290
      getTableName(tableName, pThreadInfo, tableSeq);
5291 5292 5293
      if (0 == strlen(tableName)) {
        errorPrint("[%d] %s() LN%d, getTableName return null\n",
            pThreadInfo->threadID, __func__, __LINE__);
5294
        free(buffer);
5295 5296
        return NULL;
      }
5297

5298
      uint64_t oldRemainderLen = remainderBufLen;
5299
      int64_t generated = generateInterlaceDataBuffer(
5300 5301 5302 5303
        tableName, batchPerTbl, i, batchPerTblTimes,
        tableSeq,
        pThreadInfo, pstr,
        insertRows,
5304
        startTime,
5305
        &remainderBufLen);
5306

5307 5308
      debugPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5309
      if (generated < 0) {
5310
        errorPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
5311
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5312
        goto free_of_interlace;
5313 5314
      } else if (generated == 0) {
        break;
5315 5316
      }

5317
      tableSeq ++;
5318
      recOfBatch += batchPerTbl;
5319
      pstr += (oldRemainderLen - remainderBufLen);
5320
//      startTime += batchPerTbl * superTblInfo->timeStampStep;
5321
      pThreadInfo->totalInsertRows += batchPerTbl;
5322
      verbosePrint("[%d] %s() LN%d batchPerTbl=%"PRId64" recOfBatch=%"PRId64"\n",
5323 5324
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
5325

5326 5327 5328 5329
      if (insertMode == INTERLACE_INSERT_MODE) {
          if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
            // turn to first table
            tableSeq = pThreadInfo->start_table_from;
5330
            generatedRecPerTbl += batchPerTbl;
5331 5332

            startTime = pThreadInfo->start_time
5333
              + generatedRecPerTbl * nTimeStampStep;
5334

5335 5336 5337 5338
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;

5339 5340 5341 5342
            int remainRows = insertRows - generatedRecPerTbl;
            if ((remainRows > 0) && (batchPerTbl > remainRows))
              batchPerTbl = remainRows;

5343 5344
            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
5345
          }
5346 5347
      }

5348
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n",
5349 5350 5351 5352
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
5353 5354 5355
        break;
    }

5356
    verbosePrint("[%d] %s() LN%d recOfBatch=%"PRIu64" totalInsertRows=%"PRIu64"\n",
5357 5358 5359 5360 5361
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
           pThreadInfo->threadID, __func__, __LINE__, buffer);

5362
    startTs = taosGetTimestampMs();
5363

5364 5365 5366 5367 5368 5369 5370
    if (recOfBatch == 0) {
      errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n",
              pThreadInfo->threadID, __func__, __LINE__,
              recOfBatch);
      errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n");
      goto free_of_interlace;
    }
5371
    int64_t affectedRows = execInsert(pThreadInfo, buffer, recOfBatch);
5372

5373
    endTs = taosGetTimestampMs();
5374 5375
    uint64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n",
5376
            __func__, __LINE__, delay);
5377 5378 5379
    verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5380 5381 5382 5383 5384 5385

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

5386 5387
    if (recOfBatch != affectedRows) {
        errorPrint("[%d] %s() LN%d execInsert insert %"PRIu64", affected rows: %"PRId64"\n%s\n",
5388
                pThreadInfo->threadID, __func__, __LINE__,
5389
                recOfBatch, affectedRows, buffer);
5390
        goto free_of_interlace;
5391
    }
5392

5393
    pThreadInfo->totalAffectedRows += affectedRows;
5394

5395 5396
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
5397
      printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n",
5398 5399 5400
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5401 5402
      lastPrintTime = currentPrintTime;
    }
5403

5404
    if ((insert_interval) && flagSleep) {
5405
      et = taosGetTimestampMs();
5406

5407 5408
      if (insert_interval > (et - st) ) {
        int sleepTime = insert_interval - (et -st);
5409 5410
        performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleepTime);
5411 5412 5413
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
5414
    }
5415 5416
  }

5417
free_of_interlace:
5418
  tmfree(buffer);
5419
  printStatPerThread(pThreadInfo);
5420 5421 5422
  return NULL;
}

5423 5424 5425 5426 5427 5428 5429 5430
// sync insertion
/*
   1 thread: 100 tables * 2000  rows/s
   1 thread: 10  tables * 20000 rows/s
   6 thread: 300 tables * 2000  rows/s

   2 taosinsertdata , 1 thread:  10  tables * 20000 rows/s
*/
5431
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
5432
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
5433

5434
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5435
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
5436

5437
  char* buffer = calloc(maxSqlLen, 1);
5438
  if (NULL == buffer) {
5439
    errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5440
              maxSqlLen,
5441 5442 5443
              strerror(errno));
    return NULL;
  }
5444

5445 5446 5447
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5448

5449
  int64_t timeStampStep =
5450
      superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
5451
/*  int insert_interval =
5452
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5453 5454
  uint64_t st = 0;
  uint64_t et = 0xffffffff;
5455
  */
5456

5457 5458
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
5459

5460
  pThreadInfo->samplePos = 0;
5461

5462
  for (uint64_t tableSeq =
5463
          pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to;
5464
        tableSeq ++) {
5465
    int64_t start_time = pThreadInfo->start_time;
5466

5467
    int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
5468

5469 5470
    verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows);

5471
    for (uint64_t i = 0; i < insertRows;) {
5472
        /*
5473
      if (insert_interval) {
5474
            st = taosGetTimestampMs();
5475
      }
5476
      */
5477

5478 5479
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
5480
      verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n",
5481 5482 5483
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);

5484
      int64_t remainderBufLen = maxSqlLen;
5485 5486 5487 5488 5489 5490 5491 5492
      char *pstr = buffer;
      int nInsertBufLen = strlen("insert into ");

      int len = snprintf(pstr, nInsertBufLen + 1, "%s", "insert into ");

      pstr += len;
      remainderBufLen -= len;

5493
      int64_t generated = generateProgressiveDataBuffer(
5494
              tableName, tableSeq, pThreadInfo, pstr, insertRows,
5495
            i, start_time,
5496 5497
            &(pThreadInfo->samplePos),
            &remainderBufLen);
5498 5499 5500
      if (generated > 0)
        i += generated;
      else
5501
        goto free_of_progressive;
5502

5503
      start_time +=  generated * timeStampStep;
5504
      pThreadInfo->totalInsertRows += generated;
5505

5506
      startTs = taosGetTimestampMs();
5507

5508
      int64_t affectedRows = execInsert(pThreadInfo, buffer, generated);
5509

5510
      endTs = taosGetTimestampMs();
5511
      uint64_t delay = endTs - startTs;
5512 5513
      performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n",
              __func__, __LINE__, delay);
5514 5515 5516
      verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5517

5518 5519 5520 5521
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
5522

5523 5524 5525 5526 5527
      if (affectedRows < 0) {
        errorPrint("%s() LN%d, affected rows: %"PRId64"\n",
                __func__, __LINE__, affectedRows);
        goto free_of_progressive;
      }
5528 5529 5530

      pThreadInfo->totalAffectedRows += affectedRows;

5531 5532
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
5533
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
5534 5535 5536
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5537 5538 5539
        lastPrintTime = currentPrintTime;
      }

5540
      if (i >= insertRows)
5541
        break;
5542
/*
5543
      if (insert_interval) {
5544
        et = taosGetTimestampMs();
5545

5546 5547
        if (insert_interval > ((et - st)) ) {
            int sleep_time = insert_interval - (et -st);
5548 5549
            performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleep_time);
5550 5551
            taosMsleep(sleep_time); // ms
        }
5552
      }
5553
      */
5554 5555
    }   // num_of_DPT

5556 5557
    if (g_args.verbose_print) {
      if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo &&
5558 5559
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
5560
          verbosePrint("%s() LN%d samplePos=%"PRId64"\n",
5561
                  __func__, __LINE__, pThreadInfo->samplePos);
5562
      }
5563
    }
5564
  } // tableSeq
5565

5566
free_of_progressive:
5567
  tmfree(buffer);
5568
  printStatPerThread(pThreadInfo);
5569
  return NULL;
H
Hui Li 已提交
5570 5571
}

5572 5573
static void* syncWrite(void *sarg) {

5574 5575
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5576

5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588
  int interlaceRows;

  if (superTblInfo) {
    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    interlaceRows = g_args.interlace_rows;
  }
5589

5590
  if (interlaceRows > 0) {
5591
    // interlace mode
5592
    return syncWriteInterlace(pThreadInfo);
5593 5594
  } else {
    // progressive mode
5595
    return syncWriteProgressive(pThreadInfo);
5596
  }
5597

5598 5599
}

5600
static void callBack(void *param, TAOS_RES *res, int code) {
5601 5602
  threadInfo* pThreadInfo = (threadInfo*)param;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5603

5604 5605
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5606
  if (insert_interval) {
5607 5608 5609
    pThreadInfo->et = taosGetTimestampMs();
    if ((pThreadInfo->et - pThreadInfo->st) < insert_interval) {
      taosMsleep(insert_interval - (pThreadInfo->et - pThreadInfo->st)); // ms
5610
    }
H
Hui Li 已提交
5611
  }
5612

5613
  char *buffer = calloc(1, pThreadInfo->superTblInfo->maxSqlLen);
5614
  char data[MAX_DATA_SIZE];
5615
  char *pstr = buffer;
5616 5617
  pstr += sprintf(pstr, "insert into %s.%s%"PRId64" values",
          pThreadInfo->db_name, pThreadInfo->tb_prefix,
5618 5619 5620 5621 5622 5623 5624 5625
          pThreadInfo->start_table_from);
//  if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
  if (pThreadInfo->counter >= g_args.num_of_RPR) {
    pThreadInfo->start_table_from++;
    pThreadInfo->counter = 0;
  }
  if (pThreadInfo->start_table_from > pThreadInfo->end_table_to) {
    tsem_post(&pThreadInfo->lock_sem);
5626 5627 5628 5629
    free(buffer);
    taos_free_result(res);
    return;
  }
5630

5631
  for (int i = 0; i < g_args.num_of_RPR; i++) {
5632
    int rand_num = taosRandom() % 100;
5633 5634
    if (0 != pThreadInfo->superTblInfo->disorderRatio
            && rand_num < pThreadInfo->superTblInfo->disorderRatio) {
5635 5636
      int64_t d = pThreadInfo->lastTs
          - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1);
5637
      generateRowData(data, d, pThreadInfo->superTblInfo);
5638
    } else {
5639
      generateRowData(data, pThreadInfo->lastTs += 1000, pThreadInfo->superTblInfo);
H
Hui Li 已提交
5640
    }
5641
    pstr += sprintf(pstr, "%s", data);
5642
    pThreadInfo->counter++;
H
Hui Li 已提交
5643

5644
    if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
5645
      break;
H
Hui Li 已提交
5646 5647
    }
  }
5648

5649
  if (insert_interval) {
5650
    pThreadInfo->st = taosGetTimestampMs();
5651
  }
5652
  taos_query_a(pThreadInfo->taos, buffer, callBack, pThreadInfo);
5653
  free(buffer);
H
Hui Li 已提交
5654

5655
  taos_free_result(res);
H
Hui Li 已提交
5656 5657
}

5658
static void *asyncWrite(void *sarg) {
5659 5660
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5661

5662 5663 5664
  pThreadInfo->st = 0;
  pThreadInfo->et = 0;
  pThreadInfo->lastTs = pThreadInfo->start_time;
5665

5666
  int insert_interval =
5667
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5668
  if (insert_interval) {
5669
    pThreadInfo->st = taosGetTimestampMs();
H
Hui Li 已提交
5670
  }
5671
  taos_query_a(pThreadInfo->taos, "show databases", callBack, pThreadInfo);
5672

5673
  tsem_wait(&(pThreadInfo->lock_sem));
H
Hui Li 已提交
5674 5675 5676 5677

  return NULL;
}

5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703
static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in *serv_addr)
{
  uint16_t rest_port = port + TSDB_PORT_HTTP;
  struct hostent *server = gethostbyname(host);
  if ((server == NULL) || (server->h_addr == NULL)) {
    errorPrint("%s", "ERROR, no such host");
    return -1;
  }

  debugPrint("h_name: %s\nh_addr=%p\nh_addretype: %s\nh_length: %d\n",
            server->h_name,
            server->h_addr,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

  memset(serv_addr, 0, sizeof(struct sockaddr_in));
  serv_addr->sin_family = AF_INET;
  serv_addr->sin_port = htons(rest_port);
#ifdef WINDOWS
  serv_addr->sin_addr.s_addr = inet_addr(host);
#else
  memcpy(&(serv_addr->sin_addr.s_addr), server->h_addr, server->h_length);
#endif
  return 0;
}

5704 5705
static void startMultiThreadInsertData(int threads, char* db_name,
        char* precision,SSuperTable* superTblInfo) {
5706

5707 5708
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  assert(pids != NULL);
H
Hui Li 已提交
5709

5710 5711
  threadInfo *infos = malloc(threads * sizeof(threadInfo));
  assert(infos != NULL);
5712

5713 5714
  memset(pids, 0, threads * sizeof(pthread_t));
  memset(infos, 0, threads * sizeof(threadInfo));
5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731

  //TAOS* taos;
  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
  //  if (NULL == taos) {
  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
  //    exit(-1);
  //  }
  //}

  int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
  if (0 != precision[0]) {
    if (0 == strncasecmp(precision, "ms", 2)) {
      timePrec = TSDB_TIME_PRECISION_MILLI;
    }  else if (0 == strncasecmp(precision, "us", 2)) {
      timePrec = TSDB_TIME_PRECISION_MICRO;
    }  else {
5732
      errorPrint("Not support precision: %s\n", precision);
5733 5734 5735 5736
      exit(-1);
    }
  }

5737
  int64_t start_time;
5738 5739 5740
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
        start_time = taosGetTimestamp(timePrec);
5741
    } else {
5742 5743 5744 5745 5746
      if (TSDB_CODE_SUCCESS != taosParseTime(
        superTblInfo->startTimestamp,
        &start_time,
        strlen(superTblInfo->startTimestamp),
        timePrec, 0)) {
5747
          ERROR_EXIT("failed to parse time!\n");
5748
      }
5749
    }
5750 5751
  } else {
     start_time = 1500000000000;
5752 5753
  }

5754
  int64_t start = taosGetTimestampMs();
5755

5756
  // read sample data from file first
5757
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5758
              "sample", strlen("sample")))) {
5759
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5760 5761
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
              __func__, __LINE__);
5762 5763 5764 5765
      exit(-1);
    }
  }

5766
  // read sample data from file first
5767
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5768 5769
              "sample", strlen("sample")))) {
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5770 5771
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
              __func__, __LINE__);
5772 5773 5774 5775 5776 5777 5778 5779
      exit(-1);
    }
  }

  TAOS* taos = taos_connect(
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
  if (NULL == taos) {
5780 5781
    errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
                __func__, __LINE__, taos_errstr(NULL));
5782 5783 5784
    exit(-1);
  }

5785
  int64_t ntables = 0;
5786
  uint64_t startFrom;
5787

5788
  if (superTblInfo) {
5789 5790
    int64_t limit;
    uint64_t offset;
5791

5792
    if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) &&
5793
            ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) {
5794
      printf("WARNING: offset and limit will not be used since the child tables not exists!\n");
5795 5796
    }

5797
    if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) {
5798 5799
      if ((superTblInfo->childTblLimit < 0)
          || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit)
5800
            > (superTblInfo->childTblCount))) {
5801 5802 5803 5804 5805 5806
        superTblInfo->childTblLimit =
            superTblInfo->childTblCount - superTblInfo->childTblOffset;
      }

      offset = superTblInfo->childTblOffset;
      limit = superTblInfo->childTblLimit;
5807
    } else {
5808 5809
      limit = superTblInfo->childTblCount;
      offset = 0;
5810 5811
    }

5812 5813 5814
    ntables = limit;
    startFrom = offset;

5815 5816 5817 5818
    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
        && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
            > superTblInfo->childTblCount)) {
      printf("WARNING: specified offset + limit > child table count!\n");
5819
      prompt();
5820 5821 5822 5823 5824
    }

    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
            && (0 == superTblInfo->childTblLimit)) {
      printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
5825
      prompt();
5826 5827
    }

5828 5829 5830
    superTblInfo->childTblName = (char*)calloc(1,
        limit * TSDB_TABLE_NAME_LEN);
    if (superTblInfo->childTblName == NULL) {
5831
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
5832 5833 5834 5835
      taos_close(taos);
      exit(-1);
    }

5836
    int64_t childTblCount;
5837 5838 5839 5840 5841 5842
    getChildNameOfSuperTableWithLimitAndOffset(
        taos,
        db_name, superTblInfo->sTblName,
        &superTblInfo->childTblName, &childTblCount,
        limit,
        offset);
5843 5844 5845
  } else {
    ntables = g_args.num_of_tables;
    startFrom = 0;
5846
  }
5847

5848 5849
  taos_close(taos);

5850
  int64_t a = ntables / threads;
5851 5852 5853 5854 5855
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

5856
  int64_t b = 0;
5857 5858 5859 5860
  if (threads != 0) {
    b = ntables % threads;
  }

5861 5862 5863 5864 5865 5866
  if ((superTblInfo)
      && (0 == strncasecmp(superTblInfo->insertMode, "rest", strlen("rest")))) {
    if (convertHostToServAddr(g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0)
      exit(-1);
  }

H
Hui Li 已提交
5867
  for (int i = 0; i < threads; i++) {
5868 5869 5870 5871 5872
    threadInfo *pThreadInfo = infos + i;
    pThreadInfo->threadID = i;
    tstrncpy(pThreadInfo->db_name, db_name, MAX_DB_NAME_SIZE);
    pThreadInfo->time_precision = timePrec;
    pThreadInfo->superTblInfo = superTblInfo;
5873

5874 5875
    pThreadInfo->start_time = start_time;
    pThreadInfo->minDelay = UINT64_MAX;
5876

5877 5878
    if ((NULL == superTblInfo) ||
            (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) {
5879 5880
      //pThreadInfo->taos = taos;
      pThreadInfo->taos = taos_connect(
5881
              g_Dbs.host, g_Dbs.user,
5882
              g_Dbs.password, db_name, g_Dbs.port);
5883
      if (NULL == pThreadInfo->taos) {
5884 5885
        errorPrint(
                "connect to server fail from insert sub thread, reason: %s\n",
5886
                taos_errstr(NULL));
5887 5888 5889
        exit(-1);
      }
    } else {
5890
      pThreadInfo->taos = NULL;
5891 5892
    }

5893
/*    if ((NULL == superTblInfo)
5894
            || (0 == superTblInfo->multiThreadWriteOneTbl)) {
5895
            */
5896 5897 5898 5899
      pThreadInfo->start_table_from = startFrom;
      pThreadInfo->ntables = i<b?a+1:a;
      pThreadInfo->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = pThreadInfo->end_table_to + 1;
5900
/*    } else {
5901 5902 5903
      pThreadInfo->start_table_from = 0;
      pThreadInfo->ntables = superTblInfo->childTblCount;
      pThreadInfo->start_time = pThreadInfo->start_time + rand_int() % 10000 - rand_tinyint();
5904
    }
5905
*/
5906
    tsem_init(&(pThreadInfo->lock_sem), 0, 0);
5907
    if (ASYNC_MODE == g_Dbs.asyncMode) {
5908
      pthread_create(pids + i, NULL, asyncWrite, pThreadInfo);
5909
    } else {
5910
      pthread_create(pids + i, NULL, syncWrite, pThreadInfo);
5911
    }
H
Hui Li 已提交
5912
  }
5913

H
Hui Li 已提交
5914 5915 5916 5917
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

5918 5919 5920 5921
  uint64_t totalDelay = 0;
  uint64_t maxDelay = 0;
  uint64_t minDelay = UINT64_MAX;
  uint64_t cntDelay = 1;
5922 5923
  double  avgDelay = 0;

H
Hui Li 已提交
5924
  for (int i = 0; i < threads; i++) {
5925
    threadInfo *pThreadInfo = infos + i;
5926

5927 5928
    tsem_destroy(&(pThreadInfo->lock_sem));
    taos_close(pThreadInfo->taos);
5929

5930
    debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n",
5931
            __func__, __LINE__,
5932 5933
            pThreadInfo->threadID, pThreadInfo->totalInsertRows,
            pThreadInfo->totalAffectedRows);
5934
    if (superTblInfo) {
5935 5936
        superTblInfo->totalAffectedRows += pThreadInfo->totalAffectedRows;
        superTblInfo->totalInsertRows += pThreadInfo->totalInsertRows;
5937
    } else {
5938 5939
        g_args.totalAffectedRows += pThreadInfo->totalAffectedRows;
        g_args.totalInsertRows += pThreadInfo->totalInsertRows;
5940
    }
5941

5942 5943 5944 5945
    totalDelay  += pThreadInfo->totalDelay;
    cntDelay   += pThreadInfo->cntDelay;
    if (pThreadInfo->maxDelay > maxDelay) maxDelay = pThreadInfo->maxDelay;
    if (pThreadInfo->minDelay < minDelay) minDelay = pThreadInfo->minDelay;
H
Hui Li 已提交
5946
  }
5947
  cntDelay -= 1;
H
Hui Li 已提交
5948

5949
  if (cntDelay == 0)    cntDelay = 1;
5950 5951
  avgDelay = (double)totalDelay / cntDelay;

5952 5953
  int64_t end = taosGetTimestampMs();
  int64_t t = end - start;
5954 5955

  if (superTblInfo) {
5956
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
5957
          t / 1000.0, superTblInfo->totalInsertRows,
5958 5959
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5960
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
5961 5962 5963 5964

    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
5965
          t / 1000.0, superTblInfo->totalInsertRows,
5966 5967
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5968
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
5969
    }
5970
  } else {
5971
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
5972
          t / 1000.0, g_args.totalInsertRows,
5973 5974
          g_args.totalAffectedRows,
          threads, db_name,
5975
          (double)g_args.totalInsertRows / (t / 1000.0));
5976 5977 5978
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
5979
          t * 1000.0, g_args.totalInsertRows,
5980 5981
          g_args.totalAffectedRows,
          threads, db_name,
5982
          (double)g_args.totalInsertRows / (t / 1000.0));
5983
    }
5984
  }
5985

5986
  fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
5987
          avgDelay, maxDelay, minDelay);
5988 5989
  if (g_fpOfInsertResult) {
    fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
5990
          avgDelay, maxDelay, minDelay);
5991
  }
5992

5993 5994
  //taos_close(taos);

H
Hui Li 已提交
5995
  free(pids);
5996
  free(infos);
H
Hui Li 已提交
5997 5998
}

5999
static void *readTable(void *sarg) {
6000
#if 1
6001 6002
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  TAOS *taos = pThreadInfo->taos;
H
hzcheng 已提交
6003
  char command[BUFFER_SIZE] = "\0";
6004 6005 6006
  uint64_t sTime = pThreadInfo->start_time;
  char *tb_prefix = pThreadInfo->tb_prefix;
  FILE *fp = fopen(pThreadInfo->filePath, "a");
H
Hui Li 已提交
6007
  if (NULL == fp) {
6008
    errorPrint( "fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno));
H
Hui Li 已提交
6009 6010
    return NULL;
  }
6011

6012
  int64_t num_of_DPT;
6013 6014
/*  if (pThreadInfo->superTblInfo) {
    num_of_DPT = pThreadInfo->superTblInfo->insertRows; //  nrecords_per_table;
6015
  } else {
6016
  */
6017
      num_of_DPT = g_args.num_of_DPT;
6018
//  }
6019

6020
  int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
6021
  int64_t totalData = num_of_DPT * num_of_tables;
6022
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6023 6024 6025 6026 6027

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
6028
  printf("%"PRId64" records:\n", totalData);
H
hzcheng 已提交
6029 6030
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

6031
  for (int j = 0; j < n; j++) {
H
hzcheng 已提交
6032
    double totalT = 0;
6033
    uint64_t count = 0;
6034 6035
    for (int64_t i = 0; i < num_of_tables; i++) {
      sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64,
6036
              aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
6037

6038
      double t = taosGetTimestampMs();
S
Shuaiqiang Chang 已提交
6039 6040
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
6041

S
Shuaiqiang Chang 已提交
6042
      if (code != 0) {
6043
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6044
        taos_free_result(pSql);
H
hzcheng 已提交
6045
        taos_close(taos);
6046
        fclose(fp);
6047
        return NULL;
H
hzcheng 已提交
6048 6049
      }

6050
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6051 6052 6053
        count++;
      }

6054
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6055 6056
      totalT += t;

S
Shuaiqiang Chang 已提交
6057
      taos_free_result(pSql);
H
hzcheng 已提交
6058 6059
    }

6060
    fprintf(fp, "|%10s  |   %"PRId64"   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
6061
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
6062
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
6063
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000);
H
hzcheng 已提交
6064 6065 6066
  }
  fprintf(fp, "\n");
  fclose(fp);
6067
#endif
H
hzcheng 已提交
6068 6069 6070
  return NULL;
}

6071
static void *readMetric(void *sarg) {
6072
#if 1
6073 6074
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  TAOS *taos = pThreadInfo->taos;
H
hzcheng 已提交
6075
  char command[BUFFER_SIZE] = "\0";
6076
  FILE *fp = fopen(pThreadInfo->filePath, "a");
H
Hui Li 已提交
6077
  if (NULL == fp) {
6078
    printf("fopen %s fail, reason:%s.\n", pThreadInfo->filePath, strerror(errno));
H
Hui Li 已提交
6079 6080
    return NULL;
  }
6081

6082 6083
  int64_t num_of_DPT = pThreadInfo->superTblInfo->insertRows;
  int64_t num_of_tables = pThreadInfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
6084
  int64_t totalData = num_of_DPT * num_of_tables;
6085
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6086 6087 6088 6089 6090

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
6091 6092
  printf("%"PRId64" records:\n", totalData);
  fprintf(fp, "Querying On %"PRId64" records:\n", totalData);
H
hzcheng 已提交
6093 6094

  for (int j = 0; j < n; j++) {
6095
    char condition[COND_BUF_LEN] = "\0";
B
Bomin Zhang 已提交
6096
    char tempS[64] = "\0";
H
hzcheng 已提交
6097

6098
    int64_t m = 10 < num_of_tables ? 10 : num_of_tables;
H
hzcheng 已提交
6099

6100
    for (int64_t i = 1; i <= m; i++) {
H
hzcheng 已提交
6101
      if (i == 1) {
6102
        sprintf(tempS, "t1 = %"PRId64"", i);
H
hzcheng 已提交
6103
      } else {
6104
        sprintf(tempS, " or t1 = %"PRId64" ", i);
H
hzcheng 已提交
6105
      }
6106
      strncat(condition, tempS, COND_BUF_LEN - 1);
H
hzcheng 已提交
6107

L
liu0x54 已提交
6108
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
6109 6110 6111 6112

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

6113
      double t = taosGetTimestampMs();
H
hzcheng 已提交
6114

S
Shuaiqiang Chang 已提交
6115 6116 6117 6118
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
6119
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6120
        taos_free_result(pSql);
H
hzcheng 已提交
6121
        taos_close(taos);
6122
        fclose(fp);
6123
        return NULL;
H
hzcheng 已提交
6124 6125
      }
      int count = 0;
6126
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6127 6128
        count++;
      }
6129
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6130

6131
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
6132 6133
              num_of_tables * num_of_DPT / (t * 1000.0), t);
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0);
H
hzcheng 已提交
6134

S
Shuaiqiang Chang 已提交
6135
      taos_free_result(pSql);
H
hzcheng 已提交
6136 6137 6138 6139
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
6140
#endif
H
hzcheng 已提交
6141 6142 6143
  return NULL;
}

6144 6145 6146
static void prompt()
{
  if (!g_args.answer_yes) {
6147
    printf("         Press enter key to continue or Ctrl-C to stop\n\n");
6148 6149 6150 6151
    (void)getchar();
  }
}

6152
static int insertTestProcess() {
6153

6154 6155 6156
  setupForAnsiEscape();
  int ret = printfInsertMeta();
  resetAfterAnsiEscape();
6157

6158 6159 6160
  if (ret == -1)
    exit(EXIT_FAILURE);

6161
  debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
6162 6163
  g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
  if (NULL == g_fpOfInsertResult) {
6164
    errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
6165 6166
    return -1;
  }
6167

6168 6169
  if (g_fpOfInsertResult)
    printfInsertMetaToFile(g_fpOfInsertResult);
6170

6171
  prompt();
6172

6173 6174 6175
  init_rand_data();

  // create database and super tables
6176
  if(createDatabasesAndStables() != 0) {
6177 6178
    if (g_fpOfInsertResult)
      fclose(g_fpOfInsertResult);
6179 6180
    return -1;
  }
6181 6182

  // pretreatement
6183
  prepareSampleData();
6184

6185 6186 6187 6188
  double start;
  double end;

  // create child tables
6189
  start = taosGetTimestampMs();
6190
  createChildTables();
6191
  end = taosGetTimestampMs();
6192

6193
  if (g_totalChildTables > 0) {
6194
    fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6195
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6196 6197
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
6198
            "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6199
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6200
    }
6201
  }
6202

6203
  //taosMsleep(1000);
6204
  // create sub threads for inserting data
6205
  //start = taosGetTimestampMs();
6206
  for (int i = 0; i < g_Dbs.dbCount; i++) {
6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
        for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {

          SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];

          if (superTblInfo && (superTblInfo->insertRows > 0)) {
            startMultiThreadInsertData(
              g_Dbs.threadCount,
              g_Dbs.db[i].dbName,
              g_Dbs.db[i].dbCfg.precision,
              superTblInfo);
          }
6220
        }
6221
      }
6222 6223
    } else {
        startMultiThreadInsertData(
6224 6225 6226
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
6227
          NULL);
H
Hui Li 已提交
6228
    }
6229
  }
6230
  //end = taosGetTimestampMs();
6231

6232
  //int64_t    totalInsertRows = 0;
6233
  //int64_t    totalAffectedRows = 0;
6234
  //for (int i = 0; i < g_Dbs.dbCount; i++) {
6235
  //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6236
  //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
6237 6238
  //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
  //}
6239
  //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
6240
  postFreeResource();
6241

6242 6243 6244
  return 0;
}

6245 6246
static void *specifiedTableQuery(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6247

6248
  if (pThreadInfo->taos == NULL) {
6249 6250 6251 6252 6253 6254 6255 6256
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6257
            pThreadInfo->threadID, taos_errstr(NULL));
6258 6259
      return NULL;
    } else {
6260
      pThreadInfo->taos = taos;
6261 6262 6263
    }
  }

6264 6265
  char sqlStr[MAX_DB_NAME_SIZE + 5];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6266 6267
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6268 6269 6270 6271
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
    return NULL;
  }
6272

6273 6274
  uint64_t st = 0;
  uint64_t et = 0;
6275

6276
  uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes;
6277

6278 6279 6280
  uint64_t totalQueried = 0;
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
6281

6282
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') {
6283
    sprintf(pThreadInfo->filePath, "%s-%d",
6284 6285 6286 6287
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }

6288
  while(queryTimes --) {
6289 6290 6291
    if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) <
            (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms
6292 6293
    }

6294 6295 6296
    st = taosGetTimestampMs();

    selectAndGetResult(pThreadInfo,
6297
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]);
6298

6299 6300 6301 6302
    et = taosGetTimestampMs();
    printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n",
              taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0);

6303 6304
    totalQueried ++;
    g_queryInfo.specifiedQueryInfo.totalQueried ++;
6305

6306 6307
    uint64_t  currentPrintTime = taosGetTimestampMs();
    uint64_t  endTs = taosGetTimestampMs();
6308
    if (currentPrintTime - lastPrintTime > 30*1000) {
6309
      debugPrint("%s() LN%d, endTs=%"PRIu64"ms, startTs=%"PRIu64"ms\n",
6310
          __func__, __LINE__, endTs, startTs);
6311
      printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.6f\n",
6312 6313
                    pThreadInfo->threadID,
                    totalQueried,
6314
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6315
      lastPrintTime = currentPrintTime;
6316
    }
H
Hui Li 已提交
6317
  }
6318 6319
  return NULL;
}
H
Hui Li 已提交
6320

6321
static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) {
6322 6323
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
6324 6325
  sprintf(subTblName, "%s.%s",
          g_queryInfo.dbName,
6326
          g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
6327 6328

  //printf("inSql: %s\n", inSql);
6329

6330 6331
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
6332
    return;
H
Hui Li 已提交
6333
  }
6334

6335
  tstrncpy(outSql, inSql, pos - inSql + 1);
6336
  //printf("1: %s\n", outSql);
6337
  strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1);
6338
  //printf("2: %s\n", outSql);
6339
  strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1);
6340
  //printf("3: %s\n", outSql);
H
Hui Li 已提交
6341 6342
}

6343
static void *superTableQuery(void *sarg) {
6344
  char sqlstr[MAX_QUERY_SQL_LENGTH];
6345
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6346

6347
  if (pThreadInfo->taos == NULL) {
6348 6349 6350 6351 6352 6353 6354 6355
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6356
            pThreadInfo->threadID, taos_errstr(NULL));
6357 6358
      return NULL;
    } else {
6359
      pThreadInfo->taos = taos;
6360 6361 6362
    }
  }

6363 6364
  uint64_t st = 0;
  uint64_t et = (int64_t)g_queryInfo.superQueryInfo.queryInterval;
6365

6366 6367 6368
  uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes;
  uint64_t totalQueried = 0;
  uint64_t  startTs = taosGetTimestampMs();
6369

6370
  uint64_t  lastPrintTime = taosGetTimestampMs();
6371
  while(queryTimes --) {
6372 6373 6374
    if (g_queryInfo.superQueryInfo.queryInterval
            && (et - st) < (int64_t)g_queryInfo.superQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.superQueryInfo.queryInterval - (et - st)); // ms
6375
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to);
6376
    }
H
Hui Li 已提交
6377

6378
    st = taosGetTimestampMs();
6379
    for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) {
6380
      for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
6381
        memset(sqlstr,0,sizeof(sqlstr));
6382
        replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i);
6383
        if (g_queryInfo.superQueryInfo.result[j][0] != '\0') {
6384
          sprintf(pThreadInfo->filePath, "%s-%d",
6385
                  g_queryInfo.superQueryInfo.result[j],
6386
                  pThreadInfo->threadID);
6387
        }
6388
        selectAndGetResult(pThreadInfo, sqlstr);
6389 6390 6391 6392 6393 6394 6395

        totalQueried++;
        g_queryInfo.superQueryInfo.totalQueried ++;

        int64_t  currentPrintTime = taosGetTimestampMs();
        int64_t  endTs = taosGetTimestampMs();
        if (currentPrintTime - lastPrintTime > 30*1000) {
6396
          printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.3f\n",
6397 6398
                    pThreadInfo->threadID,
                    totalQueried,
6399
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6400
          lastPrintTime = currentPrintTime;
6401
        }
6402
      }
H
Hui Li 已提交
6403
    }
6404
    et = taosGetTimestampMs();
6405
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%"PRIu64" - %"PRIu64"] once queries duration:%.4fs\n\n",
6406
            taosGetSelfPthreadId(),
6407 6408
            pThreadInfo->start_table_from,
            pThreadInfo->end_table_to,
6409
            (double)(et - st)/1000.0);
6410
  }
6411

6412 6413 6414
  return NULL;
}

6415
static int queryTestProcess() {
6416 6417 6418 6419

  setupForAnsiEscape();
  printfQueryMeta();
  resetAfterAnsiEscape();
6420 6421 6422 6423 6424 6425

  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
6426
          g_queryInfo.port);
6427
  if (taos == NULL) {
6428 6429
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6430 6431 6432
    exit(-1);
  }

6433
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6434 6435
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6436 6437 6438
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6439
  }
6440

6441
  prompt();
6442

6443 6444 6445
  if (g_args.debug_print || g_args.verbose_print) {
    printfQuerySystemInfo(taos);
  }
6446

6447 6448 6449 6450 6451 6452
  if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
    if (convertHostToServAddr(
        g_queryInfo.host, g_queryInfo.port, &g_queryInfo.serv_addr) != 0)
      exit(-1);
  }

6453 6454 6455
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
6456
  int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent;
6457
  uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount;
6458

6459
  uint64_t startTs = taosGetTimestampMs();
6460

6461 6462 6463 6464 6465 6466
  if ((nSqlCount > 0) && (nConcurrent > 0)) {

    pids  = malloc(nConcurrent * nSqlCount * sizeof(pthread_t));
    infos = malloc(nConcurrent * nSqlCount * sizeof(threadInfo));

    if ((NULL == pids) || (NULL == infos)) {
6467
      taos_close(taos);
6468
      ERROR_EXIT("memory allocation failed for create threads\n");
6469
    }
6470

6471 6472 6473
    for (uint64_t i = 0; i < nSqlCount; i++) {
        for (int j = 0; j < nConcurrent; j++) {
            uint64_t seq = i * nConcurrent + j;
6474 6475 6476
            threadInfo *pThreadInfo = infos + seq;
            pThreadInfo->threadID = seq;
            pThreadInfo->querySeq = i;
6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490

            if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {

                char sqlStr[MAX_TB_NAME_SIZE*2];
                sprintf(sqlStr, "use %s", g_queryInfo.dbName);
                if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) {
                    taos_close(taos);
                    free(infos);
                    free(pids);
                    errorPrint( "use database %s failed!\n\n",
                        g_queryInfo.dbName);
                    return -1;
                }
            }
6491

6492
            pThreadInfo->taos = NULL;// TODO: workaround to use separate taos connection;
6493

6494
            pthread_create(pids + seq, NULL, specifiedTableQuery,
6495
                pThreadInfo);
6496
        }
6497
    }
6498
  } else {
6499
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
6500
  }
6501

6502 6503
  taos_close(taos);

6504 6505 6506
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
6507 6508 6509 6510
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
    pidsOfSub  = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t));
    infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo));
6511 6512

    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
6513 6514
      free(infos);
      free(pids);
6515

6516
      ERROR_EXIT("memory allocation failed for create threads\n");
6517
    }
6518

6519
    int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
6520
    int threads = g_queryInfo.superQueryInfo.threadCnt;
6521

6522
    int64_t a = ntables / threads;
6523 6524 6525 6526
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
6527

6528
    int64_t b = 0;
6529 6530 6531
    if (threads != 0) {
      b = ntables % threads;
    }
6532

6533
    uint64_t startFrom = 0;
6534
    for (int i = 0; i < threads; i++) {
6535 6536
      threadInfo *pThreadInfo = infosOfSub + i;
      pThreadInfo->threadID = i;
6537

6538 6539 6540 6541 6542 6543
      pThreadInfo->start_table_from = startFrom;
      pThreadInfo->ntables = i<b?a+1:a;
      pThreadInfo->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = pThreadInfo->end_table_to + 1;
      pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection;
      pthread_create(pidsOfSub + i, NULL, superTableQuery, pThreadInfo);
6544 6545
    }

6546
    g_queryInfo.superQueryInfo.threadCnt = threads;
6547
  } else {
6548
    g_queryInfo.superQueryInfo.threadCnt = 0;
6549
  }
6550

6551 6552 6553 6554 6555 6556
  if ((nSqlCount > 0) && (nConcurrent > 0)) {
    for (int i = 0; i < nConcurrent; i++) {
      for (int j = 0; j < nSqlCount; j++) {
        pthread_join(pids[i * nSqlCount + j], NULL);
      }
    }
6557
  }
S
Shuaiqiang Chang 已提交
6558

6559
  tmfree((char*)pids);
6560
  tmfree((char*)infos);
6561

6562
  for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
6563
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
6564
  }
H
Hui Li 已提交
6565

6566
  tmfree((char*)pidsOfSub);
6567
  tmfree((char*)infosOfSub);
6568

6569
//  taos_close(taos);// TODO: workaround to use separate taos connection;
6570
  uint64_t endTs = taosGetTimestampMs();
6571

6572
  uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried +
6573 6574
    g_queryInfo.superQueryInfo.totalQueried;

6575
  fprintf(stderr, "==== completed total queries: %"PRIu64", the QPS of all threads: %10.3f====\n",
6576
          totalQueried,
6577
          (double)(totalQueried/((endTs-startTs)/1000.0)));
6578 6579 6580
  return 0;
}

6581 6582 6583 6584 6585 6586 6587 6588 6589
static void stable_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
  if (res == NULL || taos_errno(res) != 0) {
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
    return;
  }

  if (param)
6590
    fetchResult(res, (threadInfo *)param);
6591 6592 6593 6594 6595
  // tao_unscribe() will free result.
}

static void specified_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
6596
  if (res == NULL || taos_errno(res) != 0) {
6597 6598
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
6599 6600
    return;
  }
6601

6602
  if (param)
6603
    fetchResult(res, (threadInfo *)param);
6604
  // tao_unscribe() will free result.
H
hzcheng 已提交
6605 6606
}

6607
static TAOS_SUB* subscribeImpl(
6608
        QUERY_CLASS class,
6609
        threadInfo *pThreadInfo,
6610
        char *sql, char* topic, bool restart, uint64_t interval)
6611
{
6612
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
6613

6614 6615
  if ((SPECIFIED_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) {
6616 6617
    tsub = taos_subscribe(
            pThreadInfo->taos,
6618
            restart,
6619
            topic, sql, specified_sub_callback, (void*)pThreadInfo,
6620
            g_queryInfo.specifiedQueryInfo.subscribeInterval);
6621 6622
  } else if ((STABLE_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) {
6623 6624 6625 6626 6627
    tsub = taos_subscribe(
            pThreadInfo->taos,
            restart,
            topic, sql, stable_sub_callback, (void*)pThreadInfo,
            g_queryInfo.superQueryInfo.subscribeInterval);
6628
  } else {
6629 6630
    tsub = taos_subscribe(
            pThreadInfo->taos,
6631
            restart,
6632
            topic, sql, NULL, NULL, interval);
6633
  }
6634

6635 6636 6637
  if (tsub == NULL) {
    printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
    return NULL;
6638
  }
6639

6640 6641
  return tsub;
}
H
hzcheng 已提交
6642

6643 6644
static void *superSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6645
  char subSqlstr[MAX_QUERY_SQL_LENGTH];
6646
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT] = {0};
6647
  uint64_t tsubSeq;
H
hzcheng 已提交
6648

6649 6650
  if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
      errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n",
6651 6652 6653 6654 6655
              pThreadInfo->ntables,
              MAX_QUERY_SQL_COUNT);
      exit(-1);
  }

6656
  if (pThreadInfo->taos == NULL) {
6657
    pThreadInfo->taos = taos_connect(g_queryInfo.host,
6658 6659 6660 6661
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
6662
    if (pThreadInfo->taos == NULL) {
6663
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6664
            pThreadInfo->threadID, taos_errstr(NULL));
6665 6666 6667 6668
      return NULL;
    }
  }

6669 6670
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6671 6672
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6673 6674
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
6675 6676
    return NULL;
  }
6677

6678 6679 6680
  char topic[32] = {0};
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6681 6682

      tsubSeq = i - pThreadInfo->start_table_from;
6683 6684 6685 6686 6687
      verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n",
              __func__, __LINE__,
              pThreadInfo->threadID,
              pThreadInfo->start_table_from,
              pThreadInfo->end_table_to, i);
6688 6689
      sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"",
              i, pThreadInfo->querySeq);
6690
      memset(subSqlstr, 0, sizeof(subSqlstr));
6691 6692 6693 6694
      replaceChildTblName(
              g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq],
              subSqlstr, i);
      if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6695
        sprintf(pThreadInfo->filePath, "%s-%d",
6696 6697 6698 6699 6700 6701
                g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
      }

      debugPrint("%s() LN%d, [%d] subSqlstr: %s\n",
              __func__, __LINE__, pThreadInfo->threadID, subSqlstr);
6702
      tsub[tsubSeq] = subscribeImpl(
6703
              STABLE_CLASS,
6704
              pThreadInfo, subSqlstr, topic,
6705 6706
              g_queryInfo.superQueryInfo.subscribeRestart,
              g_queryInfo.superQueryInfo.subscribeInterval);
6707
      if (NULL == tsub[tsubSeq]) {
6708
        taos_close(pThreadInfo->taos);
6709 6710
        return NULL;
      }
6711
  }
6712 6713

  // start loop to consume result
6714 6715 6716 6717
  int consumed[MAX_QUERY_SQL_COUNT];
  for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) {
    consumed[i] = 0;
  }
6718
  TAOS_RES* res = NULL;
6719

6720 6721
  uint64_t st = 0, et = 0;

6722 6723 6724 6725
  while ((g_queryInfo.superQueryInfo.endAfterConsume == -1)
          || (g_queryInfo.superQueryInfo.endAfterConsume <
              consumed[pThreadInfo->end_table_to - pThreadInfo->start_table_from])) {

6726 6727
    for (uint64_t i = pThreadInfo->start_table_from;
            i <= pThreadInfo->end_table_to; i++) {
6728 6729 6730 6731
      tsubSeq = i - pThreadInfo->start_table_from;
      if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
          continue;
      }
6732

6733 6734
      st = taosGetTimestampMs();
      performancePrint("st: %"PRIu64" et: %"PRIu64" st-et: %"PRIu64"\n", st, et, (st - et));
6735
      res = taos_consume(tsub[tsubSeq]);
6736 6737 6738
      et = taosGetTimestampMs();
      performancePrint("st: %"PRIu64" et: %"PRIu64" delta: %"PRIu64"\n", st, et, (et - st));

6739 6740
      if (res) {
          if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6741
              sprintf(pThreadInfo->filePath, "%s-%d",
6742 6743
                      g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
6744
              fetchResult(res, pThreadInfo);
6745 6746
          }
          if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
6747
              sprintf(pThreadInfo->filePath, "%s-%d",
6748 6749
                      g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
6750
              fetchResult(res, pThreadInfo);
6751 6752 6753
          }
          consumed[tsubSeq] ++;

6754
          if ((g_queryInfo.superQueryInfo.resubAfterConsume != -1)
6755
                  && (consumed[tsubSeq] >=
6756
                      g_queryInfo.superQueryInfo.resubAfterConsume)) {
6757 6758 6759 6760
              printf("keepProgress:%d, resub super table query: %"PRIu64"\n",
                      g_queryInfo.superQueryInfo.subscribeKeepProgress,
                      pThreadInfo->querySeq);
              taos_unsubscribe(tsub[tsubSeq],
6761
                    g_queryInfo.superQueryInfo.subscribeKeepProgress);
6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774
              consumed[tsubSeq]= 0;
              tsub[tsubSeq] = subscribeImpl(
                      STABLE_CLASS,
                      pThreadInfo, subSqlstr, topic,
                      g_queryInfo.superQueryInfo.subscribeRestart,
                      g_queryInfo.superQueryInfo.subscribeInterval
                      );
              if (NULL == tsub[tsubSeq]) {
                  taos_close(pThreadInfo->taos);
                  return NULL;
              }
          }
      }
H
hzcheng 已提交
6775 6776
    }
  }
6777
  taos_free_result(res);
6778

6779 6780
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6781 6782
    tsubSeq = i - pThreadInfo->start_table_from;
    taos_unsubscribe(tsub[tsubSeq], 0);
6783
  }
6784

6785
  taos_close(pThreadInfo->taos);
H
hzcheng 已提交
6786 6787 6788
  return NULL;
}

6789 6790
static void *specifiedSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6791
//  TAOS_SUB*  tsub = NULL;
6792

6793
  if (pThreadInfo->taos == NULL) {
6794
    pThreadInfo->taos = taos_connect(g_queryInfo.host,
6795 6796 6797 6798
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
6799
    if (pThreadInfo->taos == NULL) {
6800
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6801
            pThreadInfo->threadID, taos_errstr(NULL));
6802 6803 6804 6805
      return NULL;
    }
  }

6806 6807
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6808 6809
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6810 6811
    return NULL;
  }
6812

6813 6814 6815 6816
  sprintf(g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
          "taosdemo-subscribe-%"PRIu64"-%d",
          pThreadInfo->querySeq,
          pThreadInfo->threadID);
6817
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != '\0') {
6818
      sprintf(pThreadInfo->filePath, "%s-%d",
6819
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
6820
                pThreadInfo->threadID);
6821
  }
6822 6823
  g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] = subscribeImpl(
          SPECIFIED_CLASS, pThreadInfo,
6824
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
6825
          g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
6826 6827
          g_queryInfo.specifiedQueryInfo.subscribeRestart,
          g_queryInfo.specifiedQueryInfo.subscribeInterval);
6828
  if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) {
6829 6830
      taos_close(pThreadInfo->taos);
      return NULL;
6831
  }
6832

6833
  // start loop to consume result
6834

6835
  g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0;
6836 6837 6838 6839
  while((g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq] == -1)
          || (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] <
              g_queryInfo.specifiedQueryInfo.endAfterConsume[pThreadInfo->querySeq])) {

6840
      if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) {
6841 6842
        continue;
      }
6843

6844 6845 6846
      g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID] = taos_consume(
              g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]);
      if (g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]) {
6847 6848
          if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0]
                  != 0) {
6849
              sprintf(pThreadInfo->filePath, "%s-%d",
6850 6851
                      g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                      pThreadInfo->threadID);
6852 6853 6854
              fetchResult(
                      g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID],
                      pThreadInfo);
6855 6856
          }

6857
          g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] ++;
6858
          if ((g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq] != -1)
6859
                && (g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] >=
6860 6861 6862 6863
                    g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
              printf("keepProgress:%d, resub specified query: %"PRIu64"\n",
                    g_queryInfo.specifiedQueryInfo.subscribeKeepProgress,
                    pThreadInfo->querySeq);
6864 6865
              g_queryInfo.specifiedQueryInfo.consumed[pThreadInfo->threadID] = 0;
              taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID],
6866
                      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
6867 6868 6869 6870 6871 6872 6873 6874
              g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID] =
                  subscribeImpl(
                          SPECIFIED_CLASS,
                          pThreadInfo,
                          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
                          g_queryInfo.specifiedQueryInfo.topic[pThreadInfo->threadID],
                          g_queryInfo.specifiedQueryInfo.subscribeRestart,
                          g_queryInfo.specifiedQueryInfo.subscribeInterval);
6875
              if (NULL == g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->threadID]) {
6876 6877
                  taos_close(pThreadInfo->taos);
                  return NULL;
6878 6879
              }
          }
6880 6881
      }
  }
6882 6883
  taos_free_result(g_queryInfo.specifiedQueryInfo.res[pThreadInfo->threadID]);
  taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[pThreadInfo->querySeq], 0);
6884
  taos_close(pThreadInfo->taos);
6885

H
hzcheng 已提交
6886 6887 6888
  return NULL;
}

6889
static int subscribeTestProcess() {
6890
  setupForAnsiEscape();
6891
  printfQueryMeta();
6892
  resetAfterAnsiEscape();
6893

6894
  prompt();
6895

6896 6897 6898 6899 6900
  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
6901
          g_queryInfo.port);
6902
  if (taos == NULL) {
6903 6904
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6905 6906 6907
    exit(-1);
  }

6908
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6909 6910
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6911 6912 6913
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6914 6915
  }

6916 6917
  taos_close(taos); // TODO: workaround to use separate taos connection;

6918 6919
  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
6920 6921 6922 6923 6924

  pthread_t  *pidsOfStable  = NULL;
  threadInfo *infosOfStable = NULL;

  //==== create threads for query for specified table
6925
  if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) {
6926
    debugPrint("%s() LN%d, sepcified query sqlCount %d.\n",
6927
              __func__, __LINE__,
6928 6929 6930
              g_queryInfo.specifiedQueryInfo.sqlCount);
  } else {
    if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) {
6931
        errorPrint("%s() LN%d, sepcified query sqlCount %d.\n",
6932 6933 6934 6935
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount);
        exit(-1);
    }
6936

6937
    pids  = malloc(
6938 6939 6940
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(pthread_t));
6941
    infos = malloc(
6942 6943 6944
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(threadInfo));
6945 6946 6947 6948
    if ((NULL == pids) || (NULL == infos)) {
        errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
        exit(-1);
    }
6949

6950 6951 6952
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
            uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
6953 6954 6955 6956 6957
            threadInfo *pThreadInfo = infos + seq;
            pThreadInfo->threadID = seq;
            pThreadInfo->querySeq = i;
            pThreadInfo->taos = NULL;  // TODO: workaround to use separate taos connection;
            pthread_create(pids + seq, NULL, specifiedSubscribe, pThreadInfo);
6958
        }
6959
    }
H
hzcheng 已提交
6960
  }
6961

6962
  //==== create threads for super table query
6963
  if (g_queryInfo.superQueryInfo.sqlCount <= 0) {
6964
    debugPrint("%s() LN%d, super table query sqlCount %d.\n",
6965
              __func__, __LINE__,
6966
              g_queryInfo.superQueryInfo.sqlCount);
6967 6968
  } else {
    if ((g_queryInfo.superQueryInfo.sqlCount > 0)
6969
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
6970 6971 6972
        pidsOfStable  = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
6973
            sizeof(pthread_t));
6974 6975 6976
        infosOfStable = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
6977
            sizeof(threadInfo));
6978 6979
        if ((NULL == pidsOfStable) || (NULL == infosOfStable)) {
            errorPrint("%s() LN%d, malloc failed for create threads\n",
6980
              __func__, __LINE__);
6981 6982 6983
            // taos_close(taos);
            exit(-1);
        }
6984

6985 6986
        int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
        int threads = g_queryInfo.superQueryInfo.threadCnt;
6987

6988 6989 6990 6991 6992
        int64_t a = ntables / threads;
        if (a < 1) {
            threads = ntables;
            a = 1;
        }
6993

6994 6995 6996 6997
        int64_t b = 0;
        if (threads != 0) {
            b = ntables % threads;
        }
6998

6999
        for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
7000
            uint64_t startFrom = 0;
7001 7002
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
7003 7004 7005 7006 7007 7008 7009 7010 7011
                threadInfo *pThreadInfo = infosOfStable + seq;
                pThreadInfo->threadID = seq;
                pThreadInfo->querySeq = i;

                pThreadInfo->start_table_from = startFrom;
                pThreadInfo->ntables = j<b?a+1:a;
                pThreadInfo->end_table_to = j<b?startFrom+a:startFrom+a-1;
                startFrom = pThreadInfo->end_table_to + 1;
                pThreadInfo->taos = NULL; // TODO: workaround to use separate taos connection;
7012
                pthread_create(pidsOfStable + seq,
7013
                        NULL, superSubscribe, pThreadInfo);
7014 7015
            }
        }
7016

7017
        g_queryInfo.superQueryInfo.threadCnt = threads;
7018

7019 7020 7021 7022 7023 7024
        for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                pthread_join(pidsOfStable[seq], NULL);
            }
        }
7025
    }
H
hzcheng 已提交
7026
  }
7027

7028 7029 7030 7031 7032
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
    for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
        uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
        pthread_join(pids[seq], NULL);
    }
7033
  }
H
hzcheng 已提交
7034

7035
  tmfree((char*)pids);
7036
  tmfree((char*)infos);
H
hzcheng 已提交
7037

7038 7039
  tmfree((char*)pidsOfStable);
  tmfree((char*)infosOfStable);
7040
//   taos_close(taos);
7041
  return 0;
H
hzcheng 已提交
7042 7043
}

7044
static void initOfInsertMeta() {
7045
  memset(&g_Dbs, 0, sizeof(SDbs));
7046

7047
  // set default values
7048
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7049
  g_Dbs.port = 6030;
7050 7051
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
7052
  g_Dbs.threadCount = 2;
7053 7054

  g_Dbs.use_metric = g_args.use_metric;
L
Liu Tao 已提交
7055 7056
}

7057
static void initOfQueryMeta() {
7058
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
7059

7060
  // set default values
7061
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7062
  g_queryInfo.port = 6030;
7063 7064
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
L
Liu Tao 已提交
7065 7066
}

7067
static void setParaFromArg(){
7068
  if (g_args.host) {
7069
    tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE);
7070
  } else {
7071
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7072
  }
L
Liu Tao 已提交
7073

7074
  if (g_args.user) {
7075
    tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE);
7076
  }
7077 7078

  if (g_args.password) {
7079
    tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE);
7080 7081
  }

7082 7083
  if (g_args.port) {
    g_Dbs.port = g_args.port;
7084
  }
L
Liu Tao 已提交
7085

7086 7087 7088
  g_Dbs.threadCount = g_args.num_of_threads;
  g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;

7089
  g_Dbs.dbCount = 1;
7090
  g_Dbs.db[0].drop = true;
7091

7092
  tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE);
7093
  g_Dbs.db[0].dbCfg.replica = g_args.replica;
7094
  tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
7095

7096
  tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
7097 7098 7099 7100 7101

  g_Dbs.use_metric = g_args.use_metric;
  g_Dbs.insert_only = g_args.insert_only;

  g_Dbs.do_aggreFunc = true;
L
Liu Tao 已提交
7102

7103 7104
  char dataString[STRING_LEN];
  char **data_type = g_args.datatype;
7105

7106
  memset(dataString, 0, STRING_LEN);
L
Liu Tao 已提交
7107

7108 7109
  if (strcasecmp(data_type[0], "BINARY") == 0
          || strcasecmp(data_type[0], "BOOL") == 0
7110
          || strcasecmp(data_type[0], "NCHAR") == 0 ) {
7111
    g_Dbs.do_aggreFunc = false;
L
Liu Tao 已提交
7112
  }
H
hzcheng 已提交
7113

7114 7115 7116 7117 7118
  if (g_args.use_metric) {
    g_Dbs.db[0].superTblCount = 1;
    tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE);
    g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
    g_Dbs.threadCount = g_args.num_of_threads;
7119
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
7120
    g_Dbs.asyncMode = g_args.async_mode;
7121

7122 7123 7124 7125
    g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
    g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
    g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
7126
    tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix,
7127 7128 7129
            g_args.tb_prefix, MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE);
7130
    tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp,
7131
            "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
7132
    g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
7133

7134
    g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
7135
    g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len;
7136

7137 7138 7139 7140 7141
    g_Dbs.db[0].superTbls[0].columnCount = 0;
    for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
      if (data_type[i] == NULL) {
        break;
      }
7142

7143
      tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
7144
              data_type[i], MAX_TB_NAME_SIZE);
7145
      g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;
7146 7147
      g_Dbs.db[0].superTbls[0].columnCount++;
    }
7148

7149 7150 7151
    if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
      g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
    } else {
7152 7153 7154 7155
      for (int i = g_Dbs.db[0].superTbls[0].columnCount;
              i < g_args.num_of_CPR; i++) {
        tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
                "INT", MAX_TB_NAME_SIZE);
7156
        g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;
7157 7158 7159
        g_Dbs.db[0].superTbls[0].columnCount++;
      }
    }
7160

7161 7162
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType,
            "INT", MAX_TB_NAME_SIZE);
7163
    g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;
7164

7165 7166
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType,
            "BINARY", MAX_TB_NAME_SIZE);
7167 7168
    g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;
    g_Dbs.db[0].superTbls[0].tagCount = 2;
7169
  } else {
7170 7171
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
    g_Dbs.db[0].superTbls[0].tagCount = 0;
H
hzcheng 已提交
7172
  }
7173 7174 7175 7176 7177 7178
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
7179

7180 7181 7182
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
7183 7184 7185
    exit(-1);
  }

7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

7210
static void querySqlFile(TAOS* taos, char* sqlFile)
7211 7212 7213 7214 7215 7216
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
7217

7218 7219 7220 7221 7222 7223
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

7224
  double t = taosGetTimestampMs();
7225

7226
  while((read_len = tgetline(&line, &line_len, fp)) != -1) {
7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
7239
    }
7240 7241

    memcpy(cmd + cmd_len, line, read_len);
7242 7243 7244
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) {
        errorPrint("%s() LN%d, queryDbExec %s failed!\n",
               __func__, __LINE__, cmd);
7245 7246 7247 7248 7249
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
7250 7251
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
7252 7253
  }

7254
  t = taosGetTimestampMs() - t;
7255
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
7256

7257 7258 7259 7260
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
7261 7262
}

7263
static void testMetaFile() {
7264
    if (INSERT_TEST == g_args.test_mode) {
7265 7266
      if (g_Dbs.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
7267

7268
      insertTestProcess();
7269

7270
    } else if (QUERY_TEST == g_args.test_mode) {
7271 7272
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7273

7274
      queryTestProcess();
7275

7276
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
7277 7278
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7279

7280
      subscribeTestProcess();
7281

7282 7283 7284
    }  else {
      ;
    }
7285
}
7286

7287
static void queryResult() {
7288
  // query data
7289

7290
  pthread_t read_id;
7291 7292 7293 7294
  threadInfo *pThreadInfo = malloc(sizeof(threadInfo));
  assert(pThreadInfo);
  pThreadInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
  pThreadInfo->start_table_from = 0;
7295

7296
  //pThreadInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
7297
  if (g_args.use_metric) {
7298 7299 7300 7301
    pThreadInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
    pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
    pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
    tstrncpy(pThreadInfo->tb_prefix,
7302 7303
          g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE);
  } else {
7304 7305 7306
    pThreadInfo->ntables = g_args.num_of_tables;
    pThreadInfo->end_table_to = g_args.num_of_tables -1;
    tstrncpy(pThreadInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE);
7307 7308
  }

7309
  pThreadInfo->taos = taos_connect(
7310 7311 7312 7313 7314
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
          g_Dbs.port);
7315
  if (pThreadInfo->taos == NULL) {
7316 7317
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
7318
    free(pThreadInfo);
7319 7320 7321
    exit(-1);
  }

7322
  tstrncpy(pThreadInfo->filePath, g_Dbs.resultFile, MAX_FILE_NAME_LEN);
7323 7324

  if (!g_Dbs.use_metric) {
7325
    pthread_create(&read_id, NULL, readTable, pThreadInfo);
7326
  } else {
7327
    pthread_create(&read_id, NULL, readMetric, pThreadInfo);
7328 7329
  }
  pthread_join(read_id, NULL);
7330 7331
  taos_close(pThreadInfo->taos);
  free(pThreadInfo);
7332 7333
}

7334 7335
static void testCmdLine() {

7336 7337 7338 7339 7340 7341 7342 7343 7344 7345
  if (strlen(configDir)) {
    wordexp_t full_path;
    if (wordexp(configDir, &full_path, 0) != 0) {
      errorPrint( "Invalid path %s\n", configDir);
      return;
    }
    taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
    wordfree(&full_path);
  }

7346 7347
  g_args.test_mode = INSERT_TEST;
  insertTestProcess();
7348

7349
  if (false == g_Dbs.insert_only)
7350
    queryResult();
7351 7352
}

7353 7354 7355
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

7356
  debugPrint("meta file: %s\n", g_args.metaFile);
7357 7358 7359

  if (g_args.metaFile) {
    initOfInsertMeta();
7360
    initOfQueryMeta();
7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371 7372 7373

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
7374 7375 7376 7377
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
7378
          g_Dbs.port);
7379
      querySqlFile(qtaos, g_args.sqlFile);
7380 7381 7382 7383 7384
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
7385 7386 7387

    if (g_dupstr)
        free(g_dupstr);
H
hzcheng 已提交
7388
  }
7389 7390

  return 0;
H
hzcheng 已提交
7391
}
7392