taosdemo.c 244.3 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

21
#include <stdint.h>
S
slguan 已提交
22
#define _GNU_SOURCE
23 24
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
41
#else
S
TD-1057  
Shengliang Guan 已提交
42 43
  #include <regex.h>
  #include <stdio.h>
44
#endif
H
hzcheng 已提交
45

46
#include <assert.h>
47
#include <stdlib.h>
48 49
#include "cJSON.h"

50
#include "os.h"
H
hzcheng 已提交
51
#include "taos.h"
52
#include "taoserror.h"
H
Hui Li 已提交
53
#include "tutil.h"
S
slguan 已提交
54

55 56 57
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
58 59
extern char configDir[];

60 61 62 63
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

64 65 66 67 68 69
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};
70

71 72
#define MAX_RECORDS_PER_REQ     32766

73 74
#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
75
#define COND_BUF_LEN        BUFFER_SIZE - 30
76 77
#define MAX_USERNAME_SIZE  64
#define MAX_PASSWORD_SIZE  64
78
#define MAX_DB_NAME_SIZE   64
79
#define MAX_HOSTNAME_SIZE  64
80
#define MAX_TB_NAME_SIZE   64
81
#define MAX_DATA_SIZE      (16*1024)+20     // max record len: 16*1024, timestamp string and ,('') need extra space
82 83 84 85 86 87 88 89 90 91
#define MAX_NUM_DATATYPE   10
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
#define MAX_FILE_NAME_LEN  256

#define   MAX_SAMPLES_ONCE_FROM_FILE   10000
#define   MAX_NUM_DATATYPE 10

#define   MAX_DB_COUNT           8
92
#define   MAX_SUPER_TABLE_COUNT  200
93 94 95
#define   MAX_COLUMN_COUNT       1024
#define   MAX_TAG_COUNT          128

96
#define   MAX_QUERY_SQL_COUNT    100
97
#define   MAX_QUERY_SQL_LENGTH   1024
98 99

#define   MAX_DATABASE_COUNT     256
100
#define INPUT_BUF_LEN   256
101

102
#define DEFAULT_TIMESTAMP_STEP  1
103

104

105 106 107 108 109
typedef enum CREATE_SUB_TALBE_MOD_EN {
  PRE_CREATE_SUBTBL,
  AUTO_CREATE_SUBTBL,
  NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
110

111 112
typedef enum TALBE_EXISTS_EN {
  TBL_NO_EXISTS,
113
  TBL_ALREADY_EXISTS,
114 115 116
  TBL_EXISTS_BUTT
} TALBE_EXISTS_EN;

117
enum enumSYNC_MODE {
118 119
  SYNC_MODE,
  ASYNC_MODE,
120 121
  MODE_BUT
};
122

S
Shuduo Sang 已提交
123
enum enum_TAOS_INTERFACE {
124 125 126
    TAOSC_IFACE,
    REST_IFACE,
    STMT_IFACE,
127 128 129
    INTERFACE_BUT
};

130 131 132 133 134 135
typedef enum enumQUERY_CLASS {
    SPECIFIED_CLASS,
    STABLE_CLASS,
    CLASS_BUT
} QUERY_CLASS;

S
Shuduo Sang 已提交
136
typedef enum enum_PROGRESSIVE_OR_INTERLACE {
137 138 139
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
140
} PROG_OR_INTERLACE_MODE;
141

142
typedef enum enumQUERY_TYPE {
143
  NO_INSERT_TYPE,
144
  INSERT_TYPE,
145
  QUERY_TYPE_BUT
146
} QUERY_TYPE;
147 148 149 150 151 152 153

enum _show_db_index {
  TSDB_SHOW_DB_NAME_INDEX,
  TSDB_SHOW_DB_CREATED_TIME_INDEX,
  TSDB_SHOW_DB_NTABLES_INDEX,
  TSDB_SHOW_DB_VGROUPS_INDEX,
  TSDB_SHOW_DB_REPLICA_INDEX,
154
  TSDB_SHOW_DB_QUORUM_INDEX,
155 156 157 158 159 160 161 162 163 164
  TSDB_SHOW_DB_DAYS_INDEX,
  TSDB_SHOW_DB_KEEP_INDEX,
  TSDB_SHOW_DB_CACHE_INDEX,
  TSDB_SHOW_DB_BLOCKS_INDEX,
  TSDB_SHOW_DB_MINROWS_INDEX,
  TSDB_SHOW_DB_MAXROWS_INDEX,
  TSDB_SHOW_DB_WALLEVEL_INDEX,
  TSDB_SHOW_DB_FSYNC_INDEX,
  TSDB_SHOW_DB_COMP_INDEX,
  TSDB_SHOW_DB_CACHELAST_INDEX,
165
  TSDB_SHOW_DB_PRECISION_INDEX,
166 167 168 169 170 171 172 173 174 175
  TSDB_SHOW_DB_UPDATE_INDEX,
  TSDB_SHOW_DB_STATUS_INDEX,
  TSDB_MAX_SHOW_DB
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
  TSDB_SHOW_STABLES_NAME_INDEX,
  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
  TSDB_SHOW_STABLES_COLUMNS_INDEX,
176 177
  TSDB_SHOW_STABLES_METRIC_INDEX,
  TSDB_SHOW_STABLES_UID_INDEX,
178
  TSDB_SHOW_STABLES_TID_INDEX,
179
  TSDB_SHOW_STABLES_VGID_INDEX,
180
  TSDB_MAX_SHOW_STABLES
181 182
};

183 184 185 186 187 188 189 190 191 192 193 194 195 196
enum _describe_table_index {
  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
  TSDB_MAX_DESCRIBE_METRIC
};

typedef struct {
  char field[TSDB_COL_NAME_LEN + 1];
  char type[16];
  int length;
  char note[128];
} SColDes;
H
hzcheng 已提交
197 198

/* Used by main to communicate with parse_opt. */
199 200
typedef struct SArguments_S {
  char *   metaFile;
201
  uint32_t test_mode;
202 203
  char *   host;
  uint16_t port;
S
Shuduo Sang 已提交
204
  uint16_t iface;
205 206 207 208 209 210 211
  char *   user;
  char *   password;
  char *   database;
  int      replica;
  char *   tb_prefix;
  char *   sqlFile;
  bool     use_metric;
212
  bool     drop_database;
213
  bool     insert_only;
214
  bool     answer_yes;
215
  bool     debug_print;
216
  bool     verbose_print;
217
  bool     performance_print;
218
  char *   output_file;
219
  bool     async_mode;
220
  char *   datatype[MAX_NUM_DATATYPE + 1];
221 222 223 224
  uint32_t len_of_binary;
  uint32_t num_of_CPR;
  uint32_t num_of_threads;
  uint64_t insert_interval;
225
  int64_t  query_times;
226 227 228
  uint64_t interlace_rows;
  uint64_t num_of_RPR;                  // num_of_records_per_req
  uint64_t max_sql_len;
229 230
  int64_t  num_of_tables;
  int64_t  num_of_DPT;
231
  int      abort;
232 233
  int      disorderRatio;               // 0: no disorder, >0: x%
  int      disorderRange;               // ms or us by database precision
234
  uint32_t method_of_delete;
235
  char **  arg_list;
236 237
  uint64_t totalInsertRows;
  uint64_t totalAffectedRows;
238 239 240
} SArguments;

typedef struct SColumn_S {
241 242 243
  char      field[TSDB_COL_NAME_LEN + 1];
  char      dataType[MAX_TB_NAME_SIZE];
  uint32_t  dataLen;
244
  char  note[128];
245 246 247
} StrColumn;

typedef struct SSuperTable_S {
248
  char         sTblName[MAX_TB_NAME_SIZE+1];
249
  int64_t      childTblCount;
250
  bool         childTblExists;          // 0: no, 1: yes
251 252
  uint64_t     batchCreateTableNum;     // 0: no batch,  > 0: batch table number in one sql
  uint8_t      autoCreateTable;         // 0: create sub table, 1: auto create sub table
253
  char         childTblPrefix[MAX_TB_NAME_SIZE];
254
  char         dataSource[MAX_TB_NAME_SIZE+1];  // rand_gen or sample
255
  uint16_t     insertMode;              // 0: taosc, 1: rest, 2: stmt
256
  int64_t      childTblLimit;
257
  uint64_t     childTblOffset;
258

259
//  int          multiThreadWriteOneTbl;  // 0: no, 1: yes
260
  uint64_t     interlaceRows;           //
261 262
  int          disorderRatio;           // 0: no disorder, >0: x%
  int          disorderRange;           // ms or us by database precision
263
  uint64_t     maxSqlLen;               //
264

265
  uint64_t     insertInterval;          // insert interval, will override global insert interval
266
  int64_t      insertRows;
267
  int64_t      timeStampStep;
268
  char         startTimestamp[MAX_TB_NAME_SIZE];
269
  char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
270 271
  char         sampleFile[MAX_FILE_NAME_LEN+1];
  char         tagsFile[MAX_FILE_NAME_LEN+1];
272

273
  uint32_t     columnCount;
274
  StrColumn    columns[MAX_COLUMN_COUNT];
275
  uint32_t     tagCount;
276 277 278
  StrColumn    tags[MAX_TAG_COUNT];

  char*        childTblName;
279
  char*        colsOfCreateChildTable;
280 281
  uint64_t     lenOfOneRow;
  uint64_t     lenOfTagOfOneRow;
282 283 284 285 286

  char*        sampleDataBuf;
  //int          sampleRowCount;
  //int          sampleUsePos;

287
  uint32_t     tagSource;    // 0: rand, 1: tag sample
288
  char*        tagDataBuf;
289 290
  uint32_t     tagSampleCount;
  uint32_t     tagUsePos;
291

292
  // statistics
293 294
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
295 296 297 298 299
} SSuperTable;

typedef struct {
  char     name[TSDB_DB_NAME_LEN + 1];
  char     create_time[32];
S
Shuduo Sang 已提交
300
  int64_t  ntables;
301
  int32_t  vgroups;
302 303
  int16_t  replica;
  int16_t  quorum;
304
  int16_t  days;
305 306 307 308 309 310 311 312 313 314 315 316 317 318
  char     keeplist[32];
  int32_t  cache; //MB
  int32_t  blocks;
  int32_t  minrows;
  int32_t  maxrows;
  int8_t   wallevel;
  int32_t  fsync;
  int8_t   comp;
  int8_t   cachelast;
  char     precision[8];   // time resolution
  int8_t   update;
  char     status[16];
} SDbInfo;

319
typedef struct SDbCfg_S {
320
//  int       maxtablesPerVnode;
321 322
  uint32_t  minRows;        // 0 means default
  uint32_t  maxRows;        // 0 means default
323 324
  int       comp;
  int       walLevel;
325
  int       cacheLast;
326
  int       fsync;
327 328 329 330 331 332 333
  int       replica;
  int       update;
  int       keep;
  int       days;
  int       cache;
  int       blocks;
  int       quorum;
334
  char      precision[MAX_TB_NAME_SIZE];
335 336 337 338
} SDbCfg;

typedef struct SDataBase_S {
  char         dbName[MAX_DB_NAME_SIZE];
339
  bool         drop;  // 0: use exists, 1: if exists, drop then new create
340
  SDbCfg       dbCfg;
341
  uint64_t     superTblCount;
342 343 344 345
  SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
} SDataBase;

typedef struct SDbs_S {
346
  char         cfgDir[MAX_FILE_NAME_LEN+1];
347
  char         host[MAX_HOSTNAME_SIZE];
348 349
  struct sockaddr_in serv_addr;

350
  uint16_t     port;
351 352
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
353
  char         resultFile[MAX_FILE_NAME_LEN+1];
354 355 356
  bool         use_metric;
  bool         insert_only;
  bool         do_aggreFunc;
357
  bool         asyncMode;
358

359 360 361
  uint32_t     threadCount;
  uint32_t     threadCountByCreateTbl;
  uint32_t     dbCount;
362 363 364
  SDataBase    db[MAX_DB_COUNT];

  // statistics
365 366
  uint64_t     totalInsertRows;
  uint64_t     totalAffectedRows;
367

368 369
} SDbs;

370
typedef struct SpecifiedQueryInfo_S {
371 372 373
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint64_t     concurrent;
  uint64_t     sqlCount;
374
  uint32_t     asyncMode; // 0: sync, 1: async
375 376
  uint64_t     subscribeInterval; // ms
  uint64_t     queryTimes;
377 378
  int          subscribeRestart;
  int          subscribeKeepProgress;
379
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
380
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
381
  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
382
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
383
  uint64_t     totalQueried;
384
} SpecifiedQueryInfo;
385

386
typedef struct SuperQueryInfo_S {
387
  char         sTblName[MAX_TB_NAME_SIZE+1];
388 389
  uint64_t     queryInterval;  // 0: unlimit  > 0   loop/s
  uint32_t     threadCnt;
390
  uint32_t     asyncMode; // 0: sync, 1: async
391
  uint64_t     subscribeInterval; // ms
392 393
  int          subscribeRestart;
  int          subscribeKeepProgress;
394
  uint64_t     queryTimes;
395
  int64_t      childTblCount;
396
  char         childTblPrefix[MAX_TB_NAME_SIZE];
397
  uint64_t     sqlCount;
398
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
399
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
400
  int          resubAfterConsume[MAX_QUERY_SQL_COUNT];
401
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
402

403
  char*        childTblName;
404
  uint64_t     totalQueried;
405
} SuperQueryInfo;
406 407

typedef struct SQueryMetaInfo_S {
408
  char         cfgDir[MAX_FILE_NAME_LEN+1];
409
  char         host[MAX_HOSTNAME_SIZE];
410
  uint16_t     port;
411
  struct       sockaddr_in serv_addr;
412 413
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
414
  char         dbName[MAX_DB_NAME_SIZE+1];
415
  char         queryMode[MAX_TB_NAME_SIZE];  // taosc, rest
416

417
  SpecifiedQueryInfo  specifiedQueryInfo;
418
  SuperQueryInfo      superQueryInfo;
419
  uint64_t     totalQueried;
420 421 422
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
423
  TAOS *    taos;
424
  TAOS_STMT *stmt;
425 426 427 428 429 430 431
  int       threadID;
  char      db_name[MAX_DB_NAME_SIZE+1];
  uint32_t  time_precision;
  char      fp[4096];
  char      tb_prefix[MAX_TB_NAME_SIZE];
  uint64_t  start_table_from;
  uint64_t  end_table_to;
S
Shuduo Sang 已提交
432
  int64_t   ntables;
433 434 435 436
  uint64_t  data_of_rate;
  int64_t   start_time;
  char*     cols;
  bool      use_metric;
437
  SSuperTable* superTblInfo;
S
Shuduo Sang 已提交
438
  char      *buffer;    // sql cmd buffer
439 440

  // for async insert
441 442
  tsem_t    lock_sem;
  int64_t   counter;
443 444
  uint64_t  st;
  uint64_t  et;
445
  uint64_t  lastTs;
446

447
  // sample data
448
  int64_t   samplePos;
449
  // statistics
450 451
  uint64_t  totalInsertRows;
  uint64_t  totalAffectedRows;
452 453

  // insert delay statistics
454 455 456 457 458
  uint64_t  cntDelay;
  uint64_t  totalDelay;
  uint64_t  avgDelay;
  uint64_t  maxDelay;
  uint64_t  minDelay;
459

460
  // seq of query or subscribe
461
  uint64_t  querySeq;   // sequence number of sql command
462

463 464
} threadInfo;

465
#ifdef WINDOWS
466 467
#define _CRT_RAND_S

468
#include <windows.h>
469 470 471 472 473
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
474 475
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
476 477
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
478 479 480 481

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

482
static void setupForAnsiEscape(void) {
483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
501
  }
502 503
}

504
static void resetAfterAnsiEscape(void) {
505
  // Reset colors
506
  printf("\x1b[0m");
507 508 509 510 511 512

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
513

514
static int taosRandom()
515 516 517 518 519 520
{
    int number;
    rand_s(&number);

    return number;
}
521
#else   // Not windows
522
static void setupForAnsiEscape(void) {}
523

524
static void resetAfterAnsiEscape(void) {
525 526 527
  // Reset colors
  printf("\x1b[0m");
}
528

529 530
#include <time.h>

531
static int taosRandom()
532
{
533
  return rand();
534 535
}

536
#endif // ifdef Windows
537

538
static int createDatabasesAndStables();
539
static void createChildTables();
540
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet);
541 542
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
        char* sqlstr, char *resultFile);
543 544 545 546 547 548 549

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
550
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)",
551 552
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

553
SArguments g_args = {
554 555 556 557
                     NULL,            // metaFile
                     0,               // test_mode
                     "127.0.0.1",     // host
                     6030,            // port
S
Shuduo Sang 已提交
558
                     TAOSC_IFACE,     // iface
559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590
                     "root",          // user
                     #ifdef _TD_POWER_
                     "powerdb",      // password
                     #else
                     "taosdata",      // password
                     #endif
                     "test",          // database
                     1,               // replica
                     "t",             // tb_prefix
                     NULL,            // sqlFile
                     true,            // use_metric
                     true,            // drop_database
                     true,            // insert_only
                     false,           // debug_print
                     false,           // verbose_print
                     false,           // performance statistic print
                     false,           // answer_yes;
                     "./output.txt",  // output_file
                     0,               // mode : sync or async
                     {
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     "INT",           // datatype
                     },
                     16,              // len_of_binary
                     4,               // num_of_CPR
                     10,              // num_of_connections/thread
                     0,               // insert_interval
                     1,               // query_times
                     0,               // interlace_rows;
                     30000,           // num_of_RPR
sangshuduo's avatar
sangshuduo 已提交
591
                     (1024*1024),         // max_sql_len
592 593 594 595 596 597 598
                     10000,           // num_of_tables
                     10000,           // num_of_DPT
                     0,               // abort
                     0,               // disorderRatio
                     1000,            // disorderRange
                     1,               // method_of_delete
                     NULL             // arg_list
599 600 601 602 603
};



static SDbs            g_Dbs;
604
static int64_t         g_totalChildTables = 0;
605 606 607 608
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

#define debugPrint(fmt, ...) \
609
    do { if (g_args.debug_print || g_args.verbose_print) \
610
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
611

612
#define verbosePrint(fmt, ...) \
613 614
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
615

616 617 618 619
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)

620 621 622 623
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)


624
///////////////////////////////////////////////////
H
hzcheng 已提交
625

626 627
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

628 629 630 631 632 633 634 635
#ifndef TAOSDEMO_COMMIT_SHA1
#define TAOSDEMO_COMMIT_SHA1 "unknown"
#endif

#ifndef TD_VERNUMBER
#define TD_VERNUMBER    "unknown"
#endif

636
#ifndef TAOSDEMO_STATUS
637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653
#define TAOSDEMO_STATUS "unknown"
#endif

static void printVersion() {
    char tdengine_ver[] = TD_VERNUMBER;
    char taosdemo_ver[] = TAOSDEMO_COMMIT_SHA1;
    char taosdemo_status[] = TAOSDEMO_STATUS;

    if (strlen(taosdemo_status) == 0) {
        printf("taosdemo verison %s-%s\n",
                tdengine_ver, taosdemo_ver);
    } else {
        printf("taosdemo verison %s-%s, status:%s\n",
                tdengine_ver, taosdemo_ver, taosdemo_status);
    }
}

654
static void printHelp() {
655
  char indent[10] = "        ";
656
  printf("%s%s%s%s\n", indent, "-f", indent,
657
          "The meta file to the execution procedure. Default is './meta.json'.");
658
  printf("%s%s%s%s\n", indent, "-u", indent,
659 660
          "The TDengine user name to use when connecting to the server. Default is 'root'.");
#ifdef _TD_POWER_
661
  printf("%s%s%s%s\n", indent, "-P", indent,
662
          "The password to use when connecting to the server. Default is 'powerdb'.");
663
  printf("%s%s%s%s\n", indent, "-c", indent,
664 665
          "Configuration directory. Default is '/etc/power/'.");
#else
666
  printf("%s%s%s%s\n", indent, "-P", indent,
667
          "The password to use when connecting to the server. Default is 'taosdata'.");
668
  printf("%s%s%s%s\n", indent, "-c", indent,
669
          "Configuration directory. Default is '/etc/taos/'.");
670
#endif
671
  printf("%s%s%s%s\n", indent, "-h", indent,
672
          "The host to connect to TDengine. Default is localhost.");
673
  printf("%s%s%s%s\n", indent, "-p", indent,
674
          "The TCP/IP port number to use for the connection. Default is 0.");
675 676
  printf("%s%s%s%s\n", indent, "-I", indent,
          "The interface (taosc, rest, and stmt) taosdemo uses. Default is 'taosc'.");
677
  printf("%s%s%s%s\n", indent, "-d", indent,
678
          "Destination database. Default is 'test'.");
679
  printf("%s%s%s%s\n", indent, "-a", indent,
680
          "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
681
  printf("%s%s%s%s\n", indent, "-m", indent,
682 683
          "Table prefix name. Default is 't'.");
  printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
684
  printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
685
  printf("%s%s%s%s\n", indent, "-o", indent,
686
          "Direct output to the named file. Default is './output.txt'.");
687
  printf("%s%s%s%s\n", indent, "-q", indent,
688
          "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC.");
689
  printf("%s%s%s%s\n", indent, "-b", indent,
sangshuduo's avatar
sangshuduo 已提交
690
          "The data_type of columns, default: INT,INT,INT,INT.");
691
  printf("%s%s%s%s\n", indent, "-w", indent,
692
          "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
693
  printf("%s%s%s%s\n", indent, "-l", indent,
sangshuduo's avatar
sangshuduo 已提交
694
          "The number of columns per record. Default is 4.");
695
  printf("%s%s%s%s\n", indent, "-T", indent,
696
          "The number of threads. Default is 10.");
697
  printf("%s%s%s%s\n", indent, "-i", indent,
698
          "The sleep time (ms) between insertion. Default is 0.");
699
  printf("%s%s%s%s\n", indent, "-r", indent,
700
          "The number of records per request. Default is 30000.");
701
  printf("%s%s%s%s\n", indent, "-t", indent,
702
          "The number of tables. Default is 10000.");
703
  printf("%s%s%s%s\n", indent, "-n", indent,
704 705 706
          "The number of records per table. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
  printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
707
  printf("%s%s%s%s\n", indent, "-O", indent,
708
          "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order.");
709
  printf("%s%s%s%s\n", indent, "-R", indent,
710
          "Out of order data's range, ms, default is 1000.");
711
  printf("%s%s%s%s\n", indent, "-g", indent,
712
          "Print debug info.");
713
  printf("%s%s%s\n", indent, "-V, --version\t",
714
          "Print version info.");
715 716
  printf("%s%s%s%s\n", indent, "--help\t", indent,
          "Print command line arguments list info.");
717
/*    printf("%s%s%s%s\n", indent, "-D", indent,
718 719 720 721
          "if elete database if exists. 0: no, 1: yes, default is 1");
          */
}

722 723 724 725 726 727 728 729 730 731 732 733 734 735 736
static bool isStringNumber(char *input)
{
  int len = strlen(input);
  if (0 == len) {
    return false;
  }

  for (int i = 0; i < len; i++) {
    if (!isdigit(input[i]))
      return false;
  }

  return true;
}

737
static void parse_args(int argc, char *argv[], SArguments *arguments) {
738
  char **sptr;
739 740 741 742 743

  for (int i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-f") == 0) {
      arguments->metaFile = argv[++i];
    } else if (strcmp(argv[i], "-c") == 0) {
H
Haojun Liao 已提交
744 745 746 747 748
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-c need a valid path following!\n");
        exit(EXIT_FAILURE);
      }
749
      tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN);
750
    } else if (strcmp(argv[i], "-h") == 0) {
H
Haojun Liao 已提交
751 752 753 754 755
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-h need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
756 757
      arguments->host = argv[++i];
    } else if (strcmp(argv[i], "-p") == 0) {
758
      if ((argc == i+1) ||
759
        (!isStringNumber(argv[i+1]))) {
760 761 762 763
        printHelp();
        errorPrint("%s", "\n\t-p need a number following!\n");
        exit(EXIT_FAILURE);
      }
764
      arguments->port = atoi(argv[++i]);
765 766 767 768 769 770 771 772
    } else if (strcmp(argv[i], "-I") == 0) {
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-I need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
      ++i;
      if (0 == strcasecmp(argv[i], "taosc")) {
S
Shuduo Sang 已提交
773
          arguments->iface = TAOSC_IFACE;
774
      } else if (0 == strcasecmp(argv[i], "rest")) {
S
Shuduo Sang 已提交
775
          arguments->iface = REST_IFACE;
776
      } else if (0 == strcasecmp(argv[i], "stmt")) {
S
Shuduo Sang 已提交
777
          arguments->iface = STMT_IFACE;
778 779 780 781
      } else {
        errorPrint("%s", "\n\t-I need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
782
    } else if (strcmp(argv[i], "-u") == 0) {
H
Haojun Liao 已提交
783 784 785 786 787
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-u need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
788 789
      arguments->user = argv[++i];
    } else if (strcmp(argv[i], "-P") == 0) {
H
Haojun Liao 已提交
790 791 792 793 794
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-P need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
795 796
      arguments->password = argv[++i];
    } else if (strcmp(argv[i], "-o") == 0) {
H
Haojun Liao 已提交
797 798 799 800 801
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-o need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
802 803
      arguments->output_file = argv[++i];
    } else if (strcmp(argv[i], "-s") == 0) {
H
Haojun Liao 已提交
804 805 806 807 808
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-s need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
809 810
      arguments->sqlFile = argv[++i];
    } else if (strcmp(argv[i], "-q") == 0) {
811 812
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
813
        printHelp();
814
        errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n");
815 816
        exit(EXIT_FAILURE);
      }
817
      arguments->async_mode = atoi(argv[++i]);
818
    } else if (strcmp(argv[i], "-T") == 0) {
819 820
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
821 822 823 824
        printHelp();
        errorPrint("%s", "\n\t-T need a number following!\n");
        exit(EXIT_FAILURE);
      }
825 826
      arguments->num_of_threads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
827 828
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
829 830 831 832
        printHelp();
        errorPrint("%s", "\n\t-i need a number following!\n");
        exit(EXIT_FAILURE);
      }
833
      arguments->insert_interval = atoi(argv[++i]);
834
    } else if (strcmp(argv[i], "-qt") == 0) {
835
      if ((argc == i+1)
836
              || (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
837
        printHelp();
838
        errorPrint("%s", "\n\t-qt need a number following!\n");
H
Haojun Liao 已提交
839 840
        exit(EXIT_FAILURE);
      }
841
      arguments->query_times = atoi(argv[++i]);
842
    } else if (strcmp(argv[i], "-B") == 0) {
843 844
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
845 846 847 848
        printHelp();
        errorPrint("%s", "\n\t-B need a number following!\n");
        exit(EXIT_FAILURE);
      }
849
      arguments->interlace_rows = atoi(argv[++i]);
850
    } else if (strcmp(argv[i], "-r") == 0) {
851 852
      if ((argc == i+1)
              || (!isStringNumber(argv[i+1]))) {
853 854 855 856
        printHelp();
        errorPrint("%s", "\n\t-r need a number following!\n");
        exit(EXIT_FAILURE);
      }
857 858
      arguments->num_of_RPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-t") == 0) {
859 860 861 862 863 864
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-t need a number following!\n");
        exit(EXIT_FAILURE);
      }
865 866
      arguments->num_of_tables = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
867 868 869 870 871 872
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-n need a number following!\n");
        exit(EXIT_FAILURE);
      }
873 874
      arguments->num_of_DPT = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-d") == 0) {
H
Haojun Liao 已提交
875 876 877 878 879
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-d need a valid string following!\n");
        exit(EXIT_FAILURE);
      }
880 881
      arguments->database = argv[++i];
    } else if (strcmp(argv[i], "-l") == 0) {
882 883 884 885 886 887
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-l need a number following!\n");
        exit(EXIT_FAILURE);
      }
888 889
      arguments->num_of_CPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-b") == 0) {
890 891 892 893 894
      if (argc == i+1) {
        printHelp();
        errorPrint("%s", "\n\t-b need valid string following!\n");
        exit(EXIT_FAILURE);
      }
895
      sptr = arguments->datatype;
896 897
      ++i;
      if (strstr(argv[i], ",") == NULL) {
898
        // only one col
899
        if (strcasecmp(argv[i], "INT")
900 901 902 903 904 905 906 907
                && strcasecmp(argv[i], "FLOAT")
                && strcasecmp(argv[i], "TINYINT")
                && strcasecmp(argv[i], "BOOL")
                && strcasecmp(argv[i], "SMALLINT")
                && strcasecmp(argv[i], "BIGINT")
                && strcasecmp(argv[i], "DOUBLE")
                && strcasecmp(argv[i], "BINARY")
                && strcasecmp(argv[i], "NCHAR")) {
908
          printHelp();
909
          errorPrint("%s", "-b: Invalid data_type!\n");
910
          exit(EXIT_FAILURE);
911
        }
912
        sptr[0] = argv[i];
913
      } else {
914
        // more than one col
915
        int index = 0;
916
        char *dupstr = strdup(argv[i]);
917 918
        char *running = dupstr;
        char *token = strsep(&running, ",");
919
        while(token != NULL) {
920
          if (strcasecmp(token, "INT")
921 922 923 924 925 926 927 928
                  && strcasecmp(token, "FLOAT")
                  && strcasecmp(token, "TINYINT")
                  && strcasecmp(token, "BOOL")
                  && strcasecmp(token, "SMALLINT")
                  && strcasecmp(token, "BIGINT")
                  && strcasecmp(token, "DOUBLE")
                  && strcasecmp(token, "BINARY")
                  && strcasecmp(token, "NCHAR")) {
929
            printHelp();
930
            free(dupstr);
931
            errorPrint("%s", "-b: Invalid data_type!\n");
932
            exit(EXIT_FAILURE);
H
hzcheng 已提交
933
          }
934 935 936
          sptr[index++] = token;
          token = strsep(&running, ",");
          if (index >= MAX_NUM_DATATYPE) break;
S
TD-1057  
Shengliang Guan 已提交
937
        }
938
        free(dupstr);
939
        sptr[index] = NULL;
940
      }
941
    } else if (strcmp(argv[i], "-w") == 0) {
942
      if ((argc == i+1) ||
943
        (!isStringNumber(argv[i+1]))) {
944 945 946 947
        printHelp();
        errorPrint("%s", "\n\t-w need a number following!\n");
        exit(EXIT_FAILURE);
      }
948 949
      arguments->len_of_binary = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-m") == 0) {
H
Haojun Liao 已提交
950
      if ((argc == i+1) ||
951
        (!isStringNumber(argv[i+1]))) {
H
Haojun Liao 已提交
952 953 954 955
        printHelp();
        errorPrint("%s", "\n\t-m need a number following!\n");
        exit(EXIT_FAILURE);
      }
956
      arguments->tb_prefix = argv[++i];
957 958
    } else if (strcmp(argv[i], "-N") == 0) {
      arguments->use_metric = false;
959
    } else if (strcmp(argv[i], "-x") == 0) {
960
      arguments->insert_only = false;
961
    } else if (strcmp(argv[i], "-y") == 0) {
962
      arguments->answer_yes = true;
963 964
    } else if (strcmp(argv[i], "-g") == 0) {
      arguments->debug_print = true;
965 966
    } else if (strcmp(argv[i], "-gg") == 0) {
      arguments->verbose_print = true;
967 968
    } else if (strcmp(argv[i], "-pp") == 0) {
      arguments->performance_print = true;
969
    } else if (strcmp(argv[i], "-O") == 0) {
970 971 972 973 974 975
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-O need a number following!\n");
        exit(EXIT_FAILURE);
      }
976

977
      arguments->disorderRatio = atoi(argv[++i]);
978

979
      if (arguments->disorderRatio > 50) {
980
        arguments->disorderRatio = 50;
981
      }
982

983
      if (arguments->disorderRatio < 0) {
984
        arguments->disorderRatio = 0;
985
      }
986

987
    } else if (strcmp(argv[i], "-R") == 0) {
988 989 990 991 992 993
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-R need a number following!\n");
        exit(EXIT_FAILURE);
      }
994

995
      arguments->disorderRange = atoi(argv[++i]);
996 997 998
      if (arguments->disorderRange < 0)
        arguments->disorderRange = 1000;

999
    } else if (strcmp(argv[i], "-a") == 0) {
1000 1001 1002 1003 1004 1005
      if ((argc == i+1) ||
        (!isStringNumber(argv[i+1]))) {
        printHelp();
        errorPrint("%s", "\n\t-a need a number following!\n");
        exit(EXIT_FAILURE);
      }
1006 1007 1008
      arguments->replica = atoi(argv[++i]);
      if (arguments->replica > 3 || arguments->replica < 1) {
          arguments->replica = 1;
1009
      }
1010 1011
    } else if (strcmp(argv[i], "-D") == 0) {
      arguments->method_of_delete = atoi(argv[++i]);
1012 1013 1014
      if (arguments->method_of_delete > 3) {
        errorPrint("%s", "\n\t-D need a valud (0~3) number following!\n");
        exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1015
      }
1016 1017 1018 1019
    } else if ((strcmp(argv[i], "--version") == 0) ||
        (strcmp(argv[i], "-V") == 0)){
      printVersion();
      exit(0);
1020 1021 1022 1023 1024
    } else if (strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else {
      printHelp();
1025
      errorPrint("%s", "ERROR: wrong options\n");
1026
      exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
1027 1028
    }
  }
1029

1030
  if (((arguments->debug_print) && (arguments->metaFile == NULL))
1031
          || arguments->verbose_print) {
1032
    printf("###################################################################\n");
1033
    printf("# meta file:                         %s\n", arguments->metaFile);
1034
    printf("# Server IP:                         %s:%hu\n",
1035 1036
            arguments->host == NULL ? "localhost" : arguments->host,
            arguments->port );
1037 1038
    printf("# User:                              %s\n", arguments->user);
    printf("# Password:                          %s\n", arguments->password);
1039 1040
    printf("# Use metric:                        %s\n",
            arguments->use_metric ? "true" : "false");
1041
    if (*(arguments->datatype)) {
1042 1043 1044 1045 1046 1047 1048
        printf("# Specified data type:               ");
        for (int i = 0; i < MAX_NUM_DATATYPE; i++)
            if (arguments->datatype[i])
               printf("%s,", arguments->datatype[i]);
            else
                break;
        printf("\n");
1049
    }
1050
    printf("# Insertion interval:                %"PRIu64"\n",
1051
            arguments->insert_interval);
1052
    printf("# Number of records per req:         %"PRIu64"\n",
1053
            arguments->num_of_RPR);
1054
    printf("# Max SQL length:                    %"PRIu64"\n",
1055
            arguments->max_sql_len);
1056
    printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
1057
    printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
1058
    printf("# Number of Tables:                  %"PRId64"\n",
1059
            arguments->num_of_tables);
1060
    printf("# Number of Data per Table:          %"PRId64"\n",
1061
            arguments->num_of_DPT);
1062 1063 1064 1065 1066
    printf("# Database name:                     %s\n", arguments->database);
    printf("# Table prefix:                      %s\n", arguments->tb_prefix);
    if (arguments->disorderRatio) {
      printf("# Data order:                        %d\n", arguments->disorderRatio);
      printf("# Data out of order rate:            %d\n", arguments->disorderRange);
1067

1068 1069 1070 1071
    }
    printf("# Delete method:                     %d\n", arguments->method_of_delete);
    printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
    printf("# Print debug info:                  %d\n", arguments->debug_print);
1072
    printf("# Print verbose info:                %d\n", arguments->verbose_print);
1073
    printf("###################################################################\n");
1074 1075 1076 1077
    if (!arguments->answer_yes) {
        printf("Press enter key to continue\n\n");
        (void) getchar();
    }
1078
  }
1079
}
H
hzcheng 已提交
1080

1081 1082 1083 1084
static bool getInfoFromJsonFile(char* file);
//static int generateOneRowDataForStb(SSuperTable* stbInfo);
//static int getDataIntoMemForStb(SSuperTable* stbInfo);
static void init_rand_data();
1085
static void tmfclose(FILE *fp) {
1086 1087 1088 1089
  if (NULL != fp) {
    fclose(fp);
  }
}
1090

1091
static void tmfree(char *buf) {
1092 1093
  if (NULL != buf) {
    free(buf);
H
Hui Li 已提交
1094
  }
1095
}
H
Hui Li 已提交
1096

1097
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
1098 1099 1100
  int i;
  TAOS_RES *res = NULL;
  int32_t   code = -1;
1101

S
Shuduo Sang 已提交
1102
  for (i = 0; i < 5 /* retry */; i++) {
1103 1104 1105
    if (NULL != res) {
      taos_free_result(res);
      res = NULL;
H
hzcheng 已提交
1106
    }
1107

1108 1109 1110 1111
    res = taos_query(taos, command);
    code = taos_errno(res);
    if (0 == code) {
      break;
1112
    }
H
hzcheng 已提交
1113 1114
  }

1115
  if (code != 0) {
1116 1117
    if (!quiet) {
      debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
1118
      errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res));
1119
    }
1120 1121 1122
    taos_free_result(res);
    //taos_close(taos);
    return -1;
1123
  }
H
hzcheng 已提交
1124

1125 1126 1127 1128
  if (INSERT_TYPE == type) {
    int affectedRows = taos_affected_rows(res);
    taos_free_result(res);
    return affectedRows;
H
hzcheng 已提交
1129
  }
1130

1131
  taos_free_result(res);
1132 1133
  return 0;
}
H
hzcheng 已提交
1134

1135 1136
static void appendResultBufToFile(char *resultBuf, char *resultFile)
{
1137
  FILE *fp = NULL;
1138 1139
  if (resultFile[0] != 0) {
    fp = fopen(resultFile, "at");
1140
    if (fp == NULL) {
1141 1142 1143 1144
      errorPrint(
              "%s() LN%d, failed to open result file: %s, result will not save to file\n",
              __func__, __LINE__, resultFile);
      return;
1145 1146
    }
  }
1147

1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
  fprintf(fp, "%s", resultBuf);
  tmfclose(fp);
}

static void appendResultToFile(TAOS_RES *res, char* resultFile) {
  TAOS_ROW    row = NULL;
  int         num_rows = 0;
  int         num_fields = taos_field_count(res);
  TAOS_FIELD *fields     = taos_fetch_fields(res);

1158 1159
  char* databuf = (char*) calloc(1, 100*1024*1024);
  if (databuf == NULL) {
1160 1161
    errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n",
            __func__, __LINE__);
1162 1163
    return ;
  }
H
hzcheng 已提交
1164

1165 1166
  int   totalLen = 0;
  char  temp[16000];
H
hzcheng 已提交
1167

1168
  // fetch the records row by row
1169
  while((row = taos_fetch_row(res))) {
1170
    if (totalLen >= 100*1024*1024 - 32000) {
1171
      appendResultBufToFile(databuf, resultFile);
1172 1173
      totalLen = 0;
      memset(databuf, 0, 100*1024*1024);
H
hzcheng 已提交
1174
    }
1175 1176 1177 1178 1179 1180
    num_rows++;
    int len = taos_print_row(temp, row, fields, num_fields);
    len += sprintf(temp + len, "\n");
    //printf("query result:%s\n", temp);
    memcpy(databuf + totalLen, temp, len);
    totalLen += len;
H
hzcheng 已提交
1181 1182
  }

S
Shengliang Guan 已提交
1183
  verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", __func__, __LINE__, databuf, resultFile);
1184
  appendResultBufToFile(databuf, resultFile);
1185 1186
  free(databuf);
}
H
hzcheng 已提交
1187

1188
static void selectAndGetResult(
1189 1190
        threadInfo *pThreadInfo, char *command)
{
1191 1192 1193 1194 1195 1196 1197 1198 1199
  if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) {
    TAOS_RES *res = taos_query(pThreadInfo->taos, command);
    if (res == NULL || taos_errno(res) != 0) {
        errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n",
            __func__, __LINE__, command, taos_errstr(res));
        taos_free_result(res);
        return;
    }

1200 1201
    if ((strlen(pThreadInfo->fp))) {
      appendResultToFile(res, pThreadInfo->fp);
1202
    }
1203
    taos_free_result(res);
1204

1205 1206 1207
  } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
      int retCode = postProceSql(
              g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port,
1208
              command,
1209
              pThreadInfo->fp);
1210 1211 1212 1213 1214 1215 1216 1217
      if (0 != retCode) {
        printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID);
      }

  } else {
      errorPrint("%s() LN%d, unknown query mode: %s\n",
        __func__, __LINE__, g_queryInfo.queryMode);
  }
1218
}
H
hzcheng 已提交
1219

1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258
static int32_t rand_bool(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 2;
}

static int32_t rand_tinyint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 128;
}

static int32_t rand_smallint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 32767;
}

static int32_t rand_int(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor];
}

static int64_t rand_bigint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randbigint[cursor];
}

static float rand_float(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
1259
  return randfloat[cursor];
1260 1261
}

1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
#if 0
static const char charNum[] = "0123456789";

static void nonrand_string(char *, int) __attribute__ ((unused));   // reserve for debugging purpose
static void nonrand_string(char *str, int size)
{
  str[0] = 0;
  if (size > 0) {
    int n;
    for (n = 0; n < size; n++) {
      str[n] = charNum[n % 10];
    }
    str[n] = 0;
  }
}
#endif

1279
static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1280

1281
static void rand_string(char *str, int size) {
1282 1283 1284 1285
  str[0] = 0;
  if (size > 0) {
    //--size;
    int n;
1286
    for (n = 0; n < size; n++) {
1287
      int key = abs(rand_tinyint()) % (int)(sizeof(charset) - 1);
1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
      str[n] = charset[key];
    }
    str[n] = 0;
  }
}

static double rand_double() {
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randdouble[cursor];

}

static void init_rand_data() {
  for (int i = 0; i < MAX_PREPARED_RAND; i++){
1304 1305 1306 1307
    randint[i] = (int)(taosRandom() % 65535);
    randbigint[i] = (int64_t)(taosRandom() % 2147483648);
    randfloat[i] = (float)(taosRandom() / 1000.0);
    randdouble[i] = (double)(taosRandom() / 1000000.0);
1308 1309 1310
  }
}

1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1331
static int printfInsertMeta() {
1332
    SHOW_PARSE_RESULT_START();
1333

1334
  printf("interface:                  \033[33m%s\033[0m\n",
S
Shuduo Sang 已提交
1335
          (g_args.iface==TAOSC_IFACE)?"taosc":(g_args.iface==REST_IFACE)?"rest":"stmt");
1336 1337 1338
  printf("host:                       \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port);
  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
1339
  printf("configDir:                  \033[33m%s\033[0m\n", configDir);
1340 1341 1342
  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
  printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl);
1343
  printf("top insert interval:        \033[33m%"PRIu64"\033[0m\n",
1344
          g_args.insert_interval);
1345
  printf("number of records per req:  \033[33m%"PRIu64"\033[0m\n",
1346
          g_args.num_of_RPR);
1347
  printf("max sql length:             \033[33m%"PRIu64"\033[0m\n",
1348
          g_args.max_sql_len);
1349 1350

  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
1351

1352 1353
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    printf("database[\033[33m%d\033[0m]:\n", i);
1354
    printf("  database[%d] name:      \033[33m%s\033[0m\n", i, g_Dbs.db[i].dbName);
1355
    if (0 == g_Dbs.db[i].drop) {
1356 1357 1358
      printf("  drop:                  \033[33mno\033[0m\n");
    } else {
      printf("  drop:                  \033[33myes\033[0m\n");
1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      printf("  blocks:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      printf("  cache:                 \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      printf("  days:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      printf("  keep:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      printf("  replica:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      printf("  update:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      printf("  minRows:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      printf("  maxRows:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      printf("  walLevel:              \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      printf("  fsync:                 \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      printf("  quorum:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1398
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
1399
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1400
        printf("  precision:             \033[33m%s\033[0m\n",
1401
            g_Dbs.db[i].dbCfg.precision);
1402
      } else {
1403
        printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
1404
                g_Dbs.db[i].dbCfg.precision);
1405
        return -1;
1406 1407 1408
      }
    }

1409
    printf("  super table count:     \033[33m%"PRIu64"\033[0m\n",
1410
        g_Dbs.db[i].superTblCount);
1411 1412
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      printf("  super table[\033[33m%"PRIu64"\033[0m]:\n", j);
1413

1414
      printf("      stbName:           \033[33m%s\033[0m\n",
1415
          g_Dbs.db[i].superTbls[j].sTblName);
1416 1417 1418 1419 1420 1421 1422 1423

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
      }
1424

1425 1426 1427 1428 1429 1430 1431
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
      }
1432

1433
      printf("      childTblCount:     \033[33m%"PRId64"\033[0m\n",
1434
              g_Dbs.db[i].superTbls[j].childTblCount);
1435
      printf("      childTblPrefix:    \033[33m%s\033[0m\n",
1436
              g_Dbs.db[i].superTbls[j].childTblPrefix);
1437
      printf("      dataSource:        \033[33m%s\033[0m\n",
1438
              g_Dbs.db[i].superTbls[j].dataSource);
1439
      printf("      insertMode:        \033[33m%s\033[0m\n",
1440 1441
          (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc":
          (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt");
1442
      if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
1443
        printf("      childTblLimit:     \033[33m%"PRId64"\033[0m\n",
1444
                g_Dbs.db[i].superTbls[j].childTblLimit);
1445
      }
1446
      if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) {
1447
        printf("      childTblOffset:    \033[33m%"PRIu64"\033[0m\n",
1448
                g_Dbs.db[i].superTbls[j].childTblOffset);
1449
      }
1450
      printf("      insertRows:        \033[33m%"PRId64"\033[0m\n",
1451
              g_Dbs.db[i].superTbls[j].insertRows);
1452
/*
1453
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1454
        printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
1455
      }else {
1456
        printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
1457
      }
1458
      */
1459
      printf("      interlaceRows:     \033[33m%"PRIu64"\033[0m\n",
1460
              g_Dbs.db[i].superTbls[j].interlaceRows);
1461 1462

      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1463
        printf("      stable insert interval:   \033[33m%"PRIu64"\033[0m\n",
1464
            g_Dbs.db[i].superTbls[j].insertInterval);
1465 1466
      }

1467
      printf("      disorderRange:     \033[33m%d\033[0m\n",
1468
              g_Dbs.db[i].superTbls[j].disorderRange);
1469
      printf("      disorderRatio:     \033[33m%d\033[0m\n",
1470
              g_Dbs.db[i].superTbls[j].disorderRatio);
1471
      printf("      maxSqlLen:         \033[33m%"PRIu64"\033[0m\n",
1472
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1473
      printf("      timeStampStep:     \033[33m%"PRId64"\033[0m\n",
1474
              g_Dbs.db[i].superTbls[j].timeStampStep);
1475
      printf("      startTimestamp:    \033[33m%s\033[0m\n",
1476
              g_Dbs.db[i].superTbls[j].startTimestamp);
1477
      printf("      sampleFormat:      \033[33m%s\033[0m\n",
1478
              g_Dbs.db[i].superTbls[j].sampleFormat);
1479
      printf("      sampleFile:        \033[33m%s\033[0m\n",
1480
              g_Dbs.db[i].superTbls[j].sampleFile);
1481
      printf("      tagsFile:          \033[33m%s\033[0m\n",
1482
              g_Dbs.db[i].superTbls[j].tagsFile);
1483
      printf("      columnCount:       \033[33m%d\033[0m\n",
1484
              g_Dbs.db[i].superTbls[j].columnCount);
1485 1486
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1487
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
1488 1489 1490
                       "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "nchar", 5))) {
1491
          printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
1492 1493
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1494
        } else {
1495
          printf("column[%d]:\033[33m%s\033[0m ", k,
1496
                  g_Dbs.db[i].superTbls[j].columns[k].dataType);
1497 1498 1499
        }
      }
      printf("\n");
1500 1501

      printf("      tagCount:            \033[33m%d\033[0m\n        ",
1502
              g_Dbs.db[i].superTbls[j].tagCount);
1503 1504
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1505
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
1506 1507 1508
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1509
          printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
1510 1511
                  g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1512
        } else {
1513
          printf("tag[%d]:\033[33m%s\033[0m ", k,
1514
                  g_Dbs.db[i].superTbls[j].tags[k].dataType);
1515
        }
1516 1517 1518 1519 1520
      }
      printf("\n");
    }
    printf("\n");
  }
1521 1522

  SHOW_PARSE_RESULT_END();
1523 1524

  return 0;
1525 1526 1527
}

static void printfInsertMetaToFile(FILE* fp) {
1528 1529

  SHOW_PARSE_RESULT_START_TO_FILE(fp);
1530

1531 1532
  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
  fprintf(fp, "user:                       %s\n", g_Dbs.user);
1533
  fprintf(fp, "configDir:                  %s\n", configDir);
1534 1535 1536
  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
1537 1538
  fprintf(fp, "number of records per req:  %"PRIu64"\n", g_args.num_of_RPR);
  fprintf(fp, "max sql length:             %"PRIu64"\n", g_args.max_sql_len);
1539
  fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
1540

1541 1542
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    fprintf(fp, "database[%d]:\n", i);
1543
    fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
1544
    if (0 == g_Dbs.db[i].drop) {
1545
      fprintf(fp, "  drop:                  no\n");
1546
    }else {
1547
      fprintf(fp, "  drop:                  yes\n");
1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1587
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
1588
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1589 1590 1591 1592 1593 1594
        fprintf(fp, "  precision:             %s\n", g_Dbs.db[i].dbCfg.precision);
      } else {
        fprintf(fp, "  precision error:       %s\n", g_Dbs.db[i].dbCfg.precision);
      }
    }

1595
    fprintf(fp, "  super table count:     %"PRIu64"\n", g_Dbs.db[i].superTblCount);
S
Shuduo Sang 已提交
1596 1597
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      fprintf(fp, "  super table[%"PRIu64"]:\n", j);
1598 1599

      fprintf(fp, "      stbName:           %s\n",  g_Dbs.db[i].superTbls[j].sTblName);
1600 1601 1602 1603 1604 1605 1606 1607

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
      } else {
        fprintf(fp, "      autoCreateTable:   %s\n",  "error");
      }
1608

1609 1610 1611 1612 1613 1614 1615
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "yes");
      } else {
        fprintf(fp, "      childTblExists:    %s\n",  "error");
      }
1616

1617
      fprintf(fp, "      childTblCount:     %"PRId64"\n",
1618 1619 1620 1621 1622 1623
              g_Dbs.db[i].superTbls[j].childTblCount);
      fprintf(fp, "      childTblPrefix:    %s\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      fprintf(fp, "      dataSource:        %s\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      fprintf(fp, "      insertMode:        %s\n",
1624 1625
          (g_Dbs.db[i].superTbls[j].insertMode==TAOSC_IFACE)?"taosc":
          (g_Dbs.db[i].superTbls[j].insertMode==REST_IFACE)?"rest":"stmt");
1626
      fprintf(fp, "      insertRows:        %"PRId64"\n",
1627
              g_Dbs.db[i].superTbls[j].insertRows);
1628
      fprintf(fp, "      interlace rows:    %"PRIu64"\n",
1629
              g_Dbs.db[i].superTbls[j].interlaceRows);
1630
      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1631
        fprintf(fp, "      stable insert interval:   %"PRIu64"\n",
1632
                g_Dbs.db[i].superTbls[j].insertInterval);
1633
      }
1634
/*
1635
      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1636
        fprintf(fp, "      multiThreadWriteOneTbl:  no\n");
1637
      }else {
1638
        fprintf(fp, "      multiThreadWriteOneTbl:  yes\n");
1639
      }
1640
      */
1641
      fprintf(fp, "      interlaceRows:     %"PRIu64"\n",
1642
              g_Dbs.db[i].superTbls[j].interlaceRows);
1643
      fprintf(fp, "      disorderRange:     %d\n",  g_Dbs.db[i].superTbls[j].disorderRange);
1644
      fprintf(fp, "      disorderRatio:     %d\n",  g_Dbs.db[i].superTbls[j].disorderRatio);
1645
      fprintf(fp, "      maxSqlLen:         %"PRIu64"\n",
1646
              g_Dbs.db[i].superTbls[j].maxSqlLen);
1647

1648 1649 1650 1651
      fprintf(fp, "      timeStampStep:     %"PRId64"\n",
              g_Dbs.db[i].superTbls[j].timeStampStep);
      fprintf(fp, "      startTimestamp:    %s\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
1652
      fprintf(fp, "      sampleFormat:      %s\n",  g_Dbs.db[i].superTbls[j].sampleFormat);
1653 1654 1655
      fprintf(fp, "      sampleFile:        %s\n",  g_Dbs.db[i].superTbls[j].sampleFile);
      fprintf(fp, "      tagsFile:          %s\n",  g_Dbs.db[i].superTbls[j].tagsFile);

1656 1657 1658
      fprintf(fp, "      columnCount:       %d\n        ",  g_Dbs.db[i].superTbls[j].columnCount);
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1659
        if ((0 == strncasecmp(
1660 1661 1662 1663
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "nchar", strlen("nchar")))) {
1664 1665 1666
          fprintf(fp, "column[%d]:%s(%d) ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1667 1668 1669 1670 1671
        } else {
          fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
        }
      }
      fprintf(fp, "\n");
1672 1673 1674

      fprintf(fp, "      tagCount:            %d\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1675 1676
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1677
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
1678 1679 1680
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1681 1682
          fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1683 1684
        } else {
          fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
1685
        }
1686 1687 1688 1689 1690
      }
      fprintf(fp, "\n");
    }
    fprintf(fp, "\n");
  }
1691

1692
  SHOW_PARSE_RESULT_END_TO_FILE(fp);
1693 1694 1695
}

static void printfQueryMeta() {
1696

1697
  SHOW_PARSE_RESULT_START();
1698

1699
  printf("host:                    \033[33m%s:%u\033[0m\n",
1700
          g_queryInfo.host, g_queryInfo.port);
1701 1702 1703 1704
  printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
  printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

  printf("\n");
1705 1706 1707 1708

  if ((SUBSCRIBE_TEST == g_args.test_mode) || (QUERY_TEST == g_args.test_mode)) {
    printf("specified table query info:                   \n");
    printf("sqlCount:       \033[33m%"PRIu64"\033[0m\n",
1709
      g_queryInfo.specifiedQueryInfo.sqlCount);
1710 1711 1712
    if (g_queryInfo.specifiedQueryInfo.sqlCount > 0) {
      printf("specified tbl query times:\n");
      printf("                \033[33m%"PRIu64"\033[0m\n",
1713
      g_queryInfo.specifiedQueryInfo.queryTimes);
1714 1715 1716 1717 1718 1719
      printf("query interval: \033[33m%"PRIu64" ms\033[0m\n",
        g_queryInfo.specifiedQueryInfo.queryInterval);
      printf("top query times:\033[33m%"PRIu64"\033[0m\n", g_args.query_times);
      printf("concurrent:     \033[33m%"PRIu64"\033[0m\n",
      g_queryInfo.specifiedQueryInfo.concurrent);
      printf("mod:            \033[33m%s\033[0m\n",
1720
        (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync");
1721
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1722
        g_queryInfo.specifiedQueryInfo.subscribeInterval);
1723
      printf("restart:        \033[33m%d\033[0m\n",
1724
        g_queryInfo.specifiedQueryInfo.subscribeRestart);
1725
      printf("keepProgress:   \033[33m%d\033[0m\n",
1726
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
1727

1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743
      for (uint64_t i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        printf("  sql[%"PRIu64"]: \033[33m%s\033[0m\n",
            i, g_queryInfo.specifiedQueryInfo.sql[i]);
      }
      printf("\n");
    }

    printf("super table query info:\n");
    printf("sqlCount:       \033[33m%"PRIu64"\033[0m\n",
      g_queryInfo.superQueryInfo.sqlCount);

    if (g_queryInfo.superQueryInfo.sqlCount > 0) {
      printf("query interval: \033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryInterval);
      printf("threadCnt:      \033[33m%d\033[0m\n",
        g_queryInfo.superQueryInfo.threadCnt);
1744
      printf("childTblCount:  \033[33m%"PRId64"\033[0m\n",
1745 1746 1747 1748 1749 1750 1751
        g_queryInfo.superQueryInfo.childTblCount);
      printf("stable name:    \033[33m%s\033[0m\n",
        g_queryInfo.superQueryInfo.sTblName);
      printf("stb query times:\033[33m%"PRIu64"\033[0m\n",
        g_queryInfo.superQueryInfo.queryTimes);

      printf("mod:            \033[33m%s\033[0m\n",
1752
        (g_queryInfo.superQueryInfo.asyncMode)?"async":"sync");
1753
      printf("interval:       \033[33m%"PRIu64"\033[0m\n",
1754
        g_queryInfo.superQueryInfo.subscribeInterval);
1755
      printf("restart:        \033[33m%d\033[0m\n",
1756
        g_queryInfo.superQueryInfo.subscribeRestart);
1757
      printf("keepProgress:   \033[33m%d\033[0m\n",
1758
        g_queryInfo.superQueryInfo.subscribeKeepProgress);
1759

1760 1761 1762 1763 1764 1765
      for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
        printf("  sql[%d]: \033[33m%s\033[0m\n",
            i, g_queryInfo.superQueryInfo.sql[i]);
      }
      printf("\n");
    }
1766
  }
1767

1768
  SHOW_PARSE_RESULT_END();
1769 1770
}

1771
static char* formatTimestamp(char* buf, int64_t val, int precision) {
1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802
  time_t tt;
  if (precision == TSDB_TIME_PRECISION_MICRO) {
    tt = (time_t)(val / 1000000);
  } else {
    tt = (time_t)(val / 1000);
  }

/* comment out as it make testcases like select_with_tags.sim fail.
  but in windows, this may cause the call to localtime crash if tt < 0,
  need to find a better solution.
  if (tt < 0) {
    tt = 0;
  }
  */

#ifdef WINDOWS
  if (tt < 0) tt = 0;
#endif

  struct tm* ptm = localtime(&tt);
  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);

  if (precision == TSDB_TIME_PRECISION_MICRO) {
    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
  } else {
    sprintf(buf + pos, ".%03d", (int)(val % 1000));
  }

  return buf;
}

1803
static void xDumpFieldToFile(FILE* fp, const char* val,
1804
        TAOS_FIELD* field, int32_t length, int precision) {
1805

1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840
  if (val == NULL) {
    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
    return;
  }

  char buf[TSDB_MAX_BYTES_PER_ROW];
  switch (field->type) {
    case TSDB_DATA_TYPE_BOOL:
      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
      break;
    case TSDB_DATA_TYPE_TINYINT:
      fprintf(fp, "%d", *((int8_t *)val));
      break;
    case TSDB_DATA_TYPE_SMALLINT:
      fprintf(fp, "%d", *((int16_t *)val));
      break;
    case TSDB_DATA_TYPE_INT:
      fprintf(fp, "%d", *((int32_t *)val));
      break;
    case TSDB_DATA_TYPE_BIGINT:
      fprintf(fp, "%" PRId64, *((int64_t *)val));
      break;
    case TSDB_DATA_TYPE_FLOAT:
      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
      break;
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      memcpy(buf, val, length);
      buf[length] = 0;
      fprintf(fp, "\'%s\'", buf);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
1841
      formatTimestamp(buf, *(int64_t*)val, precision);
1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856
      fprintf(fp, "'%s'", buf);
      break;
    default:
      break;
  }
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  FILE* fp = fopen(fname, "at");
  if (fp == NULL) {
1857
    errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871
    return -1;
  }

  int num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int precision = taos_result_precision(tres);

  for (int col = 0; col < num_fields; col++) {
    if (col > 0) {
      fprintf(fp, ",");
    }
    fprintf(fp, "%s", fields[col].name);
  }
  fputc('\n', fp);
1872

1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892
  int numOfRows = 0;
  do {
    int32_t* length = taos_fetch_lengths(tres);
    for (int i = 0; i < num_fields; i++) {
      if (i > 0) {
        fputc(',', fp);
      }
      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
    }
    fputc('\n', fp);

    numOfRows++;
    row = taos_fetch_row(tres);
  } while( row != NULL);

  fclose(fp);

  return numOfRows;
}

1893 1894
static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
  TAOS_RES * res;
1895 1896
  TAOS_ROW row = NULL;
  int count = 0;
1897 1898

  res = taos_query(taos, "show databases;");
1899
  int32_t code = taos_errno(res);
1900

1901
  if (code != 0) {
1902
    errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
1903 1904 1905 1906 1907
    return -1;
  }

  TAOS_FIELD *fields = taos_fetch_fields(res);

1908
  while((row = taos_fetch_row(res)) != NULL) {
1909
    // sys database name : 'log'
1910
    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log",
1911
                fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) {
1912 1913
      continue;
    }
1914 1915 1916

    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
    if (dbInfos[count] == NULL) {
1917
      errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
1918 1919 1920
      return -1;
    }

1921
    tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
1922
            fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
1923
    formatTimestamp(dbInfos[count]->create_time,
1924 1925
            *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
            TSDB_TIME_PRECISION_MILLI);
S
Shuduo Sang 已提交
1926
    dbInfos[count]->ntables = *((int64_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
1927
    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
1928 1929
    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
1930
    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
1931

1932
    tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
1933
            fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
1934 1935 1936 1937 1938 1939 1940
    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
1941
    dbInfos[count]->cachelast =
1942
      (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));
1943

1944
    tstrncpy(dbInfos[count]->precision,
1945 1946
            (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
            fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
1947
    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
1948
    tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
1949
            fields[TSDB_SHOW_DB_STATUS_INDEX].bytes);
1950

1951 1952
    count++;
    if (count > MAX_DATABASE_COUNT) {
1953
      errorPrint("%s() LN%d, The database count overflow than %d\n",
1954
         __func__, __LINE__, MAX_DATABASE_COUNT);
1955 1956 1957 1958 1959 1960 1961
      break;
    }
  }

  return count;
}

1962
static void printfDbInfoForQueryToFile(
1963
        char* filename, SDbInfo* dbInfos, int index) {
1964

1965
  if (filename[0] == 0)
1966
      return;
1967 1968 1969

  FILE *fp = fopen(filename, "at");
  if (fp == NULL) {
1970
    errorPrint( "failed to open file: %s\n", filename);
1971
    return;
1972 1973 1974 1975 1976
  }

  fprintf(fp, "================ database[%d] ================\n", index);
  fprintf(fp, "name: %s\n", dbInfos->name);
  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
S
Shuduo Sang 已提交
1977
  fprintf(fp, "ntables: %"PRId64"\n", dbInfos->ntables);
1978
  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);
1979 1980
  fprintf(fp, "replica: %d\n", dbInfos->replica);
  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
1981 1982
  fprintf(fp, "days: %d\n", dbInfos->days);
  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);
1983 1984 1985 1986 1987 1988 1989
  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
  fprintf(fp, "comp: %d\n", dbInfos->comp);
1990 1991
  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);
  fprintf(fp, "precision: %s\n", dbInfos->precision);
1992
  fprintf(fp, "update: %d\n", dbInfos->update);
1993
  fprintf(fp, "status: %s\n", dbInfos->status);
1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007
  fprintf(fp, "\n");

  fclose(fp);
}

static void printfQuerySystemInfo(TAOS * taos) {
  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
  TAOS_RES* res;

  time_t t;
  struct tm* lt;
  time(&t);
  lt = localtime(&t);
2008
  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
2009 2010
          lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
          lt->tm_sec);
2011

2012 2013
  // show variables
  res = taos_query(taos, "show variables;");
2014
  //appendResultToFile(res, filename);
2015 2016 2017 2018 2019
  xDumpResultToFile(filename, res);

  // show dnodes
  res = taos_query(taos, "show dnodes;");
  xDumpResultToFile(filename, res);
2020
  //appendResultToFile(res, filename);
2021

2022 2023 2024 2025
  // show databases
  res = taos_query(taos, "show databases;");
  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
  if (dbInfos == NULL) {
2026
    errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
2027 2028 2029
    return;
  }
  int dbCount = getDbFromServer(taos, dbInfos);
2030
  if (dbCount <= 0) {
2031 2032
      free(dbInfos);
      return;
2033
  }
2034 2035

  for (int i = 0; i < dbCount; i++) {
2036
    // printf database info
2037
    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
2038

2039 2040 2041 2042
    // show db.vgroups
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);
2043

2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054
    // show db.stables
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);

    free(dbInfos[i]);
  }

  free(dbInfos);
}

2055 2056
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
        char* sqlstr, char *resultFile)
2057
{
2058
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
2059

2060
    char *url = "/rest/sql";
2061

2062 2063 2064 2065
    int bytes, sent, received, req_str_len, resp_len;
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
2066

2067
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;
2068

2069 2070 2071 2072 2073
    request_buf = malloc(req_buf_len);
    if (NULL == request_buf) {
      errorPrint("%s", "ERROR, cannot allocate memory.\n");
      exit(EXIT_FAILURE);
    }
2074

2075 2076
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};
2077

2078 2079 2080 2081 2082 2083 2084 2085
    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
2086

2087 2088 2089 2090
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);
2091

2092
    char base64_buf[INPUT_BUF_LEN];
2093
#ifdef WINDOWS
2094
    WSADATA wsaData;
2095
    WSAStartup(MAKEWORD(2, 2), &wsaData);
2096 2097
    SOCKET sockfd;
#else
2098
    int sockfd;
2099
#endif
2100 2101
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (sockfd < 0) {
2102
#ifdef WINDOWS
2103
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
2104
#endif
2105 2106 2107 2108
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
        free(request_buf);
        ERROR_EXIT("ERROR opening socket");
    }
2109

2110
    int retConn = connect(sockfd, (struct sockaddr *)pServAddr, sizeof(struct sockaddr));
2111 2112 2113 2114 2115
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
        free(request_buf);
        ERROR_EXIT("ERROR connecting");
    }
2116

2117
    memset(base64_buf, 0, INPUT_BUF_LEN);
2118

2119 2120 2121 2122 2123 2124 2125 2126
    for (int n = 0, m = 0; n < userpass_buf_len;) {
      uint32_t oct_a = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t oct_b = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t oct_c = n < userpass_buf_len ?
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;
2127

2128 2129 2130 2131 2132
      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }
2133

2134 2135
    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';
2136

2137 2138 2139
    debugPrint("%s() LN%d: auth string base64 encoded: %s\n",
            __func__, __LINE__, base64_buf);
    char *auth = base64_buf;
2140

2141 2142 2143 2144 2145 2146 2147 2148 2149
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
2150

2151 2152 2153
    req_str_len = strlen(request_buf);
    sent = 0;
    do {
2154
#ifdef WINDOWS
2155
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
2156
#else
2157
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
2158
#endif
2159 2160 2161 2162 2163 2164
        if (bytes < 0)
            ERROR_EXIT("ERROR writing message to socket");
        if (bytes == 0)
            break;
        sent+=bytes;
    } while(sent < req_str_len);
2165

2166 2167 2168 2169
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
    received = 0;
    do {
2170
#ifdef WINDOWS
2171
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
2172
#else
2173
        bytes = read(sockfd, response_buf + received, resp_len - received);
2174
#endif
2175 2176 2177 2178 2179 2180 2181 2182
        if (bytes < 0) {
            free(request_buf);
            ERROR_EXIT("ERROR reading response from socket");
        }
        if (bytes == 0)
            break;
        received += bytes;
    } while(received < resp_len);
2183

2184 2185 2186 2187
    if (received == resp_len) {
        free(request_buf);
        ERROR_EXIT("ERROR storing complete response from socket");
    }
H
Haojun Liao 已提交
2188

2189 2190
    response_buf[RESP_BUF_LEN - 1] = '\0';
    printf("Response:\n%s\n", response_buf);
H
Haojun Liao 已提交
2191

2192 2193 2194 2195
    if (resultFile) {
       appendResultBufToFile(response_buf, resultFile);
    }

2196
    free(request_buf);
2197
#ifdef WINDOWS
2198
    closesocket(sockfd);
2199 2200
    WSACleanup();
#else
2201
    close(sockfd);
2202
#endif
2203

2204
    return 0;
2205 2206
}

2207
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
2208 2209
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
2210
    errorPrint("%s() LN%d, calloc failed! size:%d\n",
2211
        __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
2212 2213
    return NULL;
  }
2214

2215
  int    dataLen = 0;
2216
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2217
          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);
2218

2219 2220 2221
  return dataBuf;
}

2222
static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) {
2223 2224 2225 2226 2227 2228 2229 2230 2231
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
    printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
    return NULL;
  }

  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
  for (int i = 0; i < stbInfo->tagCount; i++) {
2232
    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
2233
            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
2234
      if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
2235
        printf("binary or nchar length overflow, max size:%u\n",
2236
                (uint32_t)TSDB_MAX_BINARY_LEN);
2237 2238 2239
        tmfree(dataBuf);
        return NULL;
      }
2240

2241 2242
      int tagBufLen = stbInfo->tags[i].dataLen + 1;
      char* buf = (char*)calloc(tagBufLen, 1);
2243 2244 2245 2246 2247
      if (NULL == buf) {
        printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
        tmfree(dataBuf);
        return NULL;
      }
2248 2249 2250 2251 2252 2253 2254

      if (tableSeq % 2) {
        tstrncpy(buf, "beijing", tagBufLen);
      } else {
        tstrncpy(buf, "shanghai", tagBufLen);
      }
      //rand_string(buf, stbInfo->tags[i].dataLen);
2255
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2256
              "\'%s\', ", buf);
2257
      tmfree(buf);
2258
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2259
                "int", strlen("int"))) {
2260
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2261
              "%d, ", tableSeq);
2262
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2263
                "bigint", strlen("bigint"))) {
2264
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2265
              "%"PRId64", ", rand_bigint());
2266
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2267
                "float", strlen("float"))) {
2268
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2269
              "%f, ", rand_float());
2270
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2271
                "double", strlen("double"))) {
2272
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2273
              "%f, ", rand_double());
2274
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2275
                "smallint", strlen("smallint"))) {
2276
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2277
              "%d, ", rand_smallint());
2278
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2279
                "tinyint", strlen("tinyint"))) {
2280
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2281
              "%d, ", rand_tinyint());
2282
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2283
                "bool", strlen("bool"))) {
2284
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2285
              "%d, ", rand_bool());
2286
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
2287
                "timestamp", strlen("timestamp"))) {
2288
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2289
              "%"PRId64", ", rand_bigint());
2290 2291 2292 2293 2294 2295
    }  else {
      printf("No support data type: %s\n", stbInfo->tags[i].dataType);
      tmfree(dataBuf);
      return NULL;
    }
  }
2296

2297
  dataLen -= 2;
2298
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");
2299 2300 2301
  return dataBuf;
}

2302
static int calcRowLen(SSuperTable*  superTbls) {
2303 2304
  int colIndex;
  int  lenOfOneRow = 0;
2305

2306 2307
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2308

2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
2325
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2326
      lenOfOneRow += 42;
2327
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
2341

2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
2358
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2359 2360 2361 2362 2363 2364 2365 2366
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2367

2368 2369 2370 2371
  return 0;
}


2372
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
2373
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
2374
        int64_t* childTblCountOfSuperTbl, int64_t limit, uint64_t offset) {
2375

2376
  char command[BUFFER_SIZE] = "\0";
2377 2378
  char limitBuf[100] = "\0";

2379
  TAOS_RES * res;
2380 2381 2382
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2383 2384

  if (offset >= 0) {
2385
    snprintf(limitBuf, 100, " limit %"PRId64" offset %"PRIu64"",
2386
            limit, offset);
2387 2388
  }

2389
  //get all child table name use cmd: select tbname from superTblName;
2390
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s",
2391
          dbName, sTblName, limitBuf);
2392

2393
  res = taos_query(taos, command);
2394 2395 2396 2397
  int32_t code = taos_errno(res);
  if (code != 0) {
    taos_free_result(res);
    taos_close(taos);
2398
    errorPrint("%s() LN%d, failed to run command %s\n",
2399
           __func__, __LINE__, command);
2400 2401 2402
    exit(-1);
  }

2403 2404
  int64_t childTblCount = (limit < 0)?10000:limit;
  int64_t count = 0;
2405 2406 2407
  if (childTblName == NULL) {
    childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (NULL ==  childTblName) {
2408 2409 2410 2411
    taos_free_result(res);
        taos_close(taos);
        errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__);
        exit(-1);
2412 2413 2414
    }
  }

2415
  char* pTblName = childTblName;
2416
  while((row = taos_fetch_row(res)) != NULL) {
2417
    int32_t* len = taos_fetch_lengths(res);
2418
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2419 2420 2421
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2422
      char *tmp = realloc(childTblName,
2423
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2424 2425 2426
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2427
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
2428
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2429 2430
      } else {
        // exit, if allocate more memory failed
2431
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
2432
               __func__, __LINE__, dbName, sTblName);
2433 2434 2435 2436 2437 2438
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2439
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2440
  }
2441

2442 2443 2444 2445 2446 2447 2448
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2449
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
2450
        char* sTblName, char** childTblNameOfSuperTbl,
2451
        int64_t* childTblCountOfSuperTbl) {
2452

2453 2454
    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
2455
            -1, 0);
2456 2457
}

2458
static int getSuperTableFromServer(TAOS * taos, char* dbName,
2459
        SSuperTable*  superTbls) {
2460

2461
  char command[BUFFER_SIZE] = "\0";
2462
  TAOS_RES * res;
2463 2464
  TAOS_ROW row = NULL;
  int count = 0;
2465

2466
  //get schema use cmd: describe superTblName;
2467
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2468
  res = taos_query(taos, command);
2469 2470 2471
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2472
    taos_free_result(res);
2473 2474 2475 2476 2477 2478
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
2479
  while((row = taos_fetch_row(res)) != NULL) {
2480 2481 2482
    if (0 == count) {
      count++;
      continue;
2483
    }
2484 2485

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2486
      tstrncpy(superTbls->tags[tagIndex].field,
2487 2488
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
2489
      tstrncpy(superTbls->tags[tagIndex].dataType,
2490 2491
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
2492 2493 2494
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
2495 2496
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2497
      tagIndex++;
2498
    } else {
2499
      tstrncpy(superTbls->columns[columnIndex].field,
2500 2501
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
2502
      tstrncpy(superTbls->columns[columnIndex].dataType,
2503 2504
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
2505 2506 2507
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
2508 2509
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2521
/*
2522
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
2523
    //get all child table name use cmd: select tbname from superTblName;
2524 2525 2526
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2527
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2528 2529
      return -1;
    }
2530
    getAllChildNameOfSuperTable(taos, dbName,
2531 2532 2533
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2534
  }
2535
  */
2536 2537 2538
  return 0;
}

H
Haojun Liao 已提交
2539
static int createSuperTable(
2540 2541
        TAOS * taos, char* dbName,
        SSuperTable*  superTbl) {
H
Haojun Liao 已提交
2542

2543
  char command[BUFFER_SIZE] = "\0";
2544

2545 2546 2547 2548 2549
  char cols[STRING_LEN] = "\0";
  int colIndex;
  int len = 0;

  int  lenOfOneRow = 0;
2550 2551 2552

  if (superTbl->columnCount == 0) {
    errorPrint("%s() LN%d, super table column count is %d\n",
2553
            __func__, __LINE__, superTbl->columnCount);
2554 2555 2556 2557 2558
    return -1;
  }

  for (colIndex = 0; colIndex < superTbl->columnCount; colIndex++) {
    char* dataType = superTbl->columns[colIndex].dataType;
2559

2560
    if (strcasecmp(dataType, "BINARY") == 0) {
2561
      len += snprintf(cols + len, STRING_LEN - len,
2562 2563
          ", col%d %s(%d)", colIndex, "BINARY",
          superTbl->columns[colIndex].dataLen);
2564
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2565
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
2566
      len += snprintf(cols + len, STRING_LEN - len,
2567 2568
          ", col%d %s(%d)", colIndex, "NCHAR",
          superTbl->columns[colIndex].dataLen);
2569
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
      lenOfOneRow += 22;
2588
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2589 2590
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
      lenOfOneRow += 42;
2591
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2592 2593 2594 2595
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
      lenOfOneRow += 21;
    } else {
      taos_close(taos);
2596
      errorPrint("%s() LN%d, config error data type : %s\n",
2597
         __func__, __LINE__, dataType);
2598 2599 2600 2601
      exit(-1);
    }
  }

2602 2603
  superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp
  //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbl[j].sTblName, g_Dbs.db[i].superTbl[j].columnCount, lenOfOneRow);
2604 2605

  // save for creating child table
2606 2607 2608
  superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1);
  if (NULL == superTbl->colsOfCreateChildTable) {
    errorPrint("%s() LN%d, Failed when calloc, size:%d",
2609
           __func__, __LINE__, len+1);
2610 2611 2612 2613
    taos_close(taos);
    exit(-1);
  }

2614
  snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
2615
  verbosePrint("%s() LN%d: %s\n",
2616
      __func__, __LINE__, superTbl->colsOfCreateChildTable);
2617

2618 2619
  if (superTbl->tagCount == 0) {
    errorPrint("%s() LN%d, super table tag count is %d\n",
2620
            __func__, __LINE__, superTbl->tagCount);
2621 2622
    return -1;
  }
2623

2624 2625 2626
  char tags[STRING_LEN] = "\0";
  int tagIndex;
  len = 0;
2627

2628 2629 2630 2631
  int lenOfTagOfOneRow = 0;
  len += snprintf(tags + len, STRING_LEN - len, "(");
  for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) {
    char* dataType = superTbl->tags[tagIndex].dataType;
2632

2633 2634
    if (strcasecmp(dataType, "BINARY") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
2635
              "BINARY", superTbl->tags[tagIndex].dataLen);
2636 2637 2638
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
2639
              "NCHAR", superTbl->tags[tagIndex].dataLen);
2640 2641 2642
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2643
              "INT");
2644 2645 2646
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2647
              "BIGINT");
2648 2649 2650
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2651
              "SMALLINT");
2652 2653 2654
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2655
              "TINYINT");
2656 2657 2658
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2659
              "BOOL");
2660 2661 2662
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2663
              "FLOAT");
2664 2665 2666
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
2667
              "DOUBLE");
2668 2669 2670
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42;
    } else {
      taos_close(taos);
2671
      errorPrint("%s() LN%d, config error tag type : %s\n",
2672
         __func__, __LINE__, dataType);
2673
      exit(-1);
2674 2675
    }
  }
2676 2677 2678 2679 2680 2681 2682

  len -= 2;
  len += snprintf(tags + len, STRING_LEN - len, ")");

  superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow;

  snprintf(command, BUFFER_SIZE,
2683 2684
          "create table if not exists %s.%s (ts timestamp%s) tags %s",
          dbName, superTbl->sTblName, cols, tags);
2685 2686 2687
  verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command);

  if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2688 2689 2690
      errorPrint( "create supertable %s failed!\n\n",
              superTbl->sTblName);
      return -1;
2691 2692
  }
  debugPrint("create supertable %s success!\n\n", superTbl->sTblName);
2693 2694 2695
  return 0;
}

2696
static int createDatabasesAndStables() {
2697 2698 2699 2700
  TAOS * taos = NULL;
  int    ret = 0;
  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
  if (taos == NULL) {
2701
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2702
    return -1;
2703 2704 2705
  }
  char command[BUFFER_SIZE] = "\0";

2706
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2707 2708
    if (g_Dbs.db[i].drop) {
      sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
2709
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);
2710
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2711 2712 2713 2714
        taos_close(taos);
        return -1;
      }

2715
      int dataLen = 0;
2716
      dataLen += snprintf(command + dataLen,
2717
          BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2718

2719 2720
      if (g_Dbs.db[i].dbCfg.blocks > 0) {
        dataLen += snprintf(command + dataLen,
2721
            BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
2722 2723 2724
      }
      if (g_Dbs.db[i].dbCfg.cache > 0) {
        dataLen += snprintf(command + dataLen,
2725
            BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
2726 2727 2728
      }
      if (g_Dbs.db[i].dbCfg.days > 0) {
        dataLen += snprintf(command + dataLen,
2729
            BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
2730 2731 2732
      }
      if (g_Dbs.db[i].dbCfg.keep > 0) {
        dataLen += snprintf(command + dataLen,
2733
            BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
2734 2735 2736
      }
      if (g_Dbs.db[i].dbCfg.quorum > 1) {
        dataLen += snprintf(command + dataLen,
2737
            BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
2738 2739 2740
      }
      if (g_Dbs.db[i].dbCfg.replica > 0) {
        dataLen += snprintf(command + dataLen,
2741
            BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
2742 2743 2744
      }
      if (g_Dbs.db[i].dbCfg.update > 0) {
        dataLen += snprintf(command + dataLen,
2745
            BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
2746 2747 2748 2749 2750 2751 2752
      }
      //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
      //  dataLen += snprintf(command + dataLen,
      //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
      //}
      if (g_Dbs.db[i].dbCfg.minRows > 0) {
        dataLen += snprintf(command + dataLen,
2753
            BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
2754 2755 2756
      }
      if (g_Dbs.db[i].dbCfg.maxRows > 0) {
        dataLen += snprintf(command + dataLen,
2757
            BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
2758 2759 2760
      }
      if (g_Dbs.db[i].dbCfg.comp > 0) {
        dataLen += snprintf(command + dataLen,
2761
            BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
2762 2763 2764
      }
      if (g_Dbs.db[i].dbCfg.walLevel > 0) {
        dataLen += snprintf(command + dataLen,
2765
            BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
2766 2767 2768
      }
      if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
        dataLen += snprintf(command + dataLen,
2769
            BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
2770 2771 2772
      }
      if (g_Dbs.db[i].dbCfg.fsync > 0) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2773
                " fsync %d", g_Dbs.db[i].dbCfg.fsync);
2774 2775
      }
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
2776 2777
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                      "us", strlen("us")))) {
2778
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
2779
                " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
2780 2781 2782 2783 2784 2785 2786 2787 2788
      }

      debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
        taos_close(taos);
        errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
        return -1;
      }
      printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
2789 2790
    }

2791
    debugPrint("%s() LN%d supertbl count:%"PRIu64"\n",
2792
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
2793 2794 2795

    int validStbCount = 0;

S
Shuduo Sang 已提交
2796
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
2797 2798 2799 2800 2801 2802 2803
      sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
              g_Dbs.db[i].superTbls[j].sTblName);
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);

      ret = queryDbExec(taos, command, NO_INSERT_TYPE, true);

      if ((ret != 0) || (g_Dbs.db[i].drop)) {
2804
        ret = createSuperTable(taos, g_Dbs.db[i].dbName,
2805
                &g_Dbs.db[i].superTbls[j]);
2806

2807
        if (0 != ret) {
S
Shuduo Sang 已提交
2808
          errorPrint("create super table %"PRIu64" failed!\n\n", j);
2809
          continue;
2810 2811 2812
        }
      }

2813
      ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
2814
                &g_Dbs.db[i].superTbls[j]);
2815 2816
      if (0 != ret) {
        errorPrint("\nget super table %s.%s info failed!\n\n",
2817
                g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName);
2818
        continue;
2819
      }
2820 2821

      validStbCount ++;
2822
    }
2823 2824

    g_Dbs.db[i].superTblCount = validStbCount;
2825 2826 2827 2828 2829 2830
  }

  taos_close(taos);
  return 0;
}

2831 2832
static void* createTable(void *sarg)
{
2833 2834
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
2835

S
Shuduo Sang 已提交
2836
  uint64_t  lastPrintTime = taosGetTimestampMs();
2837

2838
  int buff_len;
2839
  buff_len = BUFFER_SIZE / 8;
2840

2841
  char *buffer = calloc(buff_len, 1);
2842
  if (buffer == NULL) {
2843
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
2844 2845
    exit(-1);
  }
2846 2847 2848

  int len = 0;
  int batchNum = 0;
2849

2850
  verbosePrint("%s() LN%d: Creating table from %"PRIu64" to %"PRIu64"\n",
2851 2852
          __func__, __LINE__,
          pThreadInfo->start_table_from, pThreadInfo->end_table_to);
2853

2854
  for (uint64_t i = pThreadInfo->start_table_from;
2855
          i <= pThreadInfo->end_table_to; i++) {
2856
    if (0 == g_Dbs.use_metric) {
2857
      snprintf(buffer, buff_len,
2858
              "create table if not exists %s.%s%"PRIu64" %s;",
2859 2860 2861
              pThreadInfo->db_name,
              g_args.tb_prefix, i,
              pThreadInfo->cols);
2862
    } else {
2863 2864
      if (superTblInfo == NULL) {
        errorPrint("%s() LN%d, use metric, but super table info is NULL\n",
2865
                  __func__, __LINE__);
2866
        free(buffer);
2867 2868 2869 2870 2871 2872
        exit(-1);
      } else {
        if (0 == len) {
          batchNum = 0;
          memset(buffer, 0, buff_len);
          len += snprintf(buffer + len,
2873
                  buff_len - len, "create table ");
2874 2875 2876
        }
        char* tagsValBuf = NULL;
        if (0 == superTblInfo->tagSource) {
2877
          tagsValBuf = generateTagVaulesForStb(superTblInfo, i);
2878 2879
        } else {
          tagsValBuf = getTagValueFromTagSample(
2880 2881
                  superTblInfo,
                  i % superTblInfo->tagSampleCount);
2882 2883 2884 2885 2886 2887
        }
        if (NULL == tagsValBuf) {
          free(buffer);
          return NULL;
        }
        len += snprintf(buffer + len,
2888
                buff_len - len,
2889
                "if not exists %s.%s%"PRIu64" using %s.%s tags %s ",
2890 2891 2892
                pThreadInfo->db_name, superTblInfo->childTblPrefix,
                i, pThreadInfo->db_name,
                superTblInfo->sTblName, tagsValBuf);
2893 2894 2895
        free(tagsValBuf);
        batchNum++;
        if ((batchNum < superTblInfo->batchCreateTableNum)
2896 2897
                && ((buff_len - len)
                    >= (superTblInfo->lenOfTagOfOneRow + 256))) {
2898 2899
          continue;
        }
2900 2901 2902 2903
      }
    }

    len = 0;
2904
    verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
2905
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)){
2906
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2907
      free(buffer);
2908 2909 2910
      return NULL;
    }

S
Shuduo Sang 已提交
2911
    uint64_t  currentPrintTime = taosGetTimestampMs();
2912
    if (currentPrintTime - lastPrintTime > 30*1000) {
2913
      printf("thread[%d] already create %"PRIu64" - %"PRIu64" tables\n",
2914
              pThreadInfo->threadID, pThreadInfo->start_table_from, i);
2915 2916 2917
      lastPrintTime = currentPrintTime;
    }
  }
2918

2919
  if (0 != len) {
2920
    verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer);
2921
    if (0 != queryDbExec(pThreadInfo->taos, buffer, NO_INSERT_TYPE, false)) {
2922
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2923
    }
2924
  }
2925

2926 2927 2928 2929
  free(buffer);
  return NULL;
}

2930
static int startMultiThreadCreateChildTable(
S
Shuduo Sang 已提交
2931
        char* cols, int threads, uint64_t startFrom, int64_t ntables,
2932
        char* db_name, SSuperTable* superTblInfo) {
2933

2934
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
2935
  threadInfo *infos = calloc(1, threads * sizeof(threadInfo));
2936 2937 2938 2939 2940 2941 2942 2943 2944 2945

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

S
Shuduo Sang 已提交
2946
  int64_t a = ntables / threads;
2947 2948 2949 2950 2951
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

S
Shuduo Sang 已提交
2952
  int64_t b = 0;
2953
  b = ntables % threads;
2954

S
Shuduo Sang 已提交
2955
  for (int i = 0; i < threads; i++) {
2956 2957
    threadInfo *t_info = infos + i;
    t_info->threadID = i;
H
Hui Li 已提交
2958
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
2959
    t_info->superTblInfo = superTblInfo;
2960
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
2961
    t_info->taos = taos_connect(
2962 2963 2964 2965 2966
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
2967
    if (t_info->taos == NULL) {
2968
      errorPrint( "%s() LN%d, Failed to connect to TDengine, reason:%s\n",
2969
         __func__, __LINE__, taos_errstr(NULL));
2970
      free(pids);
2971
      free(infos);
2972 2973
      return -1;
    }
2974 2975 2976 2977 2978

    t_info->start_table_from = startFrom;
    t_info->ntables = i<b?a+1:a;
    t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
    startFrom = t_info->end_table_to + 1;
2979
    t_info->use_metric = true;
2980
    t_info->cols = cols;
2981
    t_info->minDelay = UINT64_MAX;
2982 2983
    pthread_create(pids + i, NULL, createTable, t_info);
  }
2984

2985 2986 2987 2988 2989 2990 2991 2992 2993 2994
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
    threadInfo *t_info = infos + i;
    taos_close(t_info->taos);
  }

  free(pids);
2995
  free(infos);
2996 2997

  return 0;
2998 2999
}

3000
static void createChildTables() {
3001 3002
    char tblColsBuf[MAX_SQL_SIZE];
    int len;
3003

3004
  for (int i = 0; i < g_Dbs.dbCount; i++) {
3005 3006
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
3007
          // with super table
S
Shuduo Sang 已提交
3008
        for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
3009
          if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
3010
                || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
3011 3012
            continue;
          }
3013

3014
          verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
3015
                  g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
S
Shuduo Sang 已提交
3016
          uint64_t startFrom = 0;
3017 3018
          g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;

3019
          verbosePrint("%s() LN%d: create %"PRId64" child tables from %"PRIu64"\n",
3020
                  __func__, __LINE__, g_totalChildTables, startFrom);
3021
          startMultiThreadCreateChildTable(
3022 3023 3024 3025 3026
                g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
                g_Dbs.threadCountByCreateTbl,
                startFrom,
                g_Dbs.db[i].superTbls[j].childTblCount,
                g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
3027
        }
3028 3029
      }
    } else {
3030 3031
      // normal table
      len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
H
Haojun Liao 已提交
3032
      for (int j = 0; j < g_args.num_of_CPR; j++) {
3033 3034 3035 3036 3037 3038 3039 3040 3041 3042
          if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
                  || (strncasecmp(g_args.datatype[j],
                      "NCHAR", strlen("NCHAR")) == 0)) {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                      ", COL%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary);
          } else {
              snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
                      ", COL%d %s", j, g_args.datatype[j]);
          }
          len = strlen(tblColsBuf);
3043
      }
3044

3045
      snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");
3046

3047
      verbosePrint("%s() LN%d: dbName: %s num of tb: %"PRId64" schema: %s\n",
3048 3049
              __func__, __LINE__,
              g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
3050
      startMultiThreadCreateChildTable(
3051 3052 3053 3054 3055 3056
            tblColsBuf,
            g_Dbs.threadCountByCreateTbl,
            0,
            g_args.num_of_tables,
            g_Dbs.db[i].dbName,
            NULL);
3057
    }
3058 3059 3060 3061 3062 3063
  }
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3064
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
3065 3066 3067
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
3068

3069 3070
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
3071
    printf("Failed to open tags file: %s, reason:%s\n",
3072
            superTblInfo->tagsFile, strerror(errno));
3073 3074 3075 3076 3077 3078 3079
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
3080

3081 3082 3083 3084 3085 3086 3087 3088 3089
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

3090
  while((readLen = tgetline(&line, &n, fp)) != -1) {
3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
3103
      char *tmp = realloc(tagDataBuf,
3104
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
3105 3106 3107
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
3108
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
3109
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

3129
#if 0
3130 3131 3132 3133
int readSampleFromJsonFileToMem(SSuperTable  * superTblInfo) {
  // TODO
  return 0;
}
3134
#endif
3135 3136 3137 3138

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
3139
static int readSampleFromCsvFileToMem(
3140
        SSuperTable* superTblInfo) {
3141 3142 3143 3144
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
3145

3146 3147
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
3148 3149 3150
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
              superTblInfo->sampleFile, strerror(errno));
      return -1;
3151
  }
3152

3153
  assert(superTblInfo->sampleDataBuf);
3154
  memset(superTblInfo->sampleDataBuf, 0,
3155
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
3156
  while(1) {
3157
    readLen = tgetline(&line, &n, fp);
3158 3159
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
3160
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
3161
                superTblInfo->sampleFile, strerror(errno));
3162
        fclose(fp);
3163 3164 3165 3166
        return -1;
      }
      continue;
    }
3167

3168 3169 3170 3171 3172 3173 3174 3175 3176
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
3177
      printf("sample row len[%d] overflow define schema len[%"PRIu64"], so discard this row\n",
3178
              (int32_t)readLen, superTblInfo->lenOfOneRow);
3179 3180 3181
      continue;
    }

3182
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
3183
          line, readLen);
3184 3185 3186 3187 3188 3189 3190
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

3191
  fclose(fp);
3192 3193 3194 3195
  tmfree(line);
  return 0;
}

3196
static bool getColumnAndTagTypeFromInsertJsonFile(
3197
        cJSON* stbInfo, SSuperTable* superTbls) {
3198
  bool  ret = false;
3199

3200
  // columns
3201 3202
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
3203
    printf("ERROR: failed to read json, columns not found\n");
3204 3205 3206 3207 3208 3209
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
3210

3211
  int columnSize = cJSON_GetArraySize(columns);
3212
  if ((columnSize + 1/* ts */) > MAX_COLUMN_COUNT) {
3213
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
3214
            __func__, __LINE__, MAX_COLUMN_COUNT);
3215 3216 3217 3218 3219 3220
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
3221

3222
  //superTbls->columnCount = columnSize;
3223 3224 3225 3226 3227 3228 3229
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
3230
      count = countObj->valueint;
3231
    } else if (countObj && countObj->type != cJSON_Number) {
3232
      errorPrint("%s() LN%d, failed to read json, column count not found\n",
3233
          __func__, __LINE__);
3234 3235 3236 3237 3238
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3239
    // column info
3240 3241
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
3242 3243 3244
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d: failed to read json, column type not found\n",
3245
          __func__, __LINE__);
3246 3247
      goto PARSE_OVER;
    }
3248 3249
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3250

3251 3252
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3253
      columnCase.dataLen = dataLen->valueint;
3254
    } else if (dataLen && dataLen->type != cJSON_Number) {
3255
      debugPrint("%s() LN%d: failed to read json, column len not found\n",
3256
          __func__, __LINE__);
3257 3258 3259 3260
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
3261

3262
    for (int n = 0; n < count; ++n) {
3263
      tstrncpy(superTbls->columns[index].dataType,
3264
              columnCase.dataType, MAX_TB_NAME_SIZE);
3265
      superTbls->columns[index].dataLen = columnCase.dataLen;
3266 3267
      index++;
    }
3268
  }
3269

3270 3271
  if ((index + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n",
3272
            __func__, __LINE__, MAX_COLUMN_COUNT);
3273 3274 3275
    goto PARSE_OVER;
  }

3276
  superTbls->columnCount = index;
3277

3278 3279
  count = 1;
  index = 0;
3280
  // tags
3281 3282
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
3283
    errorPrint("%s() LN%d, failed to read json, tags not found\n",
3284
        __func__, __LINE__);
3285 3286
    goto PARSE_OVER;
  }
3287

3288 3289
  int tagSize = cJSON_GetArraySize(tags);
  if (tagSize > MAX_TAG_COUNT) {
3290
    errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n",
3291
        __func__, __LINE__, MAX_TAG_COUNT);
3292 3293
    goto PARSE_OVER;
  }
3294 3295

  //superTbls->tagCount = tagSize;
3296 3297 3298
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
3299

3300 3301 3302
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
3303
      count = countObj->valueint;
3304
    } else if (countObj && countObj->type != cJSON_Number) {
3305
      printf("ERROR: failed to read json, column count not found\n");
3306 3307 3308 3309 3310
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3311
    // column info
3312 3313
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
3314 3315 3316
    if (!dataType || dataType->type != cJSON_String
        || dataType->valuestring == NULL) {
      errorPrint("%s() LN%d, failed to read json, tag type not found\n",
3317
          __func__, __LINE__);
3318 3319
      goto PARSE_OVER;
    }
3320
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3321

3322 3323
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3324
      columnCase.dataLen = dataLen->valueint;
3325
    } else if (dataLen && dataLen->type != cJSON_Number) {
3326
      errorPrint("%s() LN%d, failed to read json, column len not found\n",
3327
          __func__, __LINE__);
3328 3329 3330
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
3331 3332
    }

3333
    for (int n = 0; n < count; ++n) {
3334
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType,
3335
          MAX_TB_NAME_SIZE);
3336
      superTbls->tags[index].dataLen = columnCase.dataLen;
3337 3338
      index++;
    }
3339
  }
3340 3341

  if (index > MAX_TAG_COUNT) {
3342
    errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n",
3343
        __func__, __LINE__, MAX_TAG_COUNT);
3344 3345 3346
    goto PARSE_OVER;
  }

3347 3348
  superTbls->tagCount = index;

3349 3350
  if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > MAX_COLUMN_COUNT) {
    errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n",
3351
        __func__, __LINE__, MAX_COLUMN_COUNT);
3352 3353
    goto PARSE_OVER;
  }
3354 3355
  ret = true;

3356
PARSE_OVER:
3357 3358 3359 3360 3361 3362 3363 3364
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3365
    tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3366 3367 3368 3369
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3370
    tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE);
3371
  } else if (!host) {
3372
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3373
  } else {
3374
    printf("ERROR: failed to read json, host not found\n");
3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_Dbs.port = port->valueint;
  } else if (!port) {
    g_Dbs.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3387
    tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE);
3388
  } else if (!user) {
3389
    tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE);
3390 3391 3392 3393
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3394
    tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE);
3395
  } else if (!password) {
3396
    tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE);
3397 3398 3399 3400
  }

  cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
  if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
3401
    tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
3402
  } else if (!resultfile) {
3403
    tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
3404 3405 3406 3407 3408 3409 3410 3411
  }

  cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
  if (threads && threads->type == cJSON_Number) {
    g_Dbs.threadCount = threads->valueint;
  } else if (!threads) {
    g_Dbs.threadCount = 1;
  } else {
3412
    printf("ERROR: failed to read json, threads not found\n");
3413
    goto PARSE_OVER;
3414 3415
  }

3416 3417 3418 3419
  cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
  if (threads2 && threads2->type == cJSON_Number) {
    g_Dbs.threadCountByCreateTbl = threads2->valueint;
  } else if (!threads2) {
sangshuduo's avatar
sangshuduo 已提交
3420
    g_Dbs.threadCountByCreateTbl = 1;
3421
  } else {
3422
    errorPrint("%s() LN%d, failed to read json, threads2 not found\n",
3423
            __func__, __LINE__);
3424
    goto PARSE_OVER;
3425
  }
3426

3427 3428
  cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
  if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
3429 3430 3431 3432 3433
    if (gInsertInterval->valueint <0) {
      errorPrint("%s() LN%d, failed to read json, insert interval input mistake\n",
            __func__, __LINE__);
      goto PARSE_OVER;
    }
3434 3435 3436 3437
    g_args.insert_interval = gInsertInterval->valueint;
  } else if (!gInsertInterval) {
    g_args.insert_interval = 0;
  } else {
3438
    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
3439
        __func__, __LINE__);
3440 3441
    goto PARSE_OVER;
  }
3442

3443 3444
  cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows");
  if (interlaceRows && interlaceRows->type == cJSON_Number) {
3445 3446 3447 3448 3449 3450
    if (interlaceRows->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;

    }
3451
    g_args.interlace_rows = interlaceRows->valueint;
3452 3453 3454

    // rows per table need be less than insert batch
    if (g_args.interlace_rows > g_args.num_of_RPR) {
3455
      printf("NOTICE: interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
3456
              g_args.interlace_rows, g_args.num_of_RPR);
3457
      printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
3458
              g_args.num_of_RPR);
3459 3460 3461 3462
      if (!g_args.answer_yes) {
        printf("        press Enter key to continue or Ctrl-C to stop.");
        (void)getchar();
      }
3463 3464
      g_args.interlace_rows = g_args.num_of_RPR;
    }
3465
  } else if (!interlaceRows) {
3466
    g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3467
  } else {
3468
    errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n",
3469
        __func__, __LINE__);
3470
    goto PARSE_OVER;
3471
  }
3472

3473 3474
  cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
  if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
3475 3476 3477 3478 3479
    if (maxSqlLen->valueint < 0) {
      errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
    }
3480 3481
    g_args.max_sql_len = maxSqlLen->valueint;
  } else if (!maxSqlLen) {
sangshuduo's avatar
sangshuduo 已提交
3482
    g_args.max_sql_len = (1024*1024);
3483
  } else {
3484
    errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n",
3485
        __func__, __LINE__);
3486 3487 3488 3489 3490
    goto PARSE_OVER;
  }

  cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
  if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
3491
    if (numRecPerReq->valueint <= 0) {
3492 3493 3494
      errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n",
        __func__, __LINE__);
      goto PARSE_OVER;
3495 3496
    } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) {
      numRecPerReq->valueint = MAX_RECORDS_PER_REQ;
3497
    }
3498 3499
    g_args.num_of_RPR = numRecPerReq->valueint;
  } else if (!numRecPerReq) {
3500
    g_args.num_of_RPR = MAX_RECORDS_PER_REQ;
3501
  } else {
3502
    errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n",
3503
        __func__, __LINE__);
3504 3505 3506
    goto PARSE_OVER;
  }

3507
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3508
  if (answerPrompt
3509 3510
          && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3511 3512 3513 3514 3515 3516 3517 3518
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
    } else {
      g_args.answer_yes = false;
    }
  } else if (!answerPrompt) {
3519
    g_args.answer_yes = false;
3520
  } else {
3521
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
3522
    goto PARSE_OVER;
3523
  }
3524 3525 3526

  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (!dbs || dbs->type != cJSON_Array) {
3527
    printf("ERROR: failed to read json, databases not found\n");
3528 3529 3530 3531 3532
    goto PARSE_OVER;
  }

  int dbSize = cJSON_GetArraySize(dbs);
  if (dbSize > MAX_DB_COUNT) {
3533
    errorPrint(
3534 3535
            "ERROR: failed to read json, databases size overflow, max database is %d\n",
            MAX_DB_COUNT);
3536 3537 3538 3539 3540 3541 3542 3543
    goto PARSE_OVER;
  }

  g_Dbs.dbCount = dbSize;
  for (int i = 0; i < dbSize; ++i) {
    cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
    if (dbinfos == NULL) continue;

3544
    // dbinfo
3545 3546
    cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
    if (!dbinfo || dbinfo->type != cJSON_Object) {
3547
      printf("ERROR: failed to read json, dbinfo not found\n");
3548 3549
      goto PARSE_OVER;
    }
3550

3551 3552
    cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
    if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
3553
      printf("ERROR: failed to read json, db name not found\n");
3554 3555
      goto PARSE_OVER;
    }
3556
    tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE);
3557 3558 3559

    cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
    if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
3560 3561
      if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) {
        g_Dbs.db[i].drop = true;
3562
      } else {
3563
        g_Dbs.db[i].drop = false;
3564
      }
3565
    } else if (!drop) {
3566
      g_Dbs.db[i].drop = g_args.drop_database;
3567
    } else {
3568
      errorPrint("%s() LN%d, failed to read json, drop input mistake\n",
3569
              __func__, __LINE__);
3570 3571 3572 3573
      goto PARSE_OVER;
    }

    cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
3574
    if (precision && precision->type == cJSON_String
3575
            && precision->valuestring != NULL) {
3576
      tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
3577
              MAX_DB_NAME_SIZE);
3578
    } else if (!precision) {
3579
      //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
3580 3581
      memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE);
    } else {
3582
      printf("ERROR: failed to read json, precision not found\n");
3583 3584 3585 3586 3587 3588 3589 3590 3591
      goto PARSE_OVER;
    }

    cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
    if (update && update->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.update = update->valueint;
    } else if (!update) {
      g_Dbs.db[i].dbCfg.update = -1;
    } else {
3592
      printf("ERROR: failed to read json, update not found\n");
3593 3594 3595 3596 3597 3598 3599 3600 3601
      goto PARSE_OVER;
    }

    cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
    if (replica && replica->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.replica = replica->valueint;
    } else if (!replica) {
      g_Dbs.db[i].dbCfg.replica = -1;
    } else {
3602
      printf("ERROR: failed to read json, replica not found\n");
3603 3604 3605 3606 3607 3608 3609 3610 3611
      goto PARSE_OVER;
    }

    cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
    if (keep && keep->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.keep = keep->valueint;
    } else if (!keep) {
      g_Dbs.db[i].dbCfg.keep = -1;
    } else {
3612 3613
     printf("ERROR: failed to read json, keep not found\n");
     goto PARSE_OVER;
3614
    }
3615

3616 3617 3618 3619 3620 3621
    cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
    if (days && days->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.days = days->valueint;
    } else if (!days) {
      g_Dbs.db[i].dbCfg.days = -1;
    } else {
3622 3623
     printf("ERROR: failed to read json, days not found\n");
     goto PARSE_OVER;
3624
    }
3625

3626 3627 3628 3629 3630 3631
    cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
    if (cache && cache->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cache = cache->valueint;
    } else if (!cache) {
      g_Dbs.db[i].dbCfg.cache = -1;
    } else {
3632 3633
     printf("ERROR: failed to read json, cache not found\n");
     goto PARSE_OVER;
3634
    }
3635

3636 3637 3638 3639 3640 3641
    cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
    if (blocks && blocks->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
    } else if (!blocks) {
      g_Dbs.db[i].dbCfg.blocks = -1;
    } else {
3642 3643
     printf("ERROR: failed to read json, block not found\n");
     goto PARSE_OVER;
3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659
    }

    //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
    //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
    //} else if (!maxtablesPerVnode) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
    //} else {
    // printf("failed to read json, maxtablesPerVnode not found");
    // goto PARSE_OVER;
    //}

    cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
    if (minRows && minRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
    } else if (!minRows) {
3660
      g_Dbs.db[i].dbCfg.minRows = 0;    // 0 means default
3661
    } else {
3662 3663
     printf("ERROR: failed to read json, minRows not found\n");
     goto PARSE_OVER;
3664 3665 3666 3667 3668 3669
    }

    cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
    if (maxRows && maxRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
    } else if (!maxRows) {
3670
      g_Dbs.db[i].dbCfg.maxRows = 0;    // 0 means default
3671
    } else {
3672 3673
     printf("ERROR: failed to read json, maxRows not found\n");
     goto PARSE_OVER;
3674 3675 3676 3677 3678 3679 3680 3681
    }

    cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
    if (comp && comp->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.comp = comp->valueint;
    } else if (!comp) {
      g_Dbs.db[i].dbCfg.comp = -1;
    } else {
3682 3683
     printf("ERROR: failed to read json, comp not found\n");
     goto PARSE_OVER;
3684 3685 3686 3687 3688 3689 3690 3691
    }

    cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
    if (walLevel && walLevel->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
    } else if (!walLevel) {
      g_Dbs.db[i].dbCfg.walLevel = -1;
    } else {
3692 3693
     printf("ERROR: failed to read json, walLevel not found\n");
     goto PARSE_OVER;
3694 3695
    }

3696 3697 3698 3699 3700 3701
    cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
    if (cacheLast && cacheLast->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
    } else if (!cacheLast) {
      g_Dbs.db[i].dbCfg.cacheLast = -1;
    } else {
3702 3703
     printf("ERROR: failed to read json, cacheLast not found\n");
     goto PARSE_OVER;
3704 3705
    }

3706 3707 3708 3709
    cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
    if (quorum && quorum->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
    } else if (!quorum) {
3710
      g_Dbs.db[i].dbCfg.quorum = 1;
3711
    } else {
3712 3713
     printf("failed to read json, quorum input mistake");
     goto PARSE_OVER;
3714 3715 3716 3717 3718 3719 3720 3721
    }

    cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
    if (fsync && fsync->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
    } else if (!fsync) {
      g_Dbs.db[i].dbCfg.fsync = -1;
    } else {
3722
      errorPrint("%s() LN%d, failed to read json, fsync input mistake\n",
3723
              __func__, __LINE__);
3724
      goto PARSE_OVER;
3725
    }
3726

3727
    // super_talbes
3728 3729
    cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
    if (!stables || stables->type != cJSON_Array) {
3730
      errorPrint("%s() LN%d, failed to read json, super_tables not found\n",
3731
              __func__, __LINE__);
3732
      goto PARSE_OVER;
3733 3734
    }

3735 3736
    int stbSize = cJSON_GetArraySize(stables);
    if (stbSize > MAX_SUPER_TABLE_COUNT) {
3737
      errorPrint(
3738 3739
              "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n",
              __func__, __LINE__, MAX_SUPER_TABLE_COUNT);
3740 3741 3742 3743 3744 3745 3746
      goto PARSE_OVER;
    }

    g_Dbs.db[i].superTblCount = stbSize;
    for (int j = 0; j < stbSize; ++j) {
      cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
      if (stbInfo == NULL) continue;
3747 3748

      // dbinfo
3749 3750
      cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
      if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) {
3751
        errorPrint("%s() LN%d, failed to read json, stb name not found\n",
3752
                __func__, __LINE__);
3753 3754
        goto PARSE_OVER;
      }
3755
      tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE);
3756

3757 3758
      cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
      if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
3759
        printf("ERROR: failed to read json, childtable_prefix not found\n");
3760 3761
        goto PARSE_OVER;
      }
3762
      tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE);
3763 3764

      cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null
3765
      if (autoCreateTbl
3766 3767
              && autoCreateTbl->type == cJSON_String
              && autoCreateTbl->valuestring != NULL) {
3768 3769 3770 3771 3772 3773 3774 3775 3776 3777
        if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
        } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        } else {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        }
      } else if (!autoCreateTbl) {
        g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      } else {
3778
        printf("ERROR: failed to read json, auto_create_table not found\n");
3779 3780
        goto PARSE_OVER;
      }
3781

3782 3783 3784 3785
      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
      } else if (!batchCreateTbl) {
3786
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
3787
      } else {
3788
        printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
3789
        goto PARSE_OVER;
3790
      }
3791 3792

      cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
3793
      if (childTblExists
3794 3795
              && childTblExists->type == cJSON_String
              && childTblExists->valuestring != NULL) {
3796 3797
        if ((0 == strncasecmp(childTblExists->valuestring, "yes", 3))
            && (g_Dbs.db[i].drop == false)) {
3798
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
3799
        } else if ((0 == strncasecmp(childTblExists->valuestring, "no", 2)
3800
              || (g_Dbs.db[i].drop == true))) {
3801 3802 3803 3804 3805 3806 3807
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        } else {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        }
      } else if (!childTblExists) {
        g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
      } else {
3808
        errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n",
3809
                __func__, __LINE__);
3810 3811
        goto PARSE_OVER;
      }
3812

3813 3814
      cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
      if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
3815
        errorPrint("%s() LN%d, failed to read json, childtable_count input mistake\n",
3816
                __func__, __LINE__);
3817 3818 3819 3820 3821
        goto PARSE_OVER;
      }
      g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

      cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
3822
      if (dataSource && dataSource->type == cJSON_String
3823
              && dataSource->valuestring != NULL) {
3824
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
3825
                dataSource->valuestring, MAX_DB_NAME_SIZE);
3826
      } else if (!dataSource) {
3827
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE);
3828
      } else {
3829
        errorPrint("%s() LN%d, failed to read json, data_source not found\n",
3830
            __func__, __LINE__);
3831 3832 3833
        goto PARSE_OVER;
      }

3834
      cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , rest, stmt
3835
      if (insertMode && insertMode->type == cJSON_String
3836
              && insertMode->valuestring != NULL) {
3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847
        if (0 == strcasecmp(insertMode->valuestring, "taosc")) {
            g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE;
        } else if (0 == strcasecmp(insertMode->valuestring, "rest")) {
            g_Dbs.db[i].superTbls[j].insertMode = REST_IFACE;
        } else if (0 == strcasecmp(insertMode->valuestring, "stmt")) {
            g_Dbs.db[i].superTbls[j].insertMode = STMT_IFACE;
        } else {
            errorPrint("%s() LN%d, failed to read json, insert_mode %s not recognized\n",
                    __func__, __LINE__, insertMode->valuestring);
            goto PARSE_OVER;
        }
3848
      } else if (!insertMode) {
3849
        g_Dbs.db[i].superTbls[j].insertMode = TAOSC_IFACE;
3850
      } else {
3851
        errorPrint("%s", "failed to read json, insert_mode not found\n");
3852 3853 3854
        goto PARSE_OVER;
      }

3855
      cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
3856 3857
      if ((childTbl_limit) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3858
        if (childTbl_limit->type != cJSON_Number) {
3859 3860
            printf("ERROR: failed to read json, childtable_limit\n");
            goto PARSE_OVER;
3861 3862 3863
        }
        g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
      } else {
3864
        g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result, drop = yes mean all table need recreate, limit value is invalid.
3865 3866 3867
      }

      cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
3868 3869
      if ((childTbl_offset) && (g_Dbs.db[i].drop != true)
          && (g_Dbs.db[i].superTbls[j].childTblExists == TBL_ALREADY_EXISTS)) {
3870
        if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) {
3871 3872
            printf("ERROR: failed to read json, childtable_offset\n");
            goto PARSE_OVER;
3873 3874 3875 3876 3877 3878
        }
        g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblOffset = 0;
      }

3879 3880
      cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
      if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
3881
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
3882
                ts->valuestring, MAX_DB_NAME_SIZE);
3883
      } else if (!ts) {
3884
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
3885
                "now", MAX_DB_NAME_SIZE);
3886
      } else {
3887
        printf("ERROR: failed to read json, start_timestamp not found\n");
3888 3889
        goto PARSE_OVER;
      }
3890

3891 3892 3893 3894
      cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
      if (timestampStep && timestampStep->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
      } else if (!timestampStep) {
3895
        g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
3896
      } else {
3897
        printf("ERROR: failed to read json, timestamp_step not found\n");
3898 3899 3900 3901
        goto PARSE_OVER;
      }

      cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
3902
      if (sampleFormat && sampleFormat->type
3903
              == cJSON_String && sampleFormat->valuestring != NULL) {
3904
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
3905
                sampleFormat->valuestring, MAX_DB_NAME_SIZE);
3906
      } else if (!sampleFormat) {
3907
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE);
3908
      } else {
3909
        printf("ERROR: failed to read json, sample_format not found\n");
3910
        goto PARSE_OVER;
3911
      }
3912

3913
      cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
3914 3915
      if (sampleFile && sampleFile->type == cJSON_String
          && sampleFile->valuestring != NULL) {
3916
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
3917
                sampleFile->valuestring, MAX_FILE_NAME_LEN);
3918 3919 3920
      } else if (!sampleFile) {
        memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
      } else {
3921
        printf("ERROR: failed to read json, sample_file not found\n");
3922
        goto PARSE_OVER;
3923
      }
3924

3925 3926
      cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
      if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) {
3927
        tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
3928
                tagsFile->valuestring, MAX_FILE_NAME_LEN);
3929 3930 3931 3932 3933 3934 3935 3936 3937
        if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
          g_Dbs.db[i].superTbls[j].tagSource = 0;
        } else {
          g_Dbs.db[i].superTbls[j].tagSource = 1;
        }
      } else if (!tagsFile) {
        memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
        g_Dbs.db[i].superTbls[j].tagSource = 0;
      } else {
3938
        printf("ERROR: failed to read json, tags_file not found\n");
3939 3940
        goto PARSE_OVER;
      }
3941

sangshuduo's avatar
sangshuduo 已提交
3942 3943 3944
      cJSON* stbMaxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
      if (stbMaxSqlLen && stbMaxSqlLen->type == cJSON_Number) {
        int32_t len = stbMaxSqlLen->valueint;
3945 3946
        if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
          len = TSDB_MAX_ALLOWED_SQL_LEN;
3947 3948
        } else if (len < 5) {
          len = 5;
3949
        }
3950 3951
        g_Dbs.db[i].superTbls[j].maxSqlLen = len;
      } else if (!maxSqlLen) {
3952
        g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len;
3953
      } else {
sangshuduo's avatar
sangshuduo 已提交
3954
        errorPrint("%s() LN%d, failed to read json, stbMaxSqlLen input mistake\n",
3955
            __func__, __LINE__);
3956
        goto PARSE_OVER;
3957
      }
3958
/*
3959 3960 3961 3962 3963
      cJSON *multiThreadWriteOneTbl =
          cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
      if (multiThreadWriteOneTbl
              && multiThreadWriteOneTbl->type == cJSON_String
              && multiThreadWriteOneTbl->valuestring != NULL) {
3964 3965 3966 3967
        if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
        } else {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
3968
        }
3969 3970 3971
      } else if (!multiThreadWriteOneTbl) {
        g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
      } else {
3972
        printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
3973 3974
        goto PARSE_OVER;
      }
3975
*/
sangshuduo's avatar
sangshuduo 已提交
3976 3977 3978
      cJSON* stbInterlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows");
      if (stbInterlaceRows && stbInterlaceRows->type == cJSON_Number) {
        if (stbInterlaceRows->valueint < 0) {
3979 3980 3981 3982
          errorPrint("%s() LN%d, failed to read json, interlace rows input mistake\n",
            __func__, __LINE__);
          goto PARSE_OVER;
        }
sangshuduo's avatar
sangshuduo 已提交
3983
        g_Dbs.db[i].superTbls[j].interlaceRows = stbInterlaceRows->valueint;
3984 3985
        // rows per table need be less than insert batch
        if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) {
3986
          printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %"PRIu64" > num_of_records_per_req %"PRIu64"\n\n",
3987
                  i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR);
3988
          printf("        interlace rows value will be set to num_of_records_per_req %"PRIu64"\n\n",
3989
                  g_args.num_of_RPR);
3990 3991 3992 3993
          if (!g_args.answer_yes) {
            printf("        press Enter key to continue or Ctrl-C to stop.");
            (void)getchar();
          }
3994 3995
          g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR;
        }
sangshuduo's avatar
sangshuduo 已提交
3996
      } else if (!stbInterlaceRows) {
3997
        g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3998
      } else {
3999
        errorPrint(
4000 4001
                "%s() LN%d, failed to read json, interlace rows input mistake\n",
                __func__, __LINE__);
4002
        goto PARSE_OVER;
4003
      }
4004 4005 4006

      cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
      if (disorderRatio && disorderRatio->type == cJSON_Number) {
4007 4008 4009 4010 4011 4012
        if (disorderRatio->valueint > 50)
          disorderRatio->valueint = 50;

        if (disorderRatio->valueint < 0)
          disorderRatio->valueint = 0;

4013 4014 4015 4016
        g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
      } else if (!disorderRatio) {
        g_Dbs.db[i].superTbls[j].disorderRatio = 0;
      } else {
4017
        printf("ERROR: failed to read json, disorderRatio not found\n");
4018
        goto PARSE_OVER;
4019
      }
4020 4021 4022 4023 4024 4025 4026

      cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
      if (disorderRange && disorderRange->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
      } else if (!disorderRange) {
        g_Dbs.db[i].superTbls[j].disorderRange = 1000;
      } else {
4027
        printf("ERROR: failed to read json, disorderRange not found\n");
4028 4029
        goto PARSE_OVER;
      }
4030

4031 4032
      cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
      if (insertRows && insertRows->type == cJSON_Number) {
4033 4034 4035 4036 4037
        if (insertRows->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4038 4039 4040 4041
        g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
      } else if (!insertRows) {
        g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
      } else {
4042
        errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
4043
                __func__, __LINE__);
4044 4045 4046 4047 4048 4049
        goto PARSE_OVER;
      }

      cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
      if (insertInterval && insertInterval->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
4050 4051 4052 4053 4054
        if (insertInterval->valueint < 0) {
          errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
          goto PARSE_OVER;
        }
4055
      } else if (!insertInterval) {
4056
        verbosePrint("%s() LN%d: stable insert interval be overrided by global %"PRIu64".\n",
4057
                __func__, __LINE__, g_args.insert_interval);
4058 4059
        g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
      } else {
4060
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
4061
                __func__, __LINE__);
4062 4063 4064
        goto PARSE_OVER;
      }

4065
      int retVal = getColumnAndTagTypeFromInsertJsonFile(
4066
              stbInfo, &g_Dbs.db[i].superTbls[j]);
4067 4068
      if (false == retVal) {
        goto PARSE_OVER;
4069 4070
      }
    }
4071 4072 4073 4074
  }

  ret = true;

4075
PARSE_OVER:
4076 4077 4078 4079 4080 4081 4082 4083
  return ret;
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
4084
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
4085 4086 4087 4088
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
4089
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE);
4090
  } else if (!host) {
4091
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
4092
  } else {
4093
    printf("ERROR: failed to read json, host not found\n");
4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
4106
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE);
4107
  } else if (!user) {
4108
    tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ;
4109 4110 4111 4112
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
4113
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE);
4114
  } else if (!password) {
4115
    tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);;
4116
  }
H
hzcheng 已提交
4117

4118
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
4119
  if (answerPrompt && answerPrompt->type == cJSON_String
4120
          && answerPrompt->valuestring != NULL) {
4121 4122 4123 4124
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
4125
    } else {
4126
      g_args.answer_yes = false;
H
hzcheng 已提交
4127
    }
4128 4129 4130
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
4131
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
4132
    goto PARSE_OVER;
4133
  }
4134

4135 4136
  cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
  if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
4137
    if (gQueryTimes->valueint <= 0) {
4138 4139
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
        __func__, __LINE__);
4140 4141
      goto PARSE_OVER;
    }
4142 4143 4144 4145
    g_args.query_times = gQueryTimes->valueint;
  } else if (!gQueryTimes) {
    g_args.query_times = 1;
  } else {
4146
    errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4147
        __func__, __LINE__);
4148 4149 4150
    goto PARSE_OVER;
  }

4151 4152
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
4153
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE);
4154
  } else if (!dbs) {
4155
    printf("ERROR: failed to read json, databases not found\n");
4156
    goto PARSE_OVER;
H
hzcheng 已提交
4157
  }
4158 4159 4160

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
4161
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
4162
  } else if (!queryMode) {
4163
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
4164
  } else {
4165
    printf("ERROR: failed to read json, query_mode not found\n");
4166
    goto PARSE_OVER;
H
hzcheng 已提交
4167
  }
4168

4169
  // specified_table_query
4170 4171
  cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!specifiedQuery) {
4172
    g_queryInfo.specifiedQueryInfo.concurrent = 1;
4173 4174
    g_queryInfo.specifiedQueryInfo.sqlCount = 0;
  } else if (specifiedQuery->type != cJSON_Object) {
4175
    printf("ERROR: failed to read json, super_table_query not found\n");
4176
    goto PARSE_OVER;
4177
  } else {
4178 4179 4180 4181 4182
    cJSON* queryInterval = cJSON_GetObjectItem(specifiedQuery, "query_interval");
    if (queryInterval && queryInterval->type == cJSON_Number) {
      g_queryInfo.specifiedQueryInfo.queryInterval = queryInterval->valueint;
    } else if (!queryInterval) {
      g_queryInfo.specifiedQueryInfo.queryInterval = 0;
4183 4184
    }

4185
    cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery,
4186
        "query_times");
4187
    if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
4188 4189
      if (specifiedQueryTimes->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4190
          __func__, __LINE__, specifiedQueryTimes->valueint);
4191 4192 4193
        goto PARSE_OVER;

      }
4194 4195 4196 4197
      g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint;
    } else if (!specifiedQueryTimes) {
      g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times;
    } else {
4198
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4199
          __func__, __LINE__);
4200
      goto PARSE_OVER;
4201
    }
4202

4203
    cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent");
4204
    if (concurrent && concurrent->type == cJSON_Number) {
4205 4206 4207 4208
      if (concurrent->valueint <= 0) {
        errorPrint("%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n",
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount,
4209
              g_queryInfo.specifiedQueryInfo.concurrent);
4210 4211
        goto PARSE_OVER;
      }
4212
      g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint;
4213
    } else if (!concurrent) {
4214
      g_queryInfo.specifiedQueryInfo.concurrent = 1;
4215
    }
4216

4217 4218 4219 4220 4221 4222 4223
    cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode");
    if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String
        && specifiedAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) {
        g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE;
4224
      } else {
4225
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4226
            __func__, __LINE__);
4227 4228 4229
        goto PARSE_OVER;
      }
    } else {
4230
      g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE;
4231
    }
4232

4233
    cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval");
4234
    if (interval && interval->type == cJSON_Number) {
4235
      g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint;
4236
    } else if (!interval) {
4237 4238
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4239
      g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000;
4240
    }
4241

4242
    cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart");
4243
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
4244
      if (0 == strcmp("yes", restart->valuestring)) {
4245
        g_queryInfo.specifiedQueryInfo.subscribeRestart = 1;
4246
      } else if (0 == strcmp("no", restart->valuestring)) {
4247
        g_queryInfo.specifiedQueryInfo.subscribeRestart = 0;
4248
      } else {
4249
        printf("ERROR: failed to read json, subscribe restart error\n");
4250 4251 4252
        goto PARSE_OVER;
      }
    } else {
4253
      g_queryInfo.specifiedQueryInfo.subscribeRestart = 1;
4254
    }
4255

4256
    cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress");
4257
    if (keepProgress
4258 4259
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
4260
      if (0 == strcmp("yes", keepProgress->valuestring)) {
4261
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1;
4262
      } else if (0 == strcmp("no", keepProgress->valuestring)) {
4263
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4264
      } else {
4265
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4266 4267 4268
        goto PARSE_OVER;
      }
    } else {
4269
      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
4270 4271
    }

4272
    // sqls
4273
    cJSON* superSqls = cJSON_GetObjectItem(specifiedQuery, "sqls");
4274
    if (!superSqls) {
4275
      g_queryInfo.specifiedQueryInfo.sqlCount = 0;
4276
    } else if (superSqls->type != cJSON_Array) {
4277
      errorPrint("%s() LN%d, failed to read json, super sqls not found\n",
4278
          __func__, __LINE__);
4279
      goto PARSE_OVER;
4280
    } else {
4281 4282
      int superSqlSize = cJSON_GetArraySize(superSqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4283
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
4284
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4285 4286
        goto PARSE_OVER;
      }
4287

4288
      g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize;
4289 4290 4291
      for (int j = 0; j < superSqlSize; ++j) {
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
        if (sql == NULL) continue;
4292

4293 4294
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
4295
          printf("ERROR: failed to read json, sql not found\n");
4296 4297
          goto PARSE_OVER;
        }
4298
        tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
H
hzcheng 已提交
4299

4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311
        cJSON* resubAfterConsume =
            cJSON_GetObjectItem(specifiedQuery, "resubAfterConsume");
        if (resubAfterConsume
                && resubAfterConsume->type == cJSON_Number) {
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j]
                = resubAfterConsume->valueint;
        } else if (!resubAfterConsume) {
            //printf("failed to read json, subscribe interval no found\n");
            //goto PARSE_OVER;
            g_queryInfo.specifiedQueryInfo.resubAfterConsume[j] = 1;
        }

4312 4313
        cJSON *result = cJSON_GetObjectItem(sql, "result");
        if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) {
4314
          tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN);
4315
        } else if (NULL == result) {
4316
          memset(g_queryInfo.specifiedQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4317
        } else {
4318
          printf("ERROR: failed to read json, super query result file not found\n");
4319
          goto PARSE_OVER;
4320
        }
4321 4322 4323
      }
    }
  }
4324

4325
  // super_table_query
4326 4327
  cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!superQuery) {
4328
    g_queryInfo.superQueryInfo.threadCnt = 1;
4329 4330
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
4331
    printf("ERROR: failed to read json, sub_table_query not found\n");
4332 4333
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
4334
  } else {
4335
    cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval");
4336
    if (subrate && subrate->type == cJSON_Number) {
4337
      g_queryInfo.superQueryInfo.queryInterval = subrate->valueint;
4338
    } else if (!subrate) {
4339
      g_queryInfo.superQueryInfo.queryInterval = 0;
4340 4341 4342 4343
    }

    cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
    if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
4344 4345
      if (superQueryTimes->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
4346
          __func__, __LINE__, superQueryTimes->valueint);
4347 4348
        goto PARSE_OVER;
      }
4349 4350 4351 4352
      g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
    } else if (!superQueryTimes) {
      g_queryInfo.superQueryInfo.queryTimes = g_args.query_times;
    } else {
4353
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
4354
          __func__, __LINE__);
4355
      goto PARSE_OVER;
4356
    }
4357

4358
    cJSON* threads = cJSON_GetObjectItem(superQuery, "threads");
4359
    if (threads && threads->type == cJSON_Number) {
4360 4361 4362 4363 4364 4365
      if (threads->valueint <= 0) {
        errorPrint("%s() LN%d, failed to read json, threads input mistake\n",
          __func__, __LINE__);
        goto PARSE_OVER;

      }
4366
      g_queryInfo.superQueryInfo.threadCnt = threads->valueint;
4367
    } else if (!threads) {
4368
      g_queryInfo.superQueryInfo.threadCnt = 1;
4369
    }
4370

4371
    //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count");
4372
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
4373
    //  g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint;
4374
    //} else if (!subTblCnt) {
4375
    //  g_queryInfo.superQueryInfo.childTblCount = 0;
4376
    //}
4377

4378
    cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname");
4379 4380 4381
    if (stblname && stblname->type == cJSON_String
        && stblname->valuestring != NULL) {
      tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring,
4382
          MAX_TB_NAME_SIZE);
4383
    } else {
4384
      errorPrint("%s() LN%d, failed to read json, super table name input error\n",
4385
          __func__, __LINE__);
4386 4387
      goto PARSE_OVER;
    }
4388

4389 4390 4391 4392 4393 4394 4395
    cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode");
    if (superAsyncMode && superAsyncMode->type == cJSON_String
        && superAsyncMode->valuestring != NULL) {
      if (0 == strcmp("sync", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
      } else if (0 == strcmp("async", superAsyncMode->valuestring)) {
        g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE;
4396
      } else {
4397
        errorPrint("%s() LN%d, failed to read json, async mode input error\n",
4398
            __func__, __LINE__);
4399 4400 4401
        goto PARSE_OVER;
      }
    } else {
4402
      g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE;
4403
    }
4404

4405 4406 4407 4408 4409 4410 4411 4412 4413
    cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval");
    if (superInterval && superInterval->type == cJSON_Number) {
      if (superInterval->valueint < 0) {
        errorPrint("%s() LN%d, failed to read json, interval input mistake\n",
            __func__, __LINE__);
        goto PARSE_OVER;
      }
      g_queryInfo.superQueryInfo.subscribeInterval = superInterval->valueint;
    } else if (!superInterval) {
4414 4415
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4416
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
4417
    }
4418

4419
    cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart");
4420 4421
    if (subrestart && subrestart->type == cJSON_String
        && subrestart->valuestring != NULL) {
4422
      if (0 == strcmp("yes", subrestart->valuestring)) {
4423
        g_queryInfo.superQueryInfo.subscribeRestart = 1;
4424
      } else if (0 == strcmp("no", subrestart->valuestring)) {
4425
        g_queryInfo.superQueryInfo.subscribeRestart = 0;
4426
      } else {
4427
        printf("ERROR: failed to read json, subscribe restart error\n");
4428 4429 4430
        goto PARSE_OVER;
      }
    } else {
4431
      g_queryInfo.superQueryInfo.subscribeRestart = 1;
4432
    }
4433

4434
    cJSON* subkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
4435
    if (subkeepProgress &&
4436 4437
            subkeepProgress->type == cJSON_String
            && subkeepProgress->valuestring != NULL) {
4438
      if (0 == strcmp("yes", subkeepProgress->valuestring)) {
4439
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
4440
      } else if (0 == strcmp("no", subkeepProgress->valuestring)) {
4441
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4442
      } else {
4443
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4444 4445 4446
        goto PARSE_OVER;
      }
    } else {
4447
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4448
    }
4449

4450
    // sqls
4451
    cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls");
4452
    if (!subsqls) {
4453
      g_queryInfo.superQueryInfo.sqlCount = 0;
4454
    } else if (subsqls->type != cJSON_Array) {
4455
      errorPrint("%s() LN%d: failed to read json, super sqls not found\n",
4456
          __func__, __LINE__);
4457
      goto PARSE_OVER;
4458
    } else {
4459 4460
      int superSqlSize = cJSON_GetArraySize(subsqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4461
        errorPrint("%s() LN%d, failed to read json, query sql size overflow, max is %d\n",
4462
           __func__, __LINE__, MAX_QUERY_SQL_COUNT);
4463 4464
        goto PARSE_OVER;
      }
4465

4466
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
4467
      for (int j = 0; j < superSqlSize; ++j) {
4468 4469
        cJSON* sql = cJSON_GetArrayItem(subsqls, j);
        if (sql == NULL) continue;
4470

4471
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
4472 4473 4474
        if (!sqlStr || sqlStr->type != cJSON_String
            || sqlStr->valuestring == NULL) {
          errorPrint("%s() LN%d, failed to read json, sql not found\n",
4475
              __func__, __LINE__);
4476 4477
          goto PARSE_OVER;
        }
4478
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring,
4479
            MAX_QUERY_SQL_LENGTH);
4480

4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492
        cJSON* superResubAfterConsume =
            cJSON_GetObjectItem(sql, "resubAfterConsume");
        if (superResubAfterConsume
                && superResubAfterConsume->type == cJSON_Number) {
            g_queryInfo.superQueryInfo.resubAfterConsume[j] =
                superResubAfterConsume->valueint;
        } else if (!superResubAfterConsume) {
            //printf("failed to read json, subscribe interval no found\n");
            //goto PARSE_OVER;
            g_queryInfo.superQueryInfo.resubAfterConsume[j] = 1;
        }

4493
        cJSON *result = cJSON_GetObjectItem(sql, "result");
4494 4495 4496
        if (result != NULL && result->type == cJSON_String
            && result->valuestring != NULL){
          tstrncpy(g_queryInfo.superQueryInfo.result[j],
4497
              result->valuestring, MAX_FILE_NAME_LEN);
4498
        } else if (NULL == result) {
4499
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4500
        }  else {
4501
          errorPrint("%s() LN%d, failed to read json, sub query result file not found\n",
4502
              __func__, __LINE__);
4503
          goto PARSE_OVER;
4504
        }
4505 4506
      }
    }
H
hzcheng 已提交
4507 4508
  }

4509
  ret = true;
H
hzcheng 已提交
4510

4511
PARSE_OVER:
4512 4513
  return ret;
}
H
hzcheng 已提交
4514

4515
static bool getInfoFromJsonFile(char* file) {
4516
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
4517

4518 4519 4520 4521 4522
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
4523

4524
  bool  ret = false;
4525
  int   maxLen = 6400000;
4526 4527 4528 4529 4530 4531 4532
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
4533
  }
H
Hui Li 已提交
4534

4535 4536 4537
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
4538
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
4539 4540
    goto PARSE_OVER;
  }
H
Hui Li 已提交
4541

4542 4543 4544
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
4545
      g_args.test_mode = INSERT_TEST;
4546
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
4547
      g_args.test_mode = QUERY_TEST;
4548
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
4549
      g_args.test_mode = SUBSCRIBE_TEST;
4550
    } else {
4551
      printf("ERROR: failed to read json, filetype not support\n");
4552 4553 4554
      goto PARSE_OVER;
    }
  } else if (!filetype) {
4555
    g_args.test_mode = INSERT_TEST;
4556
  } else {
4557
    printf("ERROR: failed to read json, filetype not found\n");
4558 4559
    goto PARSE_OVER;
  }
H
hzcheng 已提交
4560

4561
  if (INSERT_TEST == g_args.test_mode) {
4562
    ret = getMetaFromInsertJsonFile(root);
4563
  } else if ((QUERY_TEST == g_args.test_mode)
4564
          || (SUBSCRIBE_TEST == g_args.test_mode)) {
4565 4566
    ret = getMetaFromQueryJsonFile(root);
  } else {
4567
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n",
4568
            __func__, __LINE__);
4569
    goto PARSE_OVER;
4570
  }
4571

4572
PARSE_OVER:
4573 4574
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
4575
  fclose(fp);
4576 4577
  return ret;
}
H
hzcheng 已提交
4578

4579
static void prepareSampleData() {
4580
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4581 4582 4583
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
        (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]);
S
Shuaiqiang Chang 已提交
4584
      }
4585 4586 4587 4588
    }
  }
}

4589
static void postFreeResource() {
4590
  tmfclose(g_fpOfInsertResult);
4591
  for (int i = 0; i < g_Dbs.dbCount; i++) {
S
Shuduo Sang 已提交
4592
    for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
4593 4594 4595
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
4596
      }
4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4609
    }
4610 4611
  }
}
S
Shuaiqiang Chang 已提交
4612

4613 4614 4615
static int getRowDataFromSample(
        char* dataBuf, int64_t maxLen, int64_t timestamp,
      SSuperTable* superTblInfo, int64_t* sampleUsePos) {
4616
  if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
4617
/*    int ret = readSampleFromCsvFileToMem(superTblInfo);
4618
    if (0 != ret) {
4619
      tmfree(superTblInfo->sampleDataBuf);
4620
      superTblInfo->sampleDataBuf = NULL;
4621 4622
      return -1;
    }
4623
*/
4624 4625
    *sampleUsePos = 0;
  }
S
Shuaiqiang Chang 已提交
4626

4627
  int    dataLen = 0;
4628

4629
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
4630
          "(%" PRId64 ", ", timestamp);
4631
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
4632
          "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
4633
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
S
Shuaiqiang Chang 已提交
4634

4635
  (*sampleUsePos)++;
4636

4637 4638
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4639

4640 4641
static int64_t generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stbInfo) {
  int64_t   dataLen = 0;
4642
  char  *pstr = recBuf;
4643
  int64_t maxLen = MAX_DATA_SIZE;
4644

4645
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "(%" PRId64 ",", timestamp);
4646

4647
  for (int i = 0; i < stbInfo->columnCount; i++) {
4648 4649
    if ((0 == strncasecmp(stbInfo->columns[i].dataType, "BINARY", strlen("BINARY")))
            || (0 == strncasecmp(stbInfo->columns[i].dataType, "NCHAR", strlen("NCHAR")))) {
4650
      if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
4651
        errorPrint( "binary or nchar length overflow, max size:%u\n",
4652
                (uint32_t)TSDB_MAX_BINARY_LEN);
4653
        return -1;
4654
      }
4655

4656 4657
      char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
      if (NULL == buf) {
4658
        errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
4659
        return -1;
4660 4661
      }
      rand_string(buf, stbInfo->columns[i].dataLen);
4662
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf);
4663
      tmfree(buf);
4664
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4665
                "INT", 3)) {
4666
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4667
              "%d,", rand_int());
4668
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4669
                "BIGINT", 6)) {
4670
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4671
              "%"PRId64",", rand_bigint());
4672
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4673
                "FLOAT", 5)) {
4674
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4675
              "%f,", rand_float());
4676
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4677
                "DOUBLE", 6)) {
4678
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4679
              "%f,", rand_double());
4680
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4681
                "SMALLINT", 8)) {
4682
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4683
          "%d,", rand_smallint());
4684
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4685
          "TINYINT", strlen("TINYINT"))) {
4686
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4687
          "%d,", rand_tinyint());
4688
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4689
          "BOOL", strlen("BOOL"))) {
4690
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4691
          "%d,", rand_bool());
4692
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
4693
          "TIMESTAMP", strlen("TIMESTAMP"))) {
4694
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4695
          "%"PRId64",", rand_bigint());
4696
    }  else {
4697
      errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType);
4698
      return -1;
4699
    }
S
Shuaiqiang Chang 已提交
4700
  }
4701

4702
  dataLen -= 1;
4703
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")");
4704

4705 4706 4707
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

  return strlen(recBuf);
4708
}
S
Shuaiqiang Chang 已提交
4709

4710
static int64_t generateData(char *recBuf, char **data_type,
4711
        int num_of_cols, int64_t timestamp, int lenOfBinary) {
4712 4713
  memset(recBuf, 0, MAX_DATA_SIZE);
  char *pstr = recBuf;
4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727
  pstr += sprintf(pstr, "(%" PRId64, timestamp);
  int c = 0;

  for (; c < MAX_NUM_DATATYPE; c++) {
    if (data_type[c] == NULL) {
      break;
    }
  }

  if (0 == c) {
    perror("data type error!");
    exit(-1);
  }

H
Haojun Liao 已提交
4728
  for (int i = 0; i < c; i++) {
4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739
    if (strcasecmp(data_type[i % c], "TINYINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_tinyint() );
    } else if (strcasecmp(data_type[i % c], "SMALLINT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_smallint());
    } else if (strcasecmp(data_type[i % c], "INT") == 0) {
      pstr += sprintf(pstr, ",%d", rand_int());
    } else if (strcasecmp(data_type[i % c], "BIGINT") == 0) {
      pstr += sprintf(pstr, ",%" PRId64, rand_bigint());
    } else if (strcasecmp(data_type[i % c], "FLOAT") == 0) {
      pstr += sprintf(pstr, ",%10.4f", rand_float());
    } else if (strcasecmp(data_type[i % c], "DOUBLE") == 0) {
4740
      double t = rand_double();
4741 4742
      pstr += sprintf(pstr, ",%20.8f", t);
    } else if (strcasecmp(data_type[i % c], "BOOL") == 0) {
4743
      bool b = rand_bool() & 1;
4744 4745
      pstr += sprintf(pstr, ",%s", b ? "true" : "false");
    } else if (strcasecmp(data_type[i % c], "BINARY") == 0) {
4746 4747
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4748
      pstr += sprintf(pstr, ",\"%s\"", s);
4749
      free(s);
4750
    } else if (strcasecmp(data_type[i % c], "NCHAR") == 0) {
4751 4752
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4753
      pstr += sprintf(pstr, ",\"%s\"", s);
4754 4755 4756
      free(s);
    }

4757
    if (strlen(recBuf) > MAX_DATA_SIZE) {
4758 4759 4760 4761 4762 4763 4764
      perror("column length too long, abort");
      exit(-1);
    }
  }

  pstr += sprintf(pstr, ")");

4765 4766
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

4767
  return (int32_t)strlen(recBuf);
4768 4769
}

4770
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
4771 4772
  char* sampleDataBuf = NULL;

4773
  sampleDataBuf = calloc(
4774
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
4775
  if (sampleDataBuf == NULL) {
4776
      errorPrint("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n",
4777 4778 4779 4780
              __func__, __LINE__,
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE,
              strerror(errno));
      return -1;
4781
  }
4782

4783 4784 4785 4786
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
4787 4788 4789 4790 4791
      errorPrint("%s() LN%d, read sample from csv file failed.\n",
          __func__, __LINE__);
      tmfree(sampleDataBuf);
      superTblInfo->sampleDataBuf = NULL;
      return -1;
H
Hui Li 已提交
4792
  }
4793

4794 4795 4796
  return 0;
}

S
Shuduo Sang 已提交
4797
static int64_t execInsert(threadInfo *pThreadInfo, uint64_t k)
4798 4799
{
  int affectedRows;
4800
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4801

4802
  verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
S
Shuduo Sang 已提交
4803
            __func__, __LINE__, pThreadInfo->buffer);
4804
  if (superTblInfo) {
4805
    if (superTblInfo->insertMode == TAOSC_IFACE) {
S
Shuduo Sang 已提交
4806 4807 4808
      affectedRows = queryDbExec(
              pThreadInfo->taos,
              pThreadInfo->buffer, INSERT_TYPE, false);
4809
    } else if (superTblInfo->insertMode == REST_IFACE) {
4810
      if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port,
S
Shuduo Sang 已提交
4811
                  pThreadInfo->buffer, NULL /* not set result file */)) {
4812
        affectedRows = -1;
4813
        printf("========restful return fail, threadID[%d]\n",
4814
            pThreadInfo->threadID);
4815 4816 4817
      } else {
        affectedRows = k;
      }
4818
    } else if (superTblInfo->insertMode == STMT_IFACE) {
S
Shuduo Sang 已提交
4819 4820 4821 4822 4823 4824
      debugPrint("%s() LN%d, stmt=%p", __func__, __LINE__, pThreadInfo->stmt);
      if (0 != taos_stmt_execute(pThreadInfo->stmt)) {
        errorPrint("%s() LN%d, failied to execute insert statement\n",
                    __func__, __LINE__);
        exit(-1);
      }
S
Shuduo Sang 已提交
4825 4826

      affectedRows = k;
4827
    } else {
4828
      errorPrint("%s() LN%d: unknown insert mode: %d\n",
4829 4830
        __func__, __LINE__, superTblInfo->insertMode);
      affectedRows = 0;
4831
    }
4832
  } else {
S
Shuduo Sang 已提交
4833
    affectedRows = queryDbExec(pThreadInfo->taos, pThreadInfo->buffer, INSERT_TYPE, false);
4834 4835 4836 4837 4838
  }

  return affectedRows;
}

4839
static void getTableName(char *pTblName, threadInfo* pThreadInfo, uint64_t tableSeq)
4840 4841
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4842
  if (superTblInfo) {
4843
    if (superTblInfo->childTblLimit > 0) {
4844 4845 4846
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName +
            (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
4847
    } else {
4848

4849
        verbosePrint("[%d] %s() LN%d: from=%"PRIu64" count=%"PRId64" seq=%"PRIu64"\n",
4850 4851 4852 4853 4854
                pThreadInfo->threadID, __func__, __LINE__,
                pThreadInfo->start_table_from,
                pThreadInfo->ntables, tableSeq);
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
4855 4856
    }
  } else {
4857
    snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%"PRIu64"",
4858
        g_args.tb_prefix, tableSeq);
4859 4860 4861
  }
}

4862
static int64_t generateDataTail(
4863
        SSuperTable* superTblInfo,
4864
        uint64_t batch, char* buffer, int64_t remainderBufLen, int64_t insertRows,
4865
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos, int64_t *dataLen) {
4866 4867
  uint64_t len = 0;
  uint32_t ncols_per_record = 1; // count first col ts
4868

4869 4870
  char *pstr = buffer;

4871
  if (superTblInfo == NULL) {
4872
    uint32_t datatypeSeq = 0;
4873
    while(g_args.datatype[datatypeSeq]) {
4874 4875
        datatypeSeq ++;
        ncols_per_record ++;
4876
    }
4877 4878
  }

4879
  verbosePrint("%s() LN%d batch=%"PRIu64"\n", __func__, __LINE__, batch);
4880

4881
  uint64_t k = 0;
4882
  for (k = 0; k < batch;) {
4883
    char data[MAX_DATA_SIZE];
4884 4885
    memset(data, 0, MAX_DATA_SIZE);

4886
    int64_t retLen = 0;
4887

4888 4889
    if (superTblInfo) {
      if (0 == strncasecmp(superTblInfo->dataSource,
4890 4891 4892 4893 4894 4895 4896
                    "sample", strlen("sample"))) {
          retLen = getRowDataFromSample(
                    data,
                    remainderBufLen,
                    startTime + superTblInfo->timeStampStep * k,
                    superTblInfo,
                    pSamplePos);
4897
      } else if (0 == strncasecmp(superTblInfo->dataSource,
4898
                   "rand", strlen("rand"))) {
4899

4900
        int64_t randTail = superTblInfo->timeStampStep * k;
4901 4902 4903 4904
        if (superTblInfo->disorderRatio > 0) {
          int rand_num = taosRandom() % 100;
          if(rand_num < superTblInfo->disorderRatio) {
            randTail = (randTail + (taosRandom() % superTblInfo->disorderRange + 1)) * (-1);
4905
            debugPrint("rand data generated, back %"PRId64"\n", randTail);
4906
          }
4907 4908
        }

4909
        int64_t d = startTime
4910
                + randTail;
4911
        retLen = generateRowData(
4912 4913 4914
                      data,
                      d,
                      superTblInfo);
4915
      }
4916

4917 4918 4919
      if (retLen > remainderBufLen) {
        break;
      }
4920

4921
      pstr += snprintf(pstr , retLen + 1, "%s", data);
4922 4923 4924
      k++;
      len += retLen;
      remainderBufLen -= retLen;
4925
    } else {
4926 4927
      char **data_type = g_args.datatype;
      int lenOfBinary = g_args.len_of_binary;
4928

4929 4930 4931 4932 4933 4934
      int64_t randTail = DEFAULT_TIMESTAMP_STEP * k;

      if (g_args.disorderRatio != 0) {
        int rand_num = taosRandom() % 100;
        if (rand_num < g_args.disorderRatio) {
          randTail = (randTail + (taosRandom() % g_args.disorderRange + 1)) * (-1);
4935

4936 4937
          debugPrint("rand data generated, back %"PRId64"\n", randTail);
        }
4938
      } else {
4939 4940 4941 4942
        randTail = DEFAULT_TIMESTAMP_STEP * k;
      }

      retLen = generateData(data, data_type,
4943 4944 4945
                  ncols_per_record,
                  startTime + randTail,
                  lenOfBinary);
4946

4947 4948 4949
      if (len > remainderBufLen)
        break;

H
Haojun Liao 已提交
4950
      pstr += sprintf(pstr, "%s", data);
4951 4952 4953
      k++;
      len += retLen;
      remainderBufLen -= retLen;
4954 4955
    }

4956
    verbosePrint("%s() LN%d len=%"PRIu64" k=%"PRIu64" \nbuffer=%s\n",
4957
            __func__, __LINE__, len, k, buffer);
4958 4959

    startFrom ++;
4960

4961
    if (startFrom >= insertRows) {
4962
      break;
4963
    }
4964
  }
4965

4966 4967 4968
  *dataLen = len;
  return k;
}
4969

4970
static int generateSQLHead(char *tableName, int32_t tableSeq,
4971 4972
        threadInfo* pThreadInfo, SSuperTable* superTblInfo,
        char *buffer, int remainderBufLen)
4973 4974
{
  int len;
4975 4976 4977 4978

#define HEAD_BUFF_LEN    1024*24  // 16*1024 + (192+32)*2 + insert into ..
  char headBuf[HEAD_BUFF_LEN];

4979 4980 4981 4982
  if (superTblInfo) {
    if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
4983
            tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq);
4984
      } else {
4985 4986 4987
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
4988 4989
      }
      if (NULL == tagsValBuf) {
4990
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n",
4991
            __func__, __LINE__);
4992 4993 4994
        return -1;
      }

4995 4996
      len = snprintf(
          headBuf,
4997 4998 4999 5000 5001 5002 5003
                  HEAD_BUFF_LEN,
                  "%s.%s using %s.%s tags %s values",
                  pThreadInfo->db_name,
                  tableName,
                  pThreadInfo->db_name,
                  superTblInfo->sTblName,
                  tagsValBuf);
5004 5005
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
5006 5007
      len = snprintf(
          headBuf,
5008 5009 5010 5011
                  HEAD_BUFF_LEN,
                  "%s.%s values",
                  pThreadInfo->db_name,
                  tableName);
5012
    } else {
5013 5014
      len = snprintf(
          headBuf,
5015 5016 5017 5018
                  HEAD_BUFF_LEN,
                  "%s.%s values",
                  pThreadInfo->db_name,
                  tableName);
5019 5020
    }
  } else {
5021 5022 5023 5024 5025 5026
      len = snprintf(
          headBuf,
                  HEAD_BUFF_LEN,
                  "%s.%s values",
                  pThreadInfo->db_name,
                  tableName);
5027 5028
  }

5029 5030 5031 5032 5033
  if (len > remainderBufLen)
    return -1;

  tstrncpy(buffer, headBuf, len + 1);

5034 5035 5036
  return len;
}

5037
static int64_t generateInterlaceDataBuffer(
5038 5039
        char *tableName, uint64_t batchPerTbl, uint64_t i, uint64_t batchPerTblTimes,
        uint64_t tableSeq,
5040
        threadInfo *pThreadInfo, char *buffer,
5041
        int64_t insertRows,
5042
        int64_t startTime,
5043
        uint64_t *pRemainderBufLen)
5044
{
5045
  assert(buffer);
5046 5047 5048 5049
  char *pstr = buffer;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  int headLen = generateSQLHead(tableName, tableSeq, pThreadInfo,
5050
            superTblInfo, pstr, *pRemainderBufLen);
5051 5052 5053 5054 5055

  if (headLen <= 0) {
    return 0;
  }
  // generate data buffer
5056
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" buffer:\n%s\n",
5057
            pThreadInfo->threadID, __func__, __LINE__, i, buffer);
5058 5059 5060 5061

  pstr += headLen;
  *pRemainderBufLen -= headLen;

5062
  int64_t dataLen = 0;
5063

5064
  verbosePrint("[%d] %s() LN%d i=%"PRIu64" batchPerTblTimes=%"PRIu64" batchPerTbl = %"PRIu64"\n",
5065 5066
            pThreadInfo->threadID, __func__, __LINE__,
            i, batchPerTblTimes, batchPerTbl);
5067 5068 5069 5070 5071 5072

  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
      startTime = taosGetTimestamp(pThreadInfo->time_precision);
    }
  } else {
5073
      startTime = 1500000000000;
5074
  }
5075

5076
  int64_t k = generateDataTail(
5077 5078 5079 5080
    superTblInfo,
    batchPerTbl, pstr, *pRemainderBufLen, insertRows, 0,
    startTime,
    &(pThreadInfo->samplePos), &dataLen);
5081

5082
  if (k == batchPerTbl) {
5083 5084 5085
    pstr += dataLen;
    *pRemainderBufLen -= dataLen;
  } else {
5086
    debugPrint("%s() LN%d, generated data tail: %"PRIu64", not equal batch per table: %"PRIu64"\n",
5087
            __func__, __LINE__, k, batchPerTbl);
5088 5089
    pstr -= headLen;
    pstr[0] = '\0';
5090
    k = 0;
5091 5092 5093 5094 5095
  }

  return k;
}

5096
static int64_t generateProgressiveDataBuffer(
5097
        char *tableName,
5098
        int64_t tableSeq,
5099 5100
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
5101
        uint64_t startFrom, int64_t startTime, int64_t *pSamplePos,
5102
        int64_t *pRemainderBufLen)
5103 5104 5105 5106 5107 5108 5109 5110
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  int ncols_per_record = 1; // count first col ts

  if (superTblInfo == NULL) {
    int datatypeSeq = 0;
    while(g_args.datatype[datatypeSeq]) {
5111 5112
        datatypeSeq ++;
        ncols_per_record ++;
5113 5114 5115 5116
    }
  }

  assert(buffer != NULL);
5117
  char *pstr = buffer;
5118

5119
  int64_t k = 0;
5120

5121
  memset(buffer, 0, *pRemainderBufLen);
5122

5123
  int64_t headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, superTblInfo,
5124
          buffer, *pRemainderBufLen);
5125 5126 5127 5128

  if (headLen <= 0) {
    return 0;
  }
5129
  pstr += headLen;
5130
  *pRemainderBufLen -= headLen;
5131

5132
  int64_t dataLen;
5133
  k = generateDataTail(superTblInfo,
5134 5135 5136
          g_args.num_of_RPR, pstr, *pRemainderBufLen, insertRows, startFrom,
          startTime,
          pSamplePos, &dataLen);
5137

5138 5139
  return k;
}
5140

5141 5142 5143 5144 5145 5146 5147 5148 5149
static void printStatPerThread(threadInfo *pThreadInfo)
{
  fprintf(stderr, "====thread[%d] completed total inserted rows: %"PRIu64 ", total affected rows: %"PRIu64". %.2f records/second====\n",
          pThreadInfo->threadID,
          pThreadInfo->totalInsertRows,
          pThreadInfo->totalAffectedRows,
          (double)(pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay/1000.0)));
}

5150
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
5151
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
5152 5153
         pThreadInfo->threadID, __func__, __LINE__);

5154
  int64_t insertRows;
5155
  uint64_t interlaceRows;
5156 5157

  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5158

5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171
  if (superTblInfo) {
    insertRows = superTblInfo->insertRows;

    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    insertRows = g_args.num_of_DPT;
    interlaceRows = g_args.interlace_rows;
  }
5172

5173
  if (interlaceRows > insertRows)
H
Haojun Liao 已提交
5174
    interlaceRows = insertRows;
5175

5176 5177
  if (interlaceRows > g_args.num_of_RPR)
    interlaceRows = g_args.num_of_RPR;
5178

5179
  int progOrInterlace;
5180 5181

  if (interlaceRows > 0) {
5182
    progOrInterlace= INTERLACE_INSERT_MODE;
5183
  } else {
5184
    progOrInterlace = PROGRESSIVE_INSERT_MODE;
5185 5186
  }

5187
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
S
Shuduo Sang 已提交
5188 5189
  pThreadInfo->buffer = calloc(maxSqlLen, 1);
  if (NULL == pThreadInfo->buffer) {
5190
    errorPrint( "%s() LN%d, Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5191
              __func__, __LINE__, maxSqlLen, strerror(errno));
5192 5193 5194
    return NULL;
  }

5195
  char tableName[TSDB_TABLE_NAME_LEN];
5196 5197 5198 5199

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

5200
  int64_t nTimeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
5201

5202
  uint64_t insert_interval =
5203
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5204 5205
  uint64_t st = 0;
  uint64_t et = UINT64_MAX;
5206

5207 5208 5209
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5210

5211
  uint64_t tableSeq = pThreadInfo->start_table_from;
5212

S
Shuduo Sang 已提交
5213
  debugPrint("[%d] %s() LN%d: start_table_from=%"PRIu64" ntables=%"PRId64" insertRows=%"PRIu64"\n",
5214 5215
          pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from,
          pThreadInfo->ntables, insertRows);
5216 5217 5218

  int64_t startTime = pThreadInfo->start_time;

5219 5220
  uint64_t batchPerTbl = interlaceRows;
  uint64_t batchPerTblTimes;
5221

5222
  if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) {
5223
    batchPerTblTimes =
5224
        g_args.num_of_RPR / interlaceRows;
5225 5226 5227 5228
  } else {
    batchPerTblTimes = 1;
  }

5229
  uint64_t generatedRecPerTbl = 0;
5230
  bool flagSleep = true;
5231
  uint64_t sleepTimeTotal = 0;
5232

5233 5234 5235
  char *strInsertInto = "insert into ";
  int nInsertBufLen = strlen(strInsertInto);

5236
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
5237
    if ((flagSleep) && (insert_interval)) {
5238 5239
        st = taosGetTimestampMs();
        flagSleep = false;
5240 5241
    }
    // generate data
S
Shuduo Sang 已提交
5242
    memset(pThreadInfo->buffer, 0, maxSqlLen);
5243
    uint64_t remainderBufLen = maxSqlLen;
5244

S
Shuduo Sang 已提交
5245
    char *pstr = pThreadInfo->buffer;
5246 5247 5248 5249 5250

    int len = snprintf(pstr, nInsertBufLen + 1, "%s", strInsertInto);
    pstr += len;
    remainderBufLen -= len;

5251
    uint64_t recOfBatch = 0;
5252

5253
    for (uint64_t i = 0; i < batchPerTblTimes; i ++) {
5254
      getTableName(tableName, pThreadInfo, tableSeq);
5255 5256
      if (0 == strlen(tableName)) {
        errorPrint("[%d] %s() LN%d, getTableName return null\n",
5257
            pThreadInfo->threadID, __func__, __LINE__);
S
Shuduo Sang 已提交
5258
        free(pThreadInfo->buffer);
5259 5260
        return NULL;
      }
5261

5262
      uint64_t oldRemainderLen = remainderBufLen;
5263
      int64_t generated = generateInterlaceDataBuffer(
5264 5265 5266 5267 5268 5269
        tableName, batchPerTbl, i, batchPerTblTimes,
        tableSeq,
        pThreadInfo, pstr,
        insertRows,
        startTime,
        &remainderBufLen);
5270

5271 5272
      debugPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5273
      if (generated < 0) {
5274
        errorPrint("[%d] %s() LN%d, generated records is %"PRId64"\n",
5275
                  pThreadInfo->threadID, __func__, __LINE__, generated);
5276
        goto free_of_interlace;
5277 5278
      } else if (generated == 0) {
        break;
5279 5280
      }

5281
      tableSeq ++;
5282
      recOfBatch += batchPerTbl;
5283
      pstr += (oldRemainderLen - remainderBufLen);
5284
//      startTime += batchPerTbl * superTblInfo->timeStampStep;
5285
      pThreadInfo->totalInsertRows += batchPerTbl;
5286
      verbosePrint("[%d] %s() LN%d batchPerTbl=%"PRId64" recOfBatch=%"PRId64"\n",
5287 5288
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
5289

5290
      if (progOrInterlace == INTERLACE_INSERT_MODE) {
5291 5292 5293 5294
          if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
            // turn to first table
            tableSeq = pThreadInfo->start_table_from;
            generatedRecPerTbl += batchPerTbl;
5295

5296 5297
            startTime = pThreadInfo->start_time
              + generatedRecPerTbl * nTimeStampStep;
5298

5299 5300 5301
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;
5302

5303 5304 5305
            int remainRows = insertRows - generatedRecPerTbl;
            if ((remainRows > 0) && (batchPerTbl > remainRows))
              batchPerTbl = remainRows;
5306

5307 5308 5309 5310
            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
          }
      }
5311

5312
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%"PRId64" insertRows=%"PRId64"\n",
5313 5314
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);
5315 5316

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
5317 5318 5319
        break;
    }

5320
    verbosePrint("[%d] %s() LN%d recOfBatch=%"PRIu64" totalInsertRows=%"PRIu64"\n",
5321 5322
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
5323
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
S
Shuduo Sang 已提交
5324
           pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->buffer);
5325

5326
    startTs = taosGetTimestampMs();
5327

5328 5329 5330 5331 5332 5333 5334
    if (recOfBatch == 0) {
      errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n",
              pThreadInfo->threadID, __func__, __LINE__,
              recOfBatch);
      errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n");
      goto free_of_interlace;
    }
S
Shuduo Sang 已提交
5335
    int64_t affectedRows = execInsert(pThreadInfo, recOfBatch);
5336

5337
    endTs = taosGetTimestampMs();
5338 5339
    uint64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %"PRIu64"ms\n",
5340
            __func__, __LINE__, delay);
5341 5342 5343
    verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5344 5345 5346 5347 5348 5349

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

5350 5351
    if (recOfBatch != affectedRows) {
        errorPrint("[%d] %s() LN%d execInsert insert %"PRIu64", affected rows: %"PRId64"\n%s\n",
5352
                pThreadInfo->threadID, __func__, __LINE__,
S
Shuduo Sang 已提交
5353
                recOfBatch, affectedRows, pThreadInfo->buffer);
5354
        goto free_of_interlace;
5355
    }
5356

5357
    pThreadInfo->totalAffectedRows += affectedRows;
5358

5359 5360
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
5361
      printf("thread[%d] has currently inserted rows: %"PRIu64 ", affected rows: %"PRIu64 "\n",
5362 5363 5364
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5365 5366
      lastPrintTime = currentPrintTime;
    }
5367

5368
    if ((insert_interval) && flagSleep) {
5369
      et = taosGetTimestampMs();
5370

5371
      if (insert_interval > (et - st) ) {
S
Shuduo Sang 已提交
5372 5373
        uint64_t sleepTime = insert_interval - (et -st);
        performancePrint("%s() LN%d sleep: %"PRId64" ms for insert interval\n",
5374
                    __func__, __LINE__, sleepTime);
5375 5376 5377
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
5378
    }
5379 5380
  }

5381
free_of_interlace:
S
Shuduo Sang 已提交
5382
  tmfree(pThreadInfo->buffer);
5383
  printStatPerThread(pThreadInfo);
5384 5385 5386
  return NULL;
}

5387 5388 5389 5390 5391 5392 5393 5394
// sync insertion
/*
   1 thread: 100 tables * 2000  rows/s
   1 thread: 10  tables * 20000 rows/s
   6 thread: 300 tables * 2000  rows/s

   2 taosinsertdata , 1 thread:  10  tables * 20000 rows/s
*/
5395
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
5396
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
5397

5398
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5399
  uint64_t maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
5400

S
Shuduo Sang 已提交
5401 5402
  pThreadInfo->buffer = calloc(maxSqlLen, 1);
  if (NULL == pThreadInfo->buffer) {
5403
    errorPrint( "Failed to alloc %"PRIu64" Bytes, reason:%s\n",
5404 5405
              maxSqlLen,
              strerror(errno));
5406 5407
    return NULL;
  }
5408

5409 5410 5411
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
  uint64_t endTs;
5412

5413
  int64_t timeStampStep =
5414
      superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
5415
/*  int insert_interval =
5416
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5417 5418
  uint64_t st = 0;
  uint64_t et = 0xffffffff;
5419
  */
5420

5421 5422
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
5423

5424
  pThreadInfo->samplePos = 0;
5425

5426
  for (uint64_t tableSeq =
5427 5428
          pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to;
        tableSeq ++) {
5429
    int64_t start_time = pThreadInfo->start_time;
5430

5431
    int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
5432 5433
    verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows);

5434
    for (uint64_t i = 0; i < insertRows;) {
5435 5436 5437 5438 5439
        /*
      if (insert_interval) {
            st = taosGetTimestampMs();
      }
      */
5440

5441 5442
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
5443
      verbosePrint("%s() LN%d: tid=%d seq=%"PRId64" tableName=%s\n",
5444 5445
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);
5446

5447
      int64_t remainderBufLen = maxSqlLen;
S
Shuduo Sang 已提交
5448
      char *pstr = pThreadInfo->buffer;
5449 5450 5451 5452 5453 5454 5455
      int nInsertBufLen = strlen("insert into ");

      int len = snprintf(pstr, nInsertBufLen + 1, "%s", "insert into ");

      pstr += len;
      remainderBufLen -= len;

5456
      int64_t generated = generateProgressiveDataBuffer(
5457 5458 5459 5460
              tableName, tableSeq, pThreadInfo, pstr, insertRows,
            i, start_time,
            &(pThreadInfo->samplePos),
            &remainderBufLen);
5461 5462 5463
      if (generated > 0)
        i += generated;
      else
5464
        goto free_of_progressive;
5465

5466
      start_time +=  generated * timeStampStep;
5467
      pThreadInfo->totalInsertRows += generated;
5468

5469
      startTs = taosGetTimestampMs();
5470

S
Shuduo Sang 已提交
5471
      int64_t affectedRows = execInsert(pThreadInfo, generated);
5472

5473
      endTs = taosGetTimestampMs();
5474
      uint64_t delay = endTs - startTs;
5475
      performancePrint("%s() LN%d, insert execution time is %"PRId64"ms\n",
5476
              __func__, __LINE__, delay);
5477 5478 5479
      verbosePrint("[%d] %s() LN%d affectedRows=%"PRId64"\n",
            pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
5480

5481 5482 5483 5484
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
5485

5486 5487 5488 5489 5490
      if (affectedRows < 0) {
        errorPrint("%s() LN%d, affected rows: %"PRId64"\n",
                __func__, __LINE__, affectedRows);
        goto free_of_progressive;
      }
5491 5492 5493

      pThreadInfo->totalAffectedRows += affectedRows;

5494 5495
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
5496
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
5497 5498 5499
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
5500 5501 5502
        lastPrintTime = currentPrintTime;
      }

5503
      if (i >= insertRows)
5504
        break;
5505
/*
5506
      if (insert_interval) {
5507
        et = taosGetTimestampMs();
5508

5509 5510
        if (insert_interval > ((et - st)) ) {
            int sleep_time = insert_interval - (et -st);
5511 5512
            performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleep_time);
5513 5514
            taosMsleep(sleep_time); // ms
        }
5515
      }
5516
      */
5517 5518
    }   // num_of_DPT

5519 5520
    if (g_args.verbose_print) {
      if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo &&
5521 5522
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
5523
          verbosePrint("%s() LN%d samplePos=%"PRId64"\n",
5524
                  __func__, __LINE__, pThreadInfo->samplePos);
5525
      }
5526
    }
5527
  } // tableSeq
5528

5529
free_of_progressive:
S
Shuduo Sang 已提交
5530
  tmfree(pThreadInfo->buffer);
5531
  printStatPerThread(pThreadInfo);
5532
  return NULL;
H
Hui Li 已提交
5533 5534
}

5535 5536
static void* syncWrite(void *sarg) {

5537 5538
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5539

5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551
  int interlaceRows;

  if (superTblInfo) {
    if ((superTblInfo->interlaceRows == 0)
        && (g_args.interlace_rows > 0)) {
      interlaceRows = g_args.interlace_rows;
    } else {
      interlaceRows = superTblInfo->interlaceRows;
    }
  } else {
    interlaceRows = g_args.interlace_rows;
  }
5552

5553
  if (interlaceRows > 0) {
5554
    // interlace mode
5555
    return syncWriteInterlace(pThreadInfo);
5556 5557
  } else {
    // progressive mode
5558
    return syncWriteProgressive(pThreadInfo);
5559
  }
5560

5561 5562
}

5563
static void callBack(void *param, TAOS_RES *res, int code) {
5564 5565
  threadInfo* pThreadInfo = (threadInfo*)param;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5566

5567 5568
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5569
  if (insert_interval) {
5570 5571 5572
    pThreadInfo->et = taosGetTimestampMs();
    if ((pThreadInfo->et - pThreadInfo->st) < insert_interval) {
      taosMsleep(insert_interval - (pThreadInfo->et - pThreadInfo->st)); // ms
5573
    }
H
Hui Li 已提交
5574
  }
5575

5576
  char *buffer = calloc(1, pThreadInfo->superTblInfo->maxSqlLen);
5577
  char data[MAX_DATA_SIZE];
5578
  char *pstr = buffer;
5579 5580
  pstr += sprintf(pstr, "insert into %s.%s%"PRId64" values",
          pThreadInfo->db_name, pThreadInfo->tb_prefix,
5581
          pThreadInfo->start_table_from);
5582 5583 5584 5585 5586 5587 5588
//  if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
  if (pThreadInfo->counter >= g_args.num_of_RPR) {
    pThreadInfo->start_table_from++;
    pThreadInfo->counter = 0;
  }
  if (pThreadInfo->start_table_from > pThreadInfo->end_table_to) {
    tsem_post(&pThreadInfo->lock_sem);
5589 5590 5591 5592
    free(buffer);
    taos_free_result(res);
    return;
  }
5593

5594
  for (int i = 0; i < g_args.num_of_RPR; i++) {
5595
    int rand_num = taosRandom() % 100;
5596
    if (0 != pThreadInfo->superTblInfo->disorderRatio
5597
            && rand_num < pThreadInfo->superTblInfo->disorderRatio) {
5598 5599
      int64_t d = pThreadInfo->lastTs - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1);
      generateRowData(data, d, pThreadInfo->superTblInfo);
5600
    } else {
5601
      generateRowData(data, pThreadInfo->lastTs += 1000, pThreadInfo->superTblInfo);
H
Hui Li 已提交
5602
    }
5603
    pstr += sprintf(pstr, "%s", data);
5604
    pThreadInfo->counter++;
H
Hui Li 已提交
5605

5606
    if (pThreadInfo->counter >= pThreadInfo->superTblInfo->insertRows) {
5607
      break;
H
Hui Li 已提交
5608 5609
    }
  }
5610

5611
  if (insert_interval) {
5612
    pThreadInfo->st = taosGetTimestampMs();
5613
  }
5614
  taos_query_a(pThreadInfo->taos, buffer, callBack, pThreadInfo);
5615
  free(buffer);
H
Hui Li 已提交
5616

5617
  taos_free_result(res);
H
Hui Li 已提交
5618 5619
}

5620
static void *asyncWrite(void *sarg) {
5621 5622
  threadInfo *pThreadInfo = (threadInfo *)sarg;
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
5623

5624 5625 5626
  pThreadInfo->st = 0;
  pThreadInfo->et = 0;
  pThreadInfo->lastTs = pThreadInfo->start_time;
5627

5628
  int insert_interval =
5629
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5630
  if (insert_interval) {
5631
    pThreadInfo->st = taosGetTimestampMs();
H
Hui Li 已提交
5632
  }
5633
  taos_query_a(pThreadInfo->taos, "show databases", callBack, pThreadInfo);
5634

5635
  tsem_wait(&(pThreadInfo->lock_sem));
H
Hui Li 已提交
5636 5637 5638 5639

  return NULL;
}

5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665
static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in *serv_addr)
{
  uint16_t rest_port = port + TSDB_PORT_HTTP;
  struct hostent *server = gethostbyname(host);
  if ((server == NULL) || (server->h_addr == NULL)) {
    errorPrint("%s", "ERROR, no such host");
    return -1;
  }

  debugPrint("h_name: %s\nh_addr=%p\nh_addretype: %s\nh_length: %d\n",
            server->h_name,
            server->h_addr,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

  memset(serv_addr, 0, sizeof(struct sockaddr_in));
  serv_addr->sin_family = AF_INET;
  serv_addr->sin_port = htons(rest_port);
#ifdef WINDOWS
  serv_addr->sin_addr.s_addr = inet_addr(host);
#else
  memcpy(&(serv_addr->sin_addr.s_addr), server->h_addr, server->h_length);
#endif
  return 0;
}

5666
static void startMultiThreadInsertData(int threads, char* db_name,
5667
        char* precision,SSuperTable* superTblInfo) {
5668

5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684
  //TAOS* taos;
  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
  //  if (NULL == taos) {
  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
  //    exit(-1);
  //  }
  //}

  int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
  if (0 != precision[0]) {
    if (0 == strncasecmp(precision, "ms", 2)) {
      timePrec = TSDB_TIME_PRECISION_MILLI;
    }  else if (0 == strncasecmp(precision, "us", 2)) {
      timePrec = TSDB_TIME_PRECISION_MICRO;
    }  else {
5685
      errorPrint("Not support precision: %s\n", precision);
5686 5687 5688 5689
      exit(-1);
    }
  }

5690
  int64_t start_time;
5691 5692
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
5693
        start_time = taosGetTimestamp(timePrec);
5694
    } else {
5695
      if (TSDB_CODE_SUCCESS != taosParseTime(
5696 5697 5698 5699 5700
        superTblInfo->startTimestamp,
        &start_time,
        strlen(superTblInfo->startTimestamp),
        timePrec, 0)) {
          ERROR_EXIT("failed to parse time!\n");
5701
      }
5702
    }
5703
  } else {
5704
     start_time = 1500000000000;
5705 5706
  }

5707
  int64_t start = taosGetTimestampMs();
5708

5709
  // read sample data from file first
5710
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5711
              "sample", strlen("sample")))) {
5712
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5713
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
5714
              __func__, __LINE__);
5715 5716 5717 5718
      exit(-1);
    }
  }

5719
  // read sample data from file first
5720
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5721
              "sample", strlen("sample")))) {
5722
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5723
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
5724
              __func__, __LINE__);
5725 5726 5727 5728
      exit(-1);
    }
  }

5729
  TAOS* taos0 = taos_connect(
5730 5731
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
5732
  if (NULL == taos0) {
5733
    errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
5734
                __func__, __LINE__, taos_errstr(NULL));
5735 5736 5737
    exit(-1);
  }

S
Shuduo Sang 已提交
5738
  int64_t ntables = 0;
S
Shuduo Sang 已提交
5739
  uint64_t startFrom;
5740

5741
  if (superTblInfo) {
5742 5743
    int64_t limit;
    uint64_t offset;
5744

H
Haojun Liao 已提交
5745
    if ((NULL != g_args.sqlFile) && (superTblInfo->childTblExists == TBL_NO_EXISTS) &&
5746
            ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit >= 0))) {
H
Haojun Liao 已提交
5747
      printf("WARNING: offset and limit will not be used since the child tables not exists!\n");
5748 5749
    }

5750
    if (superTblInfo->childTblExists == TBL_ALREADY_EXISTS) {
5751 5752
      if ((superTblInfo->childTblLimit < 0)
          || ((superTblInfo->childTblOffset + superTblInfo->childTblLimit)
5753
            > (superTblInfo->childTblCount))) {
5754 5755 5756 5757 5758 5759
        superTblInfo->childTblLimit =
            superTblInfo->childTblCount - superTblInfo->childTblOffset;
      }

      offset = superTblInfo->childTblOffset;
      limit = superTblInfo->childTblLimit;
5760
    } else {
5761 5762
      limit = superTblInfo->childTblCount;
      offset = 0;
5763 5764
    }

5765 5766 5767
    ntables = limit;
    startFrom = offset;

5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778
    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
        && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
            > superTblInfo->childTblCount)) {
      printf("WARNING: specified offset + limit > child table count!\n");
      if (!g_args.answer_yes) {
        printf("         Press enter key to continue or Ctrl-C to stop\n\n");
        (void)getchar();
      }
    }

    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
5779
            && (0 == superTblInfo->childTblLimit)) {
5780 5781 5782 5783 5784 5785 5786
      printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
      if (!g_args.answer_yes) {
        printf("         Press enter key to continue or Ctrl-C to stop\n\n");
        (void)getchar();
      }
    }

5787
    superTblInfo->childTblName = (char*)calloc(1,
5788
        limit * TSDB_TABLE_NAME_LEN);
5789
    if (superTblInfo->childTblName == NULL) {
5790
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
5791
      taos_close(taos0);
5792 5793 5794
      exit(-1);
    }

5795
    int64_t childTblCount;
5796
    getChildNameOfSuperTableWithLimitAndOffset(
5797
        taos0,
5798 5799 5800 5801
        db_name, superTblInfo->sTblName,
        &superTblInfo->childTblName, &childTblCount,
        limit,
        offset);
5802 5803 5804
  } else {
    ntables = g_args.num_of_tables;
    startFrom = 0;
5805
  }
5806

5807
  taos_close(taos0);
5808

S
Shuduo Sang 已提交
5809
  int64_t a = ntables / threads;
5810 5811 5812 5813 5814
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

S
Shuduo Sang 已提交
5815
  int64_t b = 0;
5816 5817 5818 5819
  if (threads != 0) {
    b = ntables % threads;
  }

5820
  if ((superTblInfo)
5821 5822 5823 5824
      && (superTblInfo->insertMode == REST_IFACE)) {
      if (convertHostToServAddr(g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) {
        exit(-1);
      }
5825 5826
  }

5827 5828 5829 5830 5831 5832 5833 5834 5835
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  assert(pids != NULL);

  threadInfo *infos = calloc(1, threads * sizeof(threadInfo));
  assert(infos != NULL);

  memset(pids, 0, threads * sizeof(pthread_t));
  memset(infos, 0, threads * sizeof(threadInfo));

H
Hui Li 已提交
5836
  for (int i = 0; i < threads; i++) {
5837
    threadInfo *t_info = infos + i;
H
Hui Li 已提交
5838 5839
    t_info->threadID = i;
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
5840
    t_info->time_precision = timePrec;
5841 5842 5843
    t_info->superTblInfo = superTblInfo;

    t_info->start_time = start_time;
5844
    t_info->minDelay = UINT64_MAX;
5845

5846
    if ((NULL == superTblInfo) ||
5847
            (superTblInfo->insertMode != REST_IFACE)) {
5848
      //t_info->taos = taos;
5849
      t_info->taos = taos_connect(
5850 5851
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
5852
      if (NULL == t_info->taos) {
5853
        errorPrint(
5854 5855
                "%s() LN%d, connect to server fail from insert sub thread, reason: %s\n",
                __func__, __LINE__,
5856
                taos_errstr(NULL));
5857
        free(infos);
5858 5859
        exit(-1);
      }
5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872

      if ((superTblInfo) && (superTblInfo->insertMode == STMT_IFACE)) {
        t_info->stmt = taos_stmt_init(t_info->taos);
        if (NULL == t_info->stmt) {
            errorPrint(
                "%s() LN%d, failed init stmt, reason: %s\n",
                __func__, __LINE__,
                taos_errstr(NULL));
            free(pids);
            free(infos);
            exit(-1);
        }
      }
5873 5874 5875 5876
    } else {
      t_info->taos = NULL;
    }

5877
/*    if ((NULL == superTblInfo)
5878
            || (0 == superTblInfo->multiThreadWriteOneTbl)) {
5879
            */
5880 5881 5882 5883
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
5884
/*    } else {
5885 5886
      t_info->start_table_from = 0;
      t_info->ntables = superTblInfo->childTblCount;
5887 5888
      t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint();
    }
5889
*/
5890
    tsem_init(&(t_info->lock_sem), 0, 0);
5891
    if (ASYNC_MODE == g_Dbs.asyncMode) {
5892
      pthread_create(pids + i, NULL, asyncWrite, t_info);
5893 5894
    } else {
      pthread_create(pids + i, NULL, syncWrite, t_info);
5895
    }
H
Hui Li 已提交
5896
  }
5897

H
Hui Li 已提交
5898 5899 5900 5901
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

5902 5903 5904 5905
  uint64_t totalDelay = 0;
  uint64_t maxDelay = 0;
  uint64_t minDelay = UINT64_MAX;
  uint64_t cntDelay = 1;
5906 5907
  double  avgDelay = 0;

H
Hui Li 已提交
5908
  for (int i = 0; i < threads; i++) {
5909 5910
    threadInfo *t_info = infos + i;

S
TD-1057  
Shengliang Guan 已提交
5911
    tsem_destroy(&(t_info->lock_sem));
5912 5913 5914 5915

    if (t_info->stmt) {
      taos_stmt_close(t_info->stmt);
    }
5916 5917
    taos_close(t_info->taos);

5918
    debugPrint("%s() LN%d, [%d] totalInsert=%"PRIu64" totalAffected=%"PRIu64"\n",
5919 5920 5921
            __func__, __LINE__,
            t_info->threadID, t_info->totalInsertRows,
            t_info->totalAffectedRows);
5922
    if (superTblInfo) {
5923 5924
        superTblInfo->totalAffectedRows += t_info->totalAffectedRows;
        superTblInfo->totalInsertRows += t_info->totalInsertRows;
5925
    } else {
5926 5927
        g_args.totalAffectedRows += t_info->totalAffectedRows;
        g_args.totalInsertRows += t_info->totalInsertRows;
5928
    }
5929 5930 5931 5932

    totalDelay  += t_info->totalDelay;
    cntDelay   += t_info->cntDelay;
    if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay;
5933
    if (t_info->minDelay < minDelay) minDelay = t_info->minDelay;
H
Hui Li 已提交
5934
  }
5935
  cntDelay -= 1;
H
Hui Li 已提交
5936

5937
  if (cntDelay == 0)    cntDelay = 1;
5938 5939
  avgDelay = (double)totalDelay / cntDelay;

5940 5941
  int64_t end = taosGetTimestampMs();
  int64_t t = end - start;
5942 5943

  if (superTblInfo) {
5944
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
5945 5946 5947 5948
          t / 1000.0, superTblInfo->totalInsertRows,
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
5949 5950 5951 5952

    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
5953 5954 5955 5956
          t / 1000.0, superTblInfo->totalInsertRows,
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
          (double)superTblInfo->totalInsertRows / (t / 1000.0));
5957
    }
5958
  } else {
5959
    fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
5960 5961 5962 5963
          t / 1000.0, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          (double)g_args.totalInsertRows / (t / 1000.0));
5964 5965 5966
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
          "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
5967 5968 5969 5970
          t * 1000.0, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          (double)g_args.totalInsertRows / (t / 1000.0));
5971
    }
5972
  }
5973

5974
  fprintf(stderr, "insert delay, avg: %10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
5975
          avgDelay, maxDelay, minDelay);
5976 5977
  if (g_fpOfInsertResult) {
    fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %"PRIu64"ms, min: %"PRIu64"ms\n\n",
5978
          avgDelay, maxDelay, minDelay);
5979
  }
5980

5981 5982
  //taos_close(taos);

H
Hui Li 已提交
5983
  free(pids);
5984
  free(infos);
H
Hui Li 已提交
5985 5986
}

5987
static void *readTable(void *sarg) {
5988
#if 1
5989
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
5990 5991
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
5992
  uint64_t sTime = rinfo->start_time;
H
hzcheng 已提交
5993 5994
  char *tb_prefix = rinfo->tb_prefix;
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
5995
  if (NULL == fp) {
5996
    errorPrint( "fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
H
Hui Li 已提交
5997 5998
    return NULL;
  }
5999

6000
  int64_t num_of_DPT;
6001
/*  if (rinfo->superTblInfo) {
6002 6003
    num_of_DPT = rinfo->superTblInfo->insertRows; //  nrecords_per_table;
  } else {
6004
  */
6005
      num_of_DPT = g_args.num_of_DPT;
6006
//  }
6007

S
Shuduo Sang 已提交
6008 6009
  int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
  int64_t totalData = num_of_DPT * num_of_tables;
6010
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6011 6012 6013 6014 6015

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
S
Shuduo Sang 已提交
6016
  printf("%"PRId64" records:\n", totalData);
H
hzcheng 已提交
6017 6018
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

6019
  for (int j = 0; j < n; j++) {
H
hzcheng 已提交
6020
    double totalT = 0;
6021
    uint64_t count = 0;
6022 6023
    for (int64_t i = 0; i < num_of_tables; i++) {
      sprintf(command, "select %s from %s%"PRId64" where ts>= %" PRIu64,
6024
              aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
6025

6026
      double t = taosGetTimestampMs();
S
Shuaiqiang Chang 已提交
6027 6028
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
6029

S
Shuaiqiang Chang 已提交
6030
      if (code != 0) {
6031
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6032
        taos_free_result(pSql);
H
hzcheng 已提交
6033
        taos_close(taos);
6034
        fclose(fp);
6035
        return NULL;
H
hzcheng 已提交
6036 6037
      }

6038
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6039 6040 6041
        count++;
      }

6042
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6043 6044
      totalT += t;

S
Shuaiqiang Chang 已提交
6045
      taos_free_result(pSql);
H
hzcheng 已提交
6046 6047
    }

6048
    fprintf(fp, "|%10s  |   %"PRId64"   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
6049
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
6050
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
6051
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT * 1000);
H
hzcheng 已提交
6052 6053 6054
  }
  fprintf(fp, "\n");
  fclose(fp);
6055
#endif
H
hzcheng 已提交
6056 6057 6058
  return NULL;
}

6059
static void *readMetric(void *sarg) {
6060
#if 1
6061
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
6062 6063 6064
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
6065 6066 6067 6068
  if (NULL == fp) {
    printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
    return NULL;
  }
6069

6070
  int64_t num_of_DPT = rinfo->superTblInfo->insertRows;
S
Shuduo Sang 已提交
6071 6072
  int64_t num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
  int64_t totalData = num_of_DPT * num_of_tables;
6073
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
6074 6075 6076 6077 6078

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
S
Shuduo Sang 已提交
6079 6080
  printf("%"PRId64" records:\n", totalData);
  fprintf(fp, "Querying On %"PRId64" records:\n", totalData);
H
hzcheng 已提交
6081 6082

  for (int j = 0; j < n; j++) {
6083
    char condition[COND_BUF_LEN] = "\0";
B
Bomin Zhang 已提交
6084
    char tempS[64] = "\0";
H
hzcheng 已提交
6085

6086
    int64_t m = 10 < num_of_tables ? 10 : num_of_tables;
H
hzcheng 已提交
6087

6088
    for (int64_t i = 1; i <= m; i++) {
H
hzcheng 已提交
6089
      if (i == 1) {
6090
        sprintf(tempS, "t1 = %"PRId64"", i);
H
hzcheng 已提交
6091
      } else {
6092
        sprintf(tempS, " or t1 = %"PRId64" ", i);
H
hzcheng 已提交
6093
      }
6094
      strncat(condition, tempS, COND_BUF_LEN - 1);
H
hzcheng 已提交
6095

L
liu0x54 已提交
6096
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
6097 6098 6099 6100

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

6101
      double t = taosGetTimestampMs();
H
hzcheng 已提交
6102

S
Shuaiqiang Chang 已提交
6103 6104 6105 6106
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
6107
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
6108
        taos_free_result(pSql);
H
hzcheng 已提交
6109
        taos_close(taos);
6110
        fclose(fp);
6111
        return NULL;
H
hzcheng 已提交
6112 6113
      }
      int count = 0;
6114
      while(taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
6115 6116
        count++;
      }
6117
      t = taosGetTimestampMs() - t;
H
hzcheng 已提交
6118

6119
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
6120 6121
              num_of_tables * num_of_DPT / (t * 1000.0), t);
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t * 1000.0);
H
hzcheng 已提交
6122

S
Shuaiqiang Chang 已提交
6123
      taos_free_result(pSql);
H
hzcheng 已提交
6124 6125 6126 6127
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
6128
#endif
H
hzcheng 已提交
6129 6130 6131
  return NULL;
}

H
Hui Li 已提交
6132

6133
static int insertTestProcess() {
6134

6135 6136 6137
  setupForAnsiEscape();
  int ret = printfInsertMeta();
  resetAfterAnsiEscape();
6138

6139 6140 6141
  if (ret == -1)
    exit(EXIT_FAILURE);

6142
  debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
6143 6144
  g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
  if (NULL == g_fpOfInsertResult) {
6145
    errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
6146 6147
    return -1;
  }
6148

6149 6150
  if (g_fpOfInsertResult)
    printfInsertMetaToFile(g_fpOfInsertResult);
6151

6152 6153 6154 6155
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void)getchar();
  }
6156

6157 6158 6159
  init_rand_data();

  // create database and super tables
6160
  if(createDatabasesAndStables() != 0) {
6161 6162
    if (g_fpOfInsertResult)
      fclose(g_fpOfInsertResult);
6163 6164
    return -1;
  }
6165 6166

  // pretreatement
6167
  prepareSampleData();
6168

6169 6170 6171 6172
  double start;
  double end;

  // create child tables
6173
  start = taosGetTimestampMs();
6174
  createChildTables();
6175
  end = taosGetTimestampMs();
6176

6177
  if (g_totalChildTables > 0) {
6178
    fprintf(stderr, "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6179
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6180 6181
    if (g_fpOfInsertResult) {
      fprintf(g_fpOfInsertResult,
6182
            "Spent %.4f seconds to create %"PRId64" tables with %d thread(s)\n\n",
6183
            (end - start)/1000.0, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
6184
    }
6185
  }
6186

6187
  taosMsleep(1000);
6188
  // create sub threads for inserting data
6189
  //start = taosGetTimestampMs();
6190
  for (int i = 0; i < g_Dbs.dbCount; i++) {
6191 6192
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
S
Shuduo Sang 已提交
6193
        for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6194 6195 6196 6197 6198

          SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];

          if (superTblInfo && (superTblInfo->insertRows > 0)) {
            startMultiThreadInsertData(
6199 6200 6201 6202
              g_Dbs.threadCount,
              g_Dbs.db[i].dbName,
              g_Dbs.db[i].dbCfg.precision,
              superTblInfo);
6203
          }
6204
        }
6205
      }
6206
    } else {
6207
        startMultiThreadInsertData(
6208 6209 6210
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
6211
          NULL);
H
Hui Li 已提交
6212
    }
6213
  }
6214
  //end = taosGetTimestampMs();
6215

6216
  //int64_t    totalInsertRows = 0;
6217
  //int64_t    totalAffectedRows = 0;
6218
  //for (int i = 0; i < g_Dbs.dbCount; i++) {
6219
  //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
6220
  //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
6221 6222
  //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
  //}
6223
  //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
6224
  postFreeResource();
6225

6226 6227 6228
  return 0;
}

6229 6230
static void *specifiedTableQuery(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6231

6232
  if (pThreadInfo->taos == NULL) {
6233 6234
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
6235 6236 6237 6238
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
6239 6240
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6241
            pThreadInfo->threadID, taos_errstr(NULL));
6242 6243
      return NULL;
    } else {
6244
      pThreadInfo->taos = taos;
6245 6246 6247
    }
  }

6248 6249
  char sqlStr[MAX_DB_NAME_SIZE + 5];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6250 6251
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6252 6253 6254 6255
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
    return NULL;
  }
6256

6257 6258
  uint64_t st = 0;
  uint64_t et = 0;
6259

6260
  uint64_t queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes;
6261

6262 6263 6264
  uint64_t totalQueried = 0;
  uint64_t lastPrintTime = taosGetTimestampMs();
  uint64_t startTs = taosGetTimestampMs();
6265

6266 6267 6268 6269 6270 6271
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
        sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }

6272
  while(queryTimes --) {
6273
    if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) <
6274
            (int64_t)g_queryInfo.specifiedQueryInfo.queryInterval) {
6275
      taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms
6276 6277
    }

6278 6279 6280
    st = taosGetTimestampMs();

    selectAndGetResult(pThreadInfo,
6281
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]);
6282

6283 6284 6285 6286
    et = taosGetTimestampMs();
    printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n",
              taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0);

6287 6288
    totalQueried ++;
    g_queryInfo.specifiedQueryInfo.totalQueried ++;
6289

6290 6291
    uint64_t  currentPrintTime = taosGetTimestampMs();
    uint64_t  endTs = taosGetTimestampMs();
6292
    if (currentPrintTime - lastPrintTime > 30*1000) {
6293
      debugPrint("%s() LN%d, endTs=%"PRIu64"ms, startTs=%"PRIu64"ms\n",
6294
          __func__, __LINE__, endTs, startTs);
6295
      printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.6f\n",
6296 6297
                    pThreadInfo->threadID,
                    totalQueried,
6298
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6299
      lastPrintTime = currentPrintTime;
6300
    }
H
Hui Li 已提交
6301
  }
6302 6303
  return NULL;
}
H
Hui Li 已提交
6304

6305
static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) {
6306 6307
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
6308 6309
  sprintf(subTblName, "%s.%s",
          g_queryInfo.dbName,
6310
          g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
6311 6312

  //printf("inSql: %s\n", inSql);
6313

6314 6315
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
6316
    return;
H
Hui Li 已提交
6317
  }
6318

6319
  tstrncpy(outSql, inSql, pos - inSql + 1);
6320
  //printf("1: %s\n", outSql);
6321
  strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1);
6322
  //printf("2: %s\n", outSql);
6323
  strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1);
6324
  //printf("3: %s\n", outSql);
H
Hui Li 已提交
6325 6326
}

6327
static void *superTableQuery(void *sarg) {
6328
  char sqlstr[MAX_QUERY_SQL_LENGTH];
6329
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6330

6331
  if (pThreadInfo->taos == NULL) {
6332 6333 6334 6335 6336 6337 6338 6339
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6340
            pThreadInfo->threadID, taos_errstr(NULL));
6341 6342
      return NULL;
    } else {
6343
      pThreadInfo->taos = taos;
6344 6345 6346
    }
  }

6347 6348
  uint64_t st = 0;
  uint64_t et = (int64_t)g_queryInfo.superQueryInfo.queryInterval;
6349

6350 6351 6352
  uint64_t queryTimes = g_queryInfo.superQueryInfo.queryTimes;
  uint64_t totalQueried = 0;
  uint64_t  startTs = taosGetTimestampMs();
6353

6354
  uint64_t  lastPrintTime = taosGetTimestampMs();
6355
  while(queryTimes --) {
6356 6357 6358
    if (g_queryInfo.superQueryInfo.queryInterval
            && (et - st) < (int64_t)g_queryInfo.superQueryInfo.queryInterval) {
      taosMsleep(g_queryInfo.superQueryInfo.queryInterval - (et - st)); // ms
6359
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to);
6360
    }
H
Hui Li 已提交
6361

6362
    st = taosGetTimestampMs();
6363
    for (int i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) {
6364
      for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
6365
        memset(sqlstr,0,sizeof(sqlstr));
6366
        replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i);
6367
        if (g_queryInfo.superQueryInfo.result[j][0] != 0) {
6368
          sprintf(pThreadInfo->fp, "%s-%d",
6369
                  g_queryInfo.superQueryInfo.result[j],
6370
                  pThreadInfo->threadID);
6371
        }
6372
        selectAndGetResult(pThreadInfo, sqlstr);
6373 6374 6375 6376 6377 6378 6379

        totalQueried++;
        g_queryInfo.superQueryInfo.totalQueried ++;

        int64_t  currentPrintTime = taosGetTimestampMs();
        int64_t  endTs = taosGetTimestampMs();
        if (currentPrintTime - lastPrintTime > 30*1000) {
6380
          printf("thread[%d] has currently completed queries: %"PRIu64", QPS: %10.3f\n",
6381 6382
                    pThreadInfo->threadID,
                    totalQueried,
6383
                    (double)(totalQueried/((endTs-startTs)/1000.0)));
6384
          lastPrintTime = currentPrintTime;
6385
        }
6386
      }
H
Hui Li 已提交
6387
    }
6388
    et = taosGetTimestampMs();
6389
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%"PRIu64" - %"PRIu64"] once queries duration:%.4fs\n\n",
6390
            taosGetSelfPthreadId(),
6391 6392
            pThreadInfo->start_table_from,
            pThreadInfo->end_table_to,
6393
            (double)(et - st)/1000.0);
6394
  }
6395

6396 6397 6398
  return NULL;
}

6399
static int queryTestProcess() {
6400 6401 6402 6403

  setupForAnsiEscape();
  printfQueryMeta();
  resetAfterAnsiEscape();
6404 6405 6406 6407 6408 6409

  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
6410
          g_queryInfo.port);
6411
  if (taos == NULL) {
6412 6413
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6414 6415 6416
    exit(-1);
  }

6417
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6418 6419
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6420 6421 6422
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6423
  }
6424

6425 6426 6427 6428
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void)getchar();
  }
6429

6430 6431 6432
  if (g_args.debug_print || g_args.verbose_print) {
    printfQuerySystemInfo(taos);
  }
6433

6434 6435 6436 6437 6438 6439
  if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
    if (convertHostToServAddr(
        g_queryInfo.host, g_queryInfo.port, &g_queryInfo.serv_addr) != 0)
      exit(-1);
  }

6440 6441 6442
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
6443
  int nConcurrent = g_queryInfo.specifiedQueryInfo.concurrent;
6444
  uint64_t nSqlCount = g_queryInfo.specifiedQueryInfo.sqlCount;
6445

6446
  uint64_t startTs = taosGetTimestampMs();
6447

6448 6449 6450 6451 6452 6453
  if ((nSqlCount > 0) && (nConcurrent > 0)) {

    pids  = malloc(nConcurrent * nSqlCount * sizeof(pthread_t));
    infos = malloc(nConcurrent * nSqlCount * sizeof(threadInfo));

    if ((NULL == pids) || (NULL == infos)) {
6454
      taos_close(taos);
6455
      ERROR_EXIT("memory allocation failed for create threads\n");
6456
    }
6457

6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478
    for (uint64_t i = 0; i < nSqlCount; i++) {
        for (int j = 0; j < nConcurrent; j++) {
            uint64_t seq = i * nConcurrent + j;
            threadInfo *t_info = infos + seq;
            t_info->threadID = seq;
            t_info->querySeq = i;

            if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {

                char sqlStr[MAX_TB_NAME_SIZE*2];
                sprintf(sqlStr, "use %s", g_queryInfo.dbName);
                verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
                if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) {
                    taos_close(taos);
                    free(infos);
                    free(pids);
                    errorPrint( "use database %s failed!\n\n",
                        g_queryInfo.dbName);
                    return -1;
                }
            }
6479

6480
            t_info->taos = NULL;// TODO: workaround to use separate taos connection;
6481

6482 6483
            pthread_create(pids + seq, NULL, specifiedTableQuery,
                t_info);
6484
        }
6485
    }
6486
  } else {
6487
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
6488
  }
6489

6490 6491
  taos_close(taos);

6492 6493 6494
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
6495 6496 6497 6498
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
    pidsOfSub  = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t));
    infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo));
6499 6500

    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
6501 6502
      free(infos);
      free(pids);
6503

6504
      ERROR_EXIT("memory allocation failed for create threads\n");
6505
    }
6506

S
Shuduo Sang 已提交
6507
    int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
6508
    int threads = g_queryInfo.superQueryInfo.threadCnt;
6509

S
Shuduo Sang 已提交
6510
    int64_t a = ntables / threads;
6511 6512 6513 6514
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
6515

S
Shuduo Sang 已提交
6516
    int64_t b = 0;
6517 6518 6519
    if (threads != 0) {
      b = ntables % threads;
    }
6520

6521
    uint64_t startFrom = 0;
6522
    for (int i = 0; i < threads; i++) {
6523 6524
      threadInfo *t_info = infosOfSub + i;
      t_info->threadID = i;
6525

6526 6527 6528 6529
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
6530
      t_info->taos = NULL; // TODO: workaround to use separate taos connection;
6531
      pthread_create(pidsOfSub + i, NULL, superTableQuery, t_info);
6532 6533
    }

6534
    g_queryInfo.superQueryInfo.threadCnt = threads;
6535
  } else {
6536
    g_queryInfo.superQueryInfo.threadCnt = 0;
6537
  }
6538

6539 6540 6541 6542 6543 6544
  if ((nSqlCount > 0) && (nConcurrent > 0)) {
    for (int i = 0; i < nConcurrent; i++) {
      for (int j = 0; j < nSqlCount; j++) {
        pthread_join(pids[i * nSqlCount + j], NULL);
      }
    }
6545
  }
S
Shuaiqiang Chang 已提交
6546

6547
  tmfree((char*)pids);
6548
  tmfree((char*)infos);
6549

6550
  for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
6551
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
6552
  }
H
Hui Li 已提交
6553

6554
  tmfree((char*)pidsOfSub);
6555
  tmfree((char*)infosOfSub);
6556

6557
//  taos_close(taos);// TODO: workaround to use separate taos connection;
6558
  uint64_t endTs = taosGetTimestampMs();
6559

6560
  uint64_t totalQueried = g_queryInfo.specifiedQueryInfo.totalQueried +
6561 6562
    g_queryInfo.superQueryInfo.totalQueried;

6563
  fprintf(stderr, "==== completed total queries: %"PRIu64", the QPS of all threads: %10.3f====\n",
6564
          totalQueried,
6565
          (double)(totalQueried/((endTs-startTs)/1000.0)));
6566 6567 6568
  return 0;
}

6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583
static void stable_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
  if (res == NULL || taos_errno(res) != 0) {
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
    return;
  }

  if (param)
    appendResultToFile(res, ((threadInfo *)param)->fp);
  // tao_unscribe() will free result.
}

static void specified_sub_callback(
        TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
6584
  if (res == NULL || taos_errno(res) != 0) {
6585 6586
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
6587 6588
    return;
  }
6589

6590
  if (param)
6591
    appendResultToFile(res, ((threadInfo *)param)->fp);
6592
  // tao_unscribe() will free result.
H
hzcheng 已提交
6593 6594
}

6595
static TAOS_SUB* subscribeImpl(
6596 6597 6598 6599
        QUERY_CLASS class,
        threadInfo *pThreadInfo,
        char *sql, char* topic, bool restart, uint64_t interval)
{
6600
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
6601

6602 6603 6604 6605
  if ((SPECIFIED_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode)) {
    tsub = taos_subscribe(
            pThreadInfo->taos,
6606
            restart,
6607
            topic, sql, specified_sub_callback, (void*)pThreadInfo,
6608
            g_queryInfo.specifiedQueryInfo.subscribeInterval);
6609 6610 6611 6612 6613 6614 6615
  } else if ((STABLE_CLASS == class)
          && (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode)) {
    tsub = taos_subscribe(
            pThreadInfo->taos,
            restart,
            topic, sql, stable_sub_callback, (void*)pThreadInfo,
            g_queryInfo.superQueryInfo.subscribeInterval);
6616
  } else {
6617 6618
    tsub = taos_subscribe(
            pThreadInfo->taos,
6619
            restart,
6620
            topic, sql, NULL, NULL, interval);
6621
  }
6622

6623 6624 6625
  if (tsub == NULL) {
    printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
    return NULL;
6626
  }
6627

6628 6629
  return tsub;
}
H
hzcheng 已提交
6630

6631 6632
static void *superSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6633
  char subSqlstr[MAX_QUERY_SQL_LENGTH];
6634
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT] = {0};
H
hzcheng 已提交
6635

6636 6637
  if (pThreadInfo->ntables > MAX_QUERY_SQL_COUNT) {
      errorPrint("The table number(%"PRId64") of the thread is more than max query sql count: %d\n",
6638 6639 6640 6641 6642
              pThreadInfo->ntables,
              MAX_QUERY_SQL_COUNT);
      exit(-1);
  }

6643
  if (pThreadInfo->taos == NULL) {
6644 6645 6646 6647 6648 6649 6650 6651
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6652
            pThreadInfo->threadID, taos_errstr(NULL));
6653 6654
      return NULL;
    } else {
6655
      pThreadInfo->taos = taos;
6656 6657 6658
    }
  }

6659 6660
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6661 6662
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6663 6664
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
6665 6666
    return NULL;
  }
6667

S
Shengliang Guan 已提交
6668 6669 6670
  char topic[32] = {0};
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695
      verbosePrint("%s() LN%d, [%d], start=%"PRId64" end=%"PRId64" i=%"PRIu64"\n",
              __func__, __LINE__,
              pThreadInfo->threadID,
              pThreadInfo->start_table_from,
              pThreadInfo->end_table_to, i);
      sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%"PRIu64"",
              i, pThreadInfo->querySeq);
      memset(subSqlstr, 0, sizeof(subSqlstr));
      replaceChildTblName(
              g_queryInfo.superQueryInfo.sql[pThreadInfo->querySeq],
              subSqlstr, i);
      if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
        sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
      }

      debugPrint("%s() LN%d, [%d] subSqlstr: %s\n",
              __func__, __LINE__, pThreadInfo->threadID, subSqlstr);
      tsub[i] = subscribeImpl(
              STABLE_CLASS,
              pThreadInfo, subSqlstr, topic,
              g_queryInfo.superQueryInfo.subscribeRestart,
              g_queryInfo.superQueryInfo.subscribeInterval);
      if (NULL == tsub[i]) {
6696
        taos_close(pThreadInfo->taos);
6697 6698
        return NULL;
      }
S
Shengliang Guan 已提交
6699
  }
6700 6701

  // start loop to consume result
6702 6703 6704 6705
  int consumed[MAX_QUERY_SQL_COUNT];
  for (int i = 0; i < MAX_QUERY_SQL_COUNT; i++) {
    consumed[i] = 0;
  }
6706
  TAOS_RES* res = NULL;
6707

6708
  while(1) {
6709 6710
    for (uint64_t i = pThreadInfo->start_table_from;
            i <= pThreadInfo->end_table_to; i++) {
S
Shengliang Guan 已提交
6711 6712 6713
        if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
            continue;
        }
6714

6715 6716
        taosMsleep(g_queryInfo.superQueryInfo.subscribeInterval); // ms
        res = taos_consume(tsub[i]);
S
Shengliang Guan 已提交
6717
        if (res) {
6718 6719 6720
            if (g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
                sprintf(pThreadInfo->fp, "%s-%d",
                  g_queryInfo.superQueryInfo.result[pThreadInfo->querySeq],
6721
                  pThreadInfo->threadID);
6722
                appendResultToFile(res, pThreadInfo->fp);
S
Shengliang Guan 已提交
6723
            }
6724
            consumed[i] ++;
6725 6726

            if ((g_queryInfo.superQueryInfo.subscribeKeepProgress)
6727 6728 6729 6730 6731 6732
                && (consumed[i] >=
                    g_queryInfo.superQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
                printf("keepProgress:%d, resub super table query: %"PRIu64"\n",
                    g_queryInfo.superQueryInfo.subscribeKeepProgress,
                    pThreadInfo->querySeq);
                taos_unsubscribe(tsub,
6733
                    g_queryInfo.superQueryInfo.subscribeKeepProgress);
6734 6735 6736 6737 6738 6739 6740 6741
                consumed[i]= 0;
                tsub[i] = subscribeImpl(
                        STABLE_CLASS,
                        pThreadInfo, subSqlstr, topic,
                        g_queryInfo.superQueryInfo.subscribeRestart,
                        g_queryInfo.superQueryInfo.subscribeInterval
                    );
                if (NULL == tsub[i]) {
6742 6743 6744 6745
                    taos_close(pThreadInfo->taos);
                    return NULL;
                }
            }
6746
        }
H
hzcheng 已提交
6747 6748
    }
  }
6749
  taos_free_result(res);
6750

S
Shengliang Guan 已提交
6751 6752
  for (uint64_t i = pThreadInfo->start_table_from;
          i <= pThreadInfo->end_table_to; i++) {
6753
    taos_unsubscribe(tsub[i], 0);
6754
  }
6755

6756
  taos_close(pThreadInfo->taos);
H
hzcheng 已提交
6757 6758 6759
  return NULL;
}

6760 6761
static void *specifiedSubscribe(void *sarg) {
  threadInfo *pThreadInfo = (threadInfo *)sarg;
6762
  TAOS_SUB*  tsub = NULL;
6763

6764
  if (pThreadInfo->taos == NULL) {
6765 6766 6767 6768 6769 6770 6771 6772
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
6773
            pThreadInfo->threadID, taos_errstr(NULL));
6774 6775
      return NULL;
    } else {
6776
      pThreadInfo->taos = taos;
6777 6778 6779
    }
  }

6780 6781
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6782
  debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
6783 6784
  if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(pThreadInfo->taos);
6785 6786
    return NULL;
  }
6787

S
Shengliang Guan 已提交
6788
  char topic[32] = {0};
6789 6790 6791 6792 6793 6794 6795 6796 6797 6798
  sprintf(topic, "taosdemo-subscribe-%"PRIu64"", pThreadInfo->querySeq);
  if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
      sprintf(pThreadInfo->fp, "%s-%d",
                g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
                pThreadInfo->threadID);
  }
  tsub = subscribeImpl(
                      SPECIFIED_CLASS, pThreadInfo,
          g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
          topic,
6799
          g_queryInfo.specifiedQueryInfo.subscribeRestart,
6800 6801 6802 6803
          g_queryInfo.specifiedQueryInfo.subscribeInterval);
  if (NULL == tsub) {
      taos_close(pThreadInfo->taos);
      return NULL;
S
Shengliang Guan 已提交
6804
  }
6805

6806 6807
  // start loop to consume result
  TAOS_RES* res = NULL;
6808

6809
  int consumed;
6810

6811
  while(1) {
6812
      if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) {
6813 6814
        continue;
      }
6815

6816 6817
      taosMsleep(g_queryInfo.specifiedQueryInfo.subscribeInterval); // ms
      res = taos_consume(tsub);
6818
      if (res) {
6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836
          consumed ++;
          if ((g_queryInfo.specifiedQueryInfo.subscribeKeepProgress)
                && (consumed >=
                    g_queryInfo.specifiedQueryInfo.resubAfterConsume[pThreadInfo->querySeq])) {
              printf("keepProgress:%d, resub specified query: %"PRIu64"\n",
                    g_queryInfo.specifiedQueryInfo.subscribeKeepProgress,
                    pThreadInfo->querySeq);
              consumed = 0;
              taos_unsubscribe(tsub,
                      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
              tsub = subscribeImpl(
                      SPECIFIED_CLASS,
                      pThreadInfo,
                      g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
                      topic,
                      g_queryInfo.specifiedQueryInfo.subscribeRestart,
                      g_queryInfo.specifiedQueryInfo.subscribeInterval);
              if (NULL == tsub) {
6837 6838
                taos_close(pThreadInfo->taos);
                return NULL;
6839 6840
              }
          }
6841 6842 6843
      }
  }
  taos_free_result(res);
6844
  taos_unsubscribe(tsub, 0);
6845
  taos_close(pThreadInfo->taos);
6846

H
hzcheng 已提交
6847 6848 6849
  return NULL;
}

6850
static int subscribeTestProcess() {
6851
  setupForAnsiEscape();
6852
  printfQueryMeta();
6853
  resetAfterAnsiEscape();
6854

6855 6856 6857 6858
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void) getchar();
  }
6859

6860 6861 6862 6863 6864
  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
6865
          g_queryInfo.port);
6866
  if (taos == NULL) {
6867 6868
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6869 6870 6871
    exit(-1);
  }

6872
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6873 6874
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6875 6876 6877
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6878 6879
  }

6880 6881
  taos_close(taos); // TODO: workaround to use separate taos connection;

6882 6883
  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
6884 6885 6886 6887 6888

  pthread_t  *pidsOfStable  = NULL;
  threadInfo *infosOfStable = NULL;

  //==== create threads for query for specified table
S
Shengliang Guan 已提交
6889
  if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) {
6890
    debugPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n",
6891
              __func__, __LINE__,
S
Shengliang Guan 已提交
6892 6893 6894 6895 6896 6897 6898 6899
              g_queryInfo.specifiedQueryInfo.sqlCount);
  } else {
    if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) {
        errorPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n",
              __func__, __LINE__,
              g_queryInfo.specifiedQueryInfo.sqlCount);
        exit(-1);
    }
6900

6901 6902 6903 6904 6905 6906 6907 6908
    pids  = malloc(
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(pthread_t));
    infos = malloc(
            g_queryInfo.specifiedQueryInfo.sqlCount *
            g_queryInfo.specifiedQueryInfo.concurrent *
            sizeof(threadInfo));
S
Shengliang Guan 已提交
6909 6910 6911 6912
    if ((NULL == pids) || (NULL == infos)) {
        errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
        exit(-1);
    }
6913

6914 6915 6916 6917 6918 6919 6920 6921 6922
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
        for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
            uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
            threadInfo *t_info = infos + seq;
            t_info->threadID = seq;
            t_info->querySeq = i;
            t_info->taos = NULL;  // TODO: workaround to use separate taos connection;
            pthread_create(pids + seq, NULL, specifiedSubscribe, t_info);
        }
S
Shengliang Guan 已提交
6923
    }
H
hzcheng 已提交
6924
  }
6925

6926 6927 6928 6929 6930 6931 6932
  //==== create threads for super table query
  if (g_queryInfo.superQueryInfo.sqlCount <= 0) {
    printf("%s() LN%d, super table query sqlCount %"PRIu64".\n",
              __func__, __LINE__,
              g_queryInfo.superQueryInfo.sqlCount);
  } else {
    if ((g_queryInfo.superQueryInfo.sqlCount > 0)
6933
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
6934 6935 6936
        pidsOfStable  = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
6937
            sizeof(pthread_t));
6938 6939 6940
        infosOfStable = malloc(
                g_queryInfo.superQueryInfo.sqlCount *
                g_queryInfo.superQueryInfo.threadCnt *
6941
            sizeof(threadInfo));
6942 6943
        if ((NULL == pidsOfStable) || (NULL == infosOfStable)) {
            errorPrint("%s() LN%d, malloc failed for create threads\n",
6944
              __func__, __LINE__);
6945 6946 6947
            // taos_close(taos);
            exit(-1);
        }
6948

6949 6950
        int64_t ntables = g_queryInfo.superQueryInfo.childTblCount;
        int threads = g_queryInfo.superQueryInfo.threadCnt;
6951

6952 6953 6954 6955 6956
        int64_t a = ntables / threads;
        if (a < 1) {
            threads = ntables;
            a = 1;
        }
6957

6958 6959 6960 6961
        int64_t b = 0;
        if (threads != 0) {
            b = ntables % threads;
        }
6962

6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979
        for (uint64_t i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            uint64_t startFrom = 0;
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                threadInfo *t_info = infosOfStable + seq;
                t_info->threadID = seq;
                t_info->querySeq = i;

                t_info->start_table_from = startFrom;
                t_info->ntables = j<b?a+1:a;
                t_info->end_table_to = j<b?startFrom+a:startFrom+a-1;
                startFrom = t_info->end_table_to + 1;
                t_info->taos = NULL; // TODO: workaround to use separate taos connection;
                pthread_create(pidsOfStable + seq,
                        NULL, superSubscribe, t_info);
            }
        }
6980

6981
        g_queryInfo.superQueryInfo.threadCnt = threads;
6982

6983 6984 6985 6986 6987 6988
        for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
            for (int j = 0; j < threads; j++) {
                uint64_t seq = i * threads + j;
                pthread_join(pidsOfStable[seq], NULL);
            }
        }
6989
    }
H
hzcheng 已提交
6990
  }
6991

6992 6993 6994 6995 6996
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
    for (int j = 0; j < g_queryInfo.specifiedQueryInfo.concurrent; j++) {
        uint64_t seq = i * g_queryInfo.specifiedQueryInfo.concurrent + j;
        pthread_join(pids[seq], NULL);
    }
6997
  }
H
hzcheng 已提交
6998

6999
  tmfree((char*)pids);
7000
  tmfree((char*)infos);
H
hzcheng 已提交
7001

7002 7003
  tmfree((char*)pidsOfStable);
  tmfree((char*)infosOfStable);
7004
//   taos_close(taos);
7005
  return 0;
H
hzcheng 已提交
7006 7007
}

7008
static void initOfInsertMeta() {
7009
  memset(&g_Dbs, 0, sizeof(SDbs));
7010

7011
  // set default values
7012
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7013
  g_Dbs.port = 6030;
7014 7015
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
7016
  g_Dbs.threadCount = 2;
7017 7018

  g_Dbs.use_metric = g_args.use_metric;
L
Liu Tao 已提交
7019 7020
}

7021
static void initOfQueryMeta() {
7022
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
7023

7024
  // set default values
7025
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7026
  g_queryInfo.port = 6030;
7027 7028
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
L
Liu Tao 已提交
7029 7030
}

7031
static void setParaFromArg(){
7032
  if (g_args.host) {
7033
    tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE);
7034
  } else {
7035
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
7036
  }
L
Liu Tao 已提交
7037

7038
  if (g_args.user) {
7039
    tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE);
7040
  }
7041 7042

  if (g_args.password) {
7043
    tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE);
7044 7045
  }

7046 7047
  if (g_args.port) {
    g_Dbs.port = g_args.port;
7048
  }
L
Liu Tao 已提交
7049

7050 7051 7052
  g_Dbs.threadCount = g_args.num_of_threads;
  g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;

7053 7054
  g_Dbs.dbCount = 1;
  g_Dbs.db[0].drop = 1;
7055

7056
  tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE);
7057
  g_Dbs.db[0].dbCfg.replica = g_args.replica;
7058
  tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
7059

7060
  tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
7061 7062 7063 7064 7065

  g_Dbs.use_metric = g_args.use_metric;
  g_Dbs.insert_only = g_args.insert_only;

  g_Dbs.do_aggreFunc = true;
L
Liu Tao 已提交
7066

7067 7068
  char dataString[STRING_LEN];
  char **data_type = g_args.datatype;
7069

7070
  memset(dataString, 0, STRING_LEN);
L
Liu Tao 已提交
7071

7072 7073
  if (strcasecmp(data_type[0], "BINARY") == 0
          || strcasecmp(data_type[0], "BOOL") == 0
7074
          || strcasecmp(data_type[0], "NCHAR") == 0 ) {
7075
    g_Dbs.do_aggreFunc = false;
L
Liu Tao 已提交
7076
  }
H
hzcheng 已提交
7077

7078 7079 7080 7081 7082
  if (g_args.use_metric) {
    g_Dbs.db[0].superTblCount = 1;
    tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE);
    g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
    g_Dbs.threadCount = g_args.num_of_threads;
7083
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
7084
    g_Dbs.asyncMode = g_args.async_mode;
7085

7086 7087 7088 7089
    g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
    g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
    g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
7090
    tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix,
7091 7092
            g_args.tb_prefix, MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
S
Shuduo Sang 已提交
7093
    g_Dbs.db[0].superTbls[0].insertMode = g_args.iface;
7094
    tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp,
7095
            "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
7096
    g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
7097

7098
    g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
7099
    g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len;
7100

7101 7102 7103 7104 7105
    g_Dbs.db[0].superTbls[0].columnCount = 0;
    for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
      if (data_type[i] == NULL) {
        break;
      }
7106

7107
      tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
7108
              data_type[i], MAX_TB_NAME_SIZE);
7109
      g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;
7110 7111
      g_Dbs.db[0].superTbls[0].columnCount++;
    }
7112

7113 7114 7115 7116 7117
    if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
      g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
    } else {
      for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) {
        tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE);
7118
        g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;
7119 7120 7121
        g_Dbs.db[0].superTbls[0].columnCount++;
      }
    }
7122

7123
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE);
7124
    g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;
7125

7126
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE);
7127 7128
    g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;
    g_Dbs.db[0].superTbls[0].tagCount = 2;
7129
  } else {
7130 7131
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
    g_Dbs.db[0].superTbls[0].tagCount = 0;
H
hzcheng 已提交
7132
  }
7133 7134 7135 7136 7137 7138
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
7139

7140 7141 7142
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
7143 7144 7145
    exit(-1);
  }

7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167 7168 7169
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

7170
static void querySqlFile(TAOS* taos, char* sqlFile)
7171 7172 7173 7174 7175 7176
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
7177

7178 7179 7180 7181 7182 7183
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

7184
  double t = taosGetTimestampMs();
7185

7186
  while((read_len = tgetline(&line, &line_len, fp)) != -1) {
7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
7199
    }
7200 7201

    memcpy(cmd + cmd_len, line, read_len);
7202
    verbosePrint("%s() LN%d cmd: %s\n", __func__, __LINE__, cmd);
7203 7204 7205
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) {
        errorPrint("%s() LN%d, queryDbExec %s failed!\n",
               __func__, __LINE__, cmd);
7206 7207 7208 7209 7210
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
7211 7212
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
7213 7214
  }

7215
  t = taosGetTimestampMs() - t;
7216
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
7217

7218 7219 7220 7221
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
7222 7223
}

7224
static void testMetaFile() {
7225
    if (INSERT_TEST == g_args.test_mode) {
7226 7227
      if (g_Dbs.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
7228

7229
      insertTestProcess();
7230

7231
    } else if (QUERY_TEST == g_args.test_mode) {
7232 7233
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7234

7235
      queryTestProcess();
7236

7237
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
7238 7239
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
7240

7241
      subscribeTestProcess();
7242

7243 7244 7245
    }  else {
      ;
    }
7246
}
7247

7248
static void queryResult() {
7249
  // query data
7250

7251 7252 7253 7254 7255
  pthread_t read_id;
  threadInfo *rInfo = malloc(sizeof(threadInfo));
  assert(rInfo);
  rInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
  rInfo->start_table_from = 0;
7256

7257 7258 7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281 7282 7283 7284 7285 7286 7287 7288 7289 7290 7291 7292
  //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
  if (g_args.use_metric) {
    rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
    rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
    rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
    tstrncpy(rInfo->tb_prefix,
          g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE);
  } else {
    rInfo->ntables = g_args.num_of_tables;
    rInfo->end_table_to = g_args.num_of_tables -1;
    tstrncpy(rInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE);
  }

  rInfo->taos = taos_connect(
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
          g_Dbs.port);
  if (rInfo->taos == NULL) {
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
    free(rInfo);
    exit(-1);
  }

  tstrncpy(rInfo->fp, g_Dbs.resultFile, MAX_FILE_NAME_LEN);

  if (!g_Dbs.use_metric) {
    pthread_create(&read_id, NULL, readTable, rInfo);
  } else {
    pthread_create(&read_id, NULL, readMetric, rInfo);
  }
  pthread_join(read_id, NULL);
  taos_close(rInfo->taos);
  free(rInfo);
7293 7294
}

7295 7296
static void testCmdLine() {

7297 7298 7299 7300 7301 7302 7303 7304 7305 7306
  if (strlen(configDir)) {
    wordexp_t full_path;
    if (wordexp(configDir, &full_path, 0) != 0) {
      errorPrint( "Invalid path %s\n", configDir);
      return;
    }
    taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
    wordfree(&full_path);
  }

7307 7308
  g_args.test_mode = INSERT_TEST;
  insertTestProcess();
7309

7310
  if (false == g_Dbs.insert_only)
7311
    queryResult();
7312 7313
}

7314 7315 7316
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

7317
  debugPrint("meta file: %s\n", g_args.metaFile);
7318 7319 7320

  if (g_args.metaFile) {
    initOfInsertMeta();
7321
    initOfQueryMeta();
7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333 7334

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
7335 7336 7337 7338
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
7339
          g_Dbs.port);
7340
      querySqlFile(qtaos, g_args.sqlFile);
7341 7342 7343 7344 7345
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
H
hzcheng 已提交
7346
  }
7347 7348

  return 0;
H
hzcheng 已提交
7349
}
7350