taosdemo.c 218.8 KB
Newer Older
S
TD-1057  
Shengliang Guan 已提交
1
/*
H
hzcheng 已提交
2 3 4 5 6 7 8 9 10 11 12 13 14 15
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

16 17 18 19 20

/*
   when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread.
*/

S
slguan 已提交
21
#define _GNU_SOURCE
22 23
#define CURL_STATICLIB

S
TD-1057  
Shengliang Guan 已提交
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39
#ifdef LINUX
  #include <argp.h>
  #include <inttypes.h>
  #ifndef _ALPINE
    #include <error.h>
  #endif
  #include <pthread.h>
  #include <semaphore.h>
  #include <stdbool.h>
  #include <stdio.h>
  #include <string.h>
  #include <sys/time.h>
  #include <time.h>
  #include <unistd.h>
  #include <wordexp.h>
  #include <regex.h>
40
#else
S
TD-1057  
Shengliang Guan 已提交
41 42
  #include <regex.h>
  #include <stdio.h>
43
#endif
H
hzcheng 已提交
44

45
#include <assert.h>
46
#include <stdlib.h>
47 48
#include "cJSON.h"

49
#include "os.h"
H
hzcheng 已提交
50
#include "taos.h"
51
#include "taoserror.h"
H
Hui Li 已提交
52
#include "tutil.h"
S
slguan 已提交
53

54 55 56
#define REQ_EXTRA_BUF_LEN   1024
#define RESP_BUF_LEN        4096

S
slguan 已提交
57 58
extern char configDir[];

59 60 61 62
#define INSERT_JSON_NAME      "insert.json"
#define QUERY_JSON_NAME       "query.json"
#define SUBSCRIBE_JSON_NAME   "subscribe.json"

63 64 65 66 67 68
enum TEST_MODE {
    INSERT_TEST,            // 0
    QUERY_TEST,             // 1
    SUBSCRIBE_TEST,         // 2
    INVAID_TEST
};
69 70 71

#define MAX_SQL_SIZE       65536
#define BUFFER_SIZE        (65536*2)
72 73
#define MAX_USERNAME_SIZE  64
#define MAX_PASSWORD_SIZE  64
74
#define MAX_DB_NAME_SIZE   64
75
#define MAX_HOSTNAME_SIZE  64
76 77 78 79 80 81 82 83 84 85 86 87
#define MAX_TB_NAME_SIZE   64
#define MAX_DATA_SIZE      16000
#define MAX_NUM_DATATYPE   10
#define OPT_ABORT          1 /* –abort */
#define STRING_LEN         60000
#define MAX_PREPARED_RAND  1000000
#define MAX_FILE_NAME_LEN  256

#define   MAX_SAMPLES_ONCE_FROM_FILE   10000
#define   MAX_NUM_DATATYPE 10

#define   MAX_DB_COUNT           8
88
#define   MAX_SUPER_TABLE_COUNT  200
89 90 91
#define   MAX_COLUMN_COUNT       1024
#define   MAX_TAG_COUNT          128

92
#define   MAX_QUERY_SQL_COUNT    100
93 94 95
#define   MAX_QUERY_SQL_LENGTH   256

#define   MAX_DATABASE_COUNT     256
96
#define INPUT_BUF_LEN   256
97

98
#define DEFAULT_TIMESTAMP_STEP  1
99

100

101 102 103 104 105
typedef enum CREATE_SUB_TALBE_MOD_EN {
  PRE_CREATE_SUBTBL,
  AUTO_CREATE_SUBTBL,
  NO_CREATE_SUBTBL
} CREATE_SUB_TALBE_MOD_EN;
106

107 108
typedef enum TALBE_EXISTS_EN {
  TBL_NO_EXISTS,
109
  TBL_ALREADY_EXISTS,
110 111 112 113
  TBL_EXISTS_BUTT
} TALBE_EXISTS_EN;

enum MODE {
114
  SYNC,
115 116 117
  ASYNC,
  MODE_BUT
};
118

119 120 121 122 123 124
typedef enum enum_INSERT_MODE {
    PROGRESSIVE_INSERT_MODE,
    INTERLACE_INSERT_MODE,
    INVALID_INSERT_MODE
} INSERT_MODE;

125
typedef enum enumQUERY_TYPE {
126
  NO_INSERT_TYPE,
127
  INSERT_TYPE,
128
  QUERY_TYPE_BUT
129
} QUERY_TYPE;
130 131 132 133 134 135 136

enum _show_db_index {
  TSDB_SHOW_DB_NAME_INDEX,
  TSDB_SHOW_DB_CREATED_TIME_INDEX,
  TSDB_SHOW_DB_NTABLES_INDEX,
  TSDB_SHOW_DB_VGROUPS_INDEX,
  TSDB_SHOW_DB_REPLICA_INDEX,
137
  TSDB_SHOW_DB_QUORUM_INDEX,
138 139 140 141 142 143 144 145 146 147
  TSDB_SHOW_DB_DAYS_INDEX,
  TSDB_SHOW_DB_KEEP_INDEX,
  TSDB_SHOW_DB_CACHE_INDEX,
  TSDB_SHOW_DB_BLOCKS_INDEX,
  TSDB_SHOW_DB_MINROWS_INDEX,
  TSDB_SHOW_DB_MAXROWS_INDEX,
  TSDB_SHOW_DB_WALLEVEL_INDEX,
  TSDB_SHOW_DB_FSYNC_INDEX,
  TSDB_SHOW_DB_COMP_INDEX,
  TSDB_SHOW_DB_CACHELAST_INDEX,
148
  TSDB_SHOW_DB_PRECISION_INDEX,
149 150 151 152 153 154 155 156 157 158
  TSDB_SHOW_DB_UPDATE_INDEX,
  TSDB_SHOW_DB_STATUS_INDEX,
  TSDB_MAX_SHOW_DB
};

// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
enum _show_stables_index {
  TSDB_SHOW_STABLES_NAME_INDEX,
  TSDB_SHOW_STABLES_CREATED_TIME_INDEX,
  TSDB_SHOW_STABLES_COLUMNS_INDEX,
159 160
  TSDB_SHOW_STABLES_METRIC_INDEX,
  TSDB_SHOW_STABLES_UID_INDEX,
161
  TSDB_SHOW_STABLES_TID_INDEX,
162
  TSDB_SHOW_STABLES_VGID_INDEX,
163
  TSDB_MAX_SHOW_STABLES
164 165
};

166 167 168 169 170 171 172 173 174 175 176 177 178 179
enum _describe_table_index {
  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
  TSDB_MAX_DESCRIBE_METRIC
};

typedef struct {
  char field[TSDB_COL_NAME_LEN + 1];
  char type[16];
  int length;
  char note[128];
} SColDes;
H
hzcheng 已提交
180 181

/* Used by main to communicate with parse_opt. */
182 183
typedef struct SArguments_S {
  char *   metaFile;
184
  int      test_mode;
185 186 187 188 189 190 191 192 193
  char *   host;
  uint16_t port;
  char *   user;
  char *   password;
  char *   database;
  int      replica;
  char *   tb_prefix;
  char *   sqlFile;
  bool     use_metric;
194
  bool     drop_database;
195
  bool     insert_only;
196
  bool     answer_yes;
197
  bool     debug_print;
198
  bool     verbose_print;
199
  bool     performance_print;
200 201 202 203 204 205
  char *   output_file;
  int      mode;
  char *   datatype[MAX_NUM_DATATYPE + 1];
  int      len_of_binary;
  int      num_of_CPR;
  int      num_of_threads;
206
  int      insert_interval;
207
  int      query_times;
208
  int      interlace_rows;
209
  int      num_of_RPR;
210
  int      max_sql_len;
211 212 213
  int      num_of_tables;
  int      num_of_DPT;
  int      abort;
214 215
  int      disorderRatio;               // 0: no disorder, >0: x%
  int      disorderRange;               // ms or us by database precision
216 217
  int      method_of_delete;
  char **  arg_list;
218 219
  int64_t  totalInsertRows;
  int64_t  totalAffectedRows;
220 221 222 223 224 225
} SArguments;

typedef struct SColumn_S {
  char  field[TSDB_COL_NAME_LEN + 1];
  char  dataType[MAX_TB_NAME_SIZE];
  int   dataLen;
226
  char  note[128];
227 228 229
} StrColumn;

typedef struct SSuperTable_S {
230
  char         sTblName[MAX_TB_NAME_SIZE+1];
231
  int          childTblCount;
232 233 234
  bool         childTblExists;          // 0: no, 1: yes
  int          batchCreateTableNum;     // 0: no batch,  > 0: batch table number in one sql
  int8_t       autoCreateTable;         // 0: create sub table, 1: auto create sub table
235
  char         childTblPrefix[MAX_TB_NAME_SIZE];
236
  char         dataSource[MAX_TB_NAME_SIZE+1];  // rand_gen or sample
237
  char         insertMode[MAX_TB_NAME_SIZE];    // taosc, restful
238 239
  int           childTblLimit;
  int           childTblOffset;
240

241 242 243 244 245
  int          multiThreadWriteOneTbl;  // 0: no, 1: yes
  int          interlaceRows;           //
  int          disorderRatio;           // 0: no disorder, >0: x%
  int          disorderRange;           // ms or us by database precision
  int          maxSqlLen;               //
246 247

  int          insertInterval;          // insert interval, will override global insert interval
248
  int64_t      insertRows;              // 0: no limit
249
  int          timeStampStep;
250
  char         startTimestamp[MAX_TB_NAME_SIZE];
251
  char         sampleFormat[MAX_TB_NAME_SIZE];  // csv, json
252 253
  char         sampleFile[MAX_FILE_NAME_LEN+1];
  char         tagsFile[MAX_FILE_NAME_LEN+1];
254 255 256 257 258 259 260

  int          columnCount;
  StrColumn    columns[MAX_COLUMN_COUNT];
  int          tagCount;
  StrColumn    tags[MAX_TAG_COUNT];

  char*        childTblName;
261
  char*        colsOfCreateChildTable;
262 263 264 265 266 267 268 269 270 271 272 273
  int          lenOfOneRow;
  int          lenOfTagOfOneRow;

  char*        sampleDataBuf;
  //int          sampleRowCount;
  //int          sampleUsePos;

  int          tagSource;    // 0: rand, 1: tag sample
  char*        tagDataBuf;
  int          tagSampleCount;
  int          tagUsePos;

274
  // statistics
275
  int64_t    totalInsertRows;
276 277 278 279 280 281 282
  int64_t    totalAffectedRows;
} SSuperTable;

typedef struct {
  char     name[TSDB_DB_NAME_LEN + 1];
  char     create_time[32];
  int32_t  ntables;
283
  int32_t  vgroups;
284 285
  int16_t  replica;
  int16_t  quorum;
286
  int16_t  days;
287 288 289 290 291 292 293 294 295 296 297 298 299 300
  char     keeplist[32];
  int32_t  cache; //MB
  int32_t  blocks;
  int32_t  minrows;
  int32_t  maxrows;
  int8_t   wallevel;
  int32_t  fsync;
  int8_t   comp;
  int8_t   cachelast;
  char     precision[8];   // time resolution
  int8_t   update;
  char     status[16];
} SDbInfo;

301
typedef struct SDbCfg_S {
302
//  int       maxtablesPerVnode;
303
  int       minRows;
304 305 306
  int       maxRows;
  int       comp;
  int       walLevel;
307
  int       cacheLast;
308
  int       fsync;
309 310 311 312 313 314 315
  int       replica;
  int       update;
  int       keep;
  int       days;
  int       cache;
  int       blocks;
  int       quorum;
316
  char      precision[MAX_TB_NAME_SIZE];
317 318 319 320
} SDbCfg;

typedef struct SDataBase_S {
  char         dbName[MAX_DB_NAME_SIZE];
321
  bool         drop;  // 0: use exists, 1: if exists, drop then new create
322 323 324 325 326 327
  SDbCfg       dbCfg;
  int          superTblCount;
  SSuperTable  superTbls[MAX_SUPER_TABLE_COUNT];
} SDataBase;

typedef struct SDbs_S {
328
  char         cfgDir[MAX_FILE_NAME_LEN+1];
329
  char         host[MAX_HOSTNAME_SIZE];
330
  uint16_t     port;
331 332
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
333
  char         resultFile[MAX_FILE_NAME_LEN+1];
334 335 336 337
  bool         use_metric;
  bool         insert_only;
  bool         do_aggreFunc;
  bool         queryMode;
338

339 340 341 342 343 344
  int          threadCount;
  int          threadCountByCreateTbl;
  int          dbCount;
  SDataBase    db[MAX_DB_COUNT];

  // statistics
345
  int64_t    totalInsertRows;
346
  int64_t    totalAffectedRows;
347

348 349
} SDbs;

350
typedef struct SpecifiedQueryInfo_S {
351 352 353 354 355
  int          rate;  // 0: unlimit  > 0   loop/s
  int          concurrent;
  int          sqlCount;
  int          subscribeMode; // 0: sync, 1: async
  int          subscribeInterval; // ms
356
  int          queryTimes;
357 358
  int          subscribeRestart;
  int          subscribeKeepProgress;
359
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
360
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
361
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
362
} SpecifiedQueryInfo;
363

364
typedef struct SuperQueryInfo_S {
365
  char         sTblName[MAX_TB_NAME_SIZE+1];
366
  int          rate;  // 0: unlimit  > 0   loop/s
367
  int          threadCnt;
368 369 370 371
  int          subscribeMode; // 0: sync, 1: async
  int          subscribeInterval; // ms
  int          subscribeRestart;
  int          subscribeKeepProgress;
372
  int          queryTimes;
373 374 375
  int          childTblCount;
  char         childTblPrefix[MAX_TB_NAME_SIZE];
  int          sqlCount;
376
  char         sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1];
377
  char         result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1];
378
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT];
379

380
  char*        childTblName;
381
} SuperQueryInfo;
382 383

typedef struct SQueryMetaInfo_S {
384
  char         cfgDir[MAX_FILE_NAME_LEN+1];
385
  char         host[MAX_HOSTNAME_SIZE];
386
  uint16_t     port;
387 388
  char         user[MAX_USERNAME_SIZE];
  char         password[MAX_PASSWORD_SIZE];
389
  char         dbName[MAX_DB_NAME_SIZE+1];
390 391
  char         queryMode[MAX_TB_NAME_SIZE];  // taosc, restful

392 393
  SpecifiedQueryInfo  specifiedQueryInfo;
  SuperQueryInfo    superQueryInfo;
394 395 396 397 398
} SQueryMetaInfo;

typedef struct SThreadInfo_S {
  TAOS *taos;
  int threadID;
399
  char db_name[MAX_DB_NAME_SIZE+1];
400
  uint32_t time_precision;
401 402
  char fp[4096];
  char tb_prefix[MAX_TB_NAME_SIZE];
403 404 405
  int start_table_from;
  int end_table_to;
  int ntables;
406
  int data_of_rate;
407 408 409
  uint64_t start_time;
  char* cols;
  bool  use_metric;
410 411 412 413
  SSuperTable* superTblInfo;

  // for async insert
  tsem_t lock_sem;
414
  int64_t  counter;
415 416
  uint64_t  st;
  uint64_t  et;
417 418
  int64_t  lastTs;

419 420
  // sample data
  int samplePos;
421
  // statistics
422
  int64_t totalInsertRows;
423
  int64_t totalAffectedRows;
424 425 426 427 428 429 430

  // insert delay statistics
  int64_t cntDelay;
  int64_t totalDelay;
  int64_t avgDelay;
  int64_t maxDelay;
  int64_t minDelay;
431

432 433
} threadInfo;

434
#ifdef WINDOWS
435 436
#define _CRT_RAND_S

437
#include <windows.h>
438 439 440 441 442
#include <winsock2.h>

typedef unsigned __int32 uint32_t;

#pragma comment ( lib, "ws2_32.lib" )
443 444
// Some old MinGW/CYGWIN distributions don't define this:
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
445 446
  #define ENABLE_VIRTUAL_TERMINAL_PROCESSING  0x0004
#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING
447 448 449 450

static HANDLE g_stdoutHandle;
static DWORD g_consoleMode;

451
static void setupForAnsiEscape(void) {
452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
  DWORD mode = 0;
  g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE);

  if(g_stdoutHandle == INVALID_HANDLE_VALUE) {
    exit(GetLastError());
  }

  if(!GetConsoleMode(g_stdoutHandle, &mode)) {
    exit(GetLastError());
  }

  g_consoleMode = mode;

  // Enable ANSI escape codes
  mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;

  if(!SetConsoleMode(g_stdoutHandle, mode)) {
    exit(GetLastError());
470
  }
471 472
}

473
static void resetAfterAnsiEscape(void) {
474
  // Reset colors
475
  printf("\x1b[0m");
476 477 478 479 480 481

  // Reset console mode
  if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) {
    exit(GetLastError());
  }
}
482

483
static int taosRandom()
484 485 486 487 488 489
{
    int number;
    rand_s(&number);

    return number;
}
490
#else   // Not windows
491
static void setupForAnsiEscape(void) {}
492

493
static void resetAfterAnsiEscape(void) {
494 495 496
  // Reset colors
  printf("\x1b[0m");
}
497

498 499
#include <time.h>

500
static int taosRandom()
501
{
502 503 504 505 506
  struct timeval tv;

  gettimeofday(&tv, NULL);
  srand(tv.tv_usec);

507
  return rand();
508 509
}

510
#endif // ifdef Windows
511

512
static int createDatabasesAndStables();
513
static void createChildTables();
514
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet);
515 516 517 518 519 520 521

/* ************ Global variables ************  */

int32_t  randint[MAX_PREPARED_RAND];
int64_t  randbigint[MAX_PREPARED_RAND];
float    randfloat[MAX_PREPARED_RAND];
double   randdouble[MAX_PREPARED_RAND];
522
char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)",
523 524
    "max(col0)", "min(col0)", "first(col0)", "last(col0)"};

525
SArguments g_args = {
526 527
                     NULL,            // metaFile
                     0,               // test_mode
528 529 530
                     "127.0.0.1",     // host
                     6030,            // port
                     "root",          // user
531
                     #ifdef _TD_POWER_
532 533 534 535 536 537 538 539
                     "powerdb",      // password
                     #else
                     "taosdata",      // password
                     #endif
                     "test",          // database
                     1,               // replica
                     "t",             // tb_prefix
                     NULL,            // sqlFile
540
                     true,            // use_metric
541
                     true,            // drop_database
542
                     true,            // insert_only
543
                     false,           // debug_print
544
                     false,           // verbose_print
545
                     false,           // performance statistic print
546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564
                     false,           // answer_yes;
                     "./output.txt",  // output_file
                     0,               // mode : sync or async
                     {
                     "TINYINT",           // datatype
                     "SMALLINT",
                     "INT",
                     "BIGINT",
                     "FLOAT",
                     "DOUBLE",
                     "BINARY",
                     "NCHAR",
                     "BOOL",
                     "TIMESTAMP"
                     },
                     16,              // len_of_binary
                     10,              // num_of_CPR
                     10,              // num_of_connections/thread
                     0,               // insert_interval
565
                     1,               // query_times
566
                     0,               // interlace_rows;
567
                     100,             // num_of_RPR
568
                     TSDB_PAYLOAD_SIZE,  // max_sql_len
569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585
                     10000,           // num_of_tables
                     10000,           // num_of_DPT
                     0,               // abort
                     0,               // disorderRatio
                     1000,            // disorderRange
                     1,               // method_of_delete
                     NULL             // arg_list
};



static SDbs            g_Dbs;
static int             g_totalChildTables = 0;
static SQueryMetaInfo  g_queryInfo;
static FILE *          g_fpOfInsertResult = NULL;

#define debugPrint(fmt, ...) \
586
    do { if (g_args.debug_print || g_args.verbose_print) \
587
      fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0)
588

589
#define verbosePrint(fmt, ...) \
590 591
    do { if (g_args.verbose_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)
592

593 594 595 596
#define performancePrint(fmt, ...) \
    do { if (g_args.performance_print) \
        fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0)

597 598 599 600
#define errorPrint(fmt, ...) \
    do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0)


601
///////////////////////////////////////////////////
H
hzcheng 已提交
602

603 604
static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); }

605 606 607 608 609 610 611 612
#ifndef TAOSDEMO_COMMIT_SHA1
#define TAOSDEMO_COMMIT_SHA1 "unknown"
#endif

#ifndef TD_VERNUMBER
#define TD_VERNUMBER    "unknown"
#endif

613
#ifndef TAOSDEMO_STATUS
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630
#define TAOSDEMO_STATUS "unknown"
#endif

static void printVersion() {
    char tdengine_ver[] = TD_VERNUMBER;
    char taosdemo_ver[] = TAOSDEMO_COMMIT_SHA1;
    char taosdemo_status[] = TAOSDEMO_STATUS;

    if (strlen(taosdemo_status) == 0) {
        printf("taosdemo verison %s-%s\n",
                tdengine_ver, taosdemo_ver);
    } else {
        printf("taosdemo verison %s-%s, status:%s\n",
                tdengine_ver, taosdemo_ver, taosdemo_status);
    }
}

631
static void printHelp() {
632
  char indent[10] = "        ";
633
  printf("%s%s%s%s\n", indent, "-f", indent,
634
          "The meta file to the execution procedure. Default is './meta.json'.");
635
  printf("%s%s%s%s\n", indent, "-u", indent,
636 637
          "The TDengine user name to use when connecting to the server. Default is 'root'.");
#ifdef _TD_POWER_
638
  printf("%s%s%s%s\n", indent, "-P", indent,
639
          "The password to use when connecting to the server. Default is 'powerdb'.");
640
  printf("%s%s%s%s\n", indent, "-c", indent,
641 642
          "Configuration directory. Default is '/etc/power/'.");
#else
643
  printf("%s%s%s%s\n", indent, "-P", indent,
644
          "The password to use when connecting to the server. Default is 'taosdata'.");
645
  printf("%s%s%s%s\n", indent, "-c", indent,
646
          "Configuration directory. Default is '/etc/taos/'.");
647
#endif
648
  printf("%s%s%s%s\n", indent, "-h", indent,
649
          "The host to connect to TDengine. Default is localhost.");
650
  printf("%s%s%s%s\n", indent, "-p", indent,
651
          "The TCP/IP port number to use for the connection. Default is 0.");
652
  printf("%s%s%s%s\n", indent, "-d", indent,
653
          "Destination database. Default is 'test'.");
654
  printf("%s%s%s%s\n", indent, "-a", indent,
655
          "Set the replica parameters of the database, Default 1, min: 1, max: 3.");
656
  printf("%s%s%s%s\n", indent, "-m", indent,
657 658
          "Table prefix name. Default is 't'.");
  printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file.");
659
  printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag.");
660
  printf("%s%s%s%s\n", indent, "-o", indent,
661
          "Direct output to the named file. Default is './output.txt'.");
662
  printf("%s%s%s%s\n", indent, "-q", indent,
663
          "Query mode--0: SYNC, 1: ASYNC. Default is SYNC.");
664
  printf("%s%s%s%s\n", indent, "-b", indent,
665
          "The data_type of columns, default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP.");
666
  printf("%s%s%s%s\n", indent, "-w", indent,
667
          "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
668
  printf("%s%s%s%s\n", indent, "-l", indent,
669
          "The number of columns per record. Default is 10.");
670
  printf("%s%s%s%s\n", indent, "-T", indent,
671
          "The number of threads. Default is 10.");
672
  printf("%s%s%s%s\n", indent, "-i", indent,
673
          "The sleep time (ms) between insertion. Default is 0.");
674
  printf("%s%s%s%s\n", indent, "-r", indent,
675
          "The number of records per request. Default is 100.");
676
  printf("%s%s%s%s\n", indent, "-t", indent,
677
          "The number of tables. Default is 10000.");
678
  printf("%s%s%s%s\n", indent, "-n", indent,
679 680 681
          "The number of records per table. Default is 10000.");
  printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag.");
  printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt.");
682
  printf("%s%s%s%s\n", indent, "-O", indent,
683
          "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order.");
684
  printf("%s%s%s%s\n", indent, "-R", indent,
685
          "Out of order data's range, ms, default is 1000.");
686
  printf("%s%s%s%s\n", indent, "-g", indent,
687
          "Print debug info.");
688
  printf("%s%s%s%s\n", indent, "-V, --version", indent,
689
          "Print version info.");
690
/*    printf("%s%s%s%s\n", indent, "-D", indent,
691 692 693 694
          "if elete database if exists. 0: no, 1: yes, default is 1");
          */
}

695
static void parse_args(int argc, char *argv[], SArguments *arguments) {
696
  char **sptr;
697 698 699 700 701

  for (int i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-f") == 0) {
      arguments->metaFile = argv[++i];
    } else if (strcmp(argv[i], "-c") == 0) {
702
      tstrncpy(configDir, argv[++i], MAX_FILE_NAME_LEN);
703

704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721
    } else if (strcmp(argv[i], "-h") == 0) {
      arguments->host = argv[++i];
    } else if (strcmp(argv[i], "-p") == 0) {
      arguments->port = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-u") == 0) {
      arguments->user = argv[++i];
    } else if (strcmp(argv[i], "-P") == 0) {
      arguments->password = argv[++i];
    } else if (strcmp(argv[i], "-o") == 0) {
      arguments->output_file = argv[++i];
    } else if (strcmp(argv[i], "-s") == 0) {
      arguments->sqlFile = argv[++i];
    } else if (strcmp(argv[i], "-q") == 0) {
      arguments->mode = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-T") == 0) {
      arguments->num_of_threads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
      arguments->insert_interval = atoi(argv[++i]);
722 723
    } else if (strcmp(argv[i], "-qt") == 0) {
      arguments->query_times = atoi(argv[++i]);
724
    } else if (strcmp(argv[i], "-B") == 0) {
725
      arguments->interlace_rows = atoi(argv[++i]);
726 727 728 729 730 731 732 733 734 735 736
    } else if (strcmp(argv[i], "-r") == 0) {
      arguments->num_of_RPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-t") == 0) {
      arguments->num_of_tables = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
      arguments->num_of_DPT = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-d") == 0) {
      arguments->database = argv[++i];
    } else if (strcmp(argv[i], "-l") == 0) {
      arguments->num_of_CPR = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-b") == 0) {
737
      sptr = arguments->datatype;
738 739
      ++i;
      if (strstr(argv[i], ",") == NULL) {
740
        // only one col
741 742 743 744 745 746 747
        if (strcasecmp(argv[i], "INT")
                && strcasecmp(argv[i], "FLOAT")
                && strcasecmp(argv[i], "TINYINT")
                && strcasecmp(argv[i], "BOOL")
                && strcasecmp(argv[i], "SMALLINT")
                && strcasecmp(argv[i], "BIGINT")
                && strcasecmp(argv[i], "DOUBLE")
748
                && strcasecmp(argv[i], "BINARY")
749 750
                && strcasecmp(argv[i], "NCHAR")) {
          printHelp();
751
          ERROR_EXIT( "Invalid data_type!\n");
752
          exit(EXIT_FAILURE);
753
        }
754
        sptr[0] = argv[i];
755
      } else {
756
        // more than one col
757
        int index = 0;
758
        char *dupstr = strdup(argv[i]);
759 760 761
        char *running = dupstr;
        char *token = strsep(&running, ",");
        while (token != NULL) {
762 763 764 765 766 767 768 769 770 771
          if (strcasecmp(token, "INT")
                  && strcasecmp(token, "FLOAT")
                  && strcasecmp(token, "TINYINT")
                  && strcasecmp(token, "BOOL")
                  && strcasecmp(token, "SMALLINT")
                  && strcasecmp(token, "BIGINT")
                  && strcasecmp(token, "DOUBLE")
                  && strcasecmp(token, "BINARY")
                  && strcasecmp(token, "NCHAR")) {
            printHelp();
772
            ERROR_EXIT("Invalid data_type!\n");
773
            exit(EXIT_FAILURE);
H
hzcheng 已提交
774
          }
775 776 777
          sptr[index++] = token;
          token = strsep(&running, ",");
          if (index >= MAX_NUM_DATATYPE) break;
S
TD-1057  
Shengliang Guan 已提交
778
        }
779
        sptr[index] = NULL;
780
      }
781 782 783 784
    } else if (strcmp(argv[i], "-w") == 0) {
      arguments->len_of_binary = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-m") == 0) {
      arguments->tb_prefix = argv[++i];
785 786
    } else if (strcmp(argv[i], "-N") == 0) {
      arguments->use_metric = false;
787
    } else if (strcmp(argv[i], "-x") == 0) {
788
      arguments->insert_only = false;
789
    } else if (strcmp(argv[i], "-y") == 0) {
790
      arguments->answer_yes = true;
791 792
    } else if (strcmp(argv[i], "-g") == 0) {
      arguments->debug_print = true;
793 794
    } else if (strcmp(argv[i], "-gg") == 0) {
      arguments->verbose_print = true;
795 796
    } else if (strcmp(argv[i], "-pp") == 0) {
      arguments->performance_print = true;
797
    } else if (strcmp(argv[i], "-O") == 0) {
798

799
      arguments->disorderRatio = atoi(argv[++i]);
800 801 802 803 804

      if (arguments->disorderRatio > 50)
        arguments->disorderRatio = 50;

      if (arguments->disorderRatio < 0)
805
        arguments->disorderRatio = 0;
806

807
    } else if (strcmp(argv[i], "-R") == 0) {
808

809
      arguments->disorderRange = atoi(argv[++i]);
810 811 812
      if (arguments->disorderRange < 0)
        arguments->disorderRange = 1000;

813 814 815 816
    } else if (strcmp(argv[i], "-a") == 0) {
      arguments->replica = atoi(argv[++i]);
      if (arguments->replica > 3 || arguments->replica < 1) {
          arguments->replica = 1;
817
      }
818 819 820 821 822
    } else if (strcmp(argv[i], "-D") == 0) {
      arguments->method_of_delete = atoi(argv[++i]);
      if (arguments->method_of_delete < 0
              || arguments->method_of_delete > 3) {
        arguments->method_of_delete = 0;
S
TD-1057  
Shengliang Guan 已提交
823
      }
824 825 826 827
    } else if ((strcmp(argv[i], "--version") == 0) ||
        (strcmp(argv[i], "-V") == 0)){
      printVersion();
      exit(0);
828 829 830 831 832
    } else if (strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else {
      printHelp();
833
      ERROR_EXIT("ERROR: wrong options\n");
834
      exit(EXIT_FAILURE);
S
TD-1057  
Shengliang Guan 已提交
835 836
    }
  }
837

838 839
  if (((arguments->debug_print) && (arguments->metaFile == NULL))
          || arguments->verbose_print) {
840
    printf("###################################################################\n");
841
    printf("# meta file:                         %s\n", arguments->metaFile);
842
    printf("# Server IP:                         %s:%hu\n",
843 844 845 846 847
            arguments->host == NULL ? "localhost" : arguments->host,
            arguments->port );
    printf("# User:                              %s\n", arguments->user);
    printf("# Password:                          %s\n", arguments->password);
    printf("# Use metric:                        %s\n", arguments->use_metric ? "true" : "false");
848 849 850 851 852 853 854 855 856
    if (*(arguments->datatype)) {
        printf("# Specified data type:               ");
        for (int i = 0; i < MAX_NUM_DATATYPE; i++)
            if (arguments->datatype[i])
               printf("%s,", arguments->datatype[i]);
            else
                break;
        printf("\n");
    }
857
    printf("# Insertion interval:                %d\n", arguments->insert_interval);
858
    printf("# Number of records per req:         %d\n", arguments->num_of_RPR);
859
    printf("# Max SQL length:                    %d\n", arguments->max_sql_len);
860
    printf("# Length of Binary:                  %d\n", arguments->len_of_binary);
861 862 863 864 865 866 867 868
    printf("# Number of Threads:                 %d\n", arguments->num_of_threads);
    printf("# Number of Tables:                  %d\n", arguments->num_of_tables);
    printf("# Number of Data per Table:          %d\n", arguments->num_of_DPT);
    printf("# Database name:                     %s\n", arguments->database);
    printf("# Table prefix:                      %s\n", arguments->tb_prefix);
    if (arguments->disorderRatio) {
      printf("# Data order:                        %d\n", arguments->disorderRatio);
      printf("# Data out of order rate:            %d\n", arguments->disorderRange);
869

870 871 872 873
    }
    printf("# Delete method:                     %d\n", arguments->method_of_delete);
    printf("# Answer yes when prompt:            %d\n", arguments->answer_yes);
    printf("# Print debug info:                  %d\n", arguments->debug_print);
874
    printf("# Print verbose info:                %d\n", arguments->verbose_print);
875 876 877 878 879 880
    printf("###################################################################\n");
    if (!arguments->answer_yes) {
        printf("Press enter key to continue\n\n");
        (void) getchar();
    }
  }
881
}
H
hzcheng 已提交
882

883 884 885 886
static bool getInfoFromJsonFile(char* file);
//static int generateOneRowDataForStb(SSuperTable* stbInfo);
//static int getDataIntoMemForStb(SSuperTable* stbInfo);
static void init_rand_data();
887
static void tmfclose(FILE *fp) {
888 889 890 891
  if (NULL != fp) {
    fclose(fp);
  }
}
892

893
static void tmfree(char *buf) {
894 895
  if (NULL != buf) {
    free(buf);
H
Hui Li 已提交
896
  }
897
}
H
Hui Li 已提交
898

899
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
900 901 902
  int i;
  TAOS_RES *res = NULL;
  int32_t   code = -1;
903

904 905 906 907
  for (i = 0; i < 5; i++) {
    if (NULL != res) {
      taos_free_result(res);
      res = NULL;
H
hzcheng 已提交
908
    }
909

910 911 912 913
    res = taos_query(taos, command);
    code = taos_errno(res);
    if (0 == code) {
      break;
914
    }
H
hzcheng 已提交
915 916
  }

917
  if (code != 0) {
918 919 920 921
    if (!quiet) {
      debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
      errorPrint("Failed to run %s, reason: %s\n", command, taos_errstr(res));
    }
922 923 924
    taos_free_result(res);
    //taos_close(taos);
    return -1;
925
  }
H
hzcheng 已提交
926

927 928 929 930
  if (INSERT_TYPE == type) {
    int affectedRows = taos_affected_rows(res);
    taos_free_result(res);
    return affectedRows;
H
hzcheng 已提交
931
  }
932

933
  taos_free_result(res);
934 935
  return 0;
}
H
hzcheng 已提交
936

937
static void getResult(TAOS_RES *res, char* resultFileName) {
938 939 940 941 942 943 944 945 946
  TAOS_ROW    row = NULL;
  int         num_rows = 0;
  int         num_fields = taos_field_count(res);
  TAOS_FIELD *fields     = taos_fetch_fields(res);

  FILE *fp = NULL;
  if (resultFileName[0] != 0) {
    fp = fopen(resultFileName, "at");
    if (fp == NULL) {
947 948
      errorPrint("%s() LN%d, failed to open result file: %s, result will not save to file\n",
              __func__, __LINE__, resultFileName);
949 950
    }
  }
951

952 953
  char* databuf = (char*) calloc(1, 100*1024*1024);
  if (databuf == NULL) {
954 955
    errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n",
            __func__, __LINE__);
956 957
    if (fp)
        fclose(fp);
958 959
    return ;
  }
H
hzcheng 已提交
960

961 962
  int   totalLen = 0;
  char  temp[16000];
H
hzcheng 已提交
963

964 965 966 967 968 969
  // fetch the records row by row
  while ((row = taos_fetch_row(res))) {
    if (totalLen >= 100*1024*1024 - 32000) {
      if (fp) fprintf(fp, "%s", databuf);
      totalLen = 0;
      memset(databuf, 0, 100*1024*1024);
H
hzcheng 已提交
970
    }
971 972 973 974 975 976
    num_rows++;
    int len = taos_print_row(temp, row, fields, num_fields);
    len += sprintf(temp + len, "\n");
    //printf("query result:%s\n", temp);
    memcpy(databuf + totalLen, temp, len);
    totalLen += len;
H
hzcheng 已提交
977 978
  }

979 980 981 982
  if (fp) fprintf(fp, "%s", databuf);
  tmfclose(fp);
  free(databuf);
}
H
hzcheng 已提交
983

984 985 986 987 988 989
static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) {
  TAOS_RES *res = taos_query(taos, command);
  if (res == NULL || taos_errno(res) != 0) {
    printf("failed to sql:%s, reason:%s\n", command, taos_errstr(res));
    taos_free_result(res);
    return;
H
hzcheng 已提交
990
  }
991

992 993 994
  getResult(res, resultFileName);
  taos_free_result(res);
}
H
hzcheng 已提交
995

996
static double getCurrentTimeUs() {
997
  struct timeval tv;
998

999 1000 1001
  if (gettimeofday(&tv, NULL) != 0) {
    perror("Failed to get current time in ms");
    return 0.0;
H
hzcheng 已提交
1002
  }
S
Shuduo Sang 已提交
1003

1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
  return tv.tv_sec + tv.tv_usec / 1E6;
}

static int32_t rand_bool(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 2;
}

static int32_t rand_tinyint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 128;
}

static int32_t rand_smallint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor] % 32767;
}

static int32_t rand_int(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randint[cursor];
}

static int64_t rand_bigint(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randbigint[cursor];
}

static float rand_float(){
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
1046
  return randfloat[cursor];
1047 1048 1049
}

static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
1050
static void rand_string(char *str, int size) {
1051 1052 1053 1054
  str[0] = 0;
  if (size > 0) {
    //--size;
    int n;
1055
    for (n = 0; n < size - 1; n++) {
1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072
      int key = rand_tinyint() % (int)(sizeof(charset) - 1);
      str[n] = charset[key];
    }
    str[n] = 0;
  }
}

static double rand_double() {
  static int cursor;
  cursor++;
  cursor = cursor % MAX_PREPARED_RAND;
  return randdouble[cursor];

}

static void init_rand_data() {
  for (int i = 0; i < MAX_PREPARED_RAND; i++){
1073 1074 1075 1076
    randint[i] = (int)(taosRandom() % 65535);
    randbigint[i] = (int64_t)(taosRandom() % 2147483648);
    randfloat[i] = (float)(taosRandom() / 1000.0);
    randdouble[i] = (double)(taosRandom() / 1000000.0);
1077 1078 1079
  }
}

1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099
#define SHOW_PARSE_RESULT_START()   \
    do { if (g_args.metaFile)  \
        printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END() \
    do { if (g_args.metaFile)   \
        printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_START_TO_FILE(fp)   \
    do { if (g_args.metaFile)  \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \
                g_args.metaFile); } while(0)

#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \
    do { if (g_args.metaFile)   \
        fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \
                g_args.metaFile); } while(0)

1100
static int printfInsertMeta() {
1101 1102
    SHOW_PARSE_RESULT_START();

1103 1104 1105
  printf("host:                       \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port);
  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
1106
  printf("configDir:                  \033[33m%s\033[0m\n", configDir);
1107 1108 1109
  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
  printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl);
1110
  printf("top insert interval:        \033[33m%d\033[0m\n", g_args.insert_interval);
1111
  printf("number of records per req:  \033[33m%d\033[0m\n", g_args.num_of_RPR);
1112
  printf("max sql length:             \033[33m%d\033[0m\n", g_args.max_sql_len);
1113 1114

  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
1115

1116 1117
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    printf("database[\033[33m%d\033[0m]:\n", i);
1118
    printf("  database[%d] name:      \033[33m%s\033[0m\n", i, g_Dbs.db[i].dbName);
1119
    if (0 == g_Dbs.db[i].drop) {
1120 1121 1122
      printf("  drop:                  \033[33mno\033[0m\n");
    } else {
      printf("  drop:                  \033[33myes\033[0m\n");
1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      printf("  blocks:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      printf("  cache:                 \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      printf("  days:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      printf("  keep:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      printf("  replica:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      printf("  update:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      printf("  minRows:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      printf("  maxRows:               \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      printf("  comp:                  \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      printf("  walLevel:              \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      printf("  fsync:                 \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      printf("  quorum:                \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1162 1163
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1164 1165
        printf("  precision:             \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision);
      } else {
1166 1167 1168
        printf("\033[1m\033[40;31m  precision error:       %s\033[0m\n",
                g_Dbs.db[i].dbCfg.precision);
        return -1;
1169 1170 1171 1172 1173 1174
      }
    }

    printf("  super table count:     \033[33m%d\033[0m\n", g_Dbs.db[i].superTblCount);
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      printf("  super table[\033[33m%d\033[0m]:\n", j);
1175 1176

      printf("      stbName:           \033[33m%s\033[0m\n",  g_Dbs.db[i].superTbls[j].sTblName);
1177 1178 1179 1180 1181 1182 1183 1184

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      autoCreateTable:   \033[33m%s\033[0m\n",  "error");
      }
1185

1186 1187 1188 1189 1190 1191 1192
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "yes");
      } else {
        printf("      childTblExists:    \033[33m%s\033[0m\n",  "error");
      }
1193

1194 1195 1196 1197 1198 1199 1200 1201
      printf("      childTblCount:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].childTblCount);
      printf("      childTblPrefix:    \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      printf("      dataSource:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      printf("      insertMode:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].insertMode);
1202
      if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) {
1203 1204
        printf("      childTblLimit:     \033[33m%d\033[0m\n",
                g_Dbs.db[i].superTbls[j].childTblLimit);
1205
      }
1206
      if (g_Dbs.db[i].superTbls[j].childTblOffset >= 0) {
1207 1208
        printf("      childTblOffset:    \033[33m%d\033[0m\n",
                g_Dbs.db[i].superTbls[j].childTblOffset);
1209
      }
1210 1211
      printf("      insertRows:        \033[33m%"PRId64"\033[0m\n",
              g_Dbs.db[i].superTbls[j].insertRows);
1212 1213

      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1214
        printf("      multiThreadWriteOneTbl:  \033[33mno\033[0m\n");
1215
      }else {
1216
        printf("      multiThreadWriteOneTbl:  \033[33myes\033[0m\n");
1217
      }
1218 1219
      printf("      interlaceRows:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].interlaceRows);
1220 1221 1222 1223 1224 1225

      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
        printf("      stable insert interval:   \033[33m%d\033[0m\n",
            g_Dbs.db[i].superTbls[j].insertInterval);
      }

1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
      printf("      disorderRange:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRange);
      printf("      disorderRatio:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].disorderRatio);
      printf("      maxSqlLen:         \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].maxSqlLen);
      printf("      timeStampStep:     \033[33m%d\033[0m\n",
              g_Dbs.db[i].superTbls[j].timeStampStep);
      printf("      startTimestamp:    \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].startTimestamp);
      printf("      sampleFormat:      \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFormat);
      printf("      sampleFile:        \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].sampleFile);
      printf("      tagsFile:          \033[33m%s\033[0m\n",
              g_Dbs.db[i].superTbls[j].tagsFile);
      printf("      columnCount:       \033[33m%d\033[0m\n        ",
              g_Dbs.db[i].superTbls[j].columnCount);
1244 1245
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1246 1247 1248 1249
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "binary", 6))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                       "nchar", 5))) {
1250
          printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k,
1251 1252
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1253
        } else {
1254 1255
          printf("column[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType);
1256 1257 1258
        }
      }
      printf("\n");
1259 1260 1261

      printf("      tagCount:            \033[33m%d\033[0m\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1262 1263
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1264 1265 1266 1267
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
1268
          printf("tag[%d]:\033[33m%s(%d)\033[0m ", k,
1269 1270
                  g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1271
        } else {
1272 1273
          printf("tag[%d]:\033[33m%s\033[0m ", k,
                  g_Dbs.db[i].superTbls[j].tags[k].dataType);
1274
        }
1275 1276 1277 1278 1279
      }
      printf("\n");
    }
    printf("\n");
  }
1280 1281

  SHOW_PARSE_RESULT_END();
1282 1283

  return 0;
1284 1285 1286
}

static void printfInsertMetaToFile(FILE* fp) {
1287 1288

  SHOW_PARSE_RESULT_START_TO_FILE(fp);
1289

1290 1291
  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
  fprintf(fp, "user:                       %s\n", g_Dbs.user);
1292
  fprintf(fp, "configDir:                  %s\n", configDir);
1293 1294 1295
  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
1296 1297
  fprintf(fp, "number of records per req:  %d\n", g_args.num_of_RPR);
  fprintf(fp, "max sql length:             %d\n", g_args.max_sql_len);
1298
  fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
1299

1300 1301
  for (int i = 0; i < g_Dbs.dbCount; i++) {
    fprintf(fp, "database[%d]:\n", i);
1302
    fprintf(fp, "  database[%d] name:       %s\n", i, g_Dbs.db[i].dbName);
1303
    if (0 == g_Dbs.db[i].drop) {
1304
      fprintf(fp, "  drop:                  no\n");
1305
    }else {
1306
      fprintf(fp, "  drop:                  yes\n");
1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345
    }

    if (g_Dbs.db[i].dbCfg.blocks > 0) {
      fprintf(fp, "  blocks:                %d\n", g_Dbs.db[i].dbCfg.blocks);
    }
    if (g_Dbs.db[i].dbCfg.cache > 0) {
      fprintf(fp, "  cache:                 %d\n", g_Dbs.db[i].dbCfg.cache);
    }
    if (g_Dbs.db[i].dbCfg.days > 0) {
      fprintf(fp, "  days:                  %d\n", g_Dbs.db[i].dbCfg.days);
    }
    if (g_Dbs.db[i].dbCfg.keep > 0) {
      fprintf(fp, "  keep:                  %d\n", g_Dbs.db[i].dbCfg.keep);
    }
    if (g_Dbs.db[i].dbCfg.replica > 0) {
      fprintf(fp, "  replica:               %d\n", g_Dbs.db[i].dbCfg.replica);
    }
    if (g_Dbs.db[i].dbCfg.update > 0) {
      fprintf(fp, "  update:                %d\n", g_Dbs.db[i].dbCfg.update);
    }
    if (g_Dbs.db[i].dbCfg.minRows > 0) {
      fprintf(fp, "  minRows:               %d\n", g_Dbs.db[i].dbCfg.minRows);
    }
    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
      fprintf(fp, "  maxRows:               %d\n", g_Dbs.db[i].dbCfg.maxRows);
    }
    if (g_Dbs.db[i].dbCfg.comp > 0) {
      fprintf(fp, "  comp:                  %d\n", g_Dbs.db[i].dbCfg.comp);
    }
    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
      fprintf(fp, "  walLevel:              %d\n", g_Dbs.db[i].dbCfg.walLevel);
    }
    if (g_Dbs.db[i].dbCfg.fsync > 0) {
      fprintf(fp, "  fsync:                 %d\n", g_Dbs.db[i].dbCfg.fsync);
    }
    if (g_Dbs.db[i].dbCfg.quorum > 0) {
      fprintf(fp, "  quorum:                %d\n", g_Dbs.db[i].dbCfg.quorum);
    }
    if (g_Dbs.db[i].dbCfg.precision[0] != 0) {
1346 1347
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) {
1348 1349 1350 1351 1352 1353 1354 1355 1356
        fprintf(fp, "  precision:             %s\n", g_Dbs.db[i].dbCfg.precision);
      } else {
        fprintf(fp, "  precision error:       %s\n", g_Dbs.db[i].dbCfg.precision);
      }
    }

    fprintf(fp, "  super table count:     %d\n", g_Dbs.db[i].superTblCount);
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      fprintf(fp, "  super table[%d]:\n", j);
1357 1358

      fprintf(fp, "      stbName:           %s\n",  g_Dbs.db[i].superTbls[j].sTblName);
1359 1360 1361 1362 1363 1364 1365 1366

      if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "no");
      } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) {
        fprintf(fp, "      autoCreateTable:   %s\n",  "yes");
      } else {
        fprintf(fp, "      autoCreateTable:   %s\n",  "error");
      }
1367

1368 1369 1370 1371 1372 1373 1374
      if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "no");
      } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) {
        fprintf(fp, "      childTblExists:    %s\n",  "yes");
      } else {
        fprintf(fp, "      childTblExists:    %s\n",  "error");
      }
1375

1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387
      fprintf(fp, "      childTblCount:     %d\n",
              g_Dbs.db[i].superTbls[j].childTblCount);
      fprintf(fp, "      childTblPrefix:    %s\n",
              g_Dbs.db[i].superTbls[j].childTblPrefix);
      fprintf(fp, "      dataSource:        %s\n",
              g_Dbs.db[i].superTbls[j].dataSource);
      fprintf(fp, "      insertMode:        %s\n",
              g_Dbs.db[i].superTbls[j].insertMode);
      fprintf(fp, "      insertRows:        %"PRId64"\n",
              g_Dbs.db[i].superTbls[j].insertRows);
      fprintf(fp, "      interlace rows:    %d\n",
              g_Dbs.db[i].superTbls[j].interlaceRows);
1388
      if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) {
1389 1390
        fprintf(fp, "      stable insert interval:   %d\n",
                g_Dbs.db[i].superTbls[j].insertInterval);
1391
      }
1392 1393

      if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) {
1394
        fprintf(fp, "      multiThreadWriteOneTbl:  no\n");
1395
      }else {
1396
        fprintf(fp, "      multiThreadWriteOneTbl:  yes\n");
1397
      }
1398 1399
      fprintf(fp, "      interlaceRows:     %d\n",
              g_Dbs.db[i].superTbls[j].interlaceRows);
1400
      fprintf(fp, "      disorderRange:     %d\n",  g_Dbs.db[i].superTbls[j].disorderRange);
1401
      fprintf(fp, "      disorderRatio:     %d\n",  g_Dbs.db[i].superTbls[j].disorderRatio);
1402 1403 1404 1405
      fprintf(fp, "      maxSqlLen:         %d\n",  g_Dbs.db[i].superTbls[j].maxSqlLen);

      fprintf(fp, "      timeStampStep:     %d\n",  g_Dbs.db[i].superTbls[j].timeStampStep);
      fprintf(fp, "      startTimestamp:    %s\n",  g_Dbs.db[i].superTbls[j].startTimestamp);
1406
      fprintf(fp, "      sampleFormat:      %s\n",  g_Dbs.db[i].superTbls[j].sampleFormat);
1407 1408 1409
      fprintf(fp, "      sampleFile:        %s\n",  g_Dbs.db[i].superTbls[j].sampleFile);
      fprintf(fp, "      tagsFile:          %s\n",  g_Dbs.db[i].superTbls[j].tagsFile);

1410 1411 1412
      fprintf(fp, "      columnCount:       %d\n        ",  g_Dbs.db[i].superTbls[j].columnCount);
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1413 1414 1415 1416 1417 1418 1419 1420
        if ((0 == strncasecmp(
                        g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType,
                        "nchar", strlen("nchar")))) {
          fprintf(fp, "column[%d]:%s(%d) ", k,
                  g_Dbs.db[i].superTbls[j].columns[k].dataType,
                  g_Dbs.db[i].superTbls[j].columns[k].dataLen);
1421 1422 1423 1424 1425
        } else {
          fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType);
        }
      }
      fprintf(fp, "\n");
1426 1427 1428

      fprintf(fp, "      tagCount:            %d\n        ",
              g_Dbs.db[i].superTbls[j].tagCount);
1429 1430
      for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) {
        //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1431 1432 1433 1434 1435 1436
        if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "binary", strlen("binary")))
                || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType,
                        "nchar", strlen("nchar")))) {
          fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType,
                  g_Dbs.db[i].superTbls[j].tags[k].dataLen);
1437 1438
        } else {
          fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType);
1439
        }
1440 1441 1442 1443 1444
      }
      fprintf(fp, "\n");
    }
    fprintf(fp, "\n");
  }
1445

1446
  SHOW_PARSE_RESULT_END_TO_FILE(fp);
1447 1448 1449
}

static void printfQueryMeta() {
1450

1451
  SHOW_PARSE_RESULT_START();
1452

1453 1454
  printf("host:                    \033[33m%s:%u\033[0m\n",
          g_queryInfo.host, g_queryInfo.port);
1455 1456 1457 1458
  printf("user:                    \033[33m%s\033[0m\n", g_queryInfo.user);
  printf("database name:           \033[33m%s\033[0m\n", g_queryInfo.dbName);

  printf("\n");
1459
  printf("specified table query info:                   \n");
1460
  printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.rate);
1461
  printf("top query times:\033[33m%d\033[0m\n", g_args.query_times);
1462 1463
  printf("concurrent:     \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.concurrent);
  printf("sqlCount:       \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.sqlCount);
1464 1465
  printf("specified tbl query times:\n");
  printf("                \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.queryTimes);
1466

1467
  if (SUBSCRIBE_TEST == g_args.test_mode) {
1468 1469 1470 1471
    printf("mod:            \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeMode);
    printf("interval:       \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeInterval);
    printf("restart:        \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeRestart);
    printf("keepProgress:   \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
1472 1473
  }

1474 1475
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
    printf("  sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.specifiedQueryInfo.sql[i]);
1476 1477
  }
  printf("\n");
1478
  printf("super table query info:                   \n");
1479 1480 1481 1482
  printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.rate);
  printf("threadCnt:      \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.threadCnt);
  printf("childTblCount:  \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.childTblCount);
  printf("stable name:    \033[33m%s\033[0m\n", g_queryInfo.superQueryInfo.sTblName);
1483
  printf("stb query times:\033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.queryTimes);
1484

1485
  if (SUBSCRIBE_TEST == g_args.test_mode) {
1486 1487 1488 1489
    printf("mod:            \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeMode);
    printf("interval:       \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeInterval);
    printf("restart:        \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeRestart);
    printf("keepProgress:   \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeKeepProgress);
1490
  }
1491

1492 1493 1494
  printf("sqlCount:       \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.sqlCount);
  for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
    printf("  sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.superQueryInfo.sql[i]);
1495
  }
1496
  printf("\n");
1497

1498
  SHOW_PARSE_RESULT_END();
1499 1500
}

1501
static char* formatTimestamp(char* buf, int64_t val, int precision) {
1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532
  time_t tt;
  if (precision == TSDB_TIME_PRECISION_MICRO) {
    tt = (time_t)(val / 1000000);
  } else {
    tt = (time_t)(val / 1000);
  }

/* comment out as it make testcases like select_with_tags.sim fail.
  but in windows, this may cause the call to localtime crash if tt < 0,
  need to find a better solution.
  if (tt < 0) {
    tt = 0;
  }
  */

#ifdef WINDOWS
  if (tt < 0) tt = 0;
#endif

  struct tm* ptm = localtime(&tt);
  size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm);

  if (precision == TSDB_TIME_PRECISION_MICRO) {
    sprintf(buf + pos, ".%06d", (int)(val % 1000000));
  } else {
    sprintf(buf + pos, ".%03d", (int)(val % 1000));
  }

  return buf;
}

1533 1534 1535
static void xDumpFieldToFile(FILE* fp, const char* val,
        TAOS_FIELD* field, int32_t length, int precision) {

1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
  if (val == NULL) {
    fprintf(fp, "%s", TSDB_DATA_NULL_STR);
    return;
  }

  char buf[TSDB_MAX_BYTES_PER_ROW];
  switch (field->type) {
    case TSDB_DATA_TYPE_BOOL:
      fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0));
      break;
    case TSDB_DATA_TYPE_TINYINT:
      fprintf(fp, "%d", *((int8_t *)val));
      break;
    case TSDB_DATA_TYPE_SMALLINT:
      fprintf(fp, "%d", *((int16_t *)val));
      break;
    case TSDB_DATA_TYPE_INT:
      fprintf(fp, "%d", *((int32_t *)val));
      break;
    case TSDB_DATA_TYPE_BIGINT:
      fprintf(fp, "%" PRId64, *((int64_t *)val));
      break;
    case TSDB_DATA_TYPE_FLOAT:
      fprintf(fp, "%.5f", GET_FLOAT_VAL(val));
      break;
    case TSDB_DATA_TYPE_DOUBLE:
      fprintf(fp, "%.9f", GET_DOUBLE_VAL(val));
      break;
    case TSDB_DATA_TYPE_BINARY:
    case TSDB_DATA_TYPE_NCHAR:
      memcpy(buf, val, length);
      buf[length] = 0;
      fprintf(fp, "\'%s\'", buf);
      break;
    case TSDB_DATA_TYPE_TIMESTAMP:
1571
      formatTimestamp(buf, *(int64_t*)val, precision);
1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
      fprintf(fp, "'%s'", buf);
      break;
    default:
      break;
  }
}

static int xDumpResultToFile(const char* fname, TAOS_RES* tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  FILE* fp = fopen(fname, "at");
  if (fp == NULL) {
1587
    errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname);
1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601
    return -1;
  }

  int num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int precision = taos_result_precision(tres);

  for (int col = 0; col < num_fields; col++) {
    if (col > 0) {
      fprintf(fp, ",");
    }
    fprintf(fp, "%s", fields[col].name);
  }
  fputc('\n', fp);
1602

1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622
  int numOfRows = 0;
  do {
    int32_t* length = taos_fetch_lengths(tres);
    for (int i = 0; i < num_fields; i++) {
      if (i > 0) {
        fputc(',', fp);
      }
      xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision);
    }
    fputc('\n', fp);

    numOfRows++;
    row = taos_fetch_row(tres);
  } while( row != NULL);

  fclose(fp);

  return numOfRows;
}

1623 1624
static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) {
  TAOS_RES * res;
1625 1626
  TAOS_ROW row = NULL;
  int count = 0;
1627 1628

  res = taos_query(taos, "show databases;");
1629
  int32_t code = taos_errno(res);
1630

1631
  if (code != 0) {
1632
    errorPrint( "failed to run <show databases>, reason: %s\n", taos_errstr(res));
1633 1634 1635 1636 1637 1638 1639
    return -1;
  }

  TAOS_FIELD *fields = taos_fetch_fields(res);

  while ((row = taos_fetch_row(res)) != NULL) {
    // sys database name : 'log'
1640 1641 1642 1643
    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log",
                fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) {
      continue;
    }
1644 1645 1646

    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
    if (dbInfos[count] == NULL) {
1647
      errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count);
1648 1649 1650
      return -1;
    }

1651 1652
    tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
            fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
1653
    formatTimestamp(dbInfos[count]->create_time,
1654 1655
            *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX],
            TSDB_TIME_PRECISION_MILLI);
1656
    dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
1657
    dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
1658 1659
    dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]);
    dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
1660
    dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
1661

1662
    tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
1663
            fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
1664 1665 1666 1667 1668 1669 1670 1671 1672
    dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]);
    dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]);
    dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]);
    dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]);
    dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]);
    dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]);
    dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
    dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));

1673
    tstrncpy(dbInfos[count]->precision,
1674
            (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
1675
            fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
1676
    dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
1677
    tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX],
1678
            fields[TSDB_SHOW_DB_STATUS_INDEX].bytes);
1679

1680 1681
    count++;
    if (count > MAX_DATABASE_COUNT) {
1682
      errorPrint( "The database count overflow than %d\n", MAX_DATABASE_COUNT);
1683 1684 1685 1686 1687 1688 1689
      break;
    }
  }

  return count;
}

1690 1691
static void printfDbInfoForQueryToFile(
        char* filename, SDbInfo* dbInfos, int index) {
1692
  if (filename[0] == 0)
1693
      return;
1694 1695 1696

  FILE *fp = fopen(filename, "at");
  if (fp == NULL) {
1697
    errorPrint( "failed to open file: %s\n", filename);
1698
    return;
1699 1700 1701 1702 1703 1704
  }

  fprintf(fp, "================ database[%d] ================\n", index);
  fprintf(fp, "name: %s\n", dbInfos->name);
  fprintf(fp, "created_time: %s\n", dbInfos->create_time);
  fprintf(fp, "ntables: %d\n", dbInfos->ntables);
1705
  fprintf(fp, "vgroups: %d\n", dbInfos->vgroups);
1706 1707
  fprintf(fp, "replica: %d\n", dbInfos->replica);
  fprintf(fp, "quorum: %d\n", dbInfos->quorum);
1708 1709
  fprintf(fp, "days: %d\n", dbInfos->days);
  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);
1710 1711 1712 1713 1714 1715 1716
  fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
  fprintf(fp, "blocks: %d\n", dbInfos->blocks);
  fprintf(fp, "minrows: %d\n", dbInfos->minrows);
  fprintf(fp, "maxrows: %d\n", dbInfos->maxrows);
  fprintf(fp, "wallevel: %d\n", dbInfos->wallevel);
  fprintf(fp, "fsync: %d\n", dbInfos->fsync);
  fprintf(fp, "comp: %d\n", dbInfos->comp);
1717 1718
  fprintf(fp, "cachelast: %d\n", dbInfos->cachelast);
  fprintf(fp, "precision: %s\n", dbInfos->precision);
1719
  fprintf(fp, "update: %d\n", dbInfos->update);
1720
  fprintf(fp, "status: %s\n", dbInfos->status);
1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734
  fprintf(fp, "\n");

  fclose(fp);
}

static void printfQuerySystemInfo(TAOS * taos) {
  char filename[MAX_QUERY_SQL_LENGTH+1] = {0};
  char buffer[MAX_QUERY_SQL_LENGTH+1] = {0};
  TAOS_RES* res;

  time_t t;
  struct tm* lt;
  time(&t);
  lt = localtime(&t);
1735 1736 1737 1738
  snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d",
          lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min,
          lt->tm_sec);

1739 1740 1741 1742 1743 1744 1745 1746 1747
  // show variables
  res = taos_query(taos, "show variables;");
  //getResult(res, filename);
  xDumpResultToFile(filename, res);

  // show dnodes
  res = taos_query(taos, "show dnodes;");
  xDumpResultToFile(filename, res);
  //getResult(res, filename);
1748

1749 1750 1751 1752
  // show databases
  res = taos_query(taos, "show databases;");
  SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *));
  if (dbInfos == NULL) {
1753
    errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__);
1754 1755 1756
    return;
  }
  int dbCount = getDbFromServer(taos, dbInfos);
1757 1758 1759 1760
  if (dbCount <= 0) {
      free(dbInfos);
      return;
  }
1761 1762

  for (int i = 0; i < dbCount; i++) {
1763
    // printf database info
1764
    printfDbInfoForQueryToFile(filename, dbInfos[i], i);
1765

1766 1767 1768 1769
    // show db.vgroups
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);
1770

1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781
    // show db.stables
    snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name);
    res = taos_query(taos, buffer);
    xDumpResultToFile(filename, res);

    free(dbInfos[i]);
  }

  free(dbInfos);
}

1782
static int postProceSql(char* host, uint16_t port, char* sqlstr)
1783
{
1784
    char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
1785 1786 1787 1788 1789

    char *url = "/rest/sql";

    struct hostent *server;
    struct sockaddr_in serv_addr;
1790
    int bytes, sent, received, req_str_len, resp_len;
1791 1792 1793
    char *request_buf;
    char response_buf[RESP_BUF_LEN];
    uint16_t rest_port = port + TSDB_PORT_HTTP;
1794

1795 1796 1797 1798
    int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN;

    request_buf = malloc(req_buf_len);
    if (NULL == request_buf)
1799
        ERROR_EXIT("ERROR, cannot allocate memory.");
1800

1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811
    char userpass_buf[INPUT_BUF_LEN];
    int mod_table[] = {0, 2, 1};

    static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
      'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
      'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
      'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
      'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
      'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z', '0', '1', '2', '3',
      '4', '5', '6', '7', '8', '9', '+', '/'};
1812

1813 1814 1815 1816 1817 1818
    snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s",
        g_Dbs.user, g_Dbs.password);
    size_t userpass_buf_len = strlen(userpass_buf);
    size_t encoded_len = 4 * ((userpass_buf_len +2) / 3);

    char base64_buf[INPUT_BUF_LEN];
1819
#ifdef WINDOWS
1820
    WSADATA wsaData;
1821
    WSAStartup(MAKEWORD(2, 2), &wsaData);
1822 1823 1824 1825
    SOCKET sockfd;
#else
    int sockfd;
#endif
1826
    sockfd = socket(AF_INET, SOCK_STREAM, 0);
1827
    if (sockfd < 0) {
1828
#ifdef WINDOWS
1829
        errorPrint( "Could not create socket : %d" , WSAGetLastError());
1830
#endif
1831
        debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd);
1832
        free(request_buf);
1833
        ERROR_EXIT("ERROR opening socket");
1834
    }
1835 1836

    server = gethostbyname(host);
1837 1838
    if (server == NULL) {
        free(request_buf);
1839
        ERROR_EXIT("ERROR, no such host");
1840
    }
1841

1842
    debugPrint("h_name: %s\nh_addretype: %s\nh_length: %d\n",
1843 1844 1845 1846
            server->h_name,
            (server->h_addrtype == AF_INET)?"ipv4":"ipv6",
            server->h_length);

1847
    memset(&serv_addr, 0, sizeof(serv_addr));
1848
    serv_addr.sin_family = AF_INET;
1849
    serv_addr.sin_port = htons(rest_port);
1850 1851 1852
#ifdef WINDOWS
    serv_addr.sin_addr.s_addr = inet_addr(host);
#else
1853
    memcpy(&serv_addr.sin_addr.s_addr,server->h_addr,server->h_length);
1854
#endif
1855

1856 1857 1858
    int retConn = connect(sockfd,(struct sockaddr *)&serv_addr,sizeof(serv_addr));
    debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn);
    if (retConn < 0) {
1859
        free(request_buf);
1860
        ERROR_EXIT("ERROR connecting");
1861
    }
1862

1863 1864 1865
    memset(base64_buf, 0, INPUT_BUF_LEN);

    for (int n = 0, m = 0; n < userpass_buf_len;) {
1866
      uint32_t oct_a = n < userpass_buf_len ?
1867
        (unsigned char) userpass_buf[n++]:0;
1868
      uint32_t oct_b = n < userpass_buf_len ?
1869
        (unsigned char) userpass_buf[n++]:0;
1870
      uint32_t oct_c = n < userpass_buf_len ?
1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882
        (unsigned char) userpass_buf[n++]:0;
      uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c;

      base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f];
      base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f];
    }

    for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++)
      base64_buf[encoded_len - 1 - l] = '=';

1883 1884
    debugPrint("%s() LN%d: auth string base64 encoded: %s\n",
            __func__, __LINE__, base64_buf);
1885 1886
    char *auth = base64_buf;

1887 1888 1889
    int r = snprintf(request_buf,
            req_buf_len,
            req_fmt, url, host, rest_port,
1890 1891 1892 1893 1894
            auth, strlen(sqlstr), sqlstr);
    if (r >= req_buf_len) {
        free(request_buf);
        ERROR_EXIT("ERROR too long request");
    }
1895
    verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf);
1896

1897
    req_str_len = strlen(request_buf);
1898 1899
    sent = 0;
    do {
1900 1901 1902
#ifdef WINDOWS
        bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0);
#else
1903
        bytes = write(sockfd, request_buf + sent, req_str_len - sent);
1904
#endif
1905
        if (bytes < 0)
1906
            ERROR_EXIT("ERROR writing message to socket");
1907 1908 1909
        if (bytes == 0)
            break;
        sent+=bytes;
1910
    } while (sent < req_str_len);
1911

1912 1913
    memset(response_buf, 0, RESP_BUF_LEN);
    resp_len = sizeof(response_buf) - 1;
1914 1915
    received = 0;
    do {
1916 1917 1918
#ifdef WINDOWS
        bytes = recv(sockfd, response_buf + received, resp_len - received, 0);
#else
1919
        bytes = read(sockfd, response_buf + received, resp_len - received);
1920
#endif
1921 1922
        if (bytes < 0) {
            free(request_buf);
1923
            ERROR_EXIT("ERROR reading response from socket");
1924
        }
1925 1926
        if (bytes == 0)
            break;
1927 1928
        received += bytes;
    } while (received < resp_len);
1929

1930 1931
    if (received == resp_len) {
        free(request_buf);
1932
        ERROR_EXIT("ERROR storing complete response from socket");
1933
    }
1934

1935
    response_buf[RESP_BUF_LEN - 1] = '\0';
1936 1937 1938
    printf("Response:\n%s\n", response_buf);

    free(request_buf);
1939 1940 1941 1942
#ifdef WINDOWS
    closesocket(sockfd);
    WSACleanup();
#else
1943
    close(sockfd);
1944
#endif
1945 1946 1947 1948

    return 0;
}

1949
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
1950 1951
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
1952
    errorPrint("%s() LN%d, calloc failed! size:%d\n", __func__, __LINE__, TSDB_MAX_SQL_LEN+1);
1953 1954
    return NULL;
  }
1955

1956
  int    dataLen = 0;
1957 1958 1959
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
          "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);

1960 1961 1962
  return dataBuf;
}

1963
static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) {
1964 1965 1966 1967 1968 1969 1970 1971 1972
  char*  dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
  if (NULL == dataBuf) {
    printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
    return NULL;
  }

  int    dataLen = 0;
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
  for (int i = 0; i < stbInfo->tagCount; i++) {
1973 1974
    if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
            || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
1975
      if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
1976 1977
        printf("binary or nchar length overflow, max size:%u\n",
                (uint32_t)TSDB_MAX_BINARY_LEN);
1978 1979 1980
        tmfree(dataBuf);
        return NULL;
      }
1981

1982 1983
      int tagBufLen = stbInfo->tags[i].dataLen + 1;
      char* buf = (char*)calloc(tagBufLen, 1);
1984 1985 1986 1987 1988
      if (NULL == buf) {
        printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen);
        tmfree(dataBuf);
        return NULL;
      }
1989 1990 1991 1992 1993 1994 1995

      if (tableSeq % 2) {
        tstrncpy(buf, "beijing", tagBufLen);
      } else {
        tstrncpy(buf, "shanghai", tagBufLen);
      }
      //rand_string(buf, stbInfo->tags[i].dataLen);
1996 1997
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "\'%s\', ", buf);
1998
      tmfree(buf);
1999 2000 2001
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "int", strlen("int"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
2002
              "%d, ", tableSeq);
2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030
    } else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bigint", strlen("bigint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "float", strlen("float"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_float());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "double", strlen("double"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%f, ", rand_double());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "smallint", strlen("smallint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_smallint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "tinyint", strlen("tinyint"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_tinyint());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "bool", strlen("bool"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%d, ", rand_bool());
    }  else if (0 == strncasecmp(stbInfo->tags[i].dataType,
                "timestamp", strlen("timestamp"))) {
      dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
              "%"PRId64", ", rand_bigint());
2031 2032 2033 2034 2035 2036
    }  else {
      printf("No support data type: %s\n", stbInfo->tags[i].dataType);
      tmfree(dataBuf);
      return NULL;
    }
  }
2037

2038
  dataLen -= 2;
2039
  dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");
2040 2041 2042
  return dataBuf;
}

2043
static int calcRowLen(SSuperTable*  superTbls) {
2044 2045
  int colIndex;
  int  lenOfOneRow = 0;
2046

2047 2048
  for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) {
    char* dataType = superTbls->columns[colIndex].dataType;
2049

2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfOneRow += superTbls->columns[colIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfOneRow += 22;
2066
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2067
      lenOfOneRow += 42;
2068
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081
      lenOfOneRow += 21;
    } else {
      printf("get error data type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp

  int tagIndex;
  int lenOfTagOfOneRow = 0;
  for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) {
    char* dataType = superTbls->tags[tagIndex].dataType;
2082

2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098
    if (strcasecmp(dataType, "BINARY") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22;
2099
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2100 2101 2102 2103 2104 2105 2106 2107
      lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42;
    } else {
      printf("get error tag type : %s\n", dataType);
      exit(-1);
    }
  }

  superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow;
2108

2109 2110 2111 2112
  return 0;
}


2113 2114 2115 2116
static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos,
        char* dbName, char* sTblName, char** childTblNameOfSuperTbl,
        int* childTblCountOfSuperTbl, int limit, int offset) {

2117
  char command[BUFFER_SIZE] = "\0";
2118 2119
  char limitBuf[100] = "\0";

2120
  TAOS_RES * res;
2121 2122 2123
  TAOS_ROW row = NULL;

  char* childTblName = *childTblNameOfSuperTbl;
2124 2125 2126 2127 2128

  if (offset >= 0) {
    snprintf(limitBuf, 100, " limit %d offset %d", limit, offset);
  }

2129
  //get all child table name use cmd: select tbname from superTblName;
2130 2131
  snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s",
          dbName, sTblName, limitBuf);
2132

2133
  res = taos_query(taos, command);
2134 2135 2136 2137
  int32_t code = taos_errno(res);
  if (code != 0) {
    taos_free_result(res);
    taos_close(taos);
2138 2139
    errorPrint("%s() LN%d, failed to run command %s\n",
           __func__, __LINE__, command);
2140 2141 2142
    exit(-1);
  }

2143
  int childTblCount = (limit < 0)?10000:limit;
2144
  int count = 0;
2145 2146 2147 2148 2149 2150 2151 2152 2153 2154
  if (childTblName == NULL) {
    childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (NULL ==  childTblName) {
    taos_free_result(res);
        taos_close(taos);
        errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__);
        exit(-1);
    }
  }

2155 2156 2157
  char* pTblName = childTblName;
  while ((row = taos_fetch_row(res)) != NULL) {
    int32_t* len = taos_fetch_lengths(res);
2158
    tstrncpy(pTblName, (char *)row[0], len[0]+1);
2159 2160 2161
    //printf("==== sub table name: %s\n", pTblName);
    count++;
    if (count >= childTblCount - 1) {
2162 2163
      char *tmp = realloc(childTblName,
              (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1);
2164 2165 2166
      if (tmp != NULL) {
        childTblName = tmp;
        childTblCount = (int)(childTblCount*1.5);
2167 2168
        memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0,
                (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN));
2169 2170
      } else {
        // exit, if allocate more memory failed
2171 2172
        errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n",
               __func__, __LINE__, dbName, sTblName);
2173 2174 2175 2176 2177 2178
        tmfree(childTblName);
        taos_free_result(res);
        taos_close(taos);
        exit(-1);
      }
    }
2179
    pTblName = childTblName + count * TSDB_TABLE_NAME_LEN;
2180
  }
2181

2182 2183 2184 2185 2186 2187 2188
  *childTblCountOfSuperTbl = count;
  *childTblNameOfSuperTbl  = childTblName;

  taos_free_result(res);
  return 0;
}

2189 2190
static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName,
        char* sTblName, char** childTblNameOfSuperTbl,
2191
        int* childTblCountOfSuperTbl) {
2192 2193 2194 2195 2196 2197

    return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName,
            childTblNameOfSuperTbl, childTblCountOfSuperTbl,
            -1, -1);
}

2198 2199
static int getSuperTableFromServer(TAOS * taos, char* dbName,
        SSuperTable*  superTbls) {
2200

2201
  char command[BUFFER_SIZE] = "\0";
2202
  TAOS_RES * res;
2203 2204
  TAOS_ROW row = NULL;
  int count = 0;
2205

2206
  //get schema use cmd: describe superTblName;
2207
  snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName);
2208
  res = taos_query(taos, command);
2209 2210 2211
  int32_t code = taos_errno(res);
  if (code != 0) {
    printf("failed to run command %s\n", command);
2212
    taos_free_result(res);
2213 2214 2215 2216 2217 2218 2219 2220 2221 2222
    return -1;
  }

  int tagIndex = 0;
  int columnIndex = 0;
  TAOS_FIELD *fields = taos_fetch_fields(res);
  while ((row = taos_fetch_row(res)) != NULL) {
    if (0 == count) {
      count++;
      continue;
2223
    }
2224 2225

    if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) {
2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236
      tstrncpy(superTbls->tags[tagIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->tags[tagIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->tags[tagIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->tags[tagIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2237
      tagIndex++;
2238
    } else {
2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249
      tstrncpy(superTbls->columns[columnIndex].field,
              (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
              fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
      tstrncpy(superTbls->columns[columnIndex].dataType,
              (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
      superTbls->columns[columnIndex].dataLen =
          *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
      tstrncpy(superTbls->columns[columnIndex].note,
              (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
              fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260
      columnIndex++;
    }
    count++;
  }

  superTbls->columnCount = columnIndex;
  superTbls->tagCount    = tagIndex;
  taos_free_result(res);

  calcRowLen(superTbls);

2261
/*
2262
  if (TBL_ALREADY_EXISTS == superTbls->childTblExists) {
2263
    //get all child table name use cmd: select tbname from superTblName;
2264 2265 2266
    int childTblCount = 10000;
    superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN);
    if (superTbls->childTblName == NULL) {
2267
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
2268 2269
      return -1;
    }
2270
    getAllChildNameOfSuperTable(taos, dbName,
2271 2272 2273
            superTbls->sTblName,
            &superTbls->childTblName,
            &superTbls->childTblCount);
2274
  }
2275
  */
2276 2277 2278
  return 0;
}

2279
static int createSuperTable(TAOS * taos, char* dbName,
2280
        SSuperTable*  superTbl) {
2281
  char command[BUFFER_SIZE] = "\0";
2282

2283 2284 2285 2286 2287
  char cols[STRING_LEN] = "\0";
  int colIndex;
  int len = 0;

  int  lenOfOneRow = 0;
2288 2289 2290 2291 2292 2293 2294 2295 2296

  if (superTbl->columnCount == 0) {
    errorPrint("%s() LN%d, super table column count is %d\n",
            __func__, __LINE__, superTbl->columnCount);
    return -1;
  }

  for (colIndex = 0; colIndex < superTbl->columnCount; colIndex++) {
    char* dataType = superTbl->columns[colIndex].dataType;
2297

2298
    if (strcasecmp(dataType, "BINARY") == 0) {
2299 2300
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "BINARY",
2301 2302
          superTbl->columns[colIndex].dataLen);
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2303
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
2304 2305
      len += snprintf(cols + len, STRING_LEN - len,
          ", col%d %s(%d)", colIndex, "NCHAR",
2306 2307
          superTbl->columns[colIndex].dataLen);
      lenOfOneRow += superTbl->columns[colIndex].dataLen + 3;
2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT");
      lenOfOneRow += 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT");
      lenOfOneRow += 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT");
      lenOfOneRow += 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL");
      lenOfOneRow += 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT");
      lenOfOneRow += 22;
2326
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
2327 2328
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE");
      lenOfOneRow += 42;
2329
    }  else if (strcasecmp(dataType, "TIMESTAMP") == 0) {
2330 2331 2332 2333 2334 2335 2336 2337 2338
      len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP");
      lenOfOneRow += 21;
    } else {
      taos_close(taos);
      printf("config error data type : %s\n", dataType);
      exit(-1);
    }
  }

2339 2340
  superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp
  //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbl[j].sTblName, g_Dbs.db[i].superTbl[j].columnCount, lenOfOneRow);
2341 2342

  // save for creating child table
2343 2344 2345 2346
  superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1);
  if (NULL == superTbl->colsOfCreateChildTable) {
    errorPrint("%s() LN%d, Failed when calloc, size:%d",
           __func__, __LINE__, len+1);
2347 2348 2349 2350
    taos_close(taos);
    exit(-1);
  }

2351 2352
  snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols);
  verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, superTbl->colsOfCreateChildTable);
2353

2354 2355 2356 2357 2358
  if (superTbl->tagCount == 0) {
    errorPrint("%s() LN%d, super table tag count is %d\n",
            __func__, __LINE__, superTbl->tagCount);
    return -1;
  }
2359

2360 2361 2362
  char tags[STRING_LEN] = "\0";
  int tagIndex;
  len = 0;
2363

2364 2365 2366 2367
  int lenOfTagOfOneRow = 0;
  len += snprintf(tags + len, STRING_LEN - len, "(");
  for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) {
    char* dataType = superTbl->tags[tagIndex].dataType;
2368

2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408
    if (strcasecmp(dataType, "BINARY") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
              "BINARY", superTbl->tags[tagIndex].dataLen);
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "NCHAR") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex,
              "NCHAR", superTbl->tags[tagIndex].dataLen);
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3;
    } else if (strcasecmp(dataType, "INT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "INT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11;
    } else if (strcasecmp(dataType, "BIGINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "BIGINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21;
    } else if (strcasecmp(dataType, "SMALLINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "SMALLINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "TINYINT") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "TINYINT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4;
    } else if (strcasecmp(dataType, "BOOL") == 0)  {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "BOOL");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6;
    } else if (strcasecmp(dataType, "FLOAT") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "FLOAT");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22;
    } else if (strcasecmp(dataType, "DOUBLE") == 0) {
      len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex,
              "DOUBLE");
      lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42;
    } else {
      taos_close(taos);
      printf("config error tag type : %s\n", dataType);
      exit(-1);
2409 2410
    }
  }
2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427

  len -= 2;
  len += snprintf(tags + len, STRING_LEN - len, ")");

  superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow;

  snprintf(command, BUFFER_SIZE,
          "create table if not exists %s.%s (ts timestamp%s) tags %s",
          dbName, superTbl->sTblName, cols, tags);
  verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command);

  if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
      errorPrint( "create supertable %s failed!\n\n",
              superTbl->sTblName);
      return -1;
  }
  debugPrint("create supertable %s success!\n\n", superTbl->sTblName);
2428 2429 2430
  return 0;
}

2431
static int createDatabasesAndStables() {
2432 2433 2434 2435
  TAOS * taos = NULL;
  int    ret = 0;
  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port);
  if (taos == NULL) {
2436
    errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2437
    return -1;
2438 2439 2440
  }
  char command[BUFFER_SIZE] = "\0";

2441
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2442 2443
    if (g_Dbs.db[i].drop) {
      sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName);
2444
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);
2445
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
2446 2447 2448 2449
        taos_close(taos);
        return -1;
      }

2450
      int dataLen = 0;
2451
      dataLen += snprintf(command + dataLen,
2452
          BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
2453

2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523
      if (g_Dbs.db[i].dbCfg.blocks > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
      }
      if (g_Dbs.db[i].dbCfg.cache > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
      }
      if (g_Dbs.db[i].dbCfg.days > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
      }
      if (g_Dbs.db[i].dbCfg.keep > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
      }
      if (g_Dbs.db[i].dbCfg.quorum > 1) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
      }
      if (g_Dbs.db[i].dbCfg.replica > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
      }
      if (g_Dbs.db[i].dbCfg.update > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
      }
      //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
      //  dataLen += snprintf(command + dataLen,
      //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
      //}
      if (g_Dbs.db[i].dbCfg.minRows > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
      }
      if (g_Dbs.db[i].dbCfg.maxRows > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
      }
      if (g_Dbs.db[i].dbCfg.comp > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
      }
      if (g_Dbs.db[i].dbCfg.walLevel > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
      }
      if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
        dataLen += snprintf(command + dataLen,
            BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
      }
      if (g_Dbs.db[i].dbCfg.fsync > 0) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
                " fsync %d", g_Dbs.db[i].dbCfg.fsync);
      }
      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
                      "us", strlen("us")))) {
        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
                " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
      }

      debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
        taos_close(taos);
        errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
        return -1;
      }
      printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
2524 2525
    }

2526 2527
    debugPrint("%s() %d supertbl count:%d\n",
            __func__, __LINE__, g_Dbs.db[i].superTblCount);
2528 2529 2530

    int validStbCount = 0;

2531
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
2532 2533 2534 2535 2536 2537 2538
      sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName,
              g_Dbs.db[i].superTbls[j].sTblName);
      verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command);

      ret = queryDbExec(taos, command, NO_INSERT_TYPE, true);

      if ((ret != 0) || (g_Dbs.db[i].drop)) {
2539
        ret = createSuperTable(taos, g_Dbs.db[i].dbName,
2540
                &g_Dbs.db[i].superTbls[j]);
2541

2542
        if (0 != ret) {
2543 2544
          errorPrint("create super table %d failed!\n\n", j);
          continue;
2545 2546 2547
        }
      }

2548
      ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName,
2549
                &g_Dbs.db[i].superTbls[j]);
2550 2551 2552
      if (0 != ret) {
        errorPrint("\nget super table %s.%s info failed!\n\n",
                g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName);
2553
        continue;
2554
      }
2555 2556

      validStbCount ++;
2557
    }
2558 2559

    g_Dbs.db[i].superTblCount = validStbCount;
2560 2561 2562 2563 2564 2565
  }

  taos_close(taos);
  return 0;
}

2566 2567 2568
static void* createTable(void *sarg)
{
  threadInfo *winfo = (threadInfo *)sarg;
2569 2570 2571 2572
  SSuperTable* superTblInfo = winfo->superTblInfo;

  int64_t  lastPrintTime = taosGetTimestampMs();

2573 2574 2575 2576 2577 2578
  int buff_len;
  if (superTblInfo)
    buff_len = superTblInfo->maxSqlLen;
  else
    buff_len = BUFFER_SIZE;

2579
  char *buffer = calloc(buff_len, 1);
2580
  if (buffer == NULL) {
2581
    errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__);
2582 2583
    exit(-1);
  }
2584 2585 2586

  int len = 0;
  int batchNum = 0;
2587

2588
  verbosePrint("%s() LN%d: Creating table from %d to %d\n",
2589 2590 2591 2592
          __func__, __LINE__,
          winfo->start_table_from, winfo->end_table_to);

  for (int i = winfo->start_table_from; i <= winfo->end_table_to; i++) {
2593
    if (0 == g_Dbs.use_metric) {
2594
      snprintf(buffer, buff_len,
2595 2596
              "create table if not exists %s.%s%d %s;",
              winfo->db_name,
2597 2598
              g_args.tb_prefix, i,
              winfo->cols);
2599
    } else {
2600 2601 2602
      if (superTblInfo == NULL) {
        errorPrint("%s() LN%d, use metric, but super table info is NULL\n",
                  __func__, __LINE__);
2603
        free(buffer);
2604 2605 2606 2607 2608 2609 2610 2611 2612 2613
        exit(-1);
      } else {
        if (0 == len) {
          batchNum = 0;
          memset(buffer, 0, buff_len);
          len += snprintf(buffer + len,
                  buff_len - len, "create table ");
        }
        char* tagsValBuf = NULL;
        if (0 == superTblInfo->tagSource) {
2614
          tagsValBuf = generateTagVaulesForStb(superTblInfo, i);
2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636
        } else {
          tagsValBuf = getTagValueFromTagSample(
                  superTblInfo,
                  i % superTblInfo->tagSampleCount);
        }
        if (NULL == tagsValBuf) {
          free(buffer);
          return NULL;
        }
        len += snprintf(buffer + len,
                superTblInfo->maxSqlLen - len,
                "if not exists %s.%s%d using %s.%s tags %s ",
                winfo->db_name, superTblInfo->childTblPrefix,
                i, winfo->db_name,
                superTblInfo->sTblName, tagsValBuf);
        free(tagsValBuf);
        batchNum++;
        if ((batchNum < superTblInfo->batchCreateTableNum)
                && ((superTblInfo->maxSqlLen - len)
                    >= (superTblInfo->lenOfTagOfOneRow + 256))) {
          continue;
        }
2637 2638 2639 2640
      }
    }

    len = 0;
2641
    verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer);
2642
    if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE, false)){
2643
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2644
      free(buffer);
2645 2646 2647 2648 2649
      return NULL;
    }

    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
2650
      printf("thread[%d] already create %d - %d tables\n",
2651
              winfo->threadID, winfo->start_table_from, i);
2652 2653 2654
      lastPrintTime = currentPrintTime;
    }
  }
2655

2656
  if (0 != len) {
2657
    verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer);
2658
    if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE, false)) {
2659
      errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer);
2660
    }
2661
  }
2662

2663 2664 2665 2666
  free(buffer);
  return NULL;
}

2667
static int startMultiThreadCreateChildTable(
2668
        char* cols, int threads, int startFrom, int ntables,
2669
        char* db_name, SSuperTable* superTblInfo) {
2670

2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  threadInfo *infos = malloc(threads * sizeof(threadInfo));

  if ((NULL == pids) || (NULL == infos)) {
    printf("malloc failed\n");
    exit(-1);
  }

  if (threads < 1) {
    threads = 1;
  }

  int a = ntables / threads;
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

  int b = 0;
  b = ntables % threads;
2691

2692 2693 2694
  for (int i = 0; i < threads; i++) {
    threadInfo *t_info = infos + i;
    t_info->threadID = i;
H
Hui Li 已提交
2695
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
2696
    t_info->superTblInfo = superTblInfo;
2697
    verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name);
2698 2699 2700 2701 2702 2703
    t_info->taos = taos_connect(
            g_Dbs.host,
            g_Dbs.user,
            g_Dbs.password,
            db_name,
            g_Dbs.port);
2704
    if (t_info->taos == NULL) {
2705
      errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL));
2706
      free(pids);
2707
      free(infos);
2708 2709
      return -1;
    }
2710 2711 2712 2713 2714

    t_info->start_table_from = startFrom;
    t_info->ntables = i<b?a+1:a;
    t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
    startFrom = t_info->end_table_to + 1;
2715
    t_info->use_metric = true;
2716
    t_info->cols = cols;
2717
    t_info->minDelay = INT16_MAX;
2718 2719
    pthread_create(pids + i, NULL, createTable, t_info);
  }
2720

2721 2722 2723 2724 2725 2726 2727 2728 2729 2730
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

  for (int i = 0; i < threads; i++) {
    threadInfo *t_info = infos + i;
    taos_close(t_info->taos);
  }

  free(pids);
2731
  free(infos);
2732 2733

  return 0;
2734 2735
}

2736 2737 2738 2739
static void createChildTables() {
    char tblColsBuf[MAX_SQL_SIZE];
    int len;

2740
  for (int i = 0; i < g_Dbs.dbCount; i++) {
2741 2742 2743 2744 2745 2746 2747 2748
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
          // with super table
        for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
          if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable)
                || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) {
            continue;
          }
2749

2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763
          verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
                  g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
          int startFrom = 0;
          g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;

          verbosePrint("%s() LN%d: create %d child tables from %d\n",
                  __func__, __LINE__, g_totalChildTables, startFrom);
          startMultiThreadCreateChildTable(
                g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
                g_Dbs.threadCountByCreateTbl,
                startFrom,
                g_Dbs.db[i].superTbls[j].childTblCount,
                g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
        }
2764 2765
      }
    } else {
2766 2767
        // normal table
        len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP");
2768 2769 2770
        int j = 0;
        while (g_args.datatype[j]) {
            if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0)
2771 2772
                    || (strncasecmp(g_args.datatype[j],
                        "NCHAR", strlen("NCHAR")) == 0)) {
2773
                snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
2774
                        ", COL%d %s(60)", j, g_args.datatype[j]);
2775
            } else {
2776
                snprintf(tblColsBuf + len, MAX_SQL_SIZE - len,
2777
                        ", COL%d %s", j, g_args.datatype[j]);
2778 2779
            }
            len = strlen(tblColsBuf);
2780
            j++;
2781 2782
        }

2783
        snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")");
2784

2785 2786
        verbosePrint("%s() LN%d: dbName: %s num of tb: %d schema: %s\n",
                __func__, __LINE__,
2787
                g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf);
2788
        startMultiThreadCreateChildTable(
2789
              tblColsBuf,
2790
              g_Dbs.threadCountByCreateTbl,
2791
              0,
2792
              g_args.num_of_tables,
2793 2794 2795
              g_Dbs.db[i].dbName,
              NULL);
    }
2796 2797 2798 2799 2800 2801
  }
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
2802
static int readTagFromCsvFileToMem(SSuperTable  * superTblInfo) {
2803 2804 2805
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
2806

2807 2808
  FILE *fp = fopen(superTblInfo->tagsFile, "r");
  if (fp == NULL) {
2809 2810
    printf("Failed to open tags file: %s, reason:%s\n",
            superTblInfo->tagsFile, strerror(errno));
2811 2812 2813 2814 2815 2816 2817
    return -1;
  }

  if (superTblInfo->tagDataBuf) {
    free(superTblInfo->tagDataBuf);
    superTblInfo->tagDataBuf = NULL;
  }
2818

2819 2820 2821 2822 2823 2824 2825 2826 2827
  int tagCount = 10000;
  int count = 0;
  char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount);
  if (tagDataBuf == NULL) {
    printf("Failed to calloc, reason:%s\n", strerror(errno));
    fclose(fp);
    return -1;
  }

2828
  while ((readLen = tgetline(&line, &n, fp)) != -1) {
2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen);
    count++;

    if (count >= tagCount - 1) {
2841 2842
      char *tmp = realloc(tagDataBuf,
              (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow);
2843 2844 2845
      if (tmp != NULL) {
        tagDataBuf = tmp;
        tagCount = (int)(tagCount*1.5);
2846 2847
        memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow,
                0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow));
2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874
      } else {
        // exit, if allocate more memory failed
        printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile);
        tmfree(tagDataBuf);
        free(line);
        fclose(fp);
        return -1;
      }
    }
  }

  superTblInfo->tagDataBuf = tagDataBuf;
  superTblInfo->tagSampleCount = count;

  free(line);
  fclose(fp);
  return 0;
}

int readSampleFromJsonFileToMem(SSuperTable  * superTblInfo) {
  // TODO
  return 0;
}

/*
  Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
2875 2876
static int readSampleFromCsvFileToMem(
        SSuperTable* superTblInfo) {
2877 2878 2879 2880
  size_t  n = 0;
  ssize_t readLen = 0;
  char *  line = NULL;
  int getRows = 0;
2881

2882 2883
  FILE*  fp = fopen(superTblInfo->sampleFile, "r");
  if (fp == NULL) {
2884
      errorPrint( "Failed to open sample file: %s, reason:%s\n",
2885 2886 2887
              superTblInfo->sampleFile, strerror(errno));
      return -1;
  }
2888

2889
  assert(superTblInfo->sampleDataBuf);
2890 2891
  memset(superTblInfo->sampleDataBuf, 0,
          MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow);
2892
  while (1) {
2893
    readLen = tgetline(&line, &n, fp);
2894 2895
    if (-1 == readLen) {
      if(0 != fseek(fp, 0, SEEK_SET)) {
2896
        errorPrint( "Failed to fseek file: %s, reason:%s\n",
2897
                superTblInfo->sampleFile, strerror(errno));
2898
        fclose(fp);
2899 2900 2901 2902
        return -1;
      }
      continue;
    }
2903

2904 2905 2906 2907 2908 2909 2910 2911 2912
    if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
      line[--readLen] = 0;
    }

    if (readLen == 0) {
      continue;
    }

    if (readLen > superTblInfo->lenOfOneRow) {
2913 2914
      printf("sample row len[%d] overflow define schema len[%d], so discard this row\n",
              (int32_t)readLen, superTblInfo->lenOfOneRow);
2915 2916 2917
      continue;
    }

sangshuduo's avatar
sangshuduo 已提交
2918
    verbosePrint("readLen=%ld stb->lenOfOneRow=%d getRows=%d\n", (long)readLen,
2919
            superTblInfo->lenOfOneRow, getRows);
2920

2921 2922
    memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow,
          line, readLen);
2923 2924 2925 2926 2927 2928 2929
    getRows++;

    if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) {
      break;
    }
  }

2930
  fclose(fp);
2931 2932 2933 2934
  tmfree(line);
  return 0;
}

2935 2936
static bool getColumnAndTagTypeFromInsertJsonFile(
        cJSON* stbInfo, SSuperTable* superTbls) {
2937
  bool  ret = false;
2938

2939
  // columns
2940 2941
  cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns");
  if (columns && columns->type != cJSON_Array) {
2942
    printf("ERROR: failed to read json, columns not found\n");
2943 2944 2945 2946 2947 2948
    goto PARSE_OVER;
  } else if (NULL == columns) {
    superTbls->columnCount = 0;
    superTbls->tagCount    = 0;
    return true;
  }
2949

2950 2951
  int columnSize = cJSON_GetArraySize(columns);
  if (columnSize > MAX_COLUMN_COUNT) {
2952 2953
    errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n",
            __func__, __LINE__, MAX_COLUMN_COUNT);
2954 2955 2956 2957 2958 2959
    goto PARSE_OVER;
  }

  int count = 1;
  int index = 0;
  StrColumn    columnCase;
2960

2961
  //superTbls->columnCount = columnSize;
2962 2963 2964 2965 2966 2967 2968
  for (int k = 0; k < columnSize; ++k) {
    cJSON* column = cJSON_GetArrayItem(columns, k);
    if (column == NULL) continue;

    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(column, "count");
    if (countObj && countObj->type == cJSON_Number) {
2969
      count = countObj->valueint;
2970
    } else if (countObj && countObj->type != cJSON_Number) {
2971
      errorPrint("%s() LN%d, failed to read json, column count not found\n", __func__, __LINE__);
2972 2973 2974 2975 2976
      goto PARSE_OVER;
    } else {
      count = 1;
    }

2977
    // column info
2978 2979 2980
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(column, "type");
    if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) {
2981
      errorPrint("%s() LN%d: failed to read json, column type not found\n", __func__, __LINE__);
2982 2983
      goto PARSE_OVER;
    }
2984 2985
    //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
2986

2987 2988
    cJSON* dataLen = cJSON_GetObjectItem(column, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
2989
      columnCase.dataLen = dataLen->valueint;
2990
    } else if (dataLen && dataLen->type != cJSON_Number) {
2991
      debugPrint("%s() LN%d: failed to read json, column len not found\n", __func__, __LINE__);
2992 2993 2994 2995
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 8;
    }
2996

2997
    for (int n = 0; n < count; ++n) {
2998 2999 3000
      tstrncpy(superTbls->columns[index].dataType,
              columnCase.dataType, MAX_TB_NAME_SIZE);
      superTbls->columns[index].dataLen = columnCase.dataLen;
3001 3002
      index++;
    }
3003 3004
  }
  superTbls->columnCount = index;
3005

3006 3007
  count = 1;
  index = 0;
3008
  // tags
3009 3010
  cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags");
  if (!tags || tags->type != cJSON_Array) {
3011
    debugPrint("%s() LN%d, failed to read json, tags not found\n", __func__, __LINE__);
3012 3013
    goto PARSE_OVER;
  }
3014

3015 3016
  int tagSize = cJSON_GetArraySize(tags);
  if (tagSize > MAX_TAG_COUNT) {
3017
    debugPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", __func__, __LINE__, MAX_TAG_COUNT);
3018 3019
    goto PARSE_OVER;
  }
3020 3021

  //superTbls->tagCount = tagSize;
3022 3023 3024
  for (int k = 0; k < tagSize; ++k) {
    cJSON* tag = cJSON_GetArrayItem(tags, k);
    if (tag == NULL) continue;
3025

3026 3027 3028
    count = 1;
    cJSON* countObj = cJSON_GetObjectItem(tag, "count");
    if (countObj && countObj->type == cJSON_Number) {
3029
      count = countObj->valueint;
3030
    } else if (countObj && countObj->type != cJSON_Number) {
3031
      printf("ERROR: failed to read json, column count not found\n");
3032 3033 3034 3035 3036
      goto PARSE_OVER;
    } else {
      count = 1;
    }

3037
    // column info
3038 3039 3040
    memset(&columnCase, 0, sizeof(StrColumn));
    cJSON *dataType = cJSON_GetObjectItem(tag, "type");
    if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) {
3041
      printf("ERROR: failed to read json, tag type not found\n");
3042 3043
      goto PARSE_OVER;
    }
3044
    tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE);
3045

3046 3047
    cJSON* dataLen = cJSON_GetObjectItem(tag, "len");
    if (dataLen && dataLen->type == cJSON_Number) {
3048
      columnCase.dataLen = dataLen->valueint;
3049
    } else if (dataLen && dataLen->type != cJSON_Number) {
3050
      printf("ERROR: failed to read json, column len not found\n");
3051 3052 3053
      goto PARSE_OVER;
    } else {
      columnCase.dataLen = 0;
3054 3055
    }

3056
    for (int n = 0; n < count; ++n) {
3057
      tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, MAX_TB_NAME_SIZE);
3058
      superTbls->tags[index].dataLen = columnCase.dataLen;
3059 3060
      index++;
    }
3061
  }
3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077
  superTbls->tagCount = index;

  ret = true;

PARSE_OVER:
  //free(content);
  //cJSON_Delete(root);
  //fclose(fp);
  return ret;
}

static bool getMetaFromInsertJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3078
    tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3079 3080 3081 3082
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3083
    tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE);
3084
  } else if (!host) {
3085
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3086
  } else {
3087
    printf("ERROR: failed to read json, host not found\n");
3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_Dbs.port = port->valueint;
  } else if (!port) {
    g_Dbs.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3100
    tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE);
3101
  } else if (!user) {
3102
    tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE);
3103 3104 3105 3106
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3107
    tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE);
3108
  } else if (!password) {
3109
    tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE);
3110 3111 3112 3113
  }

  cJSON* resultfile = cJSON_GetObjectItem(root, "result_file");
  if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) {
3114
    tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN);
3115
  } else if (!resultfile) {
3116
    tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN);
3117 3118 3119 3120 3121 3122 3123 3124
  }

  cJSON* threads = cJSON_GetObjectItem(root, "thread_count");
  if (threads && threads->type == cJSON_Number) {
    g_Dbs.threadCount = threads->valueint;
  } else if (!threads) {
    g_Dbs.threadCount = 1;
  } else {
3125
    printf("ERROR: failed to read json, threads not found\n");
3126
    goto PARSE_OVER;
3127 3128
  }

3129 3130 3131 3132
  cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl");
  if (threads2 && threads2->type == cJSON_Number) {
    g_Dbs.threadCountByCreateTbl = threads2->valueint;
  } else if (!threads2) {
3133
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
3134
  } else {
3135 3136
    errorPrint("%s() LN%d, failed to read json, threads2 not found\n",
            __func__, __LINE__);
3137
    goto PARSE_OVER;
3138
  }
3139

3140 3141 3142 3143 3144 3145 3146 3147 3148
  cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval");
  if (gInsertInterval && gInsertInterval->type == cJSON_Number) {
    g_args.insert_interval = gInsertInterval->valueint;
  } else if (!gInsertInterval) {
    g_args.insert_interval = 0;
  } else {
    errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", __func__, __LINE__);
    goto PARSE_OVER;
  }
3149

3150 3151 3152
  cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows");
  if (interlaceRows && interlaceRows->type == cJSON_Number) {
    g_args.interlace_rows = interlaceRows->valueint;
3153 3154 3155 3156 3157 3158 3159

    // rows per table need be less than insert batch
    if (g_args.interlace_rows > g_args.num_of_RPR) {
      printf("NOTICE: interlace rows value %d > num_of_records_per_request %d\n\n",
              g_args.interlace_rows, g_args.num_of_RPR);
      printf("        interlace rows value will be set to num_of_records_per_request %d\n\n",
              g_args.num_of_RPR);
3160
      printf("        press Enter key to continue or Ctrl-C to stop.");
3161 3162 3163
      (void)getchar();
      g_args.interlace_rows = g_args.num_of_RPR;
    }
3164
  } else if (!interlaceRows) {
3165
    g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3166
  } else {
3167
    errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", __func__, __LINE__);
3168
    goto PARSE_OVER;
3169
  }
3170

3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184
  cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len");
  if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
    g_args.max_sql_len = maxSqlLen->valueint;
  } else if (!maxSqlLen) {
    g_args.max_sql_len = TSDB_PAYLOAD_SIZE;
  } else {
    errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", __func__, __LINE__);
    goto PARSE_OVER;
  }

  cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req");
  if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
    g_args.num_of_RPR = numRecPerReq->valueint;
  } else if (!numRecPerReq) {
3185
    g_args.num_of_RPR = 0xffff;
3186 3187 3188 3189 3190
  } else {
    errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", __func__, __LINE__);
    goto PARSE_OVER;
  }

3191
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3192
  if (answerPrompt
3193 3194
          && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3195 3196 3197 3198 3199 3200 3201 3202 3203 3204
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
    } else {
      g_args.answer_yes = false;
    }
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
3205
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
3206
    goto PARSE_OVER;
3207
  }
3208 3209 3210

  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (!dbs || dbs->type != cJSON_Array) {
3211
    printf("ERROR: failed to read json, databases not found\n");
3212 3213 3214 3215 3216
    goto PARSE_OVER;
  }

  int dbSize = cJSON_GetArraySize(dbs);
  if (dbSize > MAX_DB_COUNT) {
3217
    errorPrint(
3218 3219
            "ERROR: failed to read json, databases size overflow, max database is %d\n",
            MAX_DB_COUNT);
3220 3221 3222 3223 3224 3225 3226 3227
    goto PARSE_OVER;
  }

  g_Dbs.dbCount = dbSize;
  for (int i = 0; i < dbSize; ++i) {
    cJSON* dbinfos = cJSON_GetArrayItem(dbs, i);
    if (dbinfos == NULL) continue;

3228
    // dbinfo
3229 3230
    cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo");
    if (!dbinfo || dbinfo->type != cJSON_Object) {
3231
      printf("ERROR: failed to read json, dbinfo not found\n");
3232 3233
      goto PARSE_OVER;
    }
3234

3235 3236
    cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name");
    if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) {
3237
      printf("ERROR: failed to read json, db name not found\n");
3238 3239
      goto PARSE_OVER;
    }
3240
    tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE);
3241 3242 3243

    cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop");
    if (drop && drop->type == cJSON_String && drop->valuestring != NULL) {
3244 3245
      if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) {
        g_Dbs.db[i].drop = true;
3246
      } else {
3247
        g_Dbs.db[i].drop = false;
3248
      }
3249
    } else if (!drop) {
3250
      g_Dbs.db[i].drop = g_args.drop_database;
3251
    } else {
3252 3253
      errorPrint("%s() LN%d, failed to read json, drop input mistake\n",
              __func__, __LINE__);
3254 3255 3256 3257
      goto PARSE_OVER;
    }

    cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision");
3258 3259 3260 3261
    if (precision && precision->type == cJSON_String
            && precision->valuestring != NULL) {
      tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring,
              MAX_DB_NAME_SIZE);
3262
    } else if (!precision) {
3263
      //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
3264 3265
      memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE);
    } else {
3266
      printf("ERROR: failed to read json, precision not found\n");
3267 3268 3269 3270 3271 3272 3273 3274 3275
      goto PARSE_OVER;
    }

    cJSON* update = cJSON_GetObjectItem(dbinfo, "update");
    if (update && update->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.update = update->valueint;
    } else if (!update) {
      g_Dbs.db[i].dbCfg.update = -1;
    } else {
3276
      printf("ERROR: failed to read json, update not found\n");
3277 3278 3279 3280 3281 3282 3283 3284 3285
      goto PARSE_OVER;
    }

    cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica");
    if (replica && replica->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.replica = replica->valueint;
    } else if (!replica) {
      g_Dbs.db[i].dbCfg.replica = -1;
    } else {
3286
      printf("ERROR: failed to read json, replica not found\n");
3287 3288 3289 3290 3291 3292 3293 3294 3295
      goto PARSE_OVER;
    }

    cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep");
    if (keep && keep->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.keep = keep->valueint;
    } else if (!keep) {
      g_Dbs.db[i].dbCfg.keep = -1;
    } else {
3296
     printf("ERROR: failed to read json, keep not found\n");
3297 3298
     goto PARSE_OVER;
    }
3299

3300 3301 3302 3303 3304 3305
    cJSON* days = cJSON_GetObjectItem(dbinfo, "days");
    if (days && days->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.days = days->valueint;
    } else if (!days) {
      g_Dbs.db[i].dbCfg.days = -1;
    } else {
3306
     printf("ERROR: failed to read json, days not found\n");
3307 3308
     goto PARSE_OVER;
    }
3309

3310 3311 3312 3313 3314 3315
    cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache");
    if (cache && cache->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cache = cache->valueint;
    } else if (!cache) {
      g_Dbs.db[i].dbCfg.cache = -1;
    } else {
3316
     printf("ERROR: failed to read json, cache not found\n");
3317 3318
     goto PARSE_OVER;
    }
3319

3320 3321 3322 3323 3324 3325
    cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks");
    if (blocks && blocks->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.blocks = blocks->valueint;
    } else if (!blocks) {
      g_Dbs.db[i].dbCfg.blocks = -1;
    } else {
3326
     printf("ERROR: failed to read json, block not found\n");
3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345
     goto PARSE_OVER;
    }

    //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode");
    //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint;
    //} else if (!maxtablesPerVnode) {
    //  g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES;
    //} else {
    // printf("failed to read json, maxtablesPerVnode not found");
    // goto PARSE_OVER;
    //}

    cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows");
    if (minRows && minRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.minRows = minRows->valueint;
    } else if (!minRows) {
      g_Dbs.db[i].dbCfg.minRows = -1;
    } else {
3346
     printf("ERROR: failed to read json, minRows not found\n");
3347 3348 3349 3350 3351 3352 3353 3354 3355
     goto PARSE_OVER;
    }

    cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows");
    if (maxRows && maxRows->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint;
    } else if (!maxRows) {
      g_Dbs.db[i].dbCfg.maxRows = -1;
    } else {
3356
     printf("ERROR: failed to read json, maxRows not found\n");
3357 3358 3359 3360 3361 3362 3363 3364 3365
     goto PARSE_OVER;
    }

    cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp");
    if (comp && comp->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.comp = comp->valueint;
    } else if (!comp) {
      g_Dbs.db[i].dbCfg.comp = -1;
    } else {
3366
     printf("ERROR: failed to read json, comp not found\n");
3367 3368 3369 3370 3371 3372 3373 3374 3375
     goto PARSE_OVER;
    }

    cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel");
    if (walLevel && walLevel->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint;
    } else if (!walLevel) {
      g_Dbs.db[i].dbCfg.walLevel = -1;
    } else {
3376
     printf("ERROR: failed to read json, walLevel not found\n");
3377 3378 3379
     goto PARSE_OVER;
    }

3380 3381 3382 3383 3384 3385
    cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast");
    if (cacheLast && cacheLast->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint;
    } else if (!cacheLast) {
      g_Dbs.db[i].dbCfg.cacheLast = -1;
    } else {
3386
     printf("ERROR: failed to read json, cacheLast not found\n");
3387 3388 3389
     goto PARSE_OVER;
    }

3390 3391 3392 3393
    cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum");
    if (quorum && quorum->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.quorum = quorum->valueint;
    } else if (!quorum) {
3394
      g_Dbs.db[i].dbCfg.quorum = 1;
3395
    } else {
3396
     printf("failed to read json, quorum input mistake");
3397 3398 3399 3400 3401 3402 3403 3404 3405
     goto PARSE_OVER;
    }

    cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync");
    if (fsync && fsync->type == cJSON_Number) {
      g_Dbs.db[i].dbCfg.fsync = fsync->valueint;
    } else if (!fsync) {
      g_Dbs.db[i].dbCfg.fsync = -1;
    } else {
3406 3407 3408
      errorPrint("%s() LN%d, failed to read json, fsync input mistake\n",
              __func__, __LINE__);
      goto PARSE_OVER;
3409
    }
3410

3411
    // super_talbes
3412 3413
    cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables");
    if (!stables || stables->type != cJSON_Array) {
3414 3415
      errorPrint("%s() LN%d, failed to read json, super_tables not found\n",
              __func__, __LINE__);
3416
      goto PARSE_OVER;
3417 3418
    }

3419 3420
    int stbSize = cJSON_GetArraySize(stables);
    if (stbSize > MAX_SUPER_TABLE_COUNT) {
3421
      errorPrint(
3422 3423
              "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n",
              __func__, __LINE__, MAX_SUPER_TABLE_COUNT);
3424 3425 3426 3427 3428 3429 3430
      goto PARSE_OVER;
    }

    g_Dbs.db[i].superTblCount = stbSize;
    for (int j = 0; j < stbSize; ++j) {
      cJSON* stbInfo = cJSON_GetArrayItem(stables, j);
      if (stbInfo == NULL) continue;
3431 3432

      // dbinfo
3433 3434
      cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name");
      if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) {
3435 3436
        errorPrint("%s() LN%d, failed to read json, stb name not found\n",
                __func__, __LINE__);
3437 3438
        goto PARSE_OVER;
      }
3439
      tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE);
3440

3441 3442
      cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix");
      if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) {
3443
        printf("ERROR: failed to read json, childtable_prefix not found\n");
3444 3445
        goto PARSE_OVER;
      }
3446
      tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE);
3447 3448

      cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null
3449 3450 3451
      if (autoCreateTbl
              && autoCreateTbl->type == cJSON_String
              && autoCreateTbl->valuestring != NULL) {
3452 3453 3454 3455 3456 3457 3458 3459 3460 3461
        if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL;
        } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        } else {
          g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
        }
      } else if (!autoCreateTbl) {
        g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL;
      } else {
3462
        printf("ERROR: failed to read json, auto_create_table not found\n");
3463 3464
        goto PARSE_OVER;
      }
3465

3466 3467 3468 3469
      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
      } else if (!batchCreateTbl) {
3470
        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000;
3471
      } else {
3472
        printf("ERROR: failed to read json, batch_create_tbl_num not found\n");
3473
        goto PARSE_OVER;
3474
      }
3475 3476

      cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
3477 3478 3479
      if (childTblExists
              && childTblExists->type == cJSON_String
              && childTblExists->valuestring != NULL) {
3480 3481 3482 3483 3484 3485 3486 3487 3488 3489
        if (0 == strncasecmp(childTblExists->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS;
        } else if (0 == strncasecmp(childTblExists->valuestring, "no", 2)) {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        } else {
          g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
        }
      } else if (!childTblExists) {
        g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS;
      } else {
3490 3491
        errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n",
                __func__, __LINE__);
3492 3493
        goto PARSE_OVER;
      }
3494

3495 3496
      cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count");
      if (!count || count->type != cJSON_Number || 0 >= count->valueint) {
3497 3498
        errorPrint("%s() LN%d, failed to read json, childtable_count not found\n",
                __func__, __LINE__);
3499 3500 3501 3502 3503
        goto PARSE_OVER;
      }
      g_Dbs.db[i].superTbls[j].childTblCount = count->valueint;

      cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source");
3504 3505 3506 3507
      if (dataSource && dataSource->type == cJSON_String
              && dataSource->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource,
                dataSource->valuestring, MAX_DB_NAME_SIZE);
3508
      } else if (!dataSource) {
3509
        tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE);
3510
      } else {
3511
        errorPrint("%s() LN%d, failed to read json, data_source not found\n", __func__, __LINE__);
3512 3513 3514 3515
        goto PARSE_OVER;
      }

      cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , restful
3516 3517 3518 3519
      if (insertMode && insertMode->type == cJSON_String
              && insertMode->valuestring != NULL) {
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode,
                insertMode->valuestring, MAX_DB_NAME_SIZE);
3520
      } else if (!insertMode) {
3521
        tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE);
3522
      } else {
3523
        printf("ERROR: failed to read json, insert_mode not found\n");
3524 3525 3526
        goto PARSE_OVER;
      }

3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548
      cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit");
      if (childTbl_limit) {
        if (childTbl_limit->type != cJSON_Number) {
            printf("ERROR: failed to read json, childtable_limit\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblLimit = -1;    // select ... limit -1 means all query result
      }

      cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset");
      if (childTbl_offset) {
        if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) {
            printf("ERROR: failed to read json, childtable_offset\n");
            goto PARSE_OVER;
        }
        g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint;
      } else {
        g_Dbs.db[i].superTbls[j].childTblOffset = 0;
      }

3549 3550
      cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp");
      if (ts && ts->type == cJSON_String && ts->valuestring != NULL) {
3551 3552
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                ts->valuestring, MAX_DB_NAME_SIZE);
3553
      } else if (!ts) {
3554 3555
        tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp,
                "now", MAX_DB_NAME_SIZE);
3556
      } else {
3557
        printf("ERROR: failed to read json, start_timestamp not found\n");
3558 3559
        goto PARSE_OVER;
      }
3560

3561 3562 3563 3564
      cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step");
      if (timestampStep && timestampStep->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint;
      } else if (!timestampStep) {
3565
        g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP;
3566
      } else {
3567
        printf("ERROR: failed to read json, timestamp_step not found\n");
3568 3569 3570 3571
        goto PARSE_OVER;
      }

      cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format");
3572 3573
      if (sampleFormat && sampleFormat->type
              == cJSON_String && sampleFormat->valuestring != NULL) {
3574 3575
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat,
                sampleFormat->valuestring, MAX_DB_NAME_SIZE);
3576
      } else if (!sampleFormat) {
3577
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE);
3578
      } else {
3579
        printf("ERROR: failed to read json, sample_format not found\n");
3580
        goto PARSE_OVER;
3581
      }
3582

3583 3584
      cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file");
      if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) {
3585 3586
        tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile,
                sampleFile->valuestring, MAX_FILE_NAME_LEN);
3587 3588 3589
      } else if (!sampleFile) {
        memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN);
      } else {
3590
        printf("ERROR: failed to read json, sample_file not found\n");
3591
        goto PARSE_OVER;
3592
      }
3593

3594 3595
      cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file");
      if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) {
3596 3597
        tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile,
                tagsFile->valuestring, MAX_FILE_NAME_LEN);
3598 3599 3600 3601 3602 3603 3604 3605 3606
        if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) {
          g_Dbs.db[i].superTbls[j].tagSource = 0;
        } else {
          g_Dbs.db[i].superTbls[j].tagSource = 1;
        }
      } else if (!tagsFile) {
        memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN);
        g_Dbs.db[i].superTbls[j].tagSource = 0;
      } else {
3607
        printf("ERROR: failed to read json, tags_file not found\n");
3608 3609
        goto PARSE_OVER;
      }
3610

3611 3612 3613 3614 3615 3616 3617
      cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len");
      if (maxSqlLen && maxSqlLen->type == cJSON_Number) {
        int32_t len = maxSqlLen->valueint;
        if (len > TSDB_MAX_ALLOWED_SQL_LEN) {
          len = TSDB_MAX_ALLOWED_SQL_LEN;
        } else if (len < TSDB_MAX_SQL_LEN) {
          len = TSDB_MAX_SQL_LEN;
3618
        }
3619 3620
        g_Dbs.db[i].superTbls[j].maxSqlLen = len;
      } else if (!maxSqlLen) {
3621
        g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len;
3622
      } else {
3623
        printf("ERROR: failed to read json, maxSqlLen not found\n");
3624
        goto PARSE_OVER;
3625
      }
3626

3627 3628 3629 3630 3631
      cJSON *multiThreadWriteOneTbl =
          cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes
      if (multiThreadWriteOneTbl
              && multiThreadWriteOneTbl->type == cJSON_String
              && multiThreadWriteOneTbl->valuestring != NULL) {
3632 3633 3634 3635
        if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1;
        } else {
          g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
3636
        }
3637 3638 3639
      } else if (!multiThreadWriteOneTbl) {
        g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0;
      } else {
3640
        printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n");
3641 3642
        goto PARSE_OVER;
      }
3643

3644 3645 3646
      cJSON* interlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows");
      if (interlaceRows && interlaceRows->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].interlaceRows = interlaceRows->valueint;
3647 3648 3649 3650 3651 3652
        // rows per table need be less than insert batch
        if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) {
          printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %d > num_of_records_per_request %d\n\n",
                  i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR);
          printf("        interlace rows value will be set to num_of_records_per_request %d\n\n",
                  g_args.num_of_RPR);
3653
          printf("        press Enter key to continue or Ctrl-C to stop.");
3654 3655 3656
          (void)getchar();
          g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR;
        }
3657 3658
      } else if (!interlaceRows) {
        g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req
3659
      } else {
3660 3661 3662
        errorPrint(
                "%s() LN%d, failed to read json, interlace rows input mistake\n",
                __func__, __LINE__);
3663
        goto PARSE_OVER;
3664
      }
3665 3666 3667

      cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio");
      if (disorderRatio && disorderRatio->type == cJSON_Number) {
3668 3669 3670 3671 3672 3673
        if (disorderRatio->valueint > 50)
          disorderRatio->valueint = 50;

        if (disorderRatio->valueint < 0)
          disorderRatio->valueint = 0;

3674 3675 3676 3677
        g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint;
      } else if (!disorderRatio) {
        g_Dbs.db[i].superTbls[j].disorderRatio = 0;
      } else {
3678
        printf("ERROR: failed to read json, disorderRatio not found\n");
3679
        goto PARSE_OVER;
3680
      }
3681 3682 3683 3684 3685 3686 3687

      cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range");
      if (disorderRange && disorderRange->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint;
      } else if (!disorderRange) {
        g_Dbs.db[i].superTbls[j].disorderRange = 1000;
      } else {
3688
        printf("ERROR: failed to read json, disorderRange not found\n");
3689 3690
        goto PARSE_OVER;
      }
3691

3692 3693 3694 3695 3696 3697
      cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows");
      if (insertRows && insertRows->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint;
      } else if (!insertRows) {
        g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF;
      } else {
3698 3699
        errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n",
                __func__, __LINE__);
3700 3701 3702 3703 3704 3705 3706
        goto PARSE_OVER;
      }

      cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval");
      if (insertInterval && insertInterval->type == cJSON_Number) {
        g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint;
      } else if (!insertInterval) {
3707
        verbosePrint("%s() LN%d: stable insert interval be overrided by global %d.\n",
3708 3709 3710
                __func__, __LINE__, g_args.insert_interval);
        g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval;
      } else {
3711 3712
        errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n",
                __func__, __LINE__);
3713 3714 3715
        goto PARSE_OVER;
      }

3716 3717
      int retVal = getColumnAndTagTypeFromInsertJsonFile(
              stbInfo, &g_Dbs.db[i].superTbls[j]);
3718 3719
      if (false == retVal) {
        goto PARSE_OVER;
3720 3721
      }
    }
3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737
  }

  ret = true;

PARSE_OVER:
  //free(content);
  //cJSON_Delete(root);
  //fclose(fp);
  return ret;
}

static bool getMetaFromQueryJsonFile(cJSON* root) {
  bool  ret = false;

  cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir");
  if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) {
3738
    tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN);
3739 3740 3741 3742
  }

  cJSON* host = cJSON_GetObjectItem(root, "host");
  if (host && host->type == cJSON_String && host->valuestring != NULL) {
3743
    tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE);
3744
  } else if (!host) {
3745
    tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
3746
  } else {
3747
    printf("ERROR: failed to read json, host not found\n");
3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759
    goto PARSE_OVER;
  }

  cJSON* port = cJSON_GetObjectItem(root, "port");
  if (port && port->type == cJSON_Number) {
    g_queryInfo.port = port->valueint;
  } else if (!port) {
    g_queryInfo.port = 6030;
  }

  cJSON* user = cJSON_GetObjectItem(root, "user");
  if (user && user->type == cJSON_String && user->valuestring != NULL) {
3760
    tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE);
3761
  } else if (!user) {
3762
    tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ;
3763 3764 3765 3766
  }

  cJSON* password = cJSON_GetObjectItem(root, "password");
  if (password && password->type == cJSON_String && password->valuestring != NULL) {
3767
    tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE);
3768
  } else if (!password) {
3769
    tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);;
3770
  }
H
hzcheng 已提交
3771

3772
  cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no,
3773 3774
  if (answerPrompt && answerPrompt->type == cJSON_String
          && answerPrompt->valuestring != NULL) {
3775 3776 3777 3778
    if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) {
      g_args.answer_yes = false;
    } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) {
      g_args.answer_yes = true;
H
hzcheng 已提交
3779
    } else {
3780
      g_args.answer_yes = false;
H
hzcheng 已提交
3781
    }
3782 3783 3784
  } else if (!answerPrompt) {
    g_args.answer_yes = false;
  } else {
3785
    printf("ERROR: failed to read json, confirm_parameter_prompt not found\n");
3786
    goto PARSE_OVER;
3787
  }
3788

3789 3790 3791 3792 3793 3794 3795 3796 3797 3798
  cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
  if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
    g_args.query_times = gQueryTimes->valueint;
  } else if (!gQueryTimes) {
    g_args.query_times = 1;
  } else {
    errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__);
    goto PARSE_OVER;
  }

3799 3800
  cJSON* dbs = cJSON_GetObjectItem(root, "databases");
  if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) {
3801
    tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE);
3802
  } else if (!dbs) {
3803
    printf("ERROR: failed to read json, databases not found\n");
3804
    goto PARSE_OVER;
H
hzcheng 已提交
3805
  }
3806 3807 3808

  cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode");
  if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) {
3809
    tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE);
3810
  } else if (!queryMode) {
3811
    tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE);
3812
  } else {
3813
    printf("ERROR: failed to read json, query_mode not found\n");
3814
    goto PARSE_OVER;
H
hzcheng 已提交
3815
  }
3816

3817
  // super_table_query
3818 3819 3820 3821 3822
  cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query");
  if (!specifiedQuery) {
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
    g_queryInfo.specifiedQueryInfo.sqlCount = 0;
  } else if (specifiedQuery->type != cJSON_Object) {
3823
    printf("ERROR: failed to read json, super_table_query not found\n");
3824
    goto PARSE_OVER;
3825
  } else {
3826
    cJSON* rate = cJSON_GetObjectItem(specifiedQuery, "query_interval");
3827
    if (rate && rate->type == cJSON_Number) {
3828
      g_queryInfo.specifiedQueryInfo.rate = rate->valueint;
3829
    } else if (!rate) {
3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840
      g_queryInfo.specifiedQueryInfo.rate = 0;
    }

    cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery, "query_times");
    if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
      g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint;
    } else if (!specifiedQueryTimes) {
      g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times;
    } else {
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__);
      goto PARSE_OVER;
3841
    }
3842

3843
    cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent");
3844
    if (concurrent && concurrent->type == cJSON_Number) {
3845
      g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint;
3846
    } else if (!concurrent) {
3847
      g_queryInfo.specifiedQueryInfo.concurrent = 1;
3848
    }
3849

3850
    cJSON* mode = cJSON_GetObjectItem(specifiedQuery, "mode");
3851
    if (mode && mode->type == cJSON_String && mode->valuestring != NULL) {
3852
      if (0 == strcmp("sync", mode->valuestring)) {
3853
        g_queryInfo.specifiedQueryInfo.subscribeMode = 0;
3854
      } else if (0 == strcmp("async", mode->valuestring)) {
3855
        g_queryInfo.specifiedQueryInfo.subscribeMode = 1;
3856
      } else {
3857
        printf("ERROR: failed to read json, subscribe mod error\n");
3858 3859 3860
        goto PARSE_OVER;
      }
    } else {
3861
      g_queryInfo.specifiedQueryInfo.subscribeMode = 0;
3862
    }
3863

3864
    cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval");
3865
    if (interval && interval->type == cJSON_Number) {
3866
      g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint;
3867
    } else if (!interval) {
3868 3869
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
3870
      g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000;
3871
    }
3872

3873
    cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart");
3874
    if (restart && restart->type == cJSON_String && restart->valuestring != NULL) {
3875
      if (0 == strcmp("yes", restart->valuestring)) {
3876
        g_queryInfo.specifiedQueryInfo.subscribeRestart = 1;
3877
      } else if (0 == strcmp("no", restart->valuestring)) {
3878
        g_queryInfo.specifiedQueryInfo.subscribeRestart = 0;
3879
      } else {
3880
        printf("ERROR: failed to read json, subscribe restart error\n");
3881 3882 3883
        goto PARSE_OVER;
      }
    } else {
3884
      g_queryInfo.specifiedQueryInfo.subscribeRestart = 1;
3885
    }
3886

3887
    cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress");
3888 3889 3890
    if (keepProgress
            && keepProgress->type == cJSON_String
            && keepProgress->valuestring != NULL) {
3891
      if (0 == strcmp("yes", keepProgress->valuestring)) {
3892
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1;
3893
      } else if (0 == strcmp("no", keepProgress->valuestring)) {
3894
        g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
3895
      } else {
3896
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
3897 3898 3899
        goto PARSE_OVER;
      }
    } else {
3900
      g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0;
3901 3902
    }

3903
    // sqls
3904
    cJSON* superSqls = cJSON_GetObjectItem(specifiedQuery, "sqls");
3905
    if (!superSqls) {
3906
      g_queryInfo.specifiedQueryInfo.sqlCount = 0;
3907
    } else if (superSqls->type != cJSON_Array) {
3908
      printf("ERROR: failed to read json, super sqls not found\n");
3909
      goto PARSE_OVER;
3910
    } else {
3911 3912
      int superSqlSize = cJSON_GetArraySize(superSqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
3913
        printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT);
3914 3915
        goto PARSE_OVER;
      }
3916

3917
      g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize;
3918 3919 3920
      for (int j = 0; j < superSqlSize; ++j) {
        cJSON* sql = cJSON_GetArrayItem(superSqls, j);
        if (sql == NULL) continue;
3921

3922 3923
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
3924
          printf("ERROR: failed to read json, sql not found\n");
3925 3926
          goto PARSE_OVER;
        }
3927
        tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
H
hzcheng 已提交
3928

3929 3930
        cJSON *result = cJSON_GetObjectItem(sql, "result");
        if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) {
3931
          tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN);
3932
        } else if (NULL == result) {
3933
          memset(g_queryInfo.specifiedQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
3934
        } else {
3935
          printf("ERROR: failed to read json, super query result file not found\n");
3936
          goto PARSE_OVER;
3937
        }
3938 3939 3940
      }
    }
  }
3941

3942
  // sub_table_query
3943 3944 3945 3946 3947
  cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query");
  if (!superQuery) {
    g_queryInfo.superQueryInfo.threadCnt = 0;
    g_queryInfo.superQueryInfo.sqlCount = 0;
  } else if (superQuery->type != cJSON_Object) {
3948
    printf("ERROR: failed to read json, sub_table_query not found\n");
3949 3950
    ret = true;
    goto PARSE_OVER;
H
hzcheng 已提交
3951
  } else {
3952
    cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval");
3953
    if (subrate && subrate->type == cJSON_Number) {
3954
      g_queryInfo.superQueryInfo.rate = subrate->valueint;
3955
    } else if (!subrate) {
3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966
      g_queryInfo.superQueryInfo.rate = 0;
    }

    cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
    if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
      g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
    } else if (!superQueryTimes) {
      g_queryInfo.superQueryInfo.queryTimes = g_args.query_times;
    } else {
      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__);
      goto PARSE_OVER;
3967
    }
3968

3969
    cJSON* threads = cJSON_GetObjectItem(superQuery, "threads");
3970
    if (threads && threads->type == cJSON_Number) {
3971
      g_queryInfo.superQueryInfo.threadCnt = threads->valueint;
3972
    } else if (!threads) {
3973
      g_queryInfo.superQueryInfo.threadCnt = 1;
3974
    }
3975

3976
    //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count");
3977
    //if (subTblCnt && subTblCnt->type == cJSON_Number) {
3978
    //  g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint;
3979
    //} else if (!subTblCnt) {
3980
    //  g_queryInfo.superQueryInfo.childTblCount = 0;
3981
    //}
3982

3983
    cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname");
3984
    if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) {
3985
      tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring, MAX_TB_NAME_SIZE);
3986
    } else {
3987
      printf("ERROR: failed to read json, super table name not found\n");
3988 3989
      goto PARSE_OVER;
    }
3990

3991
    cJSON* submode = cJSON_GetObjectItem(superQuery, "mode");
3992
    if (submode && submode->type == cJSON_String && submode->valuestring != NULL) {
3993
      if (0 == strcmp("sync", submode->valuestring)) {
3994
        g_queryInfo.superQueryInfo.subscribeMode = 0;
3995
      } else if (0 == strcmp("async", submode->valuestring)) {
3996
        g_queryInfo.superQueryInfo.subscribeMode = 1;
3997
      } else {
3998
        printf("ERROR: failed to read json, subscribe mod error\n");
3999 4000 4001
        goto PARSE_OVER;
      }
    } else {
4002
      g_queryInfo.superQueryInfo.subscribeMode = 0;
4003
    }
4004

4005
    cJSON* subinterval = cJSON_GetObjectItem(superQuery, "interval");
4006
    if (subinterval && subinterval->type == cJSON_Number) {
4007
      g_queryInfo.superQueryInfo.subscribeInterval = subinterval->valueint;
4008
    } else if (!subinterval) {
4009 4010
      //printf("failed to read json, subscribe interval no found\n");
      //goto PARSE_OVER;
4011
      g_queryInfo.superQueryInfo.subscribeInterval = 10000;
4012
    }
4013

4014
    cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart");
4015
    if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) {
4016
      if (0 == strcmp("yes", subrestart->valuestring)) {
4017
        g_queryInfo.superQueryInfo.subscribeRestart = 1;
4018
      } else if (0 == strcmp("no", subrestart->valuestring)) {
4019
        g_queryInfo.superQueryInfo.subscribeRestart = 0;
4020
      } else {
4021
        printf("ERROR: failed to read json, subscribe restart error\n");
4022 4023 4024
        goto PARSE_OVER;
      }
    } else {
4025
      g_queryInfo.superQueryInfo.subscribeRestart = 1;
4026
    }
4027

4028
    cJSON* subkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress");
4029 4030 4031
    if (subkeepProgress &&
            subkeepProgress->type == cJSON_String
            && subkeepProgress->valuestring != NULL) {
4032
      if (0 == strcmp("yes", subkeepProgress->valuestring)) {
4033
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 1;
4034
      } else if (0 == strcmp("no", subkeepProgress->valuestring)) {
4035
        g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4036
      } else {
4037
        printf("ERROR: failed to read json, subscribe keepProgress error\n");
4038 4039 4040
        goto PARSE_OVER;
      }
    } else {
4041
      g_queryInfo.superQueryInfo.subscribeKeepProgress = 0;
4042
    }
4043

4044
    // sqls
4045
    cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls");
4046
    if (!subsqls) {
4047
      g_queryInfo.superQueryInfo.sqlCount = 0;
4048
    } else if (subsqls->type != cJSON_Array) {
4049
      printf("ERROR: failed to read json, super sqls not found\n");
4050
      goto PARSE_OVER;
4051
    } else {
4052 4053
      int superSqlSize = cJSON_GetArraySize(subsqls);
      if (superSqlSize > MAX_QUERY_SQL_COUNT) {
4054
        printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT);
4055 4056
        goto PARSE_OVER;
      }
4057

4058
      g_queryInfo.superQueryInfo.sqlCount = superSqlSize;
4059
      for (int j = 0; j < superSqlSize; ++j) {
4060 4061
        cJSON* sql = cJSON_GetArrayItem(subsqls, j);
        if (sql == NULL) continue;
4062

4063 4064
        cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql");
        if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) {
4065
          printf("ERROR: failed to read json, sql not found\n");
4066 4067
          goto PARSE_OVER;
        }
4068
        tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH);
4069 4070 4071

        cJSON *result = cJSON_GetObjectItem(sql, "result");
        if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){
4072
          tstrncpy(g_queryInfo.superQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN);
4073
        } else if (NULL == result) {
4074
          memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN);
4075
        }  else {
4076
          printf("ERROR: failed to read json, sub query result file not found\n");
4077
          goto PARSE_OVER;
4078
        }
4079 4080
      }
    }
H
hzcheng 已提交
4081 4082
  }

4083
  ret = true;
H
hzcheng 已提交
4084

4085 4086 4087 4088 4089 4090
PARSE_OVER:
  //free(content);
  //cJSON_Delete(root);
  //fclose(fp);
  return ret;
}
H
hzcheng 已提交
4091

4092
static bool getInfoFromJsonFile(char* file) {
4093
    debugPrint("%s %d %s\n", __func__, __LINE__, file);
4094

4095 4096 4097 4098 4099
  FILE *fp = fopen(file, "r");
  if (!fp) {
    printf("failed to read %s, reason:%s\n", file, strerror(errno));
    return false;
  }
H
Hui Li 已提交
4100

4101
  bool  ret = false;
4102
  int   maxLen = 6400000;
4103 4104 4105 4106 4107 4108 4109
  char *content = calloc(1, maxLen + 1);
  int   len = fread(content, 1, maxLen, fp);
  if (len <= 0) {
    free(content);
    fclose(fp);
    printf("failed to read %s, content is null", file);
    return false;
H
hzcheng 已提交
4110
  }
H
Hui Li 已提交
4111

4112 4113 4114
  content[len] = 0;
  cJSON* root = cJSON_Parse(content);
  if (root == NULL) {
4115
    printf("ERROR: failed to cjson parse %s, invalid json format\n", file);
4116 4117
    goto PARSE_OVER;
  }
H
Hui Li 已提交
4118

4119 4120 4121
  cJSON* filetype = cJSON_GetObjectItem(root, "filetype");
  if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) {
    if (0 == strcasecmp("insert", filetype->valuestring)) {
4122
      g_args.test_mode = INSERT_TEST;
4123
    } else if (0 == strcasecmp("query", filetype->valuestring)) {
4124
      g_args.test_mode = QUERY_TEST;
4125
    } else if (0 == strcasecmp("subscribe", filetype->valuestring)) {
4126
      g_args.test_mode = SUBSCRIBE_TEST;
4127
    } else {
4128
      printf("ERROR: failed to read json, filetype not support\n");
4129 4130 4131
      goto PARSE_OVER;
    }
  } else if (!filetype) {
4132
    g_args.test_mode = INSERT_TEST;
4133
  } else {
4134
    printf("ERROR: failed to read json, filetype not found\n");
4135 4136
    goto PARSE_OVER;
  }
H
hzcheng 已提交
4137

4138
  if (INSERT_TEST == g_args.test_mode) {
4139
    ret = getMetaFromInsertJsonFile(root);
4140 4141
  } else if ((QUERY_TEST == g_args.test_mode)
          || (SUBSCRIBE_TEST == g_args.test_mode)) {
4142 4143
    ret = getMetaFromQueryJsonFile(root);
  } else {
4144 4145
    errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n",
            __func__, __LINE__);
4146
    goto PARSE_OVER;
4147
  }
4148 4149 4150 4151

PARSE_OVER:
  free(content);
  cJSON_Delete(root);
H
hzcheng 已提交
4152
  fclose(fp);
4153 4154
  return ret;
}
H
hzcheng 已提交
4155

4156
static void prepareSampleData() {
4157
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4158 4159 4160
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
      if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) {
        (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]);
S
Shuaiqiang Chang 已提交
4161
      }
4162 4163 4164 4165
    }
  }
}

4166
static void postFreeResource() {
4167
  tmfclose(g_fpOfInsertResult);
4168
  for (int i = 0; i < g_Dbs.dbCount; i++) {
4169
    for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
4170 4171 4172
      if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) {
        free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
        g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL;
S
Shuaiqiang Chang 已提交
4173
      }
4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185
      if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) {
        free(g_Dbs.db[i].superTbls[j].sampleDataBuf);
        g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) {
        free(g_Dbs.db[i].superTbls[j].tagDataBuf);
        g_Dbs.db[i].superTbls[j].tagDataBuf = NULL;
      }
      if (0 != g_Dbs.db[i].superTbls[j].childTblName) {
        free(g_Dbs.db[i].superTbls[j].childTblName);
        g_Dbs.db[i].superTbls[j].childTblName = NULL;
      }
S
Shuaiqiang Chang 已提交
4186
    }
4187 4188
  }
}
S
Shuaiqiang Chang 已提交
4189

4190 4191
static int getRowDataFromSample(char*  dataBuf, int maxLen, int64_t timestamp,
      SSuperTable* superTblInfo, int* sampleUsePos) {
4192
  if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) {
4193
/*    int ret = readSampleFromCsvFileToMem(superTblInfo);
4194
    if (0 != ret) {
4195
      tmfree(superTblInfo->sampleDataBuf);
4196
      superTblInfo->sampleDataBuf = NULL;
4197 4198
      return -1;
    }
4199
*/
4200 4201
    *sampleUsePos = 0;
  }
S
Shuaiqiang Chang 已提交
4202

4203
  int    dataLen = 0;
4204

4205 4206 4207 4208
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "(%" PRId64 ", ", timestamp);
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
          "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos));
4209
  dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
S
Shuaiqiang Chang 已提交
4210

4211
  (*sampleUsePos)++;
4212

4213 4214
  return dataLen;
}
S
Shuaiqiang Chang 已提交
4215

4216 4217 4218 4219 4220 4221 4222
static int generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stbInfo) {
  int   dataLen = 0;
  char  *pstr = recBuf;
  int maxLen = MAX_DATA_SIZE;

  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp);

4223
  for (int i = 0; i < stbInfo->columnCount; i++) {
4224 4225
    if ((0 == strncasecmp(stbInfo->columns[i].dataType, "binary", 6))
            || (0 == strncasecmp(stbInfo->columns[i].dataType, "nchar", 5))) {
4226
      if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
4227
        errorPrint( "binary or nchar length overflow, max size:%u\n",
4228
                (uint32_t)TSDB_MAX_BINARY_LEN);
4229
        return -1;
4230
      }
4231

4232 4233
      char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
      if (NULL == buf) {
4234
        errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
4235
        return -1;
4236 4237
      }
      rand_string(buf, stbInfo->columns[i].dataLen);
4238
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\', ", buf);
4239
      tmfree(buf);
4240 4241
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "int", 3)) {
4242
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4243 4244 4245
              "%d, ", rand_int());
    } else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "bigint", 6)) {
4246
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4247 4248 4249
              "%"PRId64", ", rand_bigint());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "float", 5)) {
4250
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4251 4252 4253
              "%f, ", rand_float());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "double", 6)) {
4254
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
4255 4256 4257
              "%f, ", rand_double());
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType,
                "smallint", 8)) {
4258
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_smallint());
4259
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType, "tinyint", 7)) {
4260
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_tinyint());
4261
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bool", 4)) {
4262
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_bool());
4263
    }  else if (0 == strncasecmp(stbInfo->columns[i].dataType, "timestamp", 9)) {
4264
      dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint());
4265
    }  else {
4266
      errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType);
4267
      return -1;
4268
    }
S
Shuaiqiang Chang 已提交
4269
  }
4270

4271
  dataLen -= 2;
4272
  dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")");
4273

4274 4275 4276
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

  return strlen(recBuf);
4277
}
S
Shuaiqiang Chang 已提交
4278

4279
static int32_t generateData(char *recBuf, char **data_type,
4280
        int num_of_cols, int64_t timestamp, int lenOfBinary) {
4281 4282
  memset(recBuf, 0, MAX_DATA_SIZE);
  char *pstr = recBuf;
4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302
  pstr += sprintf(pstr, "(%" PRId64, timestamp);
  int c = 0;

  for (; c < MAX_NUM_DATATYPE; c++) {
    if (data_type[c] == NULL) {
      break;
    }
  }

  if (0 == c) {
    perror("data type error!");
    exit(-1);
  }

  for (int i = 0; i < num_of_cols; i++) {
    if (strcasecmp(data_type[i % c], "tinyint") == 0) {
      pstr += sprintf(pstr, ", %d", rand_tinyint() );
    } else if (strcasecmp(data_type[i % c], "smallint") == 0) {
      pstr += sprintf(pstr, ", %d", rand_smallint());
    } else if (strcasecmp(data_type[i % c], "int") == 0) {
4303
      pstr += sprintf(pstr, ", %d", rand_int());
4304 4305 4306 4307 4308 4309 4310 4311
    } else if (strcasecmp(data_type[i % c], "bigint") == 0) {
      pstr += sprintf(pstr, ", %" PRId64, rand_bigint());
    } else if (strcasecmp(data_type[i % c], "float") == 0) {
      pstr += sprintf(pstr, ", %10.4f", rand_float());
    } else if (strcasecmp(data_type[i % c], "double") == 0) {
      double t = rand_double();
      pstr += sprintf(pstr, ", %20.8f", t);
    } else if (strcasecmp(data_type[i % c], "bool") == 0) {
4312
      bool b = taosRandom() & 1;
4313 4314
      pstr += sprintf(pstr, ", %s", b ? "true" : "false");
    } else if (strcasecmp(data_type[i % c], "binary") == 0) {
4315 4316
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4317 4318 4319
      pstr += sprintf(pstr, ", \"%s\"", s);
      free(s);
    }else if (strcasecmp(data_type[i % c], "nchar") == 0) {
4320 4321
      char *s = malloc(lenOfBinary);
      rand_string(s, lenOfBinary);
4322 4323 4324 4325
      pstr += sprintf(pstr, ", \"%s\"", s);
      free(s);
    }

4326
    if (strlen(recBuf) > MAX_DATA_SIZE) {
4327 4328 4329 4330 4331 4332 4333
      perror("column length too long, abort");
      exit(-1);
    }
  }

  pstr += sprintf(pstr, ")");

4334 4335
  verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);

4336
  return (int32_t)strlen(recBuf);
4337 4338
}

4339
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
4340 4341
  char* sampleDataBuf = NULL;

4342
  sampleDataBuf = calloc(
4343
            superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
4344
  if (sampleDataBuf == NULL) {
4345
      errorPrint("%s() LN%d, Failed to calloc %d Bytes, reason:%s\n",
4346
              __func__, __LINE__,
4347
              superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE,
4348
              strerror(errno));
4349
      return -1;
4350
  }
4351

4352 4353 4354 4355
  superTblInfo->sampleDataBuf = sampleDataBuf;
  int ret = readSampleFromCsvFileToMem(superTblInfo);

  if (0 != ret) {
4356
      errorPrint("%s() LN%d, read sample from csv file failed.\n", __func__, __LINE__);
4357
      tmfree(sampleDataBuf);
4358
      superTblInfo->sampleDataBuf = NULL;
4359
      return -1;
H
Hui Li 已提交
4360
  }
4361

4362 4363 4364
  return 0;
}

4365
static int execInsert(threadInfo *pThreadInfo, char *buffer, int k)
4366 4367
{
  int affectedRows;
4368
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4369

4370 4371
  verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID,
            __func__, __LINE__, buffer);
4372
  if (superTblInfo) {
4373
    if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) {
4374
      affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false);
4375
    } else {
4376
      if (0 != postProceSql(g_Dbs.host, g_Dbs.port, buffer)) {
4377
        affectedRows = -1;
4378
        printf("========restful return fail, threadID[%d]\n", pThreadInfo->threadID);
4379 4380 4381
      } else {
        affectedRows = k;
      }
4382
    }
4383
  } else {
4384
    affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false);
4385 4386 4387 4388 4389
  }

  return affectedRows;
}

4390
static void getTableName(char *pTblName, threadInfo* pThreadInfo, int tableSeq)
4391 4392
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4393 4394 4395 4396
  if (superTblInfo) {
    if ((superTblInfo->childTblOffset >= 0)
            && (superTblInfo->childTblLimit > 0)) {
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
4397 4398
            superTblInfo->childTblName +
            (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN);
4399
    } else {
4400

4401 4402 4403
        verbosePrint("[%d] %s() LN%d: from=%d count=%d seq=%d\n",
                pThreadInfo->threadID, __func__, __LINE__,
                pThreadInfo->start_table_from,
4404 4405 4406 4407 4408 4409
                pThreadInfo->ntables, tableSeq);
        snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s",
            superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN);
    }
  } else {
    snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%d",
4410
        g_args.tb_prefix, tableSeq);
4411 4412 4413 4414
  }
}

static int generateDataTail(char *tableName, int32_t tableSeq,
4415
        threadInfo* pThreadInfo, SSuperTable* superTblInfo,
4416
        int batch, char* buffer, int remainderBufLen, int64_t insertRows,
4417 4418
        int64_t startFrom, uint64_t startTime, int *pSamplePos, int *dataLen) {
  int len = 0;
4419 4420
  int ncols_per_record = 1; // count first col ts

4421
  if (superTblInfo == NULL) {
4422 4423 4424 4425 4426
    int datatypeSeq = 0;
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
4427 4428
  }

4429
  verbosePrint("%s() LN%d batch=%d\n", __func__, __LINE__, batch);
4430

4431 4432
  int k = 0;
  for (k = 0; k < batch;) {
4433
    char data[MAX_DATA_SIZE];
4434 4435
    memset(data, 0, MAX_DATA_SIZE);

4436
    int retLen = 0;
4437

4438 4439
    if (superTblInfo) {
      if (0 == strncasecmp(superTblInfo->dataSource,
4440 4441
                    "sample", strlen("sample"))) {
          retLen = getRowDataFromSample(
4442 4443
                    data,
                    remainderBufLen,
4444
                    startTime + superTblInfo->timeStampStep * k,
4445
                    superTblInfo,
4446
                    pSamplePos);
4447
      } else if (0 == strncasecmp(superTblInfo->dataSource,
4448
                   "rand", strlen("rand"))) {
4449
          int rand_num = taosRandom() % 100;
4450
          if (0 != superTblInfo->disorderRatio
4451
                    && rand_num < superTblInfo->disorderRatio) {
4452 4453 4454
            int64_t d = startTime
                + superTblInfo->timeStampStep * k
                - taosRandom() % superTblInfo->disorderRange;
4455
            retLen = generateRowData(
4456
                      data,
4457
                      d,
4458
                      superTblInfo);
4459 4460
          } else {
            retLen = generateRowData(
4461
                      data,
4462
                      startTime + superTblInfo->timeStampStep * k,
4463
                      superTblInfo);
4464 4465
        }
      }
4466

4467 4468 4469
      if (retLen > remainderBufLen) {
        break;
      }
4470

4471
      buffer += snprintf(buffer, retLen + 1, "%s", data);
4472 4473 4474
      k++;
      len += retLen;
      remainderBufLen -= retLen;
4475
    } else {
4476 4477
      char **data_type = g_args.datatype;
      int lenOfBinary = g_args.len_of_binary;
4478

4479
      int rand_num = taosRandom() % 100;
4480
      if ((g_args.disorderRatio != 0)
4481
                && (rand_num < g_args.disorderRatio)) {
4482

4483
        int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k
4484 4485
            - taosRandom() % g_args.disorderRange;

4486
        retLen = generateData(data, data_type,
4487
                  ncols_per_record, d, lenOfBinary);
4488
      } else {
4489
        retLen = generateData(data, data_type,
4490
                  ncols_per_record,
4491
                  startTime + DEFAULT_TIMESTAMP_STEP * k,
4492 4493
                  lenOfBinary);
      }
4494

4495 4496 4497
      if (len > remainderBufLen)
        break;

4498
      buffer += sprintf(buffer, " %s", data);
4499 4500 4501
      k++;
      len += retLen;
      remainderBufLen -= retLen;
4502 4503
    }

4504 4505
    verbosePrint("%s() LN%d len=%d k=%d \nbuffer=%s\n",
            __func__, __LINE__, len, k, buffer);
4506 4507

    startFrom ++;
4508

4509
    if (startFrom >= insertRows) {
4510
      break;
4511
    }
4512
  }
4513

4514 4515 4516
  *dataLen = len;
  return k;
}
4517

4518 4519
static int generateSQLHead(char *tableName, int32_t tableSeq,
        threadInfo* pThreadInfo, SSuperTable* superTblInfo, char *buffer)
4520 4521 4522 4523 4524 4525
{
  int len;
  if (superTblInfo) {
    if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
      char* tagsValBuf = NULL;
      if (0 == superTblInfo->tagSource) {
4526
            tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq);
4527 4528 4529 4530 4531 4532
      } else {
            tagsValBuf = getTagValueFromTagSample(
                    superTblInfo,
                    tableSeq % superTblInfo->tagSampleCount);
      }
      if (NULL == tagsValBuf) {
4533
        errorPrint("%s() LN%d, tag buf failed to allocate  memory\n", __func__, __LINE__);
4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550
        return -1;
      }

      len = snprintf(buffer,
                  superTblInfo->maxSqlLen,
                  "insert into %s.%s using %s.%s tags %s values",
                  pThreadInfo->db_name,
                  tableName,
                  pThreadInfo->db_name,
                  superTblInfo->sTblName,
                  tagsValBuf);
      tmfree(tagsValBuf);
    } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) {
      len = snprintf(buffer,
                  superTblInfo->maxSqlLen,
                  "insert into %s.%s values",
                  pThreadInfo->db_name,
4551
                  tableName);
4552 4553
    } else {
      len = snprintf(buffer,
4554
                  superTblInfo->maxSqlLen,
4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569
                  "insert into %s.%s values",
                  pThreadInfo->db_name,
                  tableName);
    }
  } else {
      len = snprintf(buffer,
                  g_args.max_sql_len,
                  "insert into %s.%s values",
                  pThreadInfo->db_name,
                  tableName);
  }

  return len;
}

4570
static int generateProgressiveDataBuffer(char *pTblName,
4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589
        int32_t tableSeq,
        threadInfo *pThreadInfo, char *buffer,
        int64_t insertRows,
        int64_t startFrom, int64_t startTime, int *pSamplePos)
{
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;

  int ncols_per_record = 1; // count first col ts

  if (superTblInfo == NULL) {
    int datatypeSeq = 0;
    while(g_args.datatype[datatypeSeq]) {
        datatypeSeq ++;
        ncols_per_record ++;
    }
  }

  assert(buffer != NULL);

4590 4591 4592 4593
  int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
  int remainderBufLen = maxSqlLen;

  memset(buffer, 0, maxSqlLen);
4594 4595 4596

  char *pstr = buffer;

4597 4598
  int headLen = generateSQLHead(pTblName, tableSeq, pThreadInfo, superTblInfo,
          buffer);
4599
  pstr += headLen;
4600
  remainderBufLen -= headLen;
4601 4602 4603 4604

  int k;
  int dataLen;
  k = generateDataTail(pTblName, tableSeq, pThreadInfo, superTblInfo,
4605
          g_args.num_of_RPR, pstr, remainderBufLen, insertRows, startFrom,
4606
          startTime,
4607
          pSamplePos, &dataLen);
4608

4609 4610
  return k;
}
4611

4612
static void* syncWriteInterlace(threadInfo *pThreadInfo) {
4613 4614
  debugPrint("[%d] %s() LN%d: ### interlace write\n",
         pThreadInfo->threadID, __func__, __LINE__);
4615 4616

  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628
  int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows;

  int insertMode;

  if (interlaceRows > 0) {
    insertMode = INTERLACE_INSERT_MODE;
  } else {
    insertMode = PROGRESSIVE_INSERT_MODE;
  }

  // TODO: prompt tbl count multple interlace rows and batch
  //
4629 4630 4631

  char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1);
  if (NULL == buffer) {
4632
    errorPrint( "Failed to alloc %d Bytes, reason:%s\n",
4633 4634 4635 4636 4637
              superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len,
              strerror(errno));
    return NULL;
  }

4638
  char tableName[TSDB_TABLE_NAME_LEN];
4639 4640 4641 4642

  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;

4643 4644
  int nTimeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;

4645
  int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
4646 4647
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
4648 4649 4650 4651 4652 4653 4654 4655 4656
  uint64_t st = 0;
  uint64_t et = 0xffffffff;

  int64_t lastPrintTime = taosGetTimestampMs();
  int64_t startTs = taosGetTimestampUs();
  int64_t endTs;

  int tableSeq = pThreadInfo->start_table_from;

4657 4658
  debugPrint("[%d] %s() LN%d: start_table_from=%d ntables=%d insertRows=%"PRId64"\n",
          pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from,
4659 4660 4661 4662 4663
          pThreadInfo->ntables, insertRows);

  int64_t startTime = pThreadInfo->start_time;

  int batchPerTblTimes;
4664
  int batchPerTbl;
4665

4666 4667
  assert(pThreadInfo->ntables > 0);

4668 4669
  if (interlaceRows > g_args.num_of_RPR)
        interlaceRows = g_args.num_of_RPR;
4670

4671 4672
  batchPerTbl = interlaceRows;
  if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) {
4673
    batchPerTblTimes =
4674
        (g_args.num_of_RPR / (interlaceRows * pThreadInfo->ntables)) + 1;
4675 4676 4677 4678
  } else {
    batchPerTblTimes = 1;
  }

4679
  int generatedRecPerTbl = 0;
4680 4681
  bool flagSleep = true;
  int sleepTimeTotal = 0;
4682 4683 4684 4685

  int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
  int remainderBufLen;

4686
  while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
4687
    if ((flagSleep) && (insert_interval)) {
4688
        st = taosGetTimestampUs();
4689
        flagSleep = false;
4690 4691
    }
    // generate data
4692 4693
    memset(buffer, 0, maxSqlLen);
    remainderBufLen = maxSqlLen;
4694

4695
    char *pstr = buffer;
4696
    int recOfBatch = 0;
4697

4698 4699
    for (int i = 0; i < batchPerTblTimes; i ++) {
      getTableName(tableName, pThreadInfo, tableSeq);
4700 4701 4702
      if (0 == strlen(tableName)) {
        errorPrint("[%d] %s() LN%d, getTableName return null\n",
            pThreadInfo->threadID, __func__, __LINE__);
4703
        free(buffer);
4704 4705
        return NULL;
      }
4706

4707 4708
      int headLen;
      if (i == 0) {
4709 4710
        headLen = generateSQLHead(tableName, tableSeq, pThreadInfo,
                superTblInfo, pstr);
4711
      } else {
4712
        headLen = snprintf(pstr, TSDB_TABLE_NAME_LEN, "%s.%s values",
4713 4714
                  pThreadInfo->db_name,
                  tableName);
4715
      }
4716

4717
      // generate data buffer
4718 4719
      verbosePrint("[%d] %s() LN%d i=%d buffer:\n%s\n",
                pThreadInfo->threadID, __func__, __LINE__, i, buffer);
4720

4721
      pstr += headLen;
4722 4723
      remainderBufLen -= headLen;

4724
      int dataLen = 0;
4725

4726
      verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n",
4727 4728
                pThreadInfo->threadID, __func__, __LINE__,
                i, batchPerTblTimes, batchPerTbl);
4729 4730 4731 4732 4733 4734 4735

      if (superTblInfo) {
        if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
          startTime = taosGetTimestamp(pThreadInfo->time_precision);
        }
      } else {
          startTime = 1500000000000;
4736
      }
4737
      int generated = generateDataTail(
4738
        tableName, tableSeq, pThreadInfo, superTblInfo,
4739
        batchPerTbl, pstr, remainderBufLen, insertRows, 0,
4740
        startTime,
4741
        &(pThreadInfo->samplePos), &dataLen);
4742

4743 4744 4745 4746 4747
      if (generated < 0) {
        debugPrint("[%d] %s() LN%d, generated data is %d\n",
                  pThreadInfo->threadID, __func__, __LINE__, generated);
        goto free_and_statistics_interlace;
      }
4748
      pstr += dataLen;
4749 4750
      remainderBufLen -= dataLen;

4751
      recOfBatch += batchPerTbl;
4752
//      startTime += batchPerTbl * superTblInfo->timeStampStep;
4753
      pThreadInfo->totalInsertRows += batchPerTbl;
4754
      verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n",
4755 4756
                pThreadInfo->threadID, __func__, __LINE__,
                batchPerTbl, recOfBatch);
4757

4758 4759 4760 4761 4762
      tableSeq ++;
      if (insertMode == INTERLACE_INSERT_MODE) {
          if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) {
            // turn to first table
            tableSeq = pThreadInfo->start_table_from;
4763
            generatedRecPerTbl += batchPerTbl;
4764 4765

            startTime = pThreadInfo->start_time
4766
              + generatedRecPerTbl * nTimeStampStep;
4767

4768 4769 4770 4771 4772 4773
            flagSleep = true;
            if (generatedRecPerTbl >= insertRows)
              break;

            if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR)
                break;
4774
          }
4775 4776
      }

4777
      int remainRows = insertRows - generatedRecPerTbl;
4778
      if ((remainRows > 0) && (batchPerTbl > remainRows))
4779 4780
        batchPerTbl = remainRows;

4781
      verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%d insertRows=%"PRId64"\n",
4782 4783 4784 4785
                pThreadInfo->threadID, __func__, __LINE__,
                generatedRecPerTbl, insertRows);

      if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl)
4786 4787 4788
        break;
    }

4789
    verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRId64"\n",
4790 4791 4792 4793 4794
              pThreadInfo->threadID, __func__, __LINE__, recOfBatch,
              pThreadInfo->totalInsertRows);
    verbosePrint("[%d] %s() LN%d, buffer=%s\n",
           pThreadInfo->threadID, __func__, __LINE__, buffer);

4795 4796
    startTs = taosGetTimestampUs();

4797
    int affectedRows = execInsert(pThreadInfo, buffer, recOfBatch);
4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808

    endTs = taosGetTimestampUs();
    int64_t delay = endTs - startTs;
    performancePrint("%s() LN%d, insert execution time is %10.6fms\n",
            __func__, __LINE__, delay/1000.0);

    if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
    if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
    pThreadInfo->cntDelay++;
    pThreadInfo->totalDelay += delay;

4809 4810
    verbosePrint("[%d] %s() LN%d affectedRows=%d\n", pThreadInfo->threadID,
            __func__, __LINE__, affectedRows);
4811 4812
    if ((affectedRows < 0) || (recOfBatch != affectedRows)) {
        errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %d\n%s\n",
4813
                pThreadInfo->threadID, __func__, __LINE__,
4814
                recOfBatch, affectedRows, buffer);
4815
        goto free_and_statistics_interlace;
4816
    }
4817

4818
    pThreadInfo->totalAffectedRows += affectedRows;
4819

4820 4821 4822
    int64_t  currentPrintTime = taosGetTimestampMs();
    if (currentPrintTime - lastPrintTime > 30*1000) {
      printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
4823 4824 4825
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
4826 4827
      lastPrintTime = currentPrintTime;
    }
4828

4829 4830
    if ((insert_interval) && flagSleep) {
      et = taosGetTimestampUs();
4831

4832 4833
      if (insert_interval > ((et - st)/1000) ) {
        int sleepTime = insert_interval - (et -st)/1000;
4834 4835
        performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleepTime);
4836 4837 4838
        taosMsleep(sleepTime); // ms
        sleepTimeTotal += insert_interval;
      }
4839
    }
4840 4841 4842 4843 4844
  }

free_and_statistics_interlace:
  tmfree(buffer);

4845 4846 4847
  printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n",
          pThreadInfo->threadID,
          pThreadInfo->totalInsertRows,
4848 4849 4850 4851
          pThreadInfo->totalAffectedRows);
  return NULL;
}

4852 4853 4854 4855 4856 4857 4858 4859
// sync insertion
/*
   1 thread: 100 tables * 2000  rows/s
   1 thread: 10  tables * 20000 rows/s
   6 thread: 300 tables * 2000  rows/s

   2 taosinsertdata , 1 thread:  10  tables * 20000 rows/s
*/
4860
static void* syncWriteProgressive(threadInfo *pThreadInfo) {
4861
  debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__);
4862

4863
  SSuperTable* superTblInfo = pThreadInfo->superTblInfo;
4864

4865 4866
  char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1);
  if (NULL == buffer) {
4867
    errorPrint( "Failed to alloc %d Bytes, reason:%s\n",
4868 4869 4870 4871
              superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len,
              strerror(errno));
    return NULL;
  }
4872

4873 4874 4875 4876
  int64_t lastPrintTime = taosGetTimestampMs();
  int64_t startTs = taosGetTimestampUs();
  int64_t endTs;

4877 4878
  int timeStampStep =
      superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP;
4879
/*  int insert_interval =
4880
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
4881 4882
  uint64_t st = 0;
  uint64_t et = 0xffffffff;
4883
  */
4884

4885 4886
  pThreadInfo->totalInsertRows = 0;
  pThreadInfo->totalAffectedRows = 0;
4887

4888
  pThreadInfo->samplePos = 0;
4889

4890 4891
  for (uint32_t tableSeq =
          pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to;
4892
        tableSeq ++) {
4893
    int64_t start_time = pThreadInfo->start_time;
4894 4895 4896 4897 4898

    int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
    verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows);

    for (int64_t i = 0; i < insertRows;) {
4899
        /*
4900 4901
      if (insert_interval) {
            st = taosGetTimestampUs();
4902
      }
4903
      */
4904

4905 4906 4907 4908 4909 4910
      char tableName[TSDB_TABLE_NAME_LEN];
      getTableName(tableName, pThreadInfo, tableSeq);
      verbosePrint("%s() LN%d: tid=%d seq=%d tableName=%s\n",
             __func__, __LINE__,
             pThreadInfo->threadID, tableSeq, tableName);

4911
      int generated = generateProgressiveDataBuffer(
4912
              tableName, tableSeq, pThreadInfo, buffer, insertRows,
4913
            i, start_time,
4914
            &(pThreadInfo->samplePos));
4915 4916 4917 4918 4919
      if (generated > 0)
        i += generated;
      else
        goto free_and_statistics_2;

4920
      start_time +=  generated * timeStampStep;
4921
      pThreadInfo->totalInsertRows += generated;
4922 4923 4924 4925

      startTs = taosGetTimestampUs();

      int affectedRows = execInsert(pThreadInfo, buffer, generated);
4926 4927 4928

      endTs = taosGetTimestampUs();
      int64_t delay = endTs - startTs;
4929 4930 4931
      performancePrint("%s() LN%d, insert execution time is %10.6fms\n",
              __func__, __LINE__, delay/1000.0);

4932 4933 4934 4935
      if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay;
      if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay;
      pThreadInfo->cntDelay++;
      pThreadInfo->totalDelay += delay;
4936

4937 4938 4939 4940 4941
      if (affectedRows < 0)
        goto free_and_statistics_2;

      pThreadInfo->totalAffectedRows += affectedRows;

4942 4943
      int64_t  currentPrintTime = taosGetTimestampMs();
      if (currentPrintTime - lastPrintTime > 30*1000) {
4944
        printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n",
4945 4946 4947
                    pThreadInfo->threadID,
                    pThreadInfo->totalInsertRows,
                    pThreadInfo->totalAffectedRows);
4948 4949 4950
        lastPrintTime = currentPrintTime;
      }

4951
      if (i >= insertRows)
4952
        break;
4953
/*
4954 4955 4956 4957 4958
      if (insert_interval) {
        et = taosGetTimestampUs();

        if (insert_interval > ((et - st)/1000) ) {
            int sleep_time = insert_interval - (et -st)/1000;
4959 4960
            performancePrint("%s() LN%d sleep: %d ms for insert interval\n",
                    __func__, __LINE__, sleep_time);
4961 4962
            taosMsleep(sleep_time); // ms
        }
4963
      }
4964
      */
4965 4966
    }   // num_of_DPT

4967
    if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo &&
4968 4969
        (0 == strncasecmp(
                    superTblInfo->dataSource, "sample", strlen("sample")))) {
4970 4971
          printf("%s() LN%d samplePos=%d\n",
                  __func__, __LINE__, pThreadInfo->samplePos);
4972
    }
4973
  } // tableSeq
4974

4975
free_and_statistics_2:
4976 4977
  tmfree(buffer);

4978 4979 4980
  printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n",
          pThreadInfo->threadID,
          pThreadInfo->totalInsertRows,
4981
          pThreadInfo->totalAffectedRows);
4982
  return NULL;
H
Hui Li 已提交
4983 4984
}

4985 4986
static void* syncWrite(void *sarg) {

4987
  threadInfo *winfo = (threadInfo *)sarg;
4988 4989
  SSuperTable* superTblInfo = winfo->superTblInfo;

4990
  int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows;
4991

4992
  if (interlaceRows > 0) {
4993 4994 4995 4996 4997 4998 4999 5000
    // interlace mode
    return syncWriteInterlace(winfo);
  } else {
    // progressive mode
    return syncWriteProgressive(winfo);
  }
}

5001
static void callBack(void *param, TAOS_RES *res, int code) {
5002
  threadInfo* winfo = (threadInfo*)param;
5003
  SSuperTable* superTblInfo = winfo->superTblInfo;
5004

5005 5006
  int insert_interval =
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5007 5008 5009 5010
  if (insert_interval) {
    winfo->et = taosGetTimestampUs();
    if (((winfo->et - winfo->st)/1000) < insert_interval) {
      taosMsleep(insert_interval - (winfo->et - winfo->st)/1000); // ms
5011
    }
H
Hui Li 已提交
5012
  }
5013

5014
  char *buffer = calloc(1, winfo->superTblInfo->maxSqlLen);
5015
  char data[MAX_DATA_SIZE];
5016
  char *pstr = buffer;
5017 5018
  pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix,
          winfo->start_table_from);
5019
//  if (winfo->counter >= winfo->superTblInfo->insertRows) {
5020
  if (winfo->counter >= g_args.num_of_RPR) {
5021
    winfo->start_table_from++;
5022 5023
    winfo->counter = 0;
  }
5024
  if (winfo->start_table_from > winfo->end_table_to) {
5025 5026 5027 5028 5029
    tsem_post(&winfo->lock_sem);
    free(buffer);
    taos_free_result(res);
    return;
  }
5030

5031
  for (int i = 0; i < g_args.num_of_RPR; i++) {
5032
    int rand_num = taosRandom() % 100;
5033 5034
    if (0 != winfo->superTblInfo->disorderRatio
            && rand_num < winfo->superTblInfo->disorderRatio) {
5035
      int64_t d = winfo->lastTs - taosRandom() % winfo->superTblInfo->disorderRange;
5036
      generateRowData(data, d, winfo->superTblInfo);
5037
    } else {
5038
      generateRowData(data, winfo->lastTs += 1000, winfo->superTblInfo);
H
Hui Li 已提交
5039
    }
5040 5041
    pstr += sprintf(pstr, "%s", data);
    winfo->counter++;
H
Hui Li 已提交
5042

5043 5044
    if (winfo->counter >= winfo->superTblInfo->insertRows) {
      break;
H
Hui Li 已提交
5045 5046
    }
  }
5047

5048 5049
  if (insert_interval) {
    winfo->st = taosGetTimestampUs();
5050 5051 5052
  }
  taos_query_a(winfo->taos, buffer, callBack, winfo);
  free(buffer);
H
Hui Li 已提交
5053

5054
  taos_free_result(res);
H
Hui Li 已提交
5055 5056
}

5057
static void *asyncWrite(void *sarg) {
5058
  threadInfo *winfo = (threadInfo *)sarg;
5059
  SSuperTable* superTblInfo = winfo->superTblInfo;
5060 5061 5062 5063

  winfo->st = 0;
  winfo->et = 0;
  winfo->lastTs = winfo->start_time;
5064

5065
  int insert_interval =
5066
      superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
5067 5068
  if (insert_interval) {
    winfo->st = taosGetTimestampUs();
H
Hui Li 已提交
5069
  }
5070 5071 5072
  taos_query_a(winfo->taos, "show databases", callBack, winfo);

  tsem_wait(&(winfo->lock_sem));
H
Hui Li 已提交
5073 5074 5075 5076

  return NULL;
}

5077 5078
static void startMultiThreadInsertData(int threads, char* db_name,
        char* precision,SSuperTable* superTblInfo) {
5079

5080 5081
  pthread_t *pids = malloc(threads * sizeof(pthread_t));
  assert(pids != NULL);
H
Hui Li 已提交
5082

5083 5084
  threadInfo *infos = malloc(threads * sizeof(threadInfo));
  assert(infos != NULL);
5085

5086 5087
  memset(pids, 0, threads * sizeof(pthread_t));
  memset(infos, 0, threads * sizeof(threadInfo));
5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104

  //TAOS* taos;
  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
  //  if (NULL == taos) {
  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
  //    exit(-1);
  //  }
  //}

  int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
  if (0 != precision[0]) {
    if (0 == strncasecmp(precision, "ms", 2)) {
      timePrec = TSDB_TIME_PRECISION_MILLI;
    }  else if (0 == strncasecmp(precision, "us", 2)) {
      timePrec = TSDB_TIME_PRECISION_MICRO;
    }  else {
5105
      errorPrint("Not support precision: %s\n", precision);
5106 5107 5108 5109
      exit(-1);
    }
  }

5110
  int64_t start_time;
5111 5112 5113
  if (superTblInfo) {
    if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) {
        start_time = taosGetTimestamp(timePrec);
5114
    } else {
5115 5116 5117 5118 5119
      if (TSDB_CODE_SUCCESS != taosParseTime(
        superTblInfo->startTimestamp,
        &start_time,
        strlen(superTblInfo->startTimestamp),
        timePrec, 0)) {
5120
          ERROR_EXIT("failed to parse time!\n");
5121
      }
5122
    }
5123 5124
  } else {
     start_time = 1500000000000;
5125 5126
  }

5127
  double start = getCurrentTimeUs();
5128

5129
  // read sample data from file first
5130
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5131
              "sample", strlen("sample")))) {
5132
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5133 5134
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
              __func__, __LINE__);
5135 5136 5137 5138
      exit(-1);
    }
  }

5139
  // read sample data from file first
5140
  if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource,
5141 5142
              "sample", strlen("sample")))) {
    if (0 != prepareSampleDataForSTable(superTblInfo)) {
5143 5144
      errorPrint("%s() LN%d, prepare sample data for stable failed!\n",
              __func__, __LINE__);
5145 5146 5147 5148 5149 5150 5151 5152
      exit(-1);
    }
  }

  TAOS* taos = taos_connect(
              g_Dbs.host, g_Dbs.user,
              g_Dbs.password, db_name, g_Dbs.port);
  if (NULL == taos) {
5153 5154
    errorPrint("%s() LN%d, connect to server fail , reason: %s\n",
                __func__, __LINE__, taos_errstr(NULL));
5155 5156 5157
    exit(-1);
  }

5158 5159
  int ntables = 0;
  int startFrom;
5160

5161
  if (superTblInfo) {
5162
    int limit, offset;
5163

5164 5165 5166
    if ((superTblInfo->childTblExists == TBL_NO_EXISTS) &&
            ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit != 0))) {
      printf("WARNING: offset and limit will not be used since the child tables are not exists!\n");
5167 5168
    }

5169 5170 5171
    if ((superTblInfo->childTblExists == TBL_ALREADY_EXISTS)
            && (superTblInfo->childTblOffset >= 0)) {
      if (superTblInfo->childTblLimit < 0) {
5172 5173 5174 5175 5176 5177
        superTblInfo->childTblLimit =
            superTblInfo->childTblCount - superTblInfo->childTblOffset;
      }

      offset = superTblInfo->childTblOffset;
      limit = superTblInfo->childTblLimit;
5178
    } else {
5179 5180
      limit = superTblInfo->childTblCount;
      offset = 0;
5181 5182
    }

5183 5184 5185
    ntables = limit;
    startFrom = offset;

5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204
    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
        && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
            > superTblInfo->childTblCount)) {
      printf("WARNING: specified offset + limit > child table count!\n");
      if (!g_args.answer_yes) {
        printf("         Press enter key to continue or Ctrl-C to stop\n\n");
        (void)getchar();
      }
    }

    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
            && (0 == superTblInfo->childTblLimit)) {
      printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
      if (!g_args.answer_yes) {
        printf("         Press enter key to continue or Ctrl-C to stop\n\n");
        (void)getchar();
      }
    }

5205 5206 5207
    superTblInfo->childTblName = (char*)calloc(1,
        limit * TSDB_TABLE_NAME_LEN);
    if (superTblInfo->childTblName == NULL) {
5208
      errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__);
5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219
      taos_close(taos);
      exit(-1);
    }

    int childTblCount;
    getChildNameOfSuperTableWithLimitAndOffset(
        taos,
        db_name, superTblInfo->sTblName,
        &superTblInfo->childTblName, &childTblCount,
        limit,
        offset);
5220 5221 5222
  } else {
    ntables = g_args.num_of_tables;
    startFrom = 0;
5223
  }
5224

5225 5226
  taos_close(taos);

5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237
  int a = ntables / threads;
  if (a < 1) {
    threads = ntables;
    a = 1;
  }

  int b = 0;
  if (threads != 0) {
    b = ntables % threads;
  }

H
Hui Li 已提交
5238
  for (int i = 0; i < threads; i++) {
5239
    threadInfo *t_info = infos + i;
H
Hui Li 已提交
5240 5241
    t_info->threadID = i;
    tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE);
5242
    t_info->time_precision = timePrec;
5243 5244 5245
    t_info->superTblInfo = superTblInfo;

    t_info->start_time = start_time;
5246
    t_info->minDelay = INT16_MAX;
5247

5248 5249
    if ((NULL == superTblInfo) ||
            (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) {
5250
      //t_info->taos = taos;
5251
      t_info->taos = taos_connect(
5252
              g_Dbs.host, g_Dbs.user,
5253
              g_Dbs.password, db_name, g_Dbs.port);
5254
      if (NULL == t_info->taos) {
5255 5256
        errorPrint(
                "connect to server fail from insert sub thread, reason: %s\n",
5257
                taos_errstr(NULL));
5258 5259 5260 5261 5262 5263
        exit(-1);
      }
    } else {
      t_info->taos = NULL;
    }

5264 5265
    if ((NULL == superTblInfo)
            || (0 == superTblInfo->multiThreadWriteOneTbl)) {
5266 5267 5268 5269
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
5270
    } else {
5271 5272
      t_info->start_table_from = 0;
      t_info->ntables = superTblInfo->childTblCount;
5273 5274 5275 5276 5277
      t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint();
    }

    tsem_init(&(t_info->lock_sem), 0, 0);
    if (SYNC == g_Dbs.queryMode) {
5278
      pthread_create(pids + i, NULL, syncWrite, t_info);
5279
    } else {
5280 5281
      pthread_create(pids + i, NULL, asyncWrite, t_info);
    }
H
Hui Li 已提交
5282
  }
5283

H
Hui Li 已提交
5284 5285 5286 5287
  for (int i = 0; i < threads; i++) {
    pthread_join(pids[i], NULL);
  }

5288 5289 5290
  int64_t totalDelay = 0;
  int64_t maxDelay = 0;
  int64_t minDelay = INT16_MAX;
5291
  int64_t cntDelay = 1;
5292 5293
  double  avgDelay = 0;

H
Hui Li 已提交
5294
  for (int i = 0; i < threads; i++) {
5295 5296
    threadInfo *t_info = infos + i;

S
TD-1057  
Shengliang Guan 已提交
5297
    tsem_destroy(&(t_info->lock_sem));
5298 5299
    taos_close(t_info->taos);

5300 5301 5302 5303
    debugPrint("%s() LN%d, [%d] totalInsert=%"PRId64" totalAffected=%"PRId64"\n",
            __func__, __LINE__,
            t_info->threadID, t_info->totalInsertRows,
            t_info->totalAffectedRows);
5304 5305
    if (superTblInfo) {
        superTblInfo->totalAffectedRows += t_info->totalAffectedRows;
5306
        superTblInfo->totalInsertRows += t_info->totalInsertRows;
5307 5308 5309
    } else {
        g_args.totalAffectedRows += t_info->totalAffectedRows;
        g_args.totalInsertRows += t_info->totalInsertRows;
5310
    }
5311 5312 5313 5314

    totalDelay  += t_info->totalDelay;
    cntDelay   += t_info->cntDelay;
    if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay;
5315
    if (t_info->minDelay < minDelay) minDelay = t_info->minDelay;
H
Hui Li 已提交
5316
  }
5317
  cntDelay -= 1;
H
Hui Li 已提交
5318

5319
  if (cntDelay == 0)    cntDelay = 1;
5320 5321
  avgDelay = (double)totalDelay / cntDelay;

5322
  double end = getCurrentTimeUs();
5323
  double t = end - start;
5324 5325

  if (superTblInfo) {
5326
    printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n",
5327
          t, superTblInfo->totalInsertRows,
5328 5329
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5330
          superTblInfo->totalInsertRows / t);
5331
    fprintf(g_fpOfInsertResult,
5332
          "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n",
5333
          t, superTblInfo->totalInsertRows,
5334 5335
          superTblInfo->totalAffectedRows,
          threads, db_name, superTblInfo->sTblName,
5336
          superTblInfo->totalInsertRows/ t);
5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348
  } else {
    printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s %2.f records/second\n\n",
          t, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          g_args.totalInsertRows / t);
    fprintf(g_fpOfInsertResult,
          "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s %2.f records/second\n\n",
          t, g_args.totalInsertRows,
          g_args.totalAffectedRows,
          threads, db_name,
          g_args.totalInsertRows / t);
5349
  }
5350 5351 5352 5353 5354

  printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n",
          avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0);
  fprintf(g_fpOfInsertResult, "insert delay, avg:%10.6fms, max: %10.6fms, min: %10.6fms\n\n",
          avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0);
5355

5356 5357
  //taos_close(taos);

H
Hui Li 已提交
5358
  free(pids);
5359
  free(infos);
H
Hui Li 已提交
5360 5361
}

5362
static void *readTable(void *sarg) {
5363
#if 1
5364
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
5365 5366
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
5367
  uint64_t sTime = rinfo->start_time;
H
hzcheng 已提交
5368 5369
  char *tb_prefix = rinfo->tb_prefix;
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
5370
  if (NULL == fp) {
5371
    errorPrint( "fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
H
Hui Li 已提交
5372 5373
    return NULL;
  }
5374

5375
    int num_of_DPT;
5376
/*  if (rinfo->superTblInfo) {
5377 5378
    num_of_DPT = rinfo->superTblInfo->insertRows; //  nrecords_per_table;
  } else {
5379
  */
5380
      num_of_DPT = g_args.num_of_DPT;
5381
//  }
5382

5383
  int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
H
hzcheng 已提交
5384
  int totalData = num_of_DPT * num_of_tables;
5385
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
  printf("%d records:\n", totalData);
  fprintf(fp, "| QFunctions |    QRecords    |   QSpeed(R/s)   |  QLatency(ms) |\n");

  for (int j = 0; j < n; j++) {
    double totalT = 0;
    int count = 0;
    for (int i = 0; i < num_of_tables; i++) {
5398 5399
      sprintf(command, "select %s from %s%d where ts>= %" PRId64,
              aggreFunc[j], tb_prefix, i, sTime);
H
hzcheng 已提交
5400

5401
      double t = getCurrentTimeUs();
S
Shuaiqiang Chang 已提交
5402 5403
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);
H
hzcheng 已提交
5404

S
Shuaiqiang Chang 已提交
5405
      if (code != 0) {
5406
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
5407
        taos_free_result(pSql);
H
hzcheng 已提交
5408
        taos_close(taos);
5409
        fclose(fp);
5410
        return NULL;
H
hzcheng 已提交
5411 5412
      }

S
Shuaiqiang Chang 已提交
5413
      while (taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
5414 5415 5416
        count++;
      }

5417
      t = getCurrentTimeUs() - t;
H
hzcheng 已提交
5418 5419
      totalT += t;

S
Shuaiqiang Chang 已提交
5420
      taos_free_result(pSql);
H
hzcheng 已提交
5421 5422 5423
    }

    fprintf(fp, "|%10s  |   %10d   |  %12.2f   |   %10.2f  |\n",
S
slguan 已提交
5424
            aggreFunc[j][0] == '*' ? "   *   " : aggreFunc[j], totalData,
H
hzcheng 已提交
5425 5426 5427 5428 5429
            (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000);
    printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT);
  }
  fprintf(fp, "\n");
  fclose(fp);
5430
#endif
H
hzcheng 已提交
5431 5432 5433
  return NULL;
}

5434
static void *readMetric(void *sarg) {
5435
#if 1
5436
  threadInfo *rinfo = (threadInfo *)sarg;
H
hzcheng 已提交
5437 5438 5439
  TAOS *taos = rinfo->taos;
  char command[BUFFER_SIZE] = "\0";
  FILE *fp = fopen(rinfo->fp, "a");
H
Hui Li 已提交
5440 5441 5442 5443
  if (NULL == fp) {
    printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno));
    return NULL;
  }
5444 5445

  int num_of_DPT = rinfo->superTblInfo->insertRows;
5446
  int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1;
H
hzcheng 已提交
5447
  int totalData = num_of_DPT * num_of_tables;
5448
  bool do_aggreFunc = g_Dbs.do_aggreFunc;
H
hzcheng 已提交
5449 5450 5451 5452 5453 5454 5455 5456 5457

  int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2;
  if (!do_aggreFunc) {
    printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n");
  }
  printf("%d records:\n", totalData);
  fprintf(fp, "Querying On %d records:\n", totalData);

  for (int j = 0; j < n; j++) {
sangshuduo's avatar
sangshuduo 已提交
5458
    char condition[BUFFER_SIZE - 30] = "\0";
B
Bomin Zhang 已提交
5459
    char tempS[64] = "\0";
H
hzcheng 已提交
5460 5461 5462 5463 5464

    int m = 10 < num_of_tables ? 10 : num_of_tables;

    for (int i = 1; i <= m; i++) {
      if (i == 1) {
5465
        sprintf(tempS, "t1 = %d", i);
H
hzcheng 已提交
5466
      } else {
5467
        sprintf(tempS, " or t1 = %d ", i);
H
hzcheng 已提交
5468 5469 5470
      }
      strcat(condition, tempS);

L
liu0x54 已提交
5471
      sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
H
hzcheng 已提交
5472 5473 5474 5475

      printf("Where condition: %s\n", condition);
      fprintf(fp, "%s\n", command);

5476
      double t = getCurrentTimeUs();
H
hzcheng 已提交
5477

S
Shuaiqiang Chang 已提交
5478 5479 5480 5481
      TAOS_RES *pSql = taos_query(taos, command);
      int32_t code = taos_errno(pSql);

      if (code != 0) {
5482
        errorPrint( "Failed to query:%s\n", taos_errstr(pSql));
S
Shuaiqiang Chang 已提交
5483
        taos_free_result(pSql);
H
hzcheng 已提交
5484
        taos_close(taos);
5485
        fclose(fp);
5486
        return NULL;
H
hzcheng 已提交
5487 5488
      }
      int count = 0;
S
Shuaiqiang Chang 已提交
5489
      while (taos_fetch_row(pSql) != NULL) {
H
hzcheng 已提交
5490 5491
        count++;
      }
5492
      t = getCurrentTimeUs() - t;
H
hzcheng 已提交
5493

5494 5495
      fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n",
              num_of_tables * num_of_DPT / t, t * 1000);
H
hzcheng 已提交
5496 5497
      printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t);

S
Shuaiqiang Chang 已提交
5498
      taos_free_result(pSql);
H
hzcheng 已提交
5499 5500 5501 5502
    }
    fprintf(fp, "\n");
  }
  fclose(fp);
5503
#endif
H
hzcheng 已提交
5504 5505 5506
  return NULL;
}

H
Hui Li 已提交
5507

5508
static int insertTestProcess() {
5509

5510 5511 5512
  setupForAnsiEscape();
  int ret = printfInsertMeta();
  resetAfterAnsiEscape();
5513

5514 5515 5516
  if (ret == -1)
    exit(EXIT_FAILURE);

5517
  debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile);
5518 5519
  g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a");
  if (NULL == g_fpOfInsertResult) {
5520
    errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile);
5521 5522
    return -1;
  }
5523

5524 5525
  printfInsertMetaToFile(g_fpOfInsertResult);

5526 5527 5528 5529
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void)getchar();
  }
5530

5531 5532 5533
  init_rand_data();

  // create database and super tables
5534
  if(createDatabasesAndStables() != 0) {
5535 5536 5537
    fclose(g_fpOfInsertResult);
    return -1;
  }
5538 5539

  // pretreatement
5540
  prepareSampleData();
5541

5542 5543 5544 5545
  double start;
  double end;

  // create child tables
5546
  start = getCurrentTimeUs();
5547
  createChildTables();
5548
  end = getCurrentTimeUs();
5549

5550
  if (g_totalChildTables > 0) {
5551
    printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n",
5552
            end - start, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
5553 5554
    fprintf(g_fpOfInsertResult,
            "Spent %.4f seconds to create %d tables with %d thread(s)\n\n",
5555
            end - start, g_totalChildTables, g_Dbs.threadCountByCreateTbl);
5556
  }
5557

5558
  taosMsleep(1000);
5559
  // create sub threads for inserting data
5560
  //start = getCurrentTimeUs();
5561
  for (int i = 0; i < g_Dbs.dbCount; i++) {
5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574
    if (g_Dbs.use_metric) {
      if (g_Dbs.db[i].superTblCount > 0) {
        for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {

          SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j];

          if (superTblInfo && (superTblInfo->insertRows > 0)) {
            startMultiThreadInsertData(
              g_Dbs.threadCount,
              g_Dbs.db[i].dbName,
              g_Dbs.db[i].dbCfg.precision,
              superTblInfo);
          }
5575
        }
5576
      }
5577 5578
    } else {
        startMultiThreadInsertData(
5579 5580 5581
          g_Dbs.threadCount,
          g_Dbs.db[i].dbName,
          g_Dbs.db[i].dbCfg.precision,
5582
          NULL);
H
Hui Li 已提交
5583
    }
5584
  }
5585
  //end = getCurrentTimeUs();
5586

5587
  //int64_t    totalInsertRows = 0;
5588
  //int64_t    totalAffectedRows = 0;
5589
  //for (int i = 0; i < g_Dbs.dbCount; i++) {
5590
  //  for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) {
5591
  //  totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows;
5592 5593
  //  totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows;
  //}
5594
  //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount);
5595
  postFreeResource();
5596

5597 5598 5599
  return 0;
}

5600
static void *superQueryProcess(void *sarg) {
5601
  threadInfo *winfo = (threadInfo *)sarg;
5602

5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618
  if (winfo->taos == NULL) {
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
            winfo->threadID, taos_errstr(NULL));
      return NULL;
    } else {
      winfo->taos = taos;
    }
  }

5619 5620 5621 5622 5623 5624 5625 5626
  char sqlStr[MAX_DB_NAME_SIZE + 5];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
  if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(winfo->taos);
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
    return NULL;
  }
5627

5628 5629
  int64_t st = 0;
  int64_t et = 0;
5630

5631
  int queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes;
5632 5633

  while(queryTimes --) {
5634 5635 5636
    if (g_queryInfo.specifiedQueryInfo.rate && (et - st) <
            (int64_t)g_queryInfo.specifiedQueryInfo.rate*1000) {
      taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms
5637
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5638 5639
    }

5640
    st = taosGetTimestampUs();
5641
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
5642
      if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {
5643 5644
        int64_t t1 = taosGetTimestampUs();
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
5645
        if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) {
5646
          sprintf(tmpFile, "%s-%d",
5647
                  g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID);
5648
        }
5649
        selectAndGetResult(winfo->taos, g_queryInfo.specifiedQueryInfo.sql[i], tmpFile);
5650 5651
        int64_t t2 = taosGetTimestampUs();
        printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %f s\n",
5652
                taosGetSelfPthreadId(), (t2 - t1)/1000000.0);
5653 5654
      } else {
        int64_t t1 = taosGetTimestampUs();
5655
        int retCode = postProceSql(g_queryInfo.host,
5656
                g_queryInfo.port, g_queryInfo.specifiedQueryInfo.sql[i]);
5657 5658
        int64_t t2 = taosGetTimestampUs();
        printf("=[restful] thread[%"PRId64"] complete one sql, Spent %f s\n",
5659
                taosGetSelfPthreadId(), (t2 - t1)/1000000.0);
5660

5661
        if (0 != retCode) {
5662
          printf("====restful return fail, threadID[%d]\n", winfo->threadID);
5663 5664
          return NULL;
        }
5665
      }
5666
    }
5667
    et = taosGetTimestampUs();
5668
    printf("==thread[%"PRId64"] complete all sqls to specify tables once queries duration:%.6fs\n\n",
5669
            taosGetSelfPthreadId(), (double)(et - st)/1000.0);
H
Hui Li 已提交
5670
  }
5671 5672
  return NULL;
}
H
Hui Li 已提交
5673

5674
static void replaceSubTblName(char* inSql, char* outSql, int tblIndex) {
5675 5676
  char sourceString[32] = "xxxx";
  char subTblName[MAX_TB_NAME_SIZE*3];
5677 5678
  sprintf(subTblName, "%s.%s",
          g_queryInfo.dbName,
5679
          g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN);
5680 5681

  //printf("inSql: %s\n", inSql);
5682

5683 5684
  char* pos = strstr(inSql, sourceString);
  if (0 == pos) {
5685
    return;
H
Hui Li 已提交
5686
  }
5687

5688
  tstrncpy(outSql, inSql, pos - inSql + 1);
5689
  //printf("1: %s\n", outSql);
5690 5691 5692 5693
  strcat(outSql, subTblName);
  //printf("2: %s\n", outSql);
  strcat(outSql, pos+strlen(sourceString));
  //printf("3: %s\n", outSql);
H
Hui Li 已提交
5694 5695
}

5696
static void *subQueryProcess(void *sarg) {
5697
  char sqlstr[1024];
5698
  threadInfo *winfo = (threadInfo *)sarg;
5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715

  if (winfo->taos == NULL) {
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
            winfo->threadID, taos_errstr(NULL));
      return NULL;
    } else {
      winfo->taos = taos;
    }
  }

5716
  int64_t st = 0;
5717 5718 5719
  int64_t et = (int64_t)g_queryInfo.superQueryInfo.rate*1000;

  int queryTimes = g_queryInfo.superQueryInfo.queryTimes;
5720

5721
  while(queryTimes --) {
5722 5723 5724
    if (g_queryInfo.superQueryInfo.rate
            && (et - st) < (int64_t)g_queryInfo.superQueryInfo.rate*1000) {
      taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms
5725
      //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5726
    }
H
Hui Li 已提交
5727

5728
    st = taosGetTimestampUs();
5729
    for (int i = winfo->start_table_from; i <= winfo->end_table_to; i++) {
5730
      for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
5731
        memset(sqlstr,0,sizeof(sqlstr));
5732
        replaceSubTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i);
5733
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
5734
        if (g_queryInfo.superQueryInfo.result[j][0] != 0) {
5735
          sprintf(tmpFile, "%s-%d",
5736
                  g_queryInfo.superQueryInfo.result[j],
5737
                  winfo->threadID);
5738
        }
5739
        selectAndGetResult(winfo->taos, sqlstr, tmpFile);
5740
      }
H
Hui Li 已提交
5741
    }
5742
    et = taosGetTimestampUs();
5743
    printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%d - %d] once queries duration:%.4fs\n\n",
5744 5745 5746
            taosGetSelfPthreadId(),
            winfo->start_table_from,
            winfo->end_table_to,
5747
            (double)(et - st)/1000000.0);
5748
  }
5749

5750 5751 5752
  return NULL;
}

5753
static int queryTestProcess() {
5754 5755 5756 5757

  setupForAnsiEscape();
  printfQueryMeta();
  resetAfterAnsiEscape();
5758 5759 5760 5761 5762 5763

  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          NULL,
5764
          g_queryInfo.port);
5765
  if (taos == NULL) {
5766 5767
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
5768 5769 5770
    exit(-1);
  }

5771
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
5772 5773
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
5774 5775 5776
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
5777
  }
5778

5779 5780 5781 5782
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
    (void)getchar();
  }
5783

5784
  printfQuerySystemInfo(taos);
5785

5786 5787 5788
  pthread_t  *pids  = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from specify table
5789 5790
  if (g_queryInfo.specifiedQueryInfo.sqlCount > 0
          && g_queryInfo.specifiedQueryInfo.concurrent > 0) {
5791

5792
    pids  = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t));
5793 5794 5795 5796
    if (NULL == pids) {
        taos_close(taos);
        ERROR_EXIT("memory allocation failed\n");
    }
5797
    infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo));
5798
    if (NULL == infos) {
5799
      taos_close(taos);
5800 5801
      free(pids);
      ERROR_EXIT("memory allocation failed for create threads\n");
5802
    }
5803

5804
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) {
5805
      threadInfo *t_info = infos + i;
5806
      t_info->threadID = i;
5807

5808
      if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) {
5809

5810 5811
        char sqlStr[MAX_TB_NAME_SIZE*2];
        sprintf(sqlStr, "use %s", g_queryInfo.dbName);
5812
        verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
5813 5814
        if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) {
            taos_close(taos);
5815 5816
            free(infos);
            free(pids);
5817 5818 5819 5820
            errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
            return -1;
        }
5821
      }
5822

5823 5824
      t_info->taos = NULL;// TODO: workaround to use separate taos connection;

5825 5826
      pthread_create(pids + i, NULL, superQueryProcess, t_info);
    }
5827
  } else {
5828
    g_queryInfo.specifiedQueryInfo.concurrent = 0;
5829
  }
5830

5831 5832
  taos_close(taos);

5833 5834 5835
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
  //==== create sub threads for query from all sub table of the super table
5836 5837 5838
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
    pidsOfSub  = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t));
5839 5840 5841 5842 5843 5844 5845
    if (NULL == pidsOfSub) {
      free(infos);
      free(pids);

      ERROR_EXIT("memory allocation failed for create threads\n");
    }

5846
    infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo));
5847 5848 5849 5850 5851
    if (NULL == infosOfSub) {
      free(pidsOfSub);
      free(infos);
      free(pids);
      ERROR_EXIT("memory allocation failed for create threads\n");
5852
    }
5853

5854 5855
    int ntables = g_queryInfo.superQueryInfo.childTblCount;
    int threads = g_queryInfo.superQueryInfo.threadCnt;
5856

5857 5858 5859 5860 5861
    int a = ntables / threads;
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
5862

5863 5864 5865 5866
    int b = 0;
    if (threads != 0) {
      b = ntables % threads;
    }
5867

5868
    int startFrom = 0;
5869
    for (int i = 0; i < threads; i++) {
5870 5871
      threadInfo *t_info = infosOfSub + i;
      t_info->threadID = i;
5872

5873 5874 5875 5876
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
5877
      t_info->taos = NULL; // TODO: workaround to use separate taos connection;
5878 5879 5880
      pthread_create(pidsOfSub + i, NULL, subQueryProcess, t_info);
    }

5881
    g_queryInfo.superQueryInfo.threadCnt = threads;
5882
  } else {
5883
    g_queryInfo.superQueryInfo.threadCnt = 0;
5884
  }
5885

5886
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) {
5887
    pthread_join(pids[i], NULL);
5888
  }
S
Shuaiqiang Chang 已提交
5889

5890
  tmfree((char*)pids);
5891
  tmfree((char*)infos);
5892

5893
  for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
5894
    pthread_join(pidsOfSub[i], NULL);
H
hzcheng 已提交
5895
  }
H
Hui Li 已提交
5896

5897
  tmfree((char*)pidsOfSub);
5898
  tmfree((char*)infosOfSub);
5899

5900
//  taos_close(taos);// TODO: workaround to use separate taos connection;
5901 5902 5903
  return 0;
}

5904
static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) {
5905
  if (res == NULL || taos_errno(res) != 0) {
5906 5907
    errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n",
           __func__, __LINE__, code, taos_errstr(res));
5908 5909
    return;
  }
5910

5911 5912
  getResult(res, (char*)param);
  taos_free_result(res);
H
hzcheng 已提交
5913 5914
}

5915
static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultFileName) {
5916
  TAOS_SUB* tsub = NULL;
H
hzcheng 已提交
5917

5918
  if (g_queryInfo.specifiedQueryInfo.subscribeMode) {
5919
    tsub = taos_subscribe(taos,
5920
            g_queryInfo.specifiedQueryInfo.subscribeRestart,
5921
            topic, sql, subscribe_callback, (void*)resultFileName,
5922
            g_queryInfo.specifiedQueryInfo.subscribeInterval);
5923
  } else {
5924
    tsub = taos_subscribe(taos,
5925
            g_queryInfo.specifiedQueryInfo.subscribeRestart,
5926
            topic, sql, NULL, NULL, 0);
5927
  }
5928

5929 5930 5931
  if (tsub == NULL) {
    printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql);
    return NULL;
5932
  }
5933

5934 5935
  return tsub;
}
H
hzcheng 已提交
5936

5937
static void *subSubscribeProcess(void *sarg) {
5938
  threadInfo *winfo = (threadInfo *)sarg;
5939
  char subSqlstr[1024];
5940
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT] = {0};
H
hzcheng 已提交
5941

5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957
  if (winfo->taos == NULL) {
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
            winfo->threadID, taos_errstr(NULL));
      return NULL;
    } else {
      winfo->taos = taos;
    }
  }

5958 5959
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
5960 5961
  if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
    taos_close(winfo->taos);
5962 5963
    errorPrint( "use database %s failed!\n\n",
                g_queryInfo.dbName);
5964 5965
    return NULL;
  }
5966

5967 5968 5969
  //int64_t st = 0;
  //int64_t et = 0;
  do {
5970 5971
    //if (g_queryInfo.specifiedQueryInfo.rate && (et - st) < g_queryInfo.specifiedQueryInfo.rate*1000) {
    //  taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms
5972
    //  //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
5973 5974 5975 5976
    //}

    //st = taosGetTimestampMs();
    char topic[32] = {0};
5977
    for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
5978 5979
      sprintf(topic, "taosdemo-subscribe-%d", i);
      memset(subSqlstr,0,sizeof(subSqlstr));
5980
      replaceSubTblName(g_queryInfo.superQueryInfo.sql[i], subSqlstr, i);
5981
      char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
5982
      if (g_queryInfo.superQueryInfo.result[i][0] != 0) {
5983
        sprintf(tmpFile, "%s-%d",
5984
                g_queryInfo.superQueryInfo.result[i], winfo->threadID);
5985
      }
5986
      tsub[i] = subscribeImpl(winfo->taos, subSqlstr, topic, tmpFile);
5987
      if (NULL == tsub[i]) {
5988
        taos_close(winfo->taos);
5989 5990 5991 5992
        return NULL;
      }
    }
    //et = taosGetTimestampMs();
5993
    //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0);
5994 5995 5996 5997 5998
  } while (0);

  // start loop to consume result
  TAOS_RES* res = NULL;
  while (1) {
5999 6000
    for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
      if (1 == g_queryInfo.superQueryInfo.subscribeMode) {
6001 6002
        continue;
      }
6003

6004
      res = taos_consume(tsub[i]);
6005 6006
      if (res) {
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
6007
        if (g_queryInfo.superQueryInfo.result[i][0] != 0) {
6008
          sprintf(tmpFile, "%s-%d",
6009
                  g_queryInfo.superQueryInfo.result[i],
6010
                  winfo->threadID);
6011 6012
        }
        getResult(res, tmpFile);
H
hzcheng 已提交
6013 6014 6015
      }
    }
  }
6016
  taos_free_result(res);
6017

6018
  for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
6019
    taos_unsubscribe(tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress);
6020
  }
6021 6022

  taos_close(winfo->taos);
H
hzcheng 已提交
6023 6024 6025
  return NULL;
}

6026
static void *superSubscribeProcess(void *sarg) {
6027
  threadInfo *winfo = (threadInfo *)sarg;
6028
  TAOS_SUB*    tsub[MAX_QUERY_SQL_COUNT] = {0};
H
hzcheng 已提交
6029

6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045
  if (winfo->taos == NULL) {
    TAOS * taos = NULL;
    taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
          g_queryInfo.port);
    if (taos == NULL) {
      errorPrint("[%d] Failed to connect to TDengine, reason:%s\n",
            winfo->threadID, taos_errstr(NULL));
      return NULL;
    } else {
      winfo->taos = taos;
    }
  }

6046 6047
  char sqlStr[MAX_TB_NAME_SIZE*2];
  sprintf(sqlStr, "use %s", g_queryInfo.dbName);
6048
  debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr);
6049
  if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) {
6050
    taos_close(winfo->taos);
6051 6052
    return NULL;
  }
6053

6054 6055 6056
  //int64_t st = 0;
  //int64_t et = 0;
  do {
6057 6058
    //if (g_queryInfo.specifiedQueryInfo.rate && (et - st) < g_queryInfo.specifiedQueryInfo.rate*1000) {
    //  taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms
6059
    //  //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to);
6060 6061 6062 6063
    //}

    //st = taosGetTimestampMs();
    char topic[32] = {0};
6064
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
6065 6066
      sprintf(topic, "taosdemo-subscribe-%d", i);
      char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
6067
      if (g_queryInfo.superQueryInfo.result[i][0] != 0) {
6068
        sprintf(tmpFile, "%s-%d",
6069
                g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID);
6070
      }
6071
      tsub[i] = subscribeImpl(winfo->taos, g_queryInfo.specifiedQueryInfo.sql[i], topic, tmpFile);
6072
      if (NULL == g_queryInfo.specifiedQueryInfo.tsub[i]) {
6073
        taos_close(winfo->taos);
6074 6075 6076 6077
        return NULL;
      }
    }
    //et = taosGetTimestampMs();
6078
    //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0);
6079 6080 6081 6082 6083
  } while (0);

  // start loop to consume result
  TAOS_RES* res = NULL;
  while (1) {
6084 6085
    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
      if (1 == g_queryInfo.specifiedQueryInfo.subscribeMode) {
6086 6087
        continue;
      }
6088

6089
      res = taos_consume(tsub[i]);
6090 6091
      if (res) {
        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
6092
        if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) {
6093
          sprintf(tmpFile, "%s-%d",
6094
                  g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID);
6095 6096 6097 6098 6099 6100
        }
        getResult(res, tmpFile);
      }
    }
  }
  taos_free_result(res);
6101

6102
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
6103
    taos_unsubscribe(tsub[i], g_queryInfo.specifiedQueryInfo.subscribeKeepProgress);
6104
  }
6105 6106

  taos_close(winfo->taos);
H
hzcheng 已提交
6107 6108 6109
  return NULL;
}

6110
static int subscribeTestProcess() {
6111
  setupForAnsiEscape();
6112
  printfQueryMeta();
6113
  resetAfterAnsiEscape();
6114

6115 6116
  if (!g_args.answer_yes) {
    printf("Press enter key to continue\n\n");
6117
    (void) getchar();
Y
yihaoDeng 已提交
6118
  }
6119

6120 6121 6122 6123 6124
  TAOS * taos = NULL;
  taos = taos_connect(g_queryInfo.host,
          g_queryInfo.user,
          g_queryInfo.password,
          g_queryInfo.dbName,
6125
          g_queryInfo.port);
6126
  if (taos == NULL) {
6127 6128
    errorPrint( "Failed to connect to TDengine, reason:%s\n",
            taos_errstr(NULL));
6129 6130 6131
    exit(-1);
  }

6132
  if (0 != g_queryInfo.superQueryInfo.sqlCount) {
6133 6134
    getAllChildNameOfSuperTable(taos,
            g_queryInfo.dbName,
6135 6136 6137
            g_queryInfo.superQueryInfo.sTblName,
            &g_queryInfo.superQueryInfo.childTblName,
            &g_queryInfo.superQueryInfo.childTblCount);
6138 6139
  }

6140 6141
  taos_close(taos); // TODO: workaround to use separate taos connection;

6142 6143 6144
  pthread_t  *pids = NULL;
  threadInfo *infos = NULL;
  //==== create sub threads for query from super table
6145 6146
  if ((g_queryInfo.specifiedQueryInfo.sqlCount <= 0) ||
          (g_queryInfo.specifiedQueryInfo.concurrent <= 0)) {
6147
    errorPrint("%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n",
6148 6149
              __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount,
              g_queryInfo.specifiedQueryInfo.concurrent);
6150 6151 6152
    exit(-1);
  }

6153 6154
  pids  = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t));
  infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo));
6155
  if ((NULL == pids) || (NULL == infos)) {
6156
      errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
6157
      exit(-1);
6158
  }
6159

6160
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) {
6161 6162
      threadInfo *t_info = infos + i;
      t_info->threadID = i;
6163
      t_info->taos = NULL;  // TODO: workaround to use separate taos connection;
6164
      pthread_create(pids + i, NULL, superSubscribeProcess, t_info);
H
hzcheng 已提交
6165
  }
6166

6167
  //==== create sub threads for query from sub table
6168 6169
  pthread_t  *pidsOfSub  = NULL;
  threadInfo *infosOfSub = NULL;
6170 6171 6172
  if ((g_queryInfo.superQueryInfo.sqlCount > 0)
          && (g_queryInfo.superQueryInfo.threadCnt > 0)) {
    pidsOfSub  = malloc(g_queryInfo.superQueryInfo.threadCnt *
6173
            sizeof(pthread_t));
6174
    infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt *
6175
            sizeof(threadInfo));
6176
    if ((NULL == pidsOfSub) || (NULL == infosOfSub)) {
6177 6178 6179
      errorPrint("%s() LN%d, malloc failed for create threads\n",
              __func__, __LINE__);
      // taos_close(taos);
6180
      exit(-1);
S
Shuaiqiang Chang 已提交
6181
    }
6182

6183 6184
    int ntables = g_queryInfo.superQueryInfo.childTblCount;
    int threads = g_queryInfo.superQueryInfo.threadCnt;
6185

6186 6187 6188 6189 6190
    int a = ntables / threads;
    if (a < 1) {
      threads = ntables;
      a = 1;
    }
6191

6192 6193 6194 6195
    int b = 0;
    if (threads != 0) {
      b = ntables % threads;
    }
6196

6197
    int startFrom = 0;
6198
    for (int i = 0; i < threads; i++) {
6199 6200
      threadInfo *t_info = infosOfSub + i;
      t_info->threadID = i;
6201

6202 6203 6204 6205
      t_info->start_table_from = startFrom;
      t_info->ntables = i<b?a+1:a;
      t_info->end_table_to = i < b ? startFrom + a : startFrom + a - 1;
      startFrom = t_info->end_table_to + 1;
6206
      t_info->taos = NULL; // TODO: workaround to use separate taos connection;
6207
      pthread_create(pidsOfSub + i, NULL, subSubscribeProcess, t_info);
H
hzcheng 已提交
6208
    }
6209

6210
    g_queryInfo.superQueryInfo.threadCnt = threads;
6211

6212
    for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) {
6213 6214
      pthread_join(pidsOfSub[i], NULL);
    }
H
hzcheng 已提交
6215
  }
6216

6217
  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) {
6218
    pthread_join(pids[i], NULL);
6219
  }
H
hzcheng 已提交
6220

6221
  tmfree((char*)pids);
6222
  tmfree((char*)infos);
H
hzcheng 已提交
6223

6224
  tmfree((char*)pidsOfSub);
6225
  tmfree((char*)infosOfSub);
6226
//   taos_close(taos);
6227
  return 0;
H
hzcheng 已提交
6228 6229
}

6230
static void initOfInsertMeta() {
6231
  memset(&g_Dbs, 0, sizeof(SDbs));
6232

6233
  // set default values
6234
  tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
6235
  g_Dbs.port = 6030;
6236 6237
  tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
6238
  g_Dbs.threadCount = 2;
6239 6240

  g_Dbs.use_metric = g_args.use_metric;
L
Liu Tao 已提交
6241 6242
}

6243
static void initOfQueryMeta() {
6244
  memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo));
6245

6246
  // set default values
6247
  tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
6248
  g_queryInfo.port = 6030;
6249 6250
  tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE);
  tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE);
L
Liu Tao 已提交
6251 6252
}

6253
static void setParaFromArg(){
6254
  if (g_args.host) {
6255
    tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE);
6256
  } else {
6257
    tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE);
6258
  }
L
Liu Tao 已提交
6259

6260
  if (g_args.user) {
6261
    tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE);
6262
  }
6263 6264

  if (g_args.password) {
6265
    tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE);
6266 6267
  }

6268 6269
  if (g_args.port) {
    g_Dbs.port = g_args.port;
6270
  }
L
Liu Tao 已提交
6271

6272 6273 6274
  g_Dbs.threadCount = g_args.num_of_threads;
  g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;

6275 6276
  g_Dbs.dbCount = 1;
  g_Dbs.db[0].drop = 1;
6277

6278
  tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE);
6279
  g_Dbs.db[0].dbCfg.replica = g_args.replica;
6280
  tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE);
L
Liu Tao 已提交
6281

6282
  tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN);
6283 6284 6285 6286 6287

  g_Dbs.use_metric = g_args.use_metric;
  g_Dbs.insert_only = g_args.insert_only;

  g_Dbs.do_aggreFunc = true;
L
Liu Tao 已提交
6288

6289 6290
  char dataString[STRING_LEN];
  char **data_type = g_args.datatype;
6291

6292
  memset(dataString, 0, STRING_LEN);
L
Liu Tao 已提交
6293

6294 6295
  if (strcasecmp(data_type[0], "BINARY") == 0
          || strcasecmp(data_type[0], "BOOL") == 0
6296
          || strcasecmp(data_type[0], "NCHAR") == 0 ) {
6297
    g_Dbs.do_aggreFunc = false;
L
Liu Tao 已提交
6298
  }
H
hzcheng 已提交
6299

6300 6301 6302 6303 6304
  if (g_args.use_metric) {
    g_Dbs.db[0].superTblCount = 1;
    tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE);
    g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables;
    g_Dbs.threadCount = g_args.num_of_threads;
6305
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
6306
    g_Dbs.queryMode = g_args.mode;
6307

6308 6309 6310 6311
    g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL;
    g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS;
    g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange;
    g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio;
6312
    tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix,
6313 6314 6315
            g_args.tb_prefix, MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE);
    tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE);
6316
    tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp,
6317
            "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE);
6318
    g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
6319

6320
    g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
6321
    g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len;
6322

6323 6324 6325 6326 6327
    g_Dbs.db[0].superTbls[0].columnCount = 0;
    for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
      if (data_type[i] == NULL) {
        break;
      }
6328

6329
      tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType,
6330
              data_type[i], MAX_TB_NAME_SIZE);
6331
      g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary;
6332 6333
      g_Dbs.db[0].superTbls[0].columnCount++;
    }
6334

6335 6336 6337 6338 6339
    if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) {
      g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR;
    } else {
      for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) {
        tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE);
6340
        g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0;
6341 6342 6343
        g_Dbs.db[0].superTbls[0].columnCount++;
      }
    }
6344

6345
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE);
6346
    g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0;
6347

6348
    tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE);
6349 6350
    g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary;
    g_Dbs.db[0].superTbls[0].tagCount = 2;
6351
  } else {
6352 6353
    g_Dbs.threadCountByCreateTbl = g_args.num_of_threads;
    g_Dbs.db[0].superTbls[0].tagCount = 0;
H
hzcheng 已提交
6354
  }
6355 6356 6357 6358 6359 6360
}

/* Function to do regular expression check */
static int regexMatch(const char *s, const char *reg, int cflags) {
  regex_t regex;
  char    msgbuf[100] = {0};
H
hzcheng 已提交
6361

6362 6363 6364
  /* Compile regular expression */
  if (regcomp(&regex, reg, cflags) != 0) {
    printf("Fail to compile regex\n");
H
Hui Li 已提交
6365 6366 6367
    exit(-1);
  }

6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391
  /* Execute regular expression */
  int reti = regexec(&regex, s, 0, NULL, 0);
  if (!reti) {
    regfree(&regex);
    return 1;
  } else if (reti == REG_NOMATCH) {
    regfree(&regex);
    return 0;
  } else {
    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
    printf("Regex match failed: %s\n", msgbuf);
    regfree(&regex);
    exit(-1);
  }

  return 0;
}

static int isCommentLine(char *line) {
  if (line == NULL) return 1;

  return regexMatch(line, "^\\s*#.*", REG_EXTENDED);
}

6392
static void querySqlFile(TAOS* taos, char* sqlFile)
6393 6394 6395 6396 6397 6398
{
  FILE *fp = fopen(sqlFile, "r");
  if (fp == NULL) {
    printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno));
    return;
  }
6399

6400 6401 6402 6403 6404 6405
  int       read_len = 0;
  char *    cmd = calloc(1, MAX_SQL_SIZE);
  size_t    cmd_len = 0;
  char *    line = NULL;
  size_t    line_len = 0;

6406
  double t = getCurrentTimeUs();
6407

6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420
  while ((read_len = tgetline(&line, &line_len, fp)) != -1) {
    if (read_len >= MAX_SQL_SIZE) continue;
    line[--read_len] = '\0';

    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
      continue;
    }

    if (line[read_len - 1] == '\\') {
      line[read_len - 1] = ' ';
      memcpy(cmd + cmd_len, line, read_len);
      cmd_len += read_len;
      continue;
6421
    }
6422 6423

    memcpy(cmd + cmd_len, line, read_len);
6424
    verbosePrint("%s() LN%d cmd: %s\n", __func__, __LINE__, cmd);
6425 6426 6427
    if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) {
        errorPrint("%s() LN%d, queryDbExec %s failed!\n",
               __func__, __LINE__, cmd);
6428 6429 6430 6431 6432
        tmfree(cmd);
        tmfree(line);
        tmfclose(fp);
        return;
    }
6433 6434
    memset(cmd, 0, MAX_SQL_SIZE);
    cmd_len = 0;
H
hzcheng 已提交
6435 6436
  }

6437
  t = getCurrentTimeUs() - t;
6438
  printf("run %s took %.6f second(s)\n\n", sqlFile, t);
6439

6440 6441 6442 6443
  tmfree(cmd);
  tmfree(line);
  tmfclose(fp);
  return;
H
hzcheng 已提交
6444 6445
}

6446
static void testMetaFile() {
6447
    if (INSERT_TEST == g_args.test_mode) {
6448 6449
      if (g_Dbs.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir);
6450

6451
      insertTestProcess();
6452

6453
    } else if (QUERY_TEST == g_args.test_mode) {
6454 6455
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
6456

6457
      queryTestProcess();
6458

6459
    } else if (SUBSCRIBE_TEST == g_args.test_mode) {
6460 6461
      if (g_queryInfo.cfgDir[0])
          taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir);
6462

6463
      subscribeTestProcess();
6464

6465 6466 6467
    }  else {
      ;
    }
6468
}
6469

6470
static void queryResult() {
6471 6472 6473
    // select
    if (false == g_Dbs.insert_only) {
      // query data
6474

6475 6476 6477
      pthread_t read_id;
      threadInfo *rInfo = malloc(sizeof(threadInfo));
      rInfo->start_time = 1500000000000;  // 2017-07-14 10:40:00.000
6478
      rInfo->start_table_from = 0;
6479

6480
      //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc;
6481
      if (g_args.use_metric) {
6482 6483
        rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount;
        rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1;
6484
        rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0];
6485 6486
        tstrncpy(rInfo->tb_prefix,
              g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE);
6487
      } else {
6488 6489
        rInfo->ntables = g_args.num_of_tables;
        rInfo->end_table_to = g_args.num_of_tables -1;
6490
        tstrncpy(rInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE);
6491 6492
      }

6493
      rInfo->taos = taos_connect(
6494 6495 6496 6497
              g_Dbs.host,
              g_Dbs.user,
              g_Dbs.password,
              g_Dbs.db[0].dbName,
6498
              g_Dbs.port);
6499
      if (rInfo->taos == NULL) {
6500 6501
        errorPrint( "Failed to connect to TDengine, reason:%s\n",
                taos_errstr(NULL));
6502 6503 6504 6505
        free(rInfo);
        exit(-1);
      }

6506
      tstrncpy(rInfo->fp, g_Dbs.resultFile, MAX_FILE_NAME_LEN);
6507 6508 6509 6510 6511 6512 6513 6514 6515

      if (!g_Dbs.use_metric) {
        pthread_create(&read_id, NULL, readTable, rInfo);
      } else {
        pthread_create(&read_id, NULL, readMetric, rInfo);
      }
      pthread_join(read_id, NULL);
      taos_close(rInfo->taos);
      free(rInfo);
H
hzcheng 已提交
6516
    }
6517 6518
}

6519 6520
static void testCmdLine() {

6521 6522 6523 6524 6525 6526 6527 6528 6529 6530
  if (strlen(configDir)) {
    wordexp_t full_path;
    if (wordexp(configDir, &full_path, 0) != 0) {
      errorPrint( "Invalid path %s\n", configDir);
      return;
    }
    taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]);
    wordfree(&full_path);
  }

6531 6532
  g_args.test_mode = INSERT_TEST;
  insertTestProcess();
6533

6534 6535 6536 6537
  if (g_Dbs.insert_only)
    return;
  else
    queryResult();
6538 6539
}

6540 6541 6542
int main(int argc, char *argv[]) {
  parse_args(argc, argv, &g_args);

6543
  debugPrint("meta file: %s\n", g_args.metaFile);
6544 6545 6546

  if (g_args.metaFile) {
    initOfInsertMeta();
6547
    initOfQueryMeta();
6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560

    if (false == getInfoFromJsonFile(g_args.metaFile)) {
      printf("Failed to read %s\n", g_args.metaFile);
      return 1;
    }

    testMetaFile();
  } else {
    memset(&g_Dbs, 0, sizeof(SDbs));
    setParaFromArg();

    if (NULL != g_args.sqlFile) {
      TAOS* qtaos = taos_connect(
6561 6562 6563 6564
          g_Dbs.host,
          g_Dbs.user,
          g_Dbs.password,
          g_Dbs.db[0].dbName,
6565
          g_Dbs.port);
6566
      querySqlFile(qtaos, g_args.sqlFile);
6567 6568 6569 6570 6571
      taos_close(qtaos);

    } else {
      testCmdLine();
    }
H
hzcheng 已提交
6572
  }
6573 6574

  return 0;
H
hzcheng 已提交
6575
}
6576