create_table.c 12.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#define _DEFAULT_SOURCE
#include "os.h"
S
Shengliang Guan 已提交
18
#include "taos.h"
19
#include "taoserror.h"
S
Shengliang Guan 已提交
20
#include "ulog.h"
21 22 23 24 25

#define GREEN "\033[1;32m"
#define NC "\033[0m"

char    dbName[32] = "db";
S
Shengliang Guan 已提交
26
char    stbName[64] = "st";
S
Shengliang Guan 已提交
27
int32_t numOfThreads = 1;
S
Shengliang Guan 已提交
28
int64_t numOfTables = 200000;
S
Shengliang Guan 已提交
29 30
int32_t createTable = 1;
int32_t insertData = 0;
S
Shengliang Guan 已提交
31
int32_t batchNum = 100;
S
Shengliang Guan 已提交
32
int32_t numOfVgroups = 2;
L
lihui 已提交
33
int32_t showTablesFlag = 0;
34 35

typedef struct {
S
Shengliang Guan 已提交
36 37
  int64_t   tableBeginIndex;
  int64_t   tableEndIndex;
38 39
  int32_t   threadIndex;
  char      dbName[32];
S
Shengliang Guan 已提交
40
  char      stbName[64];
41
  float     createTableSpeed;
S
Shengliang Guan 已提交
42
  float     insertDataSpeed;
S
Shengliang Guan 已提交
43
  int64_t   startMs;
L
lihui 已提交
44 45
  int64_t   maxDelay;
  int64_t   minDelay;
46 47 48
  pthread_t thread;
} SThreadInfo;

49 50 51
// void  parseArgument(int32_t argc, char *argv[]);
// void *threadFunc(void *param);
// void  createDbAndStb();
52

S
Shengliang Guan 已提交
53
void createDbAndStb() {
54 55
  pPrint("start to create db and stable");
  char qstr[64000];
S
Shengliang Guan 已提交
56

57 58
  TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0);
  if (con == NULL) {
L
lihui 已提交
59
    pError("failed to connect to DB, reason:%s", taos_errstr(NULL));
60 61 62
    exit(1);
  }

S
Shengliang Guan 已提交
63
  sprintf(qstr, "create database if not exists %s vgroups %d", dbName, numOfVgroups);
L
lihui 已提交
64 65
  TAOS_RES *pRes = taos_query(con, qstr);
  int32_t   code = taos_errno(pRes);
66
  if (code != 0) {
67 68
    pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(pRes),
           taos_errstr(pRes));
69 70
    exit(0);
  }
L
lihui 已提交
71
  taos_free_result(pRes);
72 73

  sprintf(qstr, "use %s", dbName);
L
lihui 已提交
74 75
  pRes = taos_query(con, qstr);
  code = taos_errno(pRes);
76
  if (code != 0) {
L
lihui 已提交
77
    pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes));
78 79
    exit(0);
  }
L
lihui 已提交
80
  taos_free_result(pRes);
81

S
Shengliang Guan 已提交
82
  sprintf(qstr, "create table %s (ts timestamp, i int) tags (j int)", stbName);
L
lihui 已提交
83 84
  pRes = taos_query(con, qstr);
  code = taos_errno(pRes);
S
Shengliang Guan 已提交
85
  if (code != 0) {
L
lihui 已提交
86
    pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes));
S
Shengliang Guan 已提交
87 88
    exit(0);
  }
L
lihui 已提交
89
  taos_free_result(pRes);
S
Shengliang Guan 已提交
90

91 92 93
  taos_close(con);
}

S
Shengliang Guan 已提交
94
void printCreateProgress(SThreadInfo *pInfo, int64_t t) {
S
Shengliang Guan 已提交
95
  int64_t endMs = taosGetTimestampMs();
S
Shengliang Guan 已提交
96
  int64_t totalTables = t - pInfo->tableBeginIndex;
S
Shengliang Guan 已提交
97 98 99
  float   seconds = (endMs - pInfo->startMs) / 1000.0;
  float   speed = totalTables / seconds;
  pInfo->createTableSpeed = speed;
S
Shengliang Guan 已提交
100 101
  pPrint("thread:%d, %" PRId64 " tables created, time:%.2f sec, speed:%.1f tables/second, ", pInfo->threadIndex,
         totalTables, seconds, speed);
S
Shengliang Guan 已提交
102 103
}

S
Shengliang Guan 已提交
104
void printInsertProgress(SThreadInfo *pInfo, int64_t t) {
S
Shengliang Guan 已提交
105
  int64_t endMs = taosGetTimestampMs();
S
Shengliang Guan 已提交
106
  int64_t totalTables = t - pInfo->tableBeginIndex;
S
Shengliang Guan 已提交
107 108 109
  float   seconds = (endMs - pInfo->startMs) / 1000.0;
  float   speed = totalTables / seconds;
  pInfo->insertDataSpeed = speed;
S
Shengliang Guan 已提交
110 111
  pPrint("thread:%d, %" PRId64 " rows inserted, time:%.2f sec, speed:%.1f rows/second, ", pInfo->threadIndex,
         totalTables, seconds, speed);
S
Shengliang Guan 已提交
112 113
}

L
lihui 已提交
114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
static int64_t getResult(TAOS_RES *tres) {
  TAOS_ROW row = taos_fetch_row(tres);
  if (row == NULL) {
    return 0;
  }

  int         num_fields = taos_num_fields(tres);
  TAOS_FIELD *fields = taos_fetch_fields(tres);
  int         precision = taos_result_precision(tres);

  int64_t numOfRows = 0;
  do {
    numOfRows++;
    row = taos_fetch_row(tres);
  } while (row != NULL);

  return numOfRows;
}

133
void showTables() {
L
lihui 已提交
134
  pPrint("start to show tables");
135
  char qstr[128];
L
lihui 已提交
136 137 138 139 140 141 142

  TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0);
  if (con == NULL) {
    pError("failed to connect to DB, reason:%s", taos_errstr(NULL));
    exit(1);
  }

143
  snprintf(qstr, 128, "use %s", dbName);
L
lihui 已提交
144
  TAOS_RES *pRes = taos_query(con, qstr);
145
  int       code = taos_errno(pRes);
L
lihui 已提交
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162
  if (code != 0) {
    pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes));
    exit(1);
  }
  taos_free_result(pRes);

  sprintf(qstr, "show tables");
  pRes = taos_query(con, qstr);
  code = taos_errno(pRes);
  if (code != 0) {
    pError("failed to show tables, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes));
    exit(0);
  }

  int64_t totalTableNum = getResult(pRes);
  taos_free_result(pRes);

163
  pPrint("%s database: %s, total %" PRId64 " tables %s", GREEN, dbName, totalTableNum, NC);
L
lihui 已提交
164 165 166 167

  taos_close(con);
}

168 169
void *threadFunc(void *param) {
  SThreadInfo *pInfo = (SThreadInfo *)param;
S
Shengliang Guan 已提交
170 171
  char        *qstr = malloc(2000 * 1000);
  int32_t      code = 0;
172 173 174

  TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0);
  if (con == NULL) {
L
lihui 已提交
175
    pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(NULL));
176 177 178
    exit(1);
  }

179 180
  // printf("thread:%d, table range: %"PRId64 " - %"PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex,
  // pInfo->tableEndIndex);
181
  sprintf(qstr, "use %s", pInfo->dbName);
L
lihui 已提交
182 183 184
  TAOS_RES *pRes = taos_query(con, qstr);
  taos_free_result(pRes);

S
Shengliang Guan 已提交
185
  if (createTable) {
L
lihui 已提交
186 187 188
    int64_t curMs = 0;
    int64_t beginMs = taosGetTimestampMs();
    pInfo->startMs = beginMs;
189
    int64_t t = pInfo->tableBeginIndex;
L
lihui 已提交
190
    for (; t <= pInfo->tableEndIndex;) {
191 192
      // int64_t batch = (pInfo->tableEndIndex - t);
      // batch = MIN(batch, batchNum);
S
Shengliang Guan 已提交
193 194

      int32_t len = sprintf(qstr, "create table");
L
lihui 已提交
195 196
      for (int32_t i = 0; i < batchNum;) {
        len += sprintf(qstr + len, " %s_t%" PRId64 " using %s tags(%" PRId64 ")", stbName, t, stbName, t);
197 198
        t++;
        i++;
L
lihui 已提交
199
        if (t > pInfo->tableEndIndex) {
200 201
          break;
        }
S
Shengliang Guan 已提交
202
      }
S
Shengliang Guan 已提交
203

204
      int64_t   startTs = taosGetTimestampUs();
L
lihui 已提交
205 206
      TAOS_RES *pRes = taos_query(con, qstr);
      code = taos_errno(pRes);
S
TD-1415  
Shengliang Guan 已提交
207
      if (code != 0) {
S
Shengliang Guan 已提交
208
        pError("failed to create table t%" PRId64 ", reason:%s", t, tstrerror(code));
S
TD-1415  
Shengliang Guan 已提交
209
      }
L
lihui 已提交
210
      taos_free_result(pRes);
211 212 213 214
      int64_t endTs = taosGetTimestampUs();
      int64_t delay = endTs - startTs;
      // printf("==== %"PRId64" -  %"PRId64", %"PRId64"\n", startTs, endTs, delay);
      if (delay > pInfo->maxDelay) pInfo->maxDelay = delay;
L
lihui 已提交
215 216
      if (delay < pInfo->minDelay) pInfo->minDelay = delay;

217 218 219 220
      curMs = taosGetTimestampMs();
      if (curMs - beginMs > 10000) {
        beginMs = curMs;
        // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t);
S
Shengliang Guan 已提交
221 222
        printCreateProgress(pInfo, t);
      }
S
TD-1415  
Shengliang Guan 已提交
223
    }
L
lihui 已提交
224
    printCreateProgress(pInfo, t);
S
Shengliang Guan 已提交
225 226 227
  }

  if (insertData) {
L
lihui 已提交
228
    int64_t curMs = 0;
229 230
    int64_t beginMs = taosGetTimestampMs();
    ;
L
lihui 已提交
231

S
Shengliang Guan 已提交
232
    pInfo->startMs = taosGetTimestampMs();
S
Shengliang Guan 已提交
233 234
    for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) {
      int64_t batch = (pInfo->tableEndIndex - t);
S
Shengliang Guan 已提交
235 236 237 238
      batch = MIN(batch, batchNum);

      int32_t len = sprintf(qstr, "insert into");
      for (int32_t i = 0; i < batch; ++i) {
S
Shengliang Guan 已提交
239
        len += sprintf(qstr + len, " t%" PRId64 " values(now, %" PRId64 ")", t + i, t + i);
S
Shengliang Guan 已提交
240 241
      }

L
lihui 已提交
242 243
      TAOS_RES *pRes = taos_query(con, qstr);
      code = taos_errno(pRes);
S
TD-1415  
Shengliang Guan 已提交
244
      if (code != 0) {
S
Shengliang Guan 已提交
245
        pError("failed to insert table t%" PRId64 ", reason:%s", t, tstrerror(code));
S
TD-1415  
Shengliang Guan 已提交
246
      }
L
lihui 已提交
247
      taos_free_result(pRes);
S
Shengliang Guan 已提交
248

L
lihui 已提交
249
      curMs = taosGetTimestampMs();
250
      if (curMs - beginMs > 10000) {
S
Shengliang Guan 已提交
251 252 253
        printInsertProgress(pInfo, t);
      }
      t += (batch - 1);
254
    }
S
Shengliang Guan 已提交
255
    printInsertProgress(pInfo, pInfo->tableEndIndex);
256 257 258
  }

  taos_close(con);
S
Shengliang Guan 已提交
259
  free(qstr);
260 261 262 263 264 265 266 267 268 269 270 271
  return 0;
}

void printHelp() {
  char indent[10] = "        ";
  printf("Used to test the performance while create table\n");

  printf("%s%s\n", indent, "-c");
  printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir);
  printf("%s%s\n", indent, "-d");
  printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName);
  printf("%s%s\n", indent, "-s");
S
Shengliang Guan 已提交
272
  printf("%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stbName);
273 274 275
  printf("%s%s\n", indent, "-t");
  printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", numOfThreads);
  printf("%s%s\n", indent, "-n");
S
Shengliang Guan 已提交
276
  printf("%s%s%s%" PRId64 "\n", indent, indent, "numOfTables, default is ", numOfTables);
S
Shengliang Guan 已提交
277 278 279 280 281 282 283 284
  printf("%s%s\n", indent, "-v");
  printf("%s%s%s%d\n", indent, indent, "numOfVgroups, default is ", numOfVgroups);
  printf("%s%s\n", indent, "-a");
  printf("%s%s%s%d\n", indent, indent, "createTable, default is ", createTable);
  printf("%s%s\n", indent, "-i");
  printf("%s%s%s%d\n", indent, indent, "insertData, default is ", insertData);
  printf("%s%s\n", indent, "-b");
  printf("%s%s%s%d\n", indent, indent, "batchNum, default is ", batchNum);
L
lihui 已提交
285 286
  printf("%s%s\n", indent, "-w");
  printf("%s%s%s%d\n", indent, indent, "showTablesFlag, default is ", showTablesFlag);
S
Shengliang Guan 已提交
287

288 289 290
  exit(EXIT_SUCCESS);
}

S
Shengliang Guan 已提交
291 292
void parseArgument(int32_t argc, char *argv[]) {
  for (int32_t i = 1; i < argc; i++) {
293 294 295 296 297 298 299 300
    if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
      printHelp();
      exit(0);
    } else if (strcmp(argv[i], "-d") == 0) {
      strcpy(dbName, argv[++i]);
    } else if (strcmp(argv[i], "-c") == 0) {
      strcpy(configDir, argv[++i]);
    } else if (strcmp(argv[i], "-s") == 0) {
S
Shengliang Guan 已提交
301
      strcpy(stbName, argv[++i]);
302 303 304
    } else if (strcmp(argv[i], "-t") == 0) {
      numOfThreads = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-n") == 0) {
S
Shengliang Guan 已提交
305
      numOfTables = atoll(argv[++i]);
L
lihui 已提交
306
    } else if (strcmp(argv[i], "-v") == 0) {
S
Shengliang Guan 已提交
307 308 309 310 311 312 313
      numOfVgroups = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-a") == 0) {
      createTable = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-i") == 0) {
      insertData = atoi(argv[++i]);
    } else if (strcmp(argv[i], "-b") == 0) {
      batchNum = atoi(argv[++i]);
L
lihui 已提交
314 315
    } else if (strcmp(argv[i], "-w") == 0) {
      showTablesFlag = atoi(argv[++i]);
316 317 318 319 320
    } else {
    }
  }

  pPrint("%s dbName:%s %s", GREEN, dbName, NC);
S
Shengliang Guan 已提交
321
  pPrint("%s stbName:%s %s", GREEN, stbName, NC);
322
  pPrint("%s configDir:%s %s", GREEN, configDir, NC);
S
Shengliang Guan 已提交
323
  pPrint("%s numOfTables:%" PRId64 " %s", GREEN, numOfTables, NC);
324
  pPrint("%s numOfThreads:%d %s", GREEN, numOfThreads, NC);
S
Shengliang Guan 已提交
325 326 327 328
  pPrint("%s numOfVgroups:%d %s", GREEN, numOfVgroups, NC);
  pPrint("%s createTable:%d %s", GREEN, createTable, NC);
  pPrint("%s insertData:%d %s", GREEN, insertData, NC);
  pPrint("%s batchNum:%d %s", GREEN, batchNum, NC);
L
lihui 已提交
329
  pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC);
S
Shengliang Guan 已提交
330

331 332
  pPrint("%s start create table performace test %s", GREEN, NC);
}
L
lihui 已提交
333 334 335 336 337

int32_t main(int32_t argc, char *argv[]) {
  parseArgument(argc, argv);

  if (showTablesFlag) {
338 339
    showTables();
    return 0;
L
lihui 已提交
340
  }
341

L
lihui 已提交
342 343
  createDbAndStb();

344
  pPrint("%d threads are spawned to create %" PRId64 " tables", numOfThreads, numOfTables);
L
lihui 已提交
345 346 347 348 349 350

  pthread_attr_t thattr;
  pthread_attr_init(&thattr);
  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
  SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo));

351 352
  // int64_t numOfTablesPerThread = numOfTables / numOfThreads;
  // numOfTables = numOfTablesPerThread * numOfThreads;
L
lihui 已提交
353 354 355 356

  if (numOfThreads < 1) {
    numOfThreads = 1;
  }
357

L
lihui 已提交
358 359
  int64_t a = numOfTables / numOfThreads;
  if (a < 1) {
360 361
    numOfThreads = numOfTables;
    a = 1;
L
lihui 已提交
362
  }
363

L
lihui 已提交
364 365 366 367 368 369 370 371 372
  int64_t b = 0;
  b = numOfTables % numOfThreads;

  int64_t tableFrom = 0;
  for (int32_t i = 0; i < numOfThreads; ++i) {
    pInfo[i].tableBeginIndex = tableFrom;
    pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1;
    tableFrom = pInfo[i].tableEndIndex + 1;
    pInfo[i].threadIndex = i;
373
    pInfo[i].minDelay = INT64_MAX;
L
lihui 已提交
374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391
    strcpy(pInfo[i].dbName, dbName);
    strcpy(pInfo[i].stbName, stbName);
    pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i));
  }

  taosMsleep(300);
  for (int32_t i = 0; i < numOfThreads; i++) {
    pthread_join(pInfo[i].thread, NULL);
  }

  int64_t maxDelay = 0;
  int64_t minDelay = INT64_MAX;

  float createTableSpeed = 0;
  for (int32_t i = 0; i < numOfThreads; ++i) {
    createTableSpeed += pInfo[i].createTableSpeed;

    if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay;
392
    if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay;
L
lihui 已提交
393 394 395 396 397 398 399
  }

  float insertDataSpeed = 0;
  for (int32_t i = 0; i < numOfThreads; ++i) {
    insertDataSpeed += pInfo[i].insertDataSpeed;
  }

400 401 402
  pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64
         "us %s",
         GREEN, numOfTables, createTableSpeed, numOfThreads, maxDelay, minDelay, NC);
L
lihui 已提交
403 404

  if (insertData) {
405 406
    pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed,
           numOfThreads, NC);
L
lihui 已提交
407 408 409 410 411
  }

  pthread_attr_destroy(&thattr);
  free(pInfo);
}