clientSml.c 61.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

wmmhello's avatar
wmmhello 已提交
16 17 18 19 20
#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

21 22 23 24 25 26
#include "clientSml.h"

int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL};
int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL};

X
Xiaoyu Wang 已提交
27
void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) {
wmmhello's avatar
wmmhello 已提交
28
  NodeList *tmp = list;
X
Xiaoyu Wang 已提交
29 30 31
  while (tmp) {
    if (fn == NULL) {
      if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
32 33
        return tmp->data.value;
      }
X
Xiaoyu Wang 已提交
34 35
    } else {
      if (tmp->data.used && fn(tmp->data.key, key) == 0) {
36 37
        return tmp->data.value;
      }
wmmhello's avatar
wmmhello 已提交
38
    }
39

wmmhello's avatar
wmmhello 已提交
40 41 42 43 44
    tmp = tmp->next;
  }
  return NULL;
}

X
Xiaoyu Wang 已提交
45
int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) {
wmmhello's avatar
wmmhello 已提交
46
  NodeList *tmp = *list;
X
Xiaoyu Wang 已提交
47 48 49 50
  while (tmp) {
    if (!tmp->data.used) break;
    if (fn == NULL) {
      if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
51 52
        return -1;
      }
X
Xiaoyu Wang 已提交
53 54
    } else {
      if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) {
55 56
        return -1;
      }
wmmhello's avatar
wmmhello 已提交
57
    }
58

wmmhello's avatar
wmmhello 已提交
59 60
    tmp = tmp->next;
  }
X
Xiaoyu Wang 已提交
61
  if (tmp) {
wmmhello's avatar
wmmhello 已提交
62 63 64 65
    tmp->data.key = key;
    tmp->data.keyLen = len;
    tmp->data.value = value;
    tmp->data.used = true;
X
Xiaoyu Wang 已提交
66
  } else {
67
    NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList));
X
Xiaoyu Wang 已提交
68
    if (newNode == NULL) {
wmmhello's avatar
wmmhello 已提交
69 70 71 72 73 74 75 76 77 78 79 80
      return -1;
    }
    newNode->data.key = key;
    newNode->data.keyLen = len;
    newNode->data.value = value;
    newNode->data.used = true;
    newNode->next = *list;
    *list = newNode;
  }
  return 0;
}

X
Xiaoyu Wang 已提交
81
int nodeListSize(NodeList *list) {
wmmhello's avatar
wmmhello 已提交
82
  int cnt = 0;
X
Xiaoyu Wang 已提交
83 84 85 86 87
  while (list) {
    if (list->data.used)
      cnt++;
    else
      break;
wmmhello's avatar
wmmhello 已提交
88 89 90 91
    list = list->next;
  }
  return cnt;
}
wmmhello's avatar
wmmhello 已提交
92

93
inline bool smlDoubleToInt64OverFlow(double num) {
X
Xiaoyu Wang 已提交
94
  if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true;
95 96 97
  return false;
}

98
int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) {
99
  if (pBuf->buf) {
100 101 102 103 104 105 106
    memset(pBuf->buf, 0, pBuf->len);
    if (msg1) strncat(pBuf->buf, msg1, pBuf->len);
    int32_t left = pBuf->len - strlen(pBuf->buf);
    if (left > 2 && msg2) {
      strncat(pBuf->buf, ":", left - 1);
      strncat(pBuf->buf, msg2, left - 2);
    }
wmmhello's avatar
wmmhello 已提交
107
  }
wmmhello's avatar
wmmhello 已提交
108 109 110
  return TSDB_CODE_SML_INVALID_DATA;
}

111 112 113 114 115
int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision) {
  char   *endPtr = NULL;
  int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10);
  if (unlikely(value + len != endPtr)) {
    return -1;
116
  }
wmmhello's avatar
wmmhello 已提交
117

X
Xiaoyu Wang 已提交
118
  if (unlikely(fromPrecision >= TSDB_TIME_PRECISION_HOURS)) {
119
    int64_t unit = smlToMilli[fromPrecision - TSDB_TIME_PRECISION_HOURS];
D
dapan1121 已提交
120
    if (tsInt64 != 0 && unit > INT64_MAX / tsInt64) {
121 122 123 124
      return -1;
    }
    tsInt64 *= unit;
    fromPrecision = TSDB_TIME_PRECISION_MILLI;
wmmhello's avatar
wmmhello 已提交
125
  }
wmmhello's avatar
wmmhello 已提交
126

127
  return convertTimePrecision(tsInt64, fromPrecision, toPrecision);
wmmhello's avatar
wmmhello 已提交
128 129
}

130 131 132 133 134 135 136
int8_t smlGetTsTypeByLen(int32_t len) {
  if (len == TSDB_TIME_PRECISION_SEC_DIGITS) {
    return TSDB_TIME_PRECISION_SECONDS;
  } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) {
    return TSDB_TIME_PRECISION_MILLI;
  } else {
    return -1;
137 138 139
  }
}

X
Xiaoyu Wang 已提交
140
SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) {
141 142 143
  SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1);
  if (!tag) {
    return NULL;
144 145
  }

146 147
  tag->sTableName = measure;
  tag->sTableNameLen = measureLen;
148

149 150 151 152
  tag->cols = taosArrayInit(numRows, POINTER_BYTES);
  if (tag->cols == NULL) {
    uError("SML:smlBuildTableInfo failed to allocate memory");
    goto cleanup;
153
  }
154

X
Xiaoyu Wang 已提交
155 156 157 158 159
  //  tag->tags = taosArrayInit(16, sizeof(SSmlKv));
  //  if (tag->tags == NULL) {
  //    uError("SML:smlBuildTableInfo failed to allocate memory");
  //    goto cleanup;
  //  }
160 161
  return tag;

X
Xiaoyu Wang 已提交
162
cleanup:
163 164
  taosMemoryFree(tag);
  return NULL;
165 166
}

167
static int32_t smlParseTableName(SArray *tags, char *childTableName) {
X
Xiaoyu Wang 已提交
168
  size_t childTableNameLen = strlen(tsSmlChildTableName);
169 170
  if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS;

X
Xiaoyu Wang 已提交
171
  for (int i = 0; i < taosArrayGetSize(tags); i++) {
172 173 174 175 176
    SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i);
    // handle child table name
    if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) {
      memset(childTableName, 0, TSDB_TABLE_NAME_LEN);
      strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN));
wmmhello's avatar
wmmhello 已提交
177
      taosArrayRemove(tags, i);
178
      break;
wmmhello's avatar
wmmhello 已提交
179 180
    }
  }
181

wmmhello's avatar
wmmhello 已提交
182 183 184
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
185
int32_t smlSetCTableName(SSmlTableInfo *oneTable) {
186
  smlParseTableName(oneTable->tags, oneTable->childTableName);
187

188
  if (strlen(oneTable->childTableName) == 0) {
X
Xiaoyu Wang 已提交
189
    SArray       *dst = taosArrayDup(oneTable->tags, NULL);
wmmhello's avatar
wmmhello 已提交
190
    RandTableName rName = {dst, oneTable->sTableName, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName};
wmmhello's avatar
wmmhello 已提交
191

192 193
    buildChildTableName(&rName);
    taosArrayDestroy(dst);
194
  }
195 196
  return TSDB_CODE_SUCCESS;
}
197

198 199 200 201
SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) {
  SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1);
  if (!meta) {
    return NULL;
202 203
  }

X
Xiaoyu Wang 已提交
204
  if (unlikely(!isDataFormat)) {
205 206 207 208 209
    meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
    if (meta->tagHash == NULL) {
      uError("SML:smlBuildSTableMeta failed to allocate memory");
      goto cleanup;
    }
wmmhello's avatar
wmmhello 已提交
210

211 212 213 214 215
    meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
    if (meta->colHash == NULL) {
      uError("SML:smlBuildSTableMeta failed to allocate memory");
      goto cleanup;
    }
wmmhello's avatar
wmmhello 已提交
216 217
  }

218 219 220 221
  meta->tags = taosArrayInit(32, sizeof(SSmlKv));
  if (meta->tags == NULL) {
    uError("SML:smlBuildSTableMeta failed to allocate memory");
    goto cleanup;
222 223
  }

224 225 226 227 228 229
  meta->cols = taosArrayInit(32, sizeof(SSmlKv));
  if (meta->cols == NULL) {
    uError("SML:smlBuildSTableMeta failed to allocate memory");
    goto cleanup;
  }
  return meta;
230

X
Xiaoyu Wang 已提交
231
cleanup:
232 233
  taosMemoryFree(meta);
  return NULL;
wmmhello's avatar
wmmhello 已提交
234 235
}

X
Xiaoyu Wang 已提交
236 237 238 239 240 241 242
// uint16_t smlCalTypeSum(char* endptr, int32_t left){
//   uint16_t sum = 0;
//   for(int i = 0; i < left; i++){
//     sum += endptr[i];
//   }
//   return sum;
// }
wmmhello's avatar
wmmhello 已提交
243

X
Xiaoyu Wang 已提交
244 245 246
#define RETURN_FALSE                                 \
  smlBuildInvalidDataMsg(msg, "invalid data", pVal); \
  return false;
wmmhello's avatar
wmmhello 已提交
247

X
Xiaoyu Wang 已提交
248 249 250 251 252 253 254 255 256 257
#define SET_DOUBLE                     \
  kvVal->type = TSDB_DATA_TYPE_DOUBLE; \
  kvVal->d = result;

#define SET_FLOAT                                                                              \
  if (!IS_VALID_FLOAT(result)) {                                                               \
    smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \
    return false;                                                                              \
  }                                                                                            \
  kvVal->type = TSDB_DATA_TYPE_FLOAT;                                                          \
wmmhello's avatar
wmmhello 已提交
258 259
  kvVal->f = (float)result;

X
Xiaoyu Wang 已提交
260
#define SET_BIGINT                                                                                       \
wmmhello's avatar
wmmhello 已提交
261 262 263 264 265 266 267 268
  errno = 0;                                                                                             \
  int64_t tmp = taosStr2Int64(pVal, &endptr, 10);                                                        \
  if (errno == ERANGE) {                                                                                 \
    smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
    return false;                                                                                        \
  }                                                                                                      \
  kvVal->type = TSDB_DATA_TYPE_BIGINT;                                                                   \
  kvVal->i = tmp;
wmmhello's avatar
wmmhello 已提交
269

X
Xiaoyu Wang 已提交
270 271 272 273 274 275
#define SET_INT                                                                    \
  if (!IS_VALID_INT(result)) {                                                     \
    smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \
    return false;                                                                  \
  }                                                                                \
  kvVal->type = TSDB_DATA_TYPE_INT;                                                \
wmmhello's avatar
wmmhello 已提交
276 277
  kvVal->i = result;

X
Xiaoyu Wang 已提交
278 279 280 281 282 283
#define SET_SMALL_INT                                                          \
  if (!IS_VALID_SMALLINT(result)) {                                            \
    smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \
    return false;                                                              \
  }                                                                            \
  kvVal->type = TSDB_DATA_TYPE_SMALLINT;                                       \
wmmhello's avatar
wmmhello 已提交
284 285
  kvVal->i = result;

X
Xiaoyu Wang 已提交
286
#define SET_UBIGINT                                                                             \
wmmhello's avatar
wmmhello 已提交
287 288 289 290 291 292 293 294
  errno = 0;                                                                                    \
  uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);                                             \
  if (errno == ERANGE || result < 0) {                                                          \
    smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
    return false;                                                                               \
  }                                                                                             \
  kvVal->type = TSDB_DATA_TYPE_UBIGINT;                                                         \
  kvVal->u = tmp;
wmmhello's avatar
wmmhello 已提交
295

X
Xiaoyu Wang 已提交
296 297 298 299 300 301
#define SET_UINT                                                                  \
  if (!IS_VALID_UINT(result)) {                                                   \
    smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \
    return false;                                                                 \
  }                                                                               \
  kvVal->type = TSDB_DATA_TYPE_UINT;                                              \
wmmhello's avatar
wmmhello 已提交
302 303
  kvVal->u = result;

X
Xiaoyu Wang 已提交
304 305 306 307 308 309
#define SET_USMALL_INT                                                            \
  if (!IS_VALID_USMALLINT(result)) {                                              \
    smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \
    return false;                                                                 \
  }                                                                               \
  kvVal->type = TSDB_DATA_TYPE_USMALLINT;                                         \
wmmhello's avatar
wmmhello 已提交
310 311
  kvVal->u = result;

X
Xiaoyu Wang 已提交
312 313 314 315 316 317
#define SET_TINYINT                                                       \
  if (!IS_VALID_TINYINT(result)) {                                        \
    smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \
    return false;                                                         \
  }                                                                       \
  kvVal->type = TSDB_DATA_TYPE_TINYINT;                                   \
wmmhello's avatar
wmmhello 已提交
318 319
  kvVal->i = result;

X
Xiaoyu Wang 已提交
320 321 322 323 324 325
#define SET_UTINYINT                                                            \
  if (!IS_VALID_UTINYINT(result)) {                                             \
    smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \
    return false;                                                               \
  }                                                                             \
  kvVal->type = TSDB_DATA_TYPE_UTINYINT;                                        \
wmmhello's avatar
wmmhello 已提交
326 327
  kvVal->u = result;

328
bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) {
wmmhello's avatar
wmmhello 已提交
329 330
  const char *pVal = kvVal->value;
  int32_t     len = kvVal->length;
X
Xiaoyu Wang 已提交
331
  char       *endptr = NULL;
wmmhello's avatar
wmmhello 已提交
332 333 334 335 336 337 338 339 340
  double      result = taosStr2Double(pVal, &endptr);
  if (pVal == endptr) {
    RETURN_FALSE
  }

  int32_t left = len - (endptr - pVal);
  if (left == 0) {
    SET_DOUBLE
  } else if (left == 3) {
X
Xiaoyu Wang 已提交
341 342
    if (endptr[0] == 'f' || endptr[0] == 'F') {
      if (endptr[1] == '6' && endptr[2] == '4') {
wmmhello's avatar
wmmhello 已提交
343
        SET_DOUBLE
X
Xiaoyu Wang 已提交
344
      } else if (endptr[1] == '3' && endptr[2] == '2') {
wmmhello's avatar
wmmhello 已提交
345
        SET_FLOAT
X
Xiaoyu Wang 已提交
346
      } else {
wmmhello's avatar
wmmhello 已提交
347 348
        RETURN_FALSE
      }
X
Xiaoyu Wang 已提交
349 350
    } else if (endptr[0] == 'i' || endptr[0] == 'I') {
      if (endptr[1] == '6' && endptr[2] == '4') {
wmmhello's avatar
wmmhello 已提交
351
        SET_BIGINT
X
Xiaoyu Wang 已提交
352
      } else if (endptr[1] == '3' && endptr[2] == '2') {
wmmhello's avatar
wmmhello 已提交
353
        SET_INT
X
Xiaoyu Wang 已提交
354
      } else if (endptr[1] == '1' && endptr[2] == '6') {
wmmhello's avatar
wmmhello 已提交
355
        SET_SMALL_INT
X
Xiaoyu Wang 已提交
356
      } else {
wmmhello's avatar
wmmhello 已提交
357 358
        RETURN_FALSE
      }
X
Xiaoyu Wang 已提交
359 360
    } else if (endptr[0] == 'u' || endptr[0] == 'U') {
      if (endptr[1] == '6' && endptr[2] == '4') {
wmmhello's avatar
wmmhello 已提交
361
        SET_UBIGINT
X
Xiaoyu Wang 已提交
362
      } else if (endptr[1] == '3' && endptr[2] == '2') {
wmmhello's avatar
wmmhello 已提交
363
        SET_UINT
X
Xiaoyu Wang 已提交
364
      } else if (endptr[1] == '1' && endptr[2] == '6') {
wmmhello's avatar
wmmhello 已提交
365
        SET_USMALL_INT
X
Xiaoyu Wang 已提交
366
      } else {
wmmhello's avatar
wmmhello 已提交
367 368
        RETURN_FALSE
      }
wmmhello's avatar
wmmhello 已提交
369
    } else {
wmmhello's avatar
wmmhello 已提交
370 371
      RETURN_FALSE
    }
X
Xiaoyu Wang 已提交
372 373 374
  } else if (left == 2) {
    if (endptr[0] == 'i' || endptr[0] == 'I') {
      if (endptr[1] == '8') {
wmmhello's avatar
wmmhello 已提交
375
        SET_TINYINT
X
Xiaoyu Wang 已提交
376
      } else {
wmmhello's avatar
wmmhello 已提交
377 378
        RETURN_FALSE
      }
X
Xiaoyu Wang 已提交
379
    } else if (endptr[0] == 'u' || endptr[0] == 'U') {
wmmhello's avatar
wmmhello 已提交
380 381 382 383 384
      if (endptr[1] == '8') {
        SET_UTINYINT
      } else {
        RETURN_FALSE
      }
X
Xiaoyu Wang 已提交
385
    } else {
wmmhello's avatar
wmmhello 已提交
386 387
      RETURN_FALSE
    }
X
Xiaoyu Wang 已提交
388 389
  } else if (left == 1) {
    if (endptr[0] == 'i' || endptr[0] == 'I') {
wmmhello's avatar
wmmhello 已提交
390
      SET_BIGINT
X
Xiaoyu Wang 已提交
391
    } else if (endptr[0] == 'u' || endptr[0] == 'U') {
wmmhello's avatar
wmmhello 已提交
392
      SET_UBIGINT
X
Xiaoyu Wang 已提交
393
    } else {
wmmhello's avatar
wmmhello 已提交
394 395 396 397 398 399 400 401 402
      RETURN_FALSE
    }
  } else {
    RETURN_FALSE;
  }
  return true;
}

bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) {
wmmhello's avatar
wmmhello 已提交
403
  const char *pVal = kvVal->value;
X
Xiaoyu Wang 已提交
404 405 406 407
  int32_t     len = kvVal->length;
  char       *endptr = NULL;
  double      result = taosStr2Double(pVal, &endptr);
  if (pVal == endptr) {
408
    smlBuildInvalidDataMsg(msg, "invalid data", pVal);
wmmhello's avatar
wmmhello 已提交
409 410 411
    return false;
  }

412
  int32_t left = len - (endptr - pVal);
X
Xiaoyu Wang 已提交
413
  if (left == 0 || (left == 3 && strncasecmp(endptr, "f64", left) == 0)) {
414 415
    kvVal->type = TSDB_DATA_TYPE_DOUBLE;
    kvVal->d = result;
X
Xiaoyu Wang 已提交
416 417
  } else if ((left == 3 && strncasecmp(endptr, "f32", left) == 0)) {
    if (!IS_VALID_FLOAT(result)) {
418 419
      smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal);
      return false;
wmmhello's avatar
wmmhello 已提交
420
    }
421 422
    kvVal->type = TSDB_DATA_TYPE_FLOAT;
    kvVal->f = (float)result;
X
Xiaoyu Wang 已提交
423 424
  } else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)) {
    if (smlDoubleToInt64OverFlow(result)) {
wmmhello's avatar
wmmhello 已提交
425 426
      errno = 0;
      int64_t tmp = taosStr2Int64(pVal, &endptr, 10);
X
Xiaoyu Wang 已提交
427
      if (errno == ERANGE) {
wmmhello's avatar
wmmhello 已提交
428 429 430 431 432 433
        smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal);
        return false;
      }
      kvVal->type = TSDB_DATA_TYPE_BIGINT;
      kvVal->i = tmp;
      return true;
wmmhello's avatar
wmmhello 已提交
434
    }
435
    kvVal->type = TSDB_DATA_TYPE_BIGINT;
wmmhello's avatar
wmmhello 已提交
436
    kvVal->i = (int64_t)result;
wmmhello's avatar
wmmhello 已提交
437
  } else if ((left == 1 && *endptr == 'u') || (left == 3 && strncasecmp(endptr, "u64", left) == 0)) {
X
Xiaoyu Wang 已提交
438
    if (result >= (double)UINT64_MAX || result < 0) {
wmmhello's avatar
wmmhello 已提交
439 440
      errno = 0;
      uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);
X
Xiaoyu Wang 已提交
441
      if (errno == ERANGE || result < 0) {
wmmhello's avatar
wmmhello 已提交
442 443 444 445 446 447
        smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal);
        return false;
      }
      kvVal->type = TSDB_DATA_TYPE_UBIGINT;
      kvVal->u = tmp;
      return true;
448
    }
449
    kvVal->type = TSDB_DATA_TYPE_UBIGINT;
wmmhello's avatar
wmmhello 已提交
450
    kvVal->u = result;
X
Xiaoyu Wang 已提交
451 452
  } else if (left == 3 && strncasecmp(endptr, "i32", left) == 0) {
    if (!IS_VALID_INT(result)) {
453 454
      smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal);
      return false;
wmmhello's avatar
wmmhello 已提交
455
    }
456 457
    kvVal->type = TSDB_DATA_TYPE_INT;
    kvVal->i = result;
X
Xiaoyu Wang 已提交
458 459
  } else if (left == 3 && strncasecmp(endptr, "u32", left) == 0) {
    if (!IS_VALID_UINT(result)) {
460 461
      smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal);
      return false;
wmmhello's avatar
wmmhello 已提交
462
    }
463 464
    kvVal->type = TSDB_DATA_TYPE_UINT;
    kvVal->u = result;
X
Xiaoyu Wang 已提交
465 466
  } else if (left == 3 && strncasecmp(endptr, "i16", left) == 0) {
    if (!IS_VALID_SMALLINT(result)) {
467 468
      smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal);
      return false;
wmmhello's avatar
wmmhello 已提交
469
    }
470 471
    kvVal->type = TSDB_DATA_TYPE_SMALLINT;
    kvVal->i = result;
X
Xiaoyu Wang 已提交
472 473
  } else if (left == 3 && strncasecmp(endptr, "u16", left) == 0) {
    if (!IS_VALID_USMALLINT(result)) {
474 475
      smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal);
      return false;
wmmhello's avatar
wmmhello 已提交
476
    }
477 478
    kvVal->type = TSDB_DATA_TYPE_USMALLINT;
    kvVal->u = result;
X
Xiaoyu Wang 已提交
479 480
  } else if (left == 2 && strncasecmp(endptr, "i8", left) == 0) {
    if (!IS_VALID_TINYINT(result)) {
481 482
      smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal);
      return false;
wmmhello's avatar
wmmhello 已提交
483
    }
484 485
    kvVal->type = TSDB_DATA_TYPE_TINYINT;
    kvVal->i = result;
X
Xiaoyu Wang 已提交
486 487
  } else if (left == 2 && strncasecmp(endptr, "u8", left) == 0) {
    if (!IS_VALID_UTINYINT(result)) {
488 489
      smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal);
      return false;
wmmhello's avatar
wmmhello 已提交
490
    }
491 492
    kvVal->type = TSDB_DATA_TYPE_UTINYINT;
    kvVal->u = result;
X
Xiaoyu Wang 已提交
493
  } else {
494
    smlBuildInvalidDataMsg(msg, "invalid data", pVal);
wmmhello's avatar
wmmhello 已提交
495 496
    return false;
  }
497
  return true;
wmmhello's avatar
wmmhello 已提交
498 499
}

X
Xiaoyu Wang 已提交
500
STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen) {
501
  STableMeta *pTableMeta = NULL;
wmmhello's avatar
wmmhello 已提交
502

503 504
  SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
  tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname));
wmmhello's avatar
wmmhello 已提交
505

506 507 508 509 510 511 512
  SRequestConnInfo conn = {0};
  conn.pTrans = info->taos->pAppInfo->pTransporter;
  conn.requestId = info->pRequest->requestId;
  conn.requestObjRefId = info->pRequest->self;
  conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp);
  memset(pName.tname, 0, TSDB_TABLE_NAME_LEN);
  memcpy(pName.tname, measure, measureLen);
wmmhello's avatar
wmmhello 已提交
513

514
  int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
X
Xiaoyu Wang 已提交
515
  if (code != TSDB_CODE_SUCCESS) {
516 517
    return NULL;
  }
518
  return pTableMeta;
wmmhello's avatar
wmmhello 已提交
519
}
wmmhello's avatar
wmmhello 已提交
520

521 522
static int64_t smlGenId() {
  static volatile int64_t linesSmlHandleId = 0;
wmmhello's avatar
wmmhello 已提交
523

524 525 526 527
  int64_t id = 0;
  do {
    id = atomic_add_fetch_64(&linesSmlHandleId, 1);
  } while (id == 0);
528

529
  return id;
530 531
}

532 533 534 535 536
static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSmlKv *kv, bool isTag,
                                       ESchemaAction *action, SSmlHandle *info) {
  uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL;
  if (index) {
    if (colField[*index].type != kv->type) {
537
      uError("SML:0x%" PRIx64 " point type and db type mismatch. point type: %d, db type: %d, key: %s", info->id, colField[*index].type, kv->type, kv->key);
538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556
      return TSDB_CODE_TSC_INVALID_VALUE;
    }

    if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR &&
         (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) ||
        (colField[*index].type == TSDB_DATA_TYPE_NCHAR &&
         ((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) {
      if (isTag) {
        *action = SCHEMA_ACTION_CHANGE_TAG_SIZE;
      } else {
        *action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE;
      }
    }
  } else {
    if (isTag) {
      *action = SCHEMA_ACTION_ADD_TAG;
    } else {
      *action = SCHEMA_ACTION_ADD_COLUMN;
    }
wmmhello's avatar
wmmhello 已提交
557
  }
558
  return 0;
559 560
}

561 562 563 564
static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) {
  int32_t result = 1;
  while (result <= length) {
    result *= 2;
565
  }
566 567 568 569
  if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
    result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE;
  } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
    result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
570
  }
571 572 573 574 575

  if (type == TSDB_DATA_TYPE_NCHAR) {
    result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
  } else if (type == TSDB_DATA_TYPE_BINARY) {
    result = result + VARSTR_HEADER_SIZE;
576
  }
577
  return result;
578 579
}

580 581 582 583 584 585 586 587 588 589
static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols,
                                      ESchemaAction *action, bool isTag) {
  int32_t code = TSDB_CODE_SUCCESS;
  for (int j = 0; j < taosArrayGetSize(cols); ++j) {
    if (j == 0 && !isTag) continue;
    SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j);
    code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info);
    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }
590
  }
591
  return TSDB_CODE_SUCCESS;
592 593
}

594 595 596 597 598
static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) {
  SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
  int32_t   i = 0;
  for (; i < length; i++) {
    taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES);
599 600
  }

601 602 603 604
  if (isTag) {
    i = 0;
  } else {
    i = 1;
605
  }
606 607 608 609 610 611
  for (; i < taosArrayGetSize(cols); i++) {
    SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
    if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) {
      taosHashCleanup(hashTmp);
      return -1;
    }
612
  }
613 614 615
  taosHashCleanup(hashTmp);
  return 0;
}
616

617 618 619 620 621 622
static int32_t getBytes(uint8_t type, int32_t length) {
  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
    return smlFindNearestPowerOf2(length, type);
  } else {
    return tDataTypes[type].bytes;
  }
623 624
}

625 626 627 628 629
static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols,
                                  SArray *results, int32_t numOfCols, bool isTag) {
  for (int j = 0; j < taosArrayGetSize(cols); ++j) {
    SSmlKv       *kv = (SSmlKv *)taosArrayGet(cols, j);
    ESchemaAction action = SCHEMA_ACTION_NULL;
X
Xiaoyu Wang 已提交
630 631
    int           code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info);
    if (code != 0) {
D
dapan1121 已提交
632 633
      return code;
    }
634 635 636 637 638 639 640 641
    if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) {
      SField field = {0};
      field.type = kv->type;
      field.bytes = getBytes(kv->type, kv->length);
      memcpy(field.name, kv->key, kv->keyLen);
      taosArrayPush(results, &field);
    } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) {
      uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen);
X
Xiaoyu Wang 已提交
642
      if (index == NULL) {
D
dapan1121 已提交
643 644 645
        uError("smlBuildFieldsList get error, key:%s", kv->key);
        return TSDB_CODE_SML_INVALID_DATA;
      }
X
Xiaoyu Wang 已提交
646
      uint16_t newIndex = *index;
647 648 649
      if (isTag) newIndex -= numOfCols;
      SField *field = (SField *)taosArrayGet(results, newIndex);
      field->bytes = getBytes(kv->type, kv->length);
650 651 652 653 654
    }
  }
  return TSDB_CODE_SUCCESS;
}

655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671
// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData,
//                               int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){
static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta,
                              ESchemaAction action) {
  SRequestObj   *pRequest = NULL;
  SMCreateStbReq pReq = {0};
  int32_t        code = TSDB_CODE_SUCCESS;
  SCmdMsgInfo    pCmdMsg = {0};

  // put front for free
  pReq.numOfColumns = taosArrayGetSize(pColumns);
  pReq.pColumns = pColumns;
  pReq.numOfTags = taosArrayGetSize(pTags);
  pReq.pTags = pTags;

  code = buildRequest(info->taos->id, "", 0, NULL, false, &pRequest, 0);
  if (code != TSDB_CODE_SUCCESS) {
672 673
    goto end;
  }
674 675 676 677

  pRequest->syncQuery = true;
  if (!pRequest->pDb) {
    code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
678 679 680
    goto end;
  }

681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696
  if (action == SCHEMA_ACTION_CREATE_STABLE) {
    pReq.colVer = 1;
    pReq.tagVer = 1;
    pReq.suid = 0;
    pReq.source = TD_REQ_FROM_APP;
  } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) {
    pReq.colVer = pTableMeta->sversion;
    pReq.tagVer = pTableMeta->tversion + 1;
    pReq.suid = pTableMeta->uid;
    pReq.source = TD_REQ_FROM_TAOX;
  } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) {
    pReq.colVer = pTableMeta->sversion + 1;
    pReq.tagVer = pTableMeta->tversion;
    pReq.suid = pTableMeta->uid;
    pReq.source = TD_REQ_FROM_TAOX;
  }
697

698 699 700 701
  if (pReq.numOfTags == 0) {
    pReq.numOfTags = 1;
    SField field = {0};
    field.type = TSDB_DATA_TYPE_NCHAR;
702
    field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
703 704
    strcpy(field.name, tsSmlTagName);
    taosArrayPush(pReq.pTags, &field);
705
  }
wmmhello's avatar
wmmhello 已提交
706

707 708 709
  pReq.commentLen = -1;
  pReq.igExists = true;
  tNameExtractFullName(pName, pReq.name);
710

711 712 713 714 715 716 717 718 719
  pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp);
  pCmdMsg.msgType = TDMT_MND_CREATE_STB;
  pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq);
  pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen);
  if (NULL == pCmdMsg.pMsg) {
    code = TSDB_CODE_OUT_OF_MEMORY;
    goto end;
  }
  tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq);
720

721 722 723 724 725 726
  SQuery pQuery;
  memset(&pQuery, 0, sizeof(pQuery));
  pQuery.execMode = QUERY_EXEC_MODE_RPC;
  pQuery.pCmdMsg = &pCmdMsg;
  pQuery.msgType = pQuery.pCmdMsg->msgType;
  pQuery.stableQuery = true;
727

728
  launchQueryImpl(pRequest, &pQuery, true, NULL);
729

730 731
  if (pRequest->code == TSDB_CODE_SUCCESS) {
    catalogRemoveTableMeta(info->pCatalog, pName);
732
  }
733 734
  code = pRequest->code;
  taosMemoryFree(pCmdMsg.pMsg);
735

X
Xiaoyu Wang 已提交
736
end:
737 738 739 740
  destroyRequest(pRequest);
  tFreeSMCreateStbReq(&pReq);
  return code;
}
741

742
static int32_t smlModifyDBSchemas(SSmlHandle *info) {
X
Xiaoyu Wang 已提交
743 744
  uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas start, format:%d, needModifySchema:%d", info->id, info->dataFormat,
         info->needModifySchema);
X
Xiaoyu Wang 已提交
745
  if (info->dataFormat && !info->needModifySchema) {
746
    return TSDB_CODE_SUCCESS;
747
  }
748 749 750
  int32_t     code = 0;
  SHashObj   *hashTmp = NULL;
  STableMeta *pTableMeta = NULL;
751 752 753 754 755 756 757 758 759

  SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
  tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname));

  SRequestConnInfo conn = {0};
  conn.pTrans = info->taos->pAppInfo->pTransporter;
  conn.requestId = info->pRequest->requestId;
  conn.requestObjRefId = info->pRequest->self;
  conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp);
760

761
  SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
762
  while (tmp) {
763
    SSmlSTableMeta *sTableData = *tmp;
764
    bool            needCheckMeta = false;  // for multi thread
765

766 767
    size_t superTableLen = 0;
    void  *superTable = taosHashGetKey(tmp, &superTableLen);
768 769
    memset(pName.tname, 0, TSDB_TABLE_NAME_LEN);
    memcpy(pName.tname, superTable, superTableLen);
770

771
    code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
772

773
    if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) {
D
dapan1121 已提交
774
      uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname);
775 776
      SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField));
      SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField));
D
dapan1121 已提交
777 778 779 780 781 782 783 784 785 786
      code = smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true);
      if (code != TSDB_CODE_SUCCESS) {
        uError("SML:0x%" PRIx64 " smlBuildFieldsList tag1 failed. %s", info->id, pName.tname);
        goto end;
      }
      code = smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false);
      if (code != TSDB_CODE_SUCCESS) {
        uError("SML:0x%" PRIx64 " smlBuildFieldsList col1 failed. %s", info->id, pName.tname);
        goto end;
      }
787 788 789 790
      code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE);
      if (code != TSDB_CODE_SUCCESS) {
        uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname);
        goto end;
791
      }
792 793
      info->cost.numOfCreateSTables++;
      taosMemoryFreeClear(pTableMeta);
794

795 796 797 798
      code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
      if (code != TSDB_CODE_SUCCESS) {
        uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname);
        goto end;
799
      }
800 801 802 803 804 805
    } else if (code == TSDB_CODE_SUCCESS) {
      hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true,
                             HASH_NO_LOCK);
      for (uint16_t i = pTableMeta->tableInfo.numOfColumns;
           i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) {
        taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES);
806
      }
807

808 809 810 811
      ESchemaAction action = SCHEMA_ACTION_NULL;
      code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &action, true);
      if (code != TSDB_CODE_SUCCESS) {
        goto end;
812
      }
813
      if (action != SCHEMA_ACTION_NULL) {
X
Xiaoyu Wang 已提交
814 815
        uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname,
               action);
816 817 818 819
        SArray *pColumns =
            taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
        SArray *pTags =
            taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField));
820

821 822 823 824 825 826 827 828 829
        for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) {
          SField field = {0};
          field.type = pTableMeta->schema[i].type;
          field.bytes = pTableMeta->schema[i].bytes;
          strcpy(field.name, pTableMeta->schema[i].name);
          if (i < pTableMeta->tableInfo.numOfColumns) {
            taosArrayPush(pColumns, &field);
          } else {
            taosArrayPush(pTags, &field);
830 831
          }
        }
D
dapan1121 已提交
832
        code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags,
X
Xiaoyu Wang 已提交
833
                                  pTableMeta->tableInfo.numOfColumns, true);
D
dapan1121 已提交
834 835 836 837
        if (code != TSDB_CODE_SUCCESS) {
          uError("SML:0x%" PRIx64 " smlBuildFieldsList tag2 failed. %s", info->id, pName.tname);
          goto end;
        }
838

839 840 841 842
        code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action);
        if (code != TSDB_CODE_SUCCESS) {
          uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname);
          goto end;
843
        }
844

845 846 847 848 849 850 851 852 853
        info->cost.numOfAlterTagSTables++;
        taosMemoryFreeClear(pTableMeta);
        code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1);
        if (code != TSDB_CODE_SUCCESS) {
          goto end;
        }
        code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
        if (code != TSDB_CODE_SUCCESS) {
          goto end;
wmmhello's avatar
wmmhello 已提交
854
        }
855 856
      }

857 858 859
      taosHashClear(hashTmp);
      for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) {
        taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES);
860
      }
861 862 863 864
      action = SCHEMA_ACTION_NULL;
      code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &action, false);
      if (code != TSDB_CODE_SUCCESS) {
        goto end;
865
      }
866
      if (action != SCHEMA_ACTION_NULL) {
X
Xiaoyu Wang 已提交
867 868
        uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname,
               action);
869 870 871 872
        SArray *pColumns =
            taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField));
        SArray *pTags =
            taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField));
873

874 875 876 877 878 879 880 881 882 883
        for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) {
          SField field = {0};
          field.type = pTableMeta->schema[i].type;
          field.bytes = pTableMeta->schema[i].bytes;
          strcpy(field.name, pTableMeta->schema[i].name);
          if (i < pTableMeta->tableInfo.numOfColumns) {
            taosArrayPush(pColumns, &field);
          } else {
            taosArrayPush(pTags, &field);
          }
884 885
        }

D
dapan1121 已提交
886
        code = smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns,
X
Xiaoyu Wang 已提交
887
                                  pTableMeta->tableInfo.numOfColumns, false);
D
dapan1121 已提交
888 889 890 891
        if (code != TSDB_CODE_SUCCESS) {
          uError("SML:0x%" PRIx64 " smlBuildFieldsList col2 failed. %s", info->id, pName.tname);
          goto end;
        }
892

893 894 895 896
        code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action);
        if (code != TSDB_CODE_SUCCESS) {
          uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname);
          goto end;
897
        }
898

899 900 901 902 903 904 905 906 907 908
        info->cost.numOfAlterColSTables++;
        taosMemoryFreeClear(pTableMeta);
        code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1);
        if (code != TSDB_CODE_SUCCESS) {
          goto end;
        }
        code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta);
        if (code != TSDB_CODE_SUCCESS) {
          uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname);
          goto end;
909 910
        }
      }
wmmhello's avatar
wmmhello 已提交
911

912 913 914
      needCheckMeta = true;
      taosHashCleanup(hashTmp);
      hashTmp = NULL;
X
Xiaoyu Wang 已提交
915
    } else {
916 917
      uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code));
      goto end;
wmmhello's avatar
wmmhello 已提交
918
    }
919

920 921 922 923 924 925 926 927 928 929 930
    if (needCheckMeta) {
      code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags,
                          sTableData->tags, true);
      if (code != TSDB_CODE_SUCCESS) {
        uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, pName.tname);
        goto end;
      }
      code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false);
      if (code != TSDB_CODE_SUCCESS) {
        uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, pName.tname);
        goto end;
931 932
      }
    }
933 934

    sTableData->tableMeta = pTableMeta;
X
Xiaoyu Wang 已提交
935 936
    uDebug("SML:0x%" PRIx64 "modify schema uid:%" PRIu64 ", sversion:%d, tversion:%d", info->id, pTableMeta->uid,
           pTableMeta->sversion, pTableMeta->tversion) tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp);
wmmhello's avatar
wmmhello 已提交
937
  }
X
Xiaoyu Wang 已提交
938 939
  uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas end success, format:%d, needModifySchema:%d", info->id, info->dataFormat,
         info->needModifySchema);
D
dapan1121 已提交
940

941
  return 0;
942

X
Xiaoyu Wang 已提交
943
end:
944 945
  taosHashCleanup(hashTmp);
  taosMemoryFreeClear(pTableMeta);
D
dapan1121 已提交
946
  catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1);
X
Xiaoyu Wang 已提交
947 948
  uError("SML:0x%" PRIx64 " smlModifyDBSchemas end failed:%d:%s, format:%d, needModifySchema:%d", info->id, code,
         tstrerror(code), info->dataFormat, info->needModifySchema);
D
dapan1121 已提交
949

950
  return code;
wmmhello's avatar
wmmhello 已提交
951 952
}

953 954 955 956 957 958 959
/*
static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){
  for(int i = 0; i < taosArrayGetSize(tags); i++) {
    SSmlKv *tag = taosArrayGet(tags, i);
    if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) {
      smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key);
      return TSDB_CODE_TSC_DUP_NAMES;
960
    }
wmmhello's avatar
wmmhello 已提交
961
  }
962 963 964
  return TSDB_CODE_SUCCESS;
}

965 966 967 968 969
static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) {
  SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
  int ret = smlCheckDupUnit(dumplicateKey, cols, msg);
  if(ret != TSDB_CODE_SUCCESS){
    goto end;
970
  }
971 972 973
  ret = smlCheckDupUnit(dumplicateKey, tags, msg);
  if(ret != TSDB_CODE_SUCCESS){
    goto end;
974
  }
975

976 977 978
  end:
  taosHashCleanup(dumplicateKey);
  return ret;
wmmhello's avatar
wmmhello 已提交
979
}
980
*/
wmmhello's avatar
wmmhello 已提交
981

X
Xiaoyu Wang 已提交
982
static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) {
983 984
  for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) {
    SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
X
Xiaoyu Wang 已提交
985 986
    int     ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES);
    if (ret == 0) {
987 988
      taosArrayPush(metaArray, kv);
    }
989
  }
990
}
wmmhello's avatar
wmmhello 已提交
991

992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
static void smlDestroySTableMeta(SSmlSTableMeta *meta) {
  taosHashCleanup(meta->tagHash);
  taosHashCleanup(meta->colHash);
  taosArrayDestroy(meta->tags);
  taosArrayDestroy(meta->cols);
  taosMemoryFree(meta->tableMeta);
  taosMemoryFree(meta);
}

static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg) {
  for (int i = 0; i < taosArrayGetSize(cols); ++i) {
    SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);

    int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen);
    if (index) {
      SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index);
X
Xiaoyu Wang 已提交
1008
      if (isTag) {
1009 1010
        if (kv->length > value->length) {
          value->length = kv->length;
1011
        }
1012 1013 1014 1015 1016
        continue;
      }
      if (kv->type != value->type) {
        smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key);
        return TSDB_CODE_SML_NOT_SAME_TYPE;
1017 1018
      }

1019 1020 1021 1022 1023
      if (IS_VAR_DATA_TYPE(kv->type) && (kv->length > value->length)) {  // update string len, if bigger
        value->length = kv->length;
      }
    } else {
      size_t tmp = taosArrayGetSize(metaArray);
X
Xiaoyu Wang 已提交
1024
      if (tmp > INT16_MAX) {
D
dapan1121 已提交
1025
        smlBuildInvalidDataMsg(msg, "too many cols or tags", kv->key);
1026
        uError("too many cols or tags");
D
dapan1121 已提交
1027
        return TSDB_CODE_SML_INVALID_DATA;
1028
      }
1029
      int16_t size = tmp;
X
Xiaoyu Wang 已提交
1030 1031
      int     ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES);
      if (ret == 0) {
1032
        taosArrayPush(metaArray, kv);
1033 1034 1035 1036
      }
    }
  }

1037 1038
  return TSDB_CODE_SUCCESS;
}
1039

1040
void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) {
1041 1042 1043 1044
  for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) {
    SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i);
    taosHashCleanup(kvHash);
  }
1045

X
Xiaoyu Wang 已提交
1046 1047 1048 1049 1050
  //  if (info->parseJsonByLib) {
  //    SSmlLineInfo *key = (SSmlLineInfo *)(tag->key);
  //    if (key != NULL) taosMemoryFree(key->tags);
  //  }
  //  taosMemoryFree(tag->key);
1051 1052 1053 1054
  taosArrayDestroy(tag->cols);
  taosArrayDestroy(tag->tags);
  taosMemoryFree(tag);
}
1055

X
Xiaoyu Wang 已提交
1056
void clearColValArray(SArray *pCols) {
wmmhello's avatar
wmmhello 已提交
1057 1058
  int32_t num = taosArrayGetSize(pCols);
  for (int32_t i = 0; i < num; ++i) {
X
Xiaoyu Wang 已提交
1059
    SColVal *pCol = taosArrayGet(pCols, i);
wmmhello's avatar
wmmhello 已提交
1060 1061 1062 1063 1064 1065
    if (TSDB_DATA_TYPE_NCHAR == pCol->type) {
      taosMemoryFreeClear(pCol->value.pData);
    }
  }
}

wmmhello's avatar
wmmhello 已提交
1066
void smlDestroyInfo(SSmlHandle *info) {
1067 1068
  if (!info) return;
  qDestroyQuery(info->pQuery);
1069

1070
  // destroy info->childTables
1071 1072 1073 1074
  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
  while (oneTable) {
    smlDestroyTableInfo(info, *oneTable);
    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
1075 1076
  }

1077
  // destroy info->superTables
1078 1079 1080 1081
  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
  while (oneSTable) {
    smlDestroySTableMeta(*oneSTable);
    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
1082
  }
1083

1084 1085
  // destroy info->pVgHash
  taosHashCleanup(info->pVgHash);
1086 1087
  taosHashCleanup(info->childTables);
  taosHashCleanup(info->superTables);
wmmhello's avatar
wmmhello 已提交
1088

X
Xiaoyu Wang 已提交
1089
  for (int i = 0; i < taosArrayGetSize(info->tagJsonArray); i++) {
wmmhello's avatar
wmmhello 已提交
1090 1091 1092 1093 1094
    cJSON *tags = (cJSON *)taosArrayGetP(info->tagJsonArray, i);
    cJSON_Delete(tags);
  }
  taosArrayDestroy(info->tagJsonArray);

1095 1096 1097 1098 1099 1100
  for (int i = 0; i < taosArrayGetSize(info->valueJsonArray); i++) {
    cJSON *value = (cJSON *)taosArrayGetP(info->valueJsonArray, i);
    cJSON_Delete(value);
  }
  taosArrayDestroy(info->valueJsonArray);

1101
  taosArrayDestroy(info->preLineTagKV);
wmmhello's avatar
wmmhello 已提交
1102

X
Xiaoyu Wang 已提交
1103 1104
  if (!info->dataFormat) {
    for (int i = 0; i < info->lineNum; i++) {
1105
      taosArrayDestroy(info->lines[i].colArray);
X
Xiaoyu Wang 已提交
1106
      if (info->parseJsonByLib) {
wmmhello's avatar
wmmhello 已提交
1107 1108
        taosMemoryFree(info->lines[i].tags);
      }
X
Xiaoyu Wang 已提交
1109
      if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag);
1110 1111
    }
    taosMemoryFree(info->lines);
wmmhello's avatar
wmmhello 已提交
1112
  }
1113

wmmhello's avatar
wmmhello 已提交
1114
  cJSON_Delete(info->root);
1115 1116
  taosMemoryFreeClear(info);
}
wmmhello's avatar
wmmhello 已提交
1117

wmmhello's avatar
wmmhello 已提交
1118
SSmlHandle *smlBuildSmlInfo(TAOS *taos) {
1119 1120 1121 1122
  int32_t     code = TSDB_CODE_SUCCESS;
  SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle));
  if (NULL == info) {
    return NULL;
1123
  }
X
Xiaoyu Wang 已提交
1124
  if (taos != NULL) {
1125
    info->taos = acquireTscObj(*(int64_t *)taos);
X
Xiaoyu Wang 已提交
1126
    if (info->taos == NULL) {
D
dapan1121 已提交
1127 1128
      goto cleanup;
    }
1129 1130 1131 1132 1133
    code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog);
    if (code != TSDB_CODE_SUCCESS) {
      uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code);
      goto cleanup;
    }
1134
  }
wmmhello's avatar
wmmhello 已提交
1135

1136
  info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
1137 1138 1139
  info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
  info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);

1140 1141 1142
  info->id = smlGenId();
  info->pQuery = smlInitHandle();
  info->dataFormat = true;
1143

wmmhello's avatar
wmmhello 已提交
1144
  info->tagJsonArray = taosArrayInit(8, POINTER_BYTES);
1145
  info->valueJsonArray = taosArrayInit(8, POINTER_BYTES);
1146 1147
  info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv));

wmmhello's avatar
wmmhello 已提交
1148
  if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables) {
1149 1150
    uError("create SSmlHandle failed");
    goto cleanup;
1151 1152
  }

1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168
  return info;

cleanup:
  smlDestroyInfo(info);
  return NULL;
}

static int32_t smlPushCols(SArray *colsArray, SArray *cols) {
  SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
  if (!kvHash) {
    uError("SML:smlDealCols failed to allocate memory");
    return TSDB_CODE_OUT_OF_MEMORY;
  }
  for (size_t i = 0; i < taosArrayGetSize(cols); i++) {
    SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
    taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES);
X
Xiaoyu Wang 已提交
1169 1170 1171
    if (terrno == TSDB_CODE_DUP_KEY) {
      return terrno;
    }
1172 1173
  }

1174
  taosArrayPush(colsArray, &kvHash);
1175
  return TSDB_CODE_SUCCESS;
wmmhello's avatar
wmmhello 已提交
1176
}
1177

1178
static int32_t smlParseLineBottom(SSmlHandle *info) {
X
Xiaoyu Wang 已提交
1179 1180
  uDebug("SML:0x%" PRIx64 " smlParseLineBottom start, format:%d, linenum:%d", info->id, info->dataFormat,
         info->lineNum);
X
Xiaoyu Wang 已提交
1181
  if (info->dataFormat) return TSDB_CODE_SUCCESS;
1182

X
Xiaoyu Wang 已提交
1183 1184
  for (int32_t i = 0; i < info->lineNum; i++) {
    SSmlLineInfo  *elements = info->lines + i;
1185
    SSmlTableInfo *tinfo = NULL;
X
Xiaoyu Wang 已提交
1186
    if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
X
Xiaoyu Wang 已提交
1187 1188 1189
      SSmlTableInfo **tmp =
          (SSmlTableInfo **)taosHashGet(info->childTables, elements->measure, elements->measureTagsLen);
      if (tmp) tinfo = *tmp;
X
Xiaoyu Wang 已提交
1190
    } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) {
X
Xiaoyu Wang 已提交
1191
      SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag,
wmmhello's avatar
wmmhello 已提交
1192
                                                          elements->measureLen + elements->tagsLen);
X
Xiaoyu Wang 已提交
1193
      if (tmp) tinfo = *tmp;
X
Xiaoyu Wang 已提交
1194
    } else {
X
Xiaoyu Wang 已提交
1195
      SSmlTableInfo **tmp = (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag,
wmmhello's avatar
wmmhello 已提交
1196
                                                          elements->measureLen + elements->tagsLen);
X
Xiaoyu Wang 已提交
1197
      if (tmp) tinfo = *tmp;
1198 1199
    }

X
Xiaoyu Wang 已提交
1200
    if (tinfo == NULL) {
1201 1202 1203
      uError("SML:0x%" PRIx64 "get oneTable failed, line num:%d", info->id, i);
      smlBuildInvalidDataMsg(&info->msgBuf, "get oneTable failed", elements->measure);
      return TSDB_CODE_SML_INVALID_DATA;
1204
    }
wmmhello's avatar
wmmhello 已提交
1205

1206 1207 1208 1209 1210
    if (taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS) {
      smlBuildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL);
      return TSDB_CODE_PAR_INVALID_TAGS_NUM;
    }

1211 1212 1213
    if (taosArrayGetSize(elements->colArray) + taosArrayGetSize(tinfo->tags) > TSDB_MAX_COLUMNS) {
      smlBuildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL);
      return TSDB_CODE_PAR_TOO_MANY_COLUMNS;
wmmhello's avatar
wmmhello 已提交
1214
    }
wmmhello's avatar
wmmhello 已提交
1215

1216
    int ret = smlPushCols(tinfo->cols, elements->colArray);
X
Xiaoyu Wang 已提交
1217
    if (ret != TSDB_CODE_SUCCESS) {
wmmhello's avatar
wmmhello 已提交
1218 1219 1220
      return ret;
    }

1221 1222
    SSmlSTableMeta **tableMeta =
        (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
1223
    if (tableMeta) {  // update meta
X
Xiaoyu Wang 已提交
1224 1225
      uDebug("SML:0x%" PRIx64 " smlParseLineBottom update meta, format:%d, linenum:%d", info->id, info->dataFormat,
             info->lineNum);
1226
      ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf);
1227
      if (ret == TSDB_CODE_SUCCESS) {
1228
        ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf);
1229 1230 1231 1232 1233
      }
      if (ret != TSDB_CODE_SUCCESS) {
        uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id);
        return ret;
      }
X
Xiaoyu Wang 已提交
1234
    } else {
X
Xiaoyu Wang 已提交
1235 1236 1237 1238 1239
      //      ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf);
      //      if (ret != TSDB_CODE_SUCCESS) {
      //        uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id);
      //        return ret;
      //      }
X
Xiaoyu Wang 已提交
1240 1241
      uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat,
             info->lineNum);
1242 1243
      SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
      smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags);
X
Xiaoyu Wang 已提交
1244 1245 1246
      if (terrno == TSDB_CODE_DUP_KEY) {
        return terrno;
      }
1247
      smlInsertMeta(meta->colHash, meta->cols, elements->colArray);
1248
      taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
wmmhello's avatar
wmmhello 已提交
1249
    }
wmmhello's avatar
wmmhello 已提交
1250
  }
D
dapan1121 已提交
1251
  uDebug("SML:0x%" PRIx64 " smlParseLineBottom end, format:%d, linenum:%d", info->id, info->dataFormat, info->lineNum);
1252

wmmhello's avatar
wmmhello 已提交
1253 1254 1255
  return TSDB_CODE_SUCCESS;
}

X
Xiaoyu Wang 已提交
1256
static int32_t smlInsertData(SSmlHandle *info) {
wmmhello's avatar
wmmhello 已提交
1257
  int32_t code = TSDB_CODE_SUCCESS;
D
dapan1121 已提交
1258
  uDebug("SML:0x%" PRIx64 " smlInsertData start, format:%d", info->id, info->dataFormat);
wmmhello's avatar
wmmhello 已提交
1259

X
Xiaoyu Wang 已提交
1260
  if (info->pRequest->dbList == NULL) {
wmmhello's avatar
wmmhello 已提交
1261 1262
    info->pRequest->dbList = taosArrayInit(1, TSDB_DB_FNAME_LEN);
  }
X
Xiaoyu Wang 已提交
1263 1264 1265
  void *data = taosArrayReserve(info->pRequest->dbList, 1);
  memcpy(data, info->pRequest->pDb,
         TSDB_DB_FNAME_LEN > strlen(info->pRequest->pDb) ? strlen(info->pRequest->pDb) : TSDB_DB_FNAME_LEN);
wmmhello's avatar
wmmhello 已提交
1266

1267 1268 1269
  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
  while (oneTable) {
    SSmlTableInfo *tableData = *oneTable;
wmmhello's avatar
wmmhello 已提交
1270 1271

    SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
wmmhello's avatar
wmmhello 已提交
1272
    tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname));
wmmhello's avatar
wmmhello 已提交
1273
    memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName));
D
dapan1121 已提交
1274

X
Xiaoyu Wang 已提交
1275
    if (info->pRequest->tableList == NULL) {
wmmhello's avatar
wmmhello 已提交
1276 1277 1278 1279
      info->pRequest->tableList = taosArrayInit(1, sizeof(SName));
    }
    taosArrayPush(info->pRequest->tableList, &pName);

D
dapan1121 已提交
1280 1281 1282 1283 1284
    SRequestConnInfo conn = {0};
    conn.pTrans = info->taos->pAppInfo->pTransporter;
    conn.requestId = info->pRequest->requestId;
    conn.requestObjRefId = info->pRequest->self;
    conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp);
X
Xiaoyu Wang 已提交
1285

wmmhello's avatar
wmmhello 已提交
1286
    SVgroupInfo vg;
D
dapan1121 已提交
1287
    code = catalogGetTableHashVgroup(info->pCatalog, &conn, &pName, &vg);
1288
    if (code != TSDB_CODE_SUCCESS) {
X
Xiaoyu Wang 已提交
1289
      uError("SML:0x%" PRIx64 " catalogGetTableHashVgroup failed. table name: %s", info->id, tableData->childTableName);
wmmhello's avatar
wmmhello 已提交
1290 1291
      return code;
    }
X
Xiaoyu Wang 已提交
1292
    taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg));
wmmhello's avatar
wmmhello 已提交
1293

1294 1295 1296
    SSmlSTableMeta **pMeta =
        (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen);
    if (unlikely(NULL == pMeta || NULL == (*pMeta)->tableMeta)) {
1297 1298 1299
      uError("SML:0x%" PRIx64 " NULL == pMeta. table name: %s", info->id, tableData->childTableName);
      return TSDB_CODE_SML_INTERNAL_ERROR;
    }
wmmhello's avatar
wmmhello 已提交
1300

1301
    // use tablemeta of stable to save vgid and uid of child table
1302 1303
    (*pMeta)->tableMeta->vgId = vg.vgId;
    (*pMeta)->tableMeta->uid = tableData->uid;  // one table merge data block together according uid
X
Xiaoyu Wang 已提交
1304 1305
    uDebug("SML:0x%" PRIx64 " smlInsertData table:%s, uid:%" PRIu64 ", format:%d", info->id, pName.tname,
           tableData->uid, info->dataFormat);
wmmhello's avatar
wmmhello 已提交
1306

X
Xiaoyu Wang 已提交
1307
    code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols,
1308
                       (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen,
1309
                       info->ttl, info->msgBuf.buf, info->msgBuf.len);
X
Xiaoyu Wang 已提交
1310 1311
    if (code != TSDB_CODE_SUCCESS) {
      uError("SML:0x%" PRIx64 " smlBindData failed", info->id);
wmmhello's avatar
wmmhello 已提交
1312 1313
      return code;
    }
1314
    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
wmmhello's avatar
wmmhello 已提交
1315
  }
wmmhello's avatar
wmmhello 已提交
1316

wmmhello's avatar
wmmhello 已提交
1317
  code = smlBuildOutput(info->pQuery, info->pVgHash);
1318
  if (code != TSDB_CODE_SUCCESS) {
X
Xiaoyu Wang 已提交
1319
    uError("SML:0x%" PRIx64 " smlBuildOutput failed", info->id);
1320 1321
    return code;
  }
1322 1323
  info->cost.insertRpcTime = taosGetTimestampUs();

1324 1325 1326
  SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary;
  atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1);

wmmhello's avatar
wmmhello 已提交
1327
  launchQueryImpl(info->pRequest, info->pQuery, true, NULL);
X
Xiaoyu Wang 已提交
1328 1329
  uDebug("SML:0x%" PRIx64 " smlInsertData end, format:%d, code:%d,%s", info->id, info->dataFormat, info->pRequest->code,
         tstrerror(info->pRequest->code));
D
dapan1121 已提交
1330

wmmhello's avatar
wmmhello 已提交
1331
  return info->pRequest->code;
wmmhello's avatar
wmmhello 已提交
1332 1333
}

X
Xiaoyu Wang 已提交
1334
static void smlPrintStatisticInfo(SSmlHandle *info) {
1335
  uDebug(
X
Xiaoyu Wang 已提交
1336
      "SML:0x%" PRIx64
D
dapan1121 已提交
1337
      " smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \
X
Xiaoyu Wang 已提交
1338
        parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64
X
Xiaoyu Wang 已提交
1339
      "",
X
Xiaoyu Wang 已提交
1340 1341 1342 1343 1344
      info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables,
      info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables,
      info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime,
      info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime,
      info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime);
1345 1346
}

X
Xiaoyu Wang 已提交
1347
int32_t smlClearForRerun(SSmlHandle *info) {
1348 1349
  info->reRun = false;
  // clear info->childTables
1350 1351 1352 1353
  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
  while (oneTable) {
    smlDestroyTableInfo(info, *oneTable);
    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
1354 1355 1356
  }

  // clear info->superTables
1357 1358 1359 1360
  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
  while (oneSTable) {
    smlDestroySTableMeta(*oneSTable);
    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
1361 1362
  }

wmmhello's avatar
wmmhello 已提交
1363 1364 1365
  taosHashClear(info->childTables);
  taosHashClear(info->superTables);

X
Xiaoyu Wang 已提交
1366
  if (!info->dataFormat) {
wmmhello's avatar
wmmhello 已提交
1367 1368 1369 1370 1371
    if (unlikely(info->lines != NULL)) {
      uError("SML:0x%" PRIx64 " info->lines != NULL", info->id);
      return TSDB_CODE_SML_INVALID_DATA;
    }
    info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo));
1372 1373 1374 1375 1376 1377
  }

  memset(&info->preLine, 0, sizeof(SSmlLineInfo));
  info->currSTableMeta = NULL;
  info->currTableDataCtx = NULL;

X
Xiaoyu Wang 已提交
1378
  SVnodeModifyOpStmt *stmt = (SVnodeModifyOpStmt *)(info->pQuery->pRoot);
1379 1380 1381 1382 1383
  stmt->freeHashFunc(stmt->pTableBlockHashObj);
  stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
  return TSDB_CODE_SUCCESS;
}

dengyihao's avatar
dengyihao 已提交
1384
static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) {
D
dapan1121 已提交
1385
  uDebug("SML:0x%" PRIx64 " smlParseLine start", info->id);
wmmhello's avatar
wmmhello 已提交
1386
  int32_t code = TSDB_CODE_SUCCESS;
1387
  if (info->protocol == TSDB_SML_JSON_PROTOCOL) {
dengyihao's avatar
dengyihao 已提交
1388
    if (lines) {
wmmhello's avatar
wmmhello 已提交
1389
      code = smlParseJSON(info, *lines);
dengyihao's avatar
dengyihao 已提交
1390
    } else if (rawLine) {
wmmhello's avatar
wmmhello 已提交
1391 1392
      code = smlParseJSON(info, rawLine);
    }
1393
    if (code != TSDB_CODE_SUCCESS) {
dengyihao's avatar
dengyihao 已提交
1394
      uError("SML:0x%" PRIx64 " smlParseJSON failed:%s", info->id, lines ? *lines : rawLine);
1395 1396
      return code;
    }
wmmhello's avatar
wmmhello 已提交
1397
    return code;
wmmhello's avatar
wmmhello 已提交
1398
  }
wmmhello's avatar
wmmhello 已提交
1399

X
Xiaoyu Wang 已提交
1400
  char   *oldRaw = rawLine;
1401 1402
  int32_t i = 0;
  while (i < numLines) {
wmmhello's avatar
wmmhello 已提交
1403
    char *tmp = NULL;
dengyihao's avatar
dengyihao 已提交
1404 1405
    int   len = 0;
    if (lines) {
wmmhello's avatar
wmmhello 已提交
1406 1407
      tmp = lines[i];
      len = strlen(tmp);
dengyihao's avatar
dengyihao 已提交
1408
    } else if (rawLine) {
wmmhello's avatar
wmmhello 已提交
1409
      tmp = rawLine;
dengyihao's avatar
dengyihao 已提交
1410 1411
      while (rawLine < rawLineEnd) {
        if (*(rawLine++) == '\n') {
wmmhello's avatar
wmmhello 已提交
1412 1413 1414 1415
          break;
        }
        len++;
      }
dengyihao's avatar
dengyihao 已提交
1416
      if (info->protocol == TSDB_SML_LINE_PROTOCOL && tmp[0] == '#') {  // this line is comment
wmmhello's avatar
wmmhello 已提交
1417 1418
        continue;
      }
wmmhello's avatar
wmmhello 已提交
1419 1420
    }

X
Xiaoyu Wang 已提交
1421 1422
    char cTmp = 0;  // for print tmp if is raw
    if (info->isRawLine) {
D
dapan1121 已提交
1423 1424 1425 1426
      cTmp = tmp[len - 1];
      tmp[len - 1] = '\0';
    }

X
Xiaoyu Wang 已提交
1427 1428 1429
    uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, numLines:%d, protocol:%d, len:%d, sql:%s", info->id,
           info->isRawLine, numLines, info->protocol, len, tmp);
    if (info->isRawLine) {
D
dapan1121 已提交
1430 1431
      tmp[len - 1] = cTmp;
    }
1432

X
Xiaoyu Wang 已提交
1433
    if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
X
Xiaoyu Wang 已提交
1434
      if (info->dataFormat) {
wmmhello's avatar
wmmhello 已提交
1435 1436
        SSmlLineInfo element = {0};
        code = smlParseInfluxString(info, tmp, tmp + len, &element);
X
Xiaoyu Wang 已提交
1437
      } else {
wmmhello's avatar
wmmhello 已提交
1438 1439
        code = smlParseInfluxString(info, tmp, tmp + len, info->lines + i);
      }
X
Xiaoyu Wang 已提交
1440
    } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) {
X
Xiaoyu Wang 已提交
1441
      if (info->dataFormat) {
1442 1443
        SSmlLineInfo element = {0};
        code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, &element);
X
Xiaoyu Wang 已提交
1444
        if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag);
X
Xiaoyu Wang 已提交
1445
      } else {
1446 1447
        code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, info->lines + i);
      }
X
Xiaoyu Wang 已提交
1448
    } else {
1449
      code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE;
1450
    }
wmmhello's avatar
wmmhello 已提交
1451
    if (code != TSDB_CODE_SUCCESS) {
wmmhello's avatar
wmmhello 已提交
1452
      uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp);
1453
      return code;
wmmhello's avatar
wmmhello 已提交
1454
    }
X
Xiaoyu Wang 已提交
1455
    if (info->reRun) {
D
dapan1121 已提交
1456
      uDebug("SML:0x%" PRIx64 " smlParseLine re run", info->id);
1457
      i = 0;
1458
      rawLine = oldRaw;
1459
      code = smlClearForRerun(info);
X
Xiaoyu Wang 已提交
1460
      if (code != TSDB_CODE_SUCCESS) {
1461
        return code;
1462
      }
1463
      continue;
1464
    }
1465
    i++;
wmmhello's avatar
wmmhello 已提交
1466
  }
D
dapan1121 已提交
1467
  uDebug("SML:0x%" PRIx64 " smlParseLine end", info->id);
1468

1469 1470 1471
  return code;
}

dengyihao's avatar
dengyihao 已提交
1472
static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) {
1473
  int32_t code = TSDB_CODE_SUCCESS;
1474 1475
  int32_t retryNum = 0;

1476 1477
  info->cost.parseTime = taosGetTimestampUs();

wmmhello's avatar
wmmhello 已提交
1478
  code = smlParseLine(info, lines, rawLine, rawLineEnd, numLines);
1479
  if (code != 0) {
X
Xiaoyu Wang 已提交
1480
    uError("SML:0x%" PRIx64 " smlParseLine error : %s", info->id, tstrerror(code));
wmmhello's avatar
wmmhello 已提交
1481
    return code;
1482
  }
1483 1484 1485 1486 1487 1488
  code = smlParseLineBottom(info);
  if (code != 0) {
    uError("SML:0x%" PRIx64 " smlParseLineBottom error : %s", info->id, tstrerror(code));
    return code;
  }

1489
  info->cost.lineNum = info->lineNum;
1490 1491
  info->cost.numOfSTables = taosHashGetSize(info->superTables);
  info->cost.numOfCTables = taosHashGetSize(info->childTables);
1492 1493

  info->cost.schemaTime = taosGetTimestampUs();
1494

X
Xiaoyu Wang 已提交
1495
  do {
1496 1497
    code = smlModifyDBSchemas(info);
    if (code == 0) break;
D
dapan1121 已提交
1498 1499
    taosMsleep(500);
    uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum);
1500
  } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES);
1501

wmmhello's avatar
wmmhello 已提交
1502
  if (code != 0) {
X
Xiaoyu Wang 已提交
1503
    uError("SML:0x%" PRIx64 " smlModifyDBSchemas error : %s", info->id, tstrerror(code));
wmmhello's avatar
wmmhello 已提交
1504
    return code;
wmmhello's avatar
wmmhello 已提交
1505
  }
wmmhello's avatar
wmmhello 已提交
1506

1507
  info->cost.insertBindTime = taosGetTimestampUs();
wmmhello's avatar
wmmhello 已提交
1508 1509
  code = smlInsertData(info);
  if (code != 0) {
X
Xiaoyu Wang 已提交
1510
    uError("SML:0x%" PRIx64 " smlInsertData error : %s", info->id, tstrerror(code));
wmmhello's avatar
wmmhello 已提交
1511
    return code;
wmmhello's avatar
wmmhello 已提交
1512 1513 1514 1515 1516
  }

  return code;
}

X
Xiaoyu Wang 已提交
1517 1518
TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, int numLines,
                                       int protocol, int precision, int32_t ttl, int64_t reqid) {
1519
  int32_t code = TSDB_CODE_SUCCESS;
wmmhello's avatar
wmmhello 已提交
1520 1521 1522
  if (NULL == taos) {
    terrno = TSDB_CODE_TSC_DISCONNECTED;
    return NULL;
1523
  }
wmmhello's avatar
wmmhello 已提交
1524
  SRequestObj *request = NULL;
X
Xiaoyu Wang 已提交
1525 1526 1527
  SSmlHandle  *info = NULL;
  int          cnt = 0;
  while (1) {
wmmhello's avatar
wmmhello 已提交
1528 1529 1530 1531 1532
    request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid);
    if (request == NULL) {
      uError("SML:taos_schemaless_insert error request is null");
      return NULL;
    }
1533

wmmhello's avatar
wmmhello 已提交
1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554
    info = smlBuildSmlInfo(taos);
    if (info == NULL) {
      request->code = TSDB_CODE_OUT_OF_MEMORY;
      uError("SML:taos_schemaless_insert error SSmlHandle is null");
      return (TAOS_RES *)request;
    }
    info->pRequest = request;
    info->isRawLine = rawLine != NULL;
    info->ttl = ttl;
    info->precision = precision;
    info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol;
    info->msgBuf.buf = info->pRequest->msgBuf;
    info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE;
    info->lineNum = numLines;

    SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf};
    if (request->pDb == NULL) {
      request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
      smlBuildInvalidDataMsg(&msg, "Database not specified", NULL);
      goto end;
    }
1555

wmmhello's avatar
wmmhello 已提交
1556 1557 1558 1559 1560
    if (protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL) {
      request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE;
      smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL);
      goto end;
    }
wmmhello's avatar
wmmhello 已提交
1561

wmmhello's avatar
wmmhello 已提交
1562 1563 1564 1565 1566 1567
    if (protocol == TSDB_SML_LINE_PROTOCOL &&
        (precision < TSDB_SML_TIMESTAMP_NOT_CONFIGURED || precision > TSDB_SML_TIMESTAMP_NANO_SECONDS)) {
      request->code = TSDB_CODE_SML_INVALID_PRECISION_TYPE;
      smlBuildInvalidDataMsg(&msg, "precision invalidate for line protocol", NULL);
      goto end;
    }
wmmhello's avatar
wmmhello 已提交
1568

wmmhello's avatar
wmmhello 已提交
1569 1570 1571 1572 1573 1574 1575
    if (protocol == TSDB_SML_JSON_PROTOCOL) {
      numLines = 1;
    } else if (numLines <= 0) {
      request->code = TSDB_CODE_SML_INVALID_DATA;
      smlBuildInvalidDataMsg(&msg, "line num is invalid", NULL);
      goto end;
    }
1576

wmmhello's avatar
wmmhello 已提交
1577 1578 1579 1580
    code = smlProcess(info, lines, rawLine, rawLineEnd, numLines);
    request->code = code;
    info->cost.endTime = taosGetTimestampUs();
    info->cost.code = code;
X
Xiaoyu Wang 已提交
1581 1582 1583 1584
    if (code == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || code == TSDB_CODE_SDB_OBJ_CREATING ||
        code == TSDB_CODE_PAR_VALUE_TOO_LONG || code == TSDB_CODE_MND_TRANS_CONFLICT) {
      if (cnt++ >= 10) {
        uInfo("SML:%" PRIx64 " retry:%d/10 end code:%d, msg:%s", info->id, cnt, code, tstrerror(code));
D
dapan1121 已提交
1585 1586 1587
        break;
      }
      taosMsleep(100);
wmmhello's avatar
wmmhello 已提交
1588
      refreshMeta(request->pTscObj, request);
X
Xiaoyu Wang 已提交
1589 1590
      uInfo("SML:%" PRIx64 " retry:%d/10,ver is old retry or object is creating code:%d, msg:%s", info->id, cnt, code,
            tstrerror(code));
wmmhello's avatar
wmmhello 已提交
1591 1592 1593 1594 1595 1596
      smlDestroyInfo(info);
      info = NULL;
      taos_free_result(request);
      request = NULL;
      continue;
    }
D
dapan1121 已提交
1597
    smlPrintStatisticInfo(info);
wmmhello's avatar
wmmhello 已提交
1598
    break;
wmmhello's avatar
wmmhello 已提交
1599 1600
  }

wmmhello's avatar
wmmhello 已提交
1601
end:
wmmhello's avatar
wmmhello 已提交
1602
  smlDestroyInfo(info);
1603
  return (TAOS_RES *)request;
wmmhello's avatar
wmmhello 已提交
1604
}
wmmhello's avatar
wmmhello 已提交
1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621

/**
 * taos_schemaless_insert() parse and insert data points into database according to
 * different protocol.
 *
 * @param $lines input array may contain multiple lines, each line indicates a data point.
 *               If protocol=2 is used input array should contain single JSON
 *               string(e.g. char *lines[] = {"$JSON_string"}). If need to insert
 *               multiple data points in JSON format, should include them in $JSON_string
 *               as a JSON array.
 * @param $numLines indicates how many data points in $lines.
 *                  If protocol = 2 is used this param will be ignored as $lines should
 *                  contain single JSON string.
 * @param $protocol indicates which protocol to use for parsing:
 *                  0 - influxDB line protocol
 *                  1 - OpenTSDB telnet line protocol
 *                  2 - OpenTSDB JSON format protocol
dengyihao's avatar
dengyihao 已提交
1622
 * @return TAOS_RES
wmmhello's avatar
wmmhello 已提交
1623 1624
 */

1625 1626
TAOS_RES *taos_schemaless_insert_ttl_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision,
                                                int32_t ttl, int64_t reqid) {
wmmhello's avatar
wmmhello 已提交
1627
  return taos_schemaless_insert_inner(taos, lines, NULL, NULL, numLines, protocol, precision, ttl, reqid);
dengyihao's avatar
dengyihao 已提交
1628 1629
}

1630 1631 1632
TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) {
  return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL, 0);
}
wmmhello's avatar
wmmhello 已提交
1633

X
Xiaoyu Wang 已提交
1634 1635
TAOS_RES *taos_schemaless_insert_ttl(TAOS *taos, char *lines[], int numLines, int protocol, int precision,
                                     int32_t ttl) {
1636 1637
  return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, ttl, 0);
}
wmmhello's avatar
wmmhello 已提交
1638

X
Xiaoyu Wang 已提交
1639 1640 1641 1642
TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision,
                                            int64_t reqid) {
  return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL,
                                               reqid);
wmmhello's avatar
wmmhello 已提交
1643 1644
}

1645
TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol,
1646
                                                    int precision, int32_t ttl, int64_t reqid) {
dengyihao's avatar
dengyihao 已提交
1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658
  int numLines = 0;
  *totalRows = 0;
  char *tmp = lines;
  for (int i = 0; i < len; i++) {
    if (lines[i] == '\n' || i == len - 1) {
      numLines++;
      if (tmp[0] != '#' || protocol != TSDB_SML_LINE_PROTOCOL) {  // ignore comment
        (*totalRows)++;
      }
      tmp = lines + i + 1;
    }
  }
wmmhello's avatar
wmmhello 已提交
1659
  return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid);
dengyihao's avatar
dengyihao 已提交
1660 1661
}

X
Xiaoyu Wang 已提交
1662 1663 1664 1665
TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol,
                                                int precision, int64_t reqid) {
  return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision,
                                                   TSDB_DEFAULT_TABLE_TTL, reqid);
1666
}
X
Xiaoyu Wang 已提交
1667 1668
TAOS_RES *taos_schemaless_insert_raw_ttl(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol,
                                         int precision, int32_t ttl) {
1669 1670
  return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, ttl, 0);
}
wmmhello's avatar
wmmhello 已提交
1671

X
Xiaoyu Wang 已提交
1672 1673 1674 1675
TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol,
                                     int precision) {
  return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision,
                                                   TSDB_DEFAULT_TABLE_TTL, 0);
wmmhello's avatar
wmmhello 已提交
1676
}