diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 99ed0822368cc0126f8c152506cdb139c16ceb5b..ac5b4f2ec2073c4d903cbd72aa1c149b81543588 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -486,6 +486,7 @@ bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols); char *tscGetErrorMsgPayload(SSqlCmd *pCmd); +int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql); int32_t tscInvalidOperationMsg(char *msg, const char *additionalInfo, const char *sql); int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 91ac7a2f3ace7f08bafb5d4620e65f59f0b89480..a1f2d3c7611f7d29d87cafd06c3fa55e7b1ed86c 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -361,15 +361,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { } if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { // stmt insert - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - code = tscGetTableMeta(pSql, pTableMetaInfo); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); - return; - } else { - assert(code == TSDB_CODE_SUCCESS); - } - (*pSql->fp)(pSql->param, pSql, code); } else if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { // file insert tscImportDataFromFile(pSql); diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 2a16f1aad6525a25da652f643dc7b7f91debd432..9401694897f8a66c05db6a07db3e90b01a5457e0 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -32,9 +32,6 @@ typedef struct { uint8_t type; int16_t length; char* value; - - //=================================== - uint32_t fieldSchemaIdx; } TAOS_SML_KV; typedef struct { @@ -47,9 +44,6 @@ typedef struct { // first kv must be timestamp TAOS_SML_KV* fields; int32_t fieldNum; - - //================================ - uint32_t schemaIdx; } TAOS_SML_DATA_POINT; typedef enum { @@ -62,10 +56,23 @@ typedef enum { typedef struct { uint64_t id; - + SHashObj* smlDataToSchema; } SSmlLinesInfo; + //================================================================================================= +static uint64_t linesSmlHandleId = 0; + +uint64_t genLinesSmlId() { + uint64_t id; + + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); + } while (id == 0); + + return id; +} + int compareSmlColKv(const void* p1, const void* p2) { TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; @@ -168,11 +175,46 @@ static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* arra taosHashPut(hash, field.name, tagKeyLen, &fieldIdx, sizeof(fieldIdx)); } - smlKv->fieldSchemaIdx = (uint32_t)fieldIdx; + uintptr_t valPointer = (uintptr_t)smlKv; + taosHashPut(info->smlDataToSchema, &valPointer, sizeof(uintptr_t), &fieldIdx, sizeof(fieldIdx)); return 0; } +static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen, + SSmlLinesInfo* info) { + tscDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id); + qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); + + SStringBuilder sb; memset(&sb, 0, sizeof(sb)); + char sTableName[TSDB_TABLE_NAME_LEN] = {0}; + strtolower(sTableName, point->stableName); + taosStringBuilderAppendString(&sb, sTableName); + for (int j = 0; j < point->tagNum; ++j) { + taosStringBuilderAppendChar(&sb, ','); + TAOS_SML_KV* tagKv = point->tags + j; + char tagName[TSDB_COL_NAME_LEN] = {0}; + strtolower(tagName, tagKv->key); + taosStringBuilderAppendString(&sb, tagName); + taosStringBuilderAppendChar(&sb, '='); + taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); + } + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + MD5_CTX context; + MD5Init(&context); + MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); + MD5Final(&context); + *tableNameLen = snprintf(tableName, *tableNameLen, + "t_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], + context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], + context.digest[12], context.digest[13], context.digest[14], context.digest[15]); + taosStringBuilderDestroy(&sb); + tscDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName); + return 0; +} + static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, SArray* stableSchemas, SSmlLinesInfo* info) { int32_t code = 0; SHashObj* sname2shema = taosHashInit(32, @@ -203,6 +245,15 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, for (int j = 0; j < point->tagNum; ++j) { TAOS_SML_KV* tagKv = point->tags + j; + if (!point->childTableName) { + char childTableName[TSDB_TABLE_NAME_LEN]; + int32_t tableNameLen = TSDB_TABLE_NAME_LEN; + getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info); + point->childTableName = calloc(1, tableNameLen+1); + strncpy(point->childTableName, childTableName, tableNameLen); + point->childTableName[tableNameLen] = '\0'; + } + code = buildSmlKvSchema(tagKv, pStableSchema->tagHash, pStableSchema->tags, info); if (code != 0) { tscError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, tagKv->key); @@ -219,7 +270,8 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, } } - point->schemaIdx = (uint32_t)stableIdx; + uintptr_t valPointer = (uintptr_t)point; + taosHashPut(info->smlDataToSchema, &valPointer, sizeof(uintptr_t), &stableIdx, sizeof(stableIdx)); } size_t numStables = taosArrayGetSize(stableSchemas); @@ -319,7 +371,23 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + char* errStr = taos_errstr(res); + char* begin = strstr(errStr, "duplicated column names"); + bool tscDupColNames = (begin != NULL); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error: %s", info->id, errStr); + } taos_free_result(res); + + if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || code == TSDB_CODE_MND_TAG_ALREAY_EXIST || tscDupColNames) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } break; } case SCHEMA_ACTION_ADD_TAG: { @@ -328,7 +396,23 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + char* errStr = taos_errstr(res); + char* begin = strstr(errStr, "duplicated column names"); + bool tscDupColNames = (begin != NULL); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } taos_free_result(res); + + if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST || code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || tscDupColNames) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } break; } case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { @@ -337,7 +421,20 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } taos_free_result(res); + + if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } break; } case SCHEMA_ACTION_CHANGE_TAG_SIZE: { @@ -346,7 +443,20 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } taos_free_result(res); + + if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } break; } case SCHEMA_ACTION_CREATE_STABLE: { @@ -375,7 +485,20 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf outBytes = snprintf(pos, freeBytes, ")"); TAOS_RES* res = taos_query(taos, result); code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } taos_free_result(res); + + if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + taosMsleep(500); + } break; } @@ -385,7 +508,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf free(result); if (code != 0) { - tscError("SML:0x%"PRIx64 "apply schema action failure. %s", info->id, tstrerror(code)); + tscError("SML:0x%"PRIx64 " apply schema action failure. %s", info->id, tstrerror(code)); } return code; } @@ -398,70 +521,12 @@ static int32_t destroySmlSTableSchema(SSmlSTableSchema* schema) { return 0; } -int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { - int32_t code = 0; - - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - tscDebug("SML:0x%"PRIx64" load table schema. super table name: %s", info->id, tableName); - - char tableNameLowerCase[TSDB_TABLE_NAME_LEN]; - strtolower(tableNameLowerCase, tableName); - - char sql[256]; - snprintf(sql, 256, "describe %s", tableNameLowerCase); - TAOS_RES* res = taos_query(taos, sql); - code = taos_errno(res); - if (code != 0) { - tscError("SML:0x%"PRIx64" describe table failure. %s", info->id, taos_errstr(res)); - taos_free_result(res); - return code; - } - taos_free_result(res); - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL){ - tscError("failed to allocate memory, reason:%s", strerror(errno)); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return code; - } - pSql->pTscObj = taos; - pSql->signature = pSql; - pSql->fp = NULL; - - SStrToken tableToken = {.z=tableNameLowerCase, .n=(uint32_t)strlen(tableNameLowerCase), .type=TK_ID}; - tGetToken(tableNameLowerCase, &tableToken.type); - // Check if the table name available or not - if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pSql->cmd.payload, "table name is invalid"); - tscFreeSqlObj(pSql); - return code; - } - - SName sname = {0}; - if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pSql); - return code; - } - char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; - memset(fullTableName, 0, tListLen(fullTableName)); - tNameExtractFullName(&sname, fullTableName); - tscFreeSqlObj(pSql); - +static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { schema->tags = taosArrayInit(8, sizeof(SSchema)); schema->fields = taosArrayInit(64, sizeof(SSchema)); schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - size_t size = 0; - STableMeta* tableMeta = NULL; - taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void **)&tableMeta, &size); - tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); schema->precision = tableMeta->tableInfo.precision; for (int i=0; itableInfo.numOfColumns; ++i) { @@ -484,9 +549,93 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSm size_t tagIndex = taosArrayGetSize(schema->tags) - 1; taosHashPut(schema->tagHash, field.name, strlen(field.name), &tagIndex, sizeof(tagIndex)); } - tscDebug("SML:0x%"PRIx64 " load table meta succeed. table name: %s, columns number: %d, tag number: %d, precision: %d", + tscDebug("SML:0x%"PRIx64 " load table schema succeed. table name: %s, columns number: %d, tag number: %d, precision: %d", info->id, tableName, tableMeta->tableInfo.numOfColumns, tableMeta->tableInfo.numOfTags, schema->precision); - free(tableMeta); tableMeta = NULL; + return TSDB_CODE_SUCCESS; +} + +static int32_t retrieveTableMeta(TAOS* taos, char* tableName, STableMeta** pTableMeta, SSmlLinesInfo* info) { + int32_t code = 0; + int32_t retries = 0; + STableMeta* tableMeta = NULL; + while (retries++ < TSDB_MAX_REPLICA && tableMeta == NULL) { + STscObj* pObj = (STscObj*)taos; + if (pObj == NULL || pObj->signature != pObj) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + + tscDebug("SML:0x%" PRIx64 " retrieve table meta. super table name: %s", info->id, tableName); + + char tableNameLowerCase[TSDB_TABLE_NAME_LEN]; + strtolower(tableNameLowerCase, tableName); + + char sql[256]; + snprintf(sql, 256, "describe %s", tableNameLowerCase); + TAOS_RES* res = taos_query(taos, sql); + code = taos_errno(res); + if (code != 0) { + tscError("SML:0x%" PRIx64 " describe table failure. %s", info->id, taos_errstr(res)); + taos_free_result(res); + return code; + } + taos_free_result(res); + + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + if (pSql == NULL) { + tscError("SML:0x%" PRIx64 " failed to allocate memory, reason:%s", info->id, strerror(errno)); + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return code; + } + pSql->pTscObj = taos; + pSql->signature = pSql; + pSql->fp = NULL; + + registerSqlObj(pSql); + SStrToken tableToken = {.z = tableNameLowerCase, .n = (uint32_t)strlen(tableNameLowerCase), .type = TK_ID}; + tGetToken(tableNameLowerCase, &tableToken.type); + // Check if the table name available or not + if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + sprintf(pSql->cmd.payload, "table name is invalid"); + tscFreeRegisteredSqlObj(pSql); + return code; + } + + SName sname = {0}; + if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { + tscFreeRegisteredSqlObj(pSql); + return code; + } + char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; + memset(fullTableName, 0, tListLen(fullTableName)); + tNameExtractFullName(&sname, fullTableName); + tscFreeRegisteredSqlObj(pSql); + + size_t size = 0; + taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void**)&tableMeta, &size); + } + + if (tableMeta != NULL) { + *pTableMeta = tableMeta; + return TSDB_CODE_SUCCESS; + } else { + tscError("SML:0x%" PRIx64 " failed to retrieve table meta. super table name: %s", info->id, tableName); + return TSDB_CODE_TSC_NO_META_CACHED; + } +} + +static int32_t loadTableSchemaFromDB(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { + int32_t code = 0; + STableMeta* tableMeta = NULL; + code = retrieveTableMeta(taos, tableName, &tableMeta, info); + if (code == TSDB_CODE_SUCCESS) { + assert(tableMeta != NULL); + fillDbSchema(tableMeta, tableName, schema, info); + free(tableMeta); + tableMeta = NULL; + } + return code; } @@ -498,7 +647,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* SSmlSTableSchema dbSchema; memset(&dbSchema, 0, sizeof(SSmlSTableSchema)); - code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema, info); + code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; @@ -507,7 +656,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* schemaAction.createSTable.tags = pointSchema->tags; schemaAction.createSTable.fields = pointSchema->fields; applySchemaAction(taos, &schemaAction, info); - code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema, info); + code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); if (code != 0) { tscError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, pointSchema->sTableName); return code; @@ -567,74 +716,6 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* return 0; } -static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen, - SSmlLinesInfo* info) { - tscDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id); - qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); - - SStringBuilder sb; memset(&sb, 0, sizeof(sb)); - char sTableName[TSDB_TABLE_NAME_LEN] = {0}; - strtolower(sTableName, point->stableName); - taosStringBuilderAppendString(&sb, sTableName); - for (int j = 0; j < point->tagNum; ++j) { - taosStringBuilderAppendChar(&sb, ','); - TAOS_SML_KV* tagKv = point->tags + j; - char tagName[TSDB_COL_NAME_LEN] = {0}; - strtolower(tagName, tagKv->key); - taosStringBuilderAppendString(&sb, tagName); - taosStringBuilderAppendChar(&sb, '='); - taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); - } - size_t len = 0; - char* keyJoined = taosStringBuilderGetResult(&sb, &len); - MD5_CTX context; - MD5Init(&context); - MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); - MD5Final(&context); - *tableNameLen = snprintf(tableName, *tableNameLen, - "t_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], - context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], - context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], - context.digest[12], context.digest[13], context.digest[14], context.digest[15]); - taosStringBuilderDestroy(&sb); - tscDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName); - return 0; -} - - -static int32_t changeChildTableTagValue(TAOS* taos, const char* cTableName, const char* tagName, TAOS_BIND* bind, SSmlLinesInfo* info) { - char sql[512]; - sprintf(sql, "alter table %s set tag %s=?", cTableName, tagName); - - int32_t code; - TAOS_STMT* stmt = taos_stmt_init(taos); - code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); - - if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); - return code; - } - - code = taos_stmt_bind_param(stmt, bind); - if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); - return code; - } - - code = taos_stmt_execute(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); - return code; - } - - code = taos_stmt_close(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); - return code; - } - return code; -} - static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, const char* sTableName, SArray* tagsSchema, SArray* tagsBind, SSmlLinesInfo* info) { size_t numTags = taosArrayGetSize(tagsSchema); @@ -673,113 +754,161 @@ static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, co free(sql); if (code != 0) { - tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_prepare returns %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_bind_param(stmt, TARRAY_GET_START(tagsBind)); if (code != 0) { - tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_bind_param returns %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_execute(stmt); if (code != 0) { - tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_execute returns %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_close(stmt); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_close return %d:%s", info->id, code, tstrerror(code)); return code; } return code; } -static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* colsSchema, SArray* rowsBind, SSmlLinesInfo* info) { - size_t numCols = taosArrayGetSize(colsSchema); - char* sql = malloc(tsMaxSQLStringLen+1); - if (sql == NULL) { - tscError("malloc sql memory error"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - int32_t freeBytes = tsMaxSQLStringLen + 1 ; - sprintf(sql, "insert into ? ("); - - for (int i = 0; i < numCols; ++i) { - SSchema* colSchema = taosArrayGet(colsSchema, i); - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", colSchema->name); - } - snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ") values ("); - - for (int i = 0; i < numCols; ++i) { - snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); - } - snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ")"); - sql[strlen(sql)] = '\0'; - - tscDebug("SML:0x%"PRIx64" insert rows into child table %s. num of rows: %zu", info->id, cTableName, taosArrayGetSize(rowsBind)); - +static int32_t doInsertChildTableWithStmt(TAOS* taos, char* sql, char* cTableName, SArray* batchBind, SSmlLinesInfo* info) { int32_t code = 0; - int32_t try = 0; TAOS_STMT* stmt = taos_stmt_init(taos); if (stmt == NULL) { - tfree(sql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } + code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); - tfree(sql); if (code != 0) { - tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } + bool tryAgain = false; + int32_t try = 0; do { code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { - tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_set_tbname return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } - size_t rows = taosArrayGetSize(rowsBind); + size_t rows = taosArrayGetSize(batchBind); for (int32_t i = 0; i < rows; ++i) { - TAOS_BIND* colsBinds = taosArrayGetP(rowsBind, i); + TAOS_BIND* colsBinds = taosArrayGetP(batchBind, i); code = taos_stmt_bind_param(stmt, colsBinds); if (code != 0) { - tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_add_batch(stmt); if (code != 0) { - tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_add_batch return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } } code = taos_stmt_execute(stmt); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s, try:%d", info->id, code, tstrerror(code), try); } - } while (code == TSDB_CODE_TDB_TABLE_RECONFIGURE && try++ < TSDB_MAX_REPLICA); - if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - } else { - taos_stmt_close(stmt); + tryAgain = false; + if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID + || code == TSDB_CODE_VND_INVALID_VGROUP_ID + || code == TSDB_CODE_TDB_TABLE_RECONFIGURE + || code == TSDB_CODE_APP_NOT_READY + || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) { + tryAgain = true; + } + + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + int32_t code2 = taos_errno(res2); + if (code2 != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + if (tryAgain) { + taosMsleep(100 * (2 << try)); + } + } + if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (tryAgain) { + taosMsleep( 100 * (2 << try)); + } + } + } while (tryAgain); + + + taos_stmt_close(stmt); + return code; +} + +static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* colsSchema, SArray* rowsBind, size_t rowSize, SSmlLinesInfo* info) { + size_t numCols = taosArrayGetSize(colsSchema); + char* sql = malloc(tsMaxSQLStringLen+1); + if (sql == NULL) { + tscError("malloc sql memory error"); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + int32_t freeBytes = tsMaxSQLStringLen + 1 ; + sprintf(sql, "insert into ? ("); + + for (int i = 0; i < numCols; ++i) { + SSchema* colSchema = taosArrayGet(colsSchema, i); + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", colSchema->name); + } + snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ") values ("); + + for (int i = 0; i < numCols; ++i) { + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); } + snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ")"); + sql[strlen(sql)] = '\0'; + + tscDebug("SML:0x%"PRIx64" insert rows into child table %s. num of rows: %zu", info->id, cTableName, taosArrayGetSize(rowsBind)); + size_t rows = taosArrayGetSize(rowsBind); + size_t maxBatchSize = TSDB_MAX_WAL_SIZE/rowSize * 4 / 5; + size_t batchSize = MIN(maxBatchSize, rows); + SArray* batchBind = taosArrayInit(batchSize, POINTER_BYTES); + int32_t code = TSDB_CODE_SUCCESS; + for (int i=0; i=i) { + code = doInsertChildTableWithStmt(taos, sql, cTableName, batchBind, info); + if (code != 0) { + taosArrayDestroy(batchBind); + tfree(sql); + return code; + } + taosArrayClear(batchBind); + } + i = j; + } + taosArrayDestroy(batchBind); + tfree(sql); return code; } @@ -787,16 +916,10 @@ static int32_t arrangePointsByChildTableName(TAOS_SML_DATA_POINT* points, int nu SHashObj* cname2points, SArray* stableSchemas, SSmlLinesInfo* info) { for (int32_t i = 0; i < numPoints; ++i) { TAOS_SML_DATA_POINT * point = points + i; - if (!point->childTableName) { - char childTableName[TSDB_TABLE_NAME_LEN]; - int32_t tableNameLen = TSDB_TABLE_NAME_LEN; - getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info); - point->childTableName = calloc(1, tableNameLen+1); - strncpy(point->childTableName, childTableName, tableNameLen); - point->childTableName[tableNameLen] = '\0'; - } - - SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, point->schemaIdx); + uintptr_t valPointer = (uintptr_t)point; + size_t* pSchemaIndex = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pSchemaIndex != NULL); + SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, *pSchemaIndex); for (int j = 0; j < point->tagNum; ++j) { TAOS_SML_KV* kv = point->tags + j; @@ -840,16 +963,10 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam TAOS_SML_DATA_POINT * pDataPoint = taosArrayGetP(cTablePoints, i); for (int j = 0; j < pDataPoint->tagNum; ++j) { TAOS_SML_KV* kv = pDataPoint->tags + j; - tagKVs[kv->fieldSchemaIdx] = kv; - } - } - - int32_t notNullTagsIndices[TSDB_MAX_TAGS] = {0}; - int32_t numNotNullTags = 0; - for (int32_t i = 0; i < numTags; ++i) { - if (tagKVs[i] != NULL) { - notNullTagsIndices[numNotNullTags] = i; - ++numNotNullTags; + uintptr_t valPointer = (uintptr_t)kv; + size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pFieldSchemaIdx != NULL); + tagKVs[*pFieldSchemaIdx] = kv; } } @@ -863,7 +980,10 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam for (int j = 0; j < numTags; ++j) { if (tagKVs[j] == NULL) continue; TAOS_SML_KV* kv = tagKVs[j]; - TAOS_BIND* bind = taosArrayGet(tagBinds, kv->fieldSchemaIdx); + uintptr_t valPointer = (uintptr_t)kv; + size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pFieldSchemaIdx != NULL); + TAOS_BIND* bind = taosArrayGet(tagBinds, *pFieldSchemaIdx); bind->buffer_type = kv->type; bind->length = malloc(sizeof(uintptr_t*)); *bind->length = kv->length; @@ -871,65 +991,8 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam bind->is_null = NULL; } - // select tag1,tag2,... from stable where tbname in (ctable) - char* sql = malloc(tsMaxSQLStringLen+1); - int freeBytes = tsMaxSQLStringLen + 1; - snprintf(sql, freeBytes, "select tbname, "); - for (int i = 0; i < numNotNullTags ; ++i) { - snprintf(sql + strlen(sql), freeBytes-strlen(sql), "%s,", tagKVs[notNullTagsIndices[i]]->key); - } - snprintf(sql + strlen(sql) - 1, freeBytes - strlen(sql) + 1, - " from %s where tbname in (\'%s\')", sTableName, cTableName); - sql[strlen(sql)] = '\0'; - - TAOS_RES* result = taos_query(taos, sql); - free(sql); - - int32_t code = taos_errno(result); - if (code != 0) { - tscError("SML:0x%"PRIx64" get child table %s tags failed. error string %s", info->id, cTableName, taos_errstr(result)); - goto cleanup; - } - - // check tag value and set tag values if different - TAOS_ROW row = taos_fetch_row(result); - if (row != NULL) { - int numFields = taos_field_count(result); - TAOS_FIELD* fields = taos_fetch_fields(result); - int* lengths = taos_fetch_lengths(result); - for (int i = 1; i < numFields; ++i) { - uint8_t dbType = fields[i].type; - int32_t length = lengths[i]; - char* val = row[i]; - - TAOS_SML_KV* tagKV = tagKVs[notNullTagsIndices[i-1]]; - if (tagKV->type != dbType) { - tscError("SML:0x%"PRIx64" child table %s tag %s type mismatch. point type : %d, db type : %d", - info->id, cTableName, tagKV->key, tagKV->type, dbType); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - assert(tagKV->value); - - if (val == NULL || length != tagKV->length || memcmp(tagKV->value, val, length) != 0) { - TAOS_BIND* bind = taosArrayGet(tagBinds, tagKV->fieldSchemaIdx); - code = changeChildTableTagValue(taos, cTableName, tagKV->key, bind, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" change child table tag failed. table name %s, tag %s", info->id, cTableName, tagKV->key); - goto cleanup; - } - } - } - tscDebug("SML:0x%"PRIx64" successfully applied point tags. child table: %s", info->id, cTableName); - } else { - code = creatChildTableIfNotExists(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, info); - if (code != 0) { - goto cleanup; - } - } + int32_t code = creatChildTableIfNotExists(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, info); -cleanup: - taos_free_result(result); for (int i = 0; i < taosArrayGetSize(tagBinds); ++i) { TAOS_BIND* bind = taosArrayGet(tagBinds, i); free(bind->length); @@ -939,7 +1002,7 @@ cleanup: } static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, char* cTableName, - SArray* cTablePoints, SSmlLinesInfo* info) { + SArray* cTablePoints, size_t rowSize, SSmlLinesInfo* info) { int32_t code = TSDB_CODE_SUCCESS; size_t numCols = taosArrayGetSize(sTableSchema->fields); @@ -963,7 +1026,10 @@ static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, } for (int j = 0; j < point->fieldNum; ++j) { TAOS_SML_KV* kv = point->fields + j; - TAOS_BIND* bind = colBinds + kv->fieldSchemaIdx; + uintptr_t valPointer = (uintptr_t)kv; + size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pFieldSchemaIdx != NULL); + TAOS_BIND* bind = colBinds + *pFieldSchemaIdx; bind->buffer_type = kv->type; bind->length = malloc(sizeof(uintptr_t*)); *bind->length = kv->length; @@ -973,7 +1039,7 @@ static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, taosArrayPush(rowsBind, &colBinds); } - code = insertChildTableBatch(taos, cTableName, sTableSchema->fields, rowsBind, info); + code = insertChildTableBatch(taos, cTableName, sTableSchema->fields, rowsBind, rowSize, info); if (code != 0) { tscError("SML:0x%"PRIx64" insert into child table %s failed. error %s", info->id, cTableName, tstrerror(code)); } @@ -1000,9 +1066,11 @@ static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t while (pCTablePoints) { SArray* cTablePoints = *pCTablePoints; - TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, 0); - SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, point->schemaIdx); + uintptr_t valPointer = (uintptr_t)point; + size_t* pSchemaIndex = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pSchemaIndex != NULL); + SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, *pSchemaIndex); tscDebug("SML:0x%"PRIx64" apply child table tags. child table: %s", info->id, point->childTableName); code = applyChildTableTags(taos, point->childTableName, point->stableName, sTableSchema, cTablePoints, info); @@ -1011,10 +1079,17 @@ static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t goto cleanup; } - tscDebug("SML:0x%"PRIx64" apply child table points. child table: %s", info->id, point->childTableName); - code = applyChildTableFields(taos, sTableSchema, point->childTableName, cTablePoints, info); + size_t rowSize = 0; + size_t numCols = taosArrayGetSize(sTableSchema->fields); + for (int i = 0; i < numCols; ++i) { + SSchema* colSchema = taosArrayGet(sTableSchema->fields, i); + rowSize += colSchema->bytes; + } + + tscDebug("SML:0x%"PRIx64" apply child table points. child table: %s, row size: %zu", info->id, point->childTableName, rowSize); + code = applyChildTableFields(taos, sTableSchema, point->childTableName, cTablePoints, rowSize, info); if (code != 0) { - tscError("Apply child table fields failed. child table %s, error %s", point->childTableName, tstrerror(code)); + tscError("SML:0x%"PRIx64" Apply child table fields failed. child table %s, error %s", info->id, point->childTableName, tstrerror(code)); goto cleanup; } @@ -1034,10 +1109,11 @@ cleanup: return code; } -int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info) { +int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info) { tscDebug("SML:0x%"PRIx64" taos_sml_insert. number of points: %d", info->id, numPoint); int32_t code = TSDB_CODE_SUCCESS; + info->smlDataToSchema = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, false); tscDebug("SML:0x%"PRIx64" build data point schemas", info->id); SArray* stableSchemas = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray @@ -1067,6 +1143,15 @@ clean_up: taosArrayDestroy(schema->tags); } taosArrayDestroy(stableSchemas); + taosHashCleanup(info->smlDataToSchema); + return code; +} + +int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { + SSmlLinesInfo* info = calloc(1, sizeof(SSmlLinesInfo)); + info->id = genLinesSmlId(); + int code = tscSmlInsert(taos, points, numPoint, info); + free(info); return code; } @@ -2076,18 +2161,6 @@ int32_t tscParseLine(const char* sql, TAOS_SML_DATA_POINT* smlData, SSmlLinesInf //========================================================================= -static uint64_t linesSmlHandleId = 0; - -uint64_t genLinesSmlId() { - uint64_t id; - - do { - id = atomic_add_fetch_64(&linesSmlHandleId, 1); - } while (id == 0); - - return id; -} - void destroySmlDataPoint(TAOS_SML_DATA_POINT* point) { for (int i=0; itagNum; ++i) { free((point->tags+i)->key); @@ -2157,7 +2230,7 @@ int taos_insert_lines(TAOS* taos, char* lines[], int numLines) { } TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); - code = taos_sml_insert(taos, points, (int)numPoints, info); + code = tscSmlInsert(taos, points, (int)numPoints, info); if (code != 0) { tscError("SML:0x%"PRIx64" taos_sml_insert error: %s", info->id, tstrerror((code))); } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 36dd60c4feeb3691e27164e309e43586374da13a..1d569064998583f5f975deec270431ef187adbcc 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1566,6 +1566,8 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { pRes->qId = 0; pRes->numOfRows = 1; + registerSqlObj(pSql); + strtolower(pSql->sqlstr, sql); tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); @@ -1575,8 +1577,6 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { pSql->cmd.insertParam.numOfParams = 0; pSql->cmd.batchSize = 0; - registerSqlObj(pSql); - int32_t ret = stmtParseInsertTbTags(pSql, pStmt); if (ret != TSDB_CODE_SUCCESS) { STMT_RET(ret); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index c84d9dd8a4fb83b21fc6e051a25da94e59f360e5..31448881d31a9cafa57590bb492e5f11a4ec0c1c 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -116,7 +116,7 @@ static int32_t validateColumnName(char* name); static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType); static int32_t setCompactVnodeInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); -static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField); +static int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField); static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); static bool hasNormalColumnFilter(SQueryInfo* pQueryInfo); @@ -1538,9 +1538,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC /* * tags name /column name is truncated in sql.y */ -bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { - //const char* msg1 = "timestamp not allowed in tags"; - const char* msg2 = "duplicated column names"; +int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { const char* msg3 = "tag length too long"; const char* msg4 = "invalid tag name"; const char* msg5 = "invalid binary/nchar tag length"; @@ -1555,8 +1553,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { // no more max columns if (numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } // no more than 6 tags @@ -1564,8 +1561,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { char msg[128] = {0}; sprintf(msg, "tags no more than %d", TSDB_MAX_TAGS); - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } // no timestamp allowable @@ -1575,8 +1571,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { //} if ((pTagField->type < TSDB_DATA_TYPE_BOOL) || (pTagField->type > TSDB_DATA_TYPE_UBIGINT)) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); @@ -1588,20 +1583,17 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { // length less than TSDB_MAX_TASG_LEN if (nLen + pTagField->bytes > TSDB_MAX_TAGS_LEN) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } // tags name can not be a keyword if (validateColumnName(pTagField->name) != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } // binary(val), val can not be equalled to or less than 0 if ((pTagField->type == TSDB_DATA_TYPE_BINARY || pTagField->type == TSDB_DATA_TYPE_NCHAR) && pTagField->bytes <= 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } // field name must be unique @@ -1609,17 +1601,15 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pTagField->name, pSchema[i].name, sizeof(pTagField->name) - 1) == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "duplicated column names"); } } - return true; + return TSDB_CODE_SUCCESS; } -bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { +int32_t validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { const char* msg1 = "too many columns"; - const char* msg2 = "duplicated column names"; const char* msg3 = "column length too long"; const char* msg4 = "invalid data type"; const char* msg5 = "invalid column name"; @@ -1634,18 +1624,15 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { // no more max columns if (numOfCols >= TSDB_MAX_COLUMNS || numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pColField->type < TSDB_DATA_TYPE_BOOL || pColField->type > TSDB_DATA_TYPE_UBIGINT) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } if (validateColumnName(pColField->name) != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } SSchema* pSchema = tscGetTableSchema(pTableMeta); @@ -1656,25 +1643,23 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { } if (pColField->bytes <= 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } // length less than TSDB_MAX_BYTES_PER_ROW if (nLen + pColField->bytes > TSDB_MAX_BYTES_PER_ROW) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } // field name must be unique for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pColField->name, pSchema[i].name, sizeof(pColField->name) - 1) == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - return false; + //return tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pColField->name, NULL); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "duplicated column names"); } } - return true; + return TSDB_CODE_SUCCESS; } /* is contained in pFieldList or not */ @@ -5792,7 +5777,6 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg19 = "invalid new tag name"; const char* msg20 = "table is not super table"; const char* msg21 = "only binary/nchar column length could be modified"; - const char* msg22 = "new column length should be bigger than old one"; const char* msg23 = "only column length coulbe be modified"; const char* msg24 = "invalid binary/nchar column length"; @@ -5844,8 +5828,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } TAOS_FIELD* p = taosArrayGet(pFieldList, 0); - if (!validateOneTags(pCmd, p)) { - return TSDB_CODE_TSC_INVALID_OPERATION; + int32_t ret = validateOneTag(pCmd, p); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); @@ -6021,8 +6006,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } TAOS_FIELD* p = taosArrayGet(pFieldList, 0); - if (!validateOneColumn(pCmd, p)) { - return TSDB_CODE_TSC_INVALID_OPERATION; + int32_t ret = validateOneColumn(pCmd, p); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); @@ -6085,7 +6071,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (pItem->bytes <= pColSchema->bytes) { - return invalidOperationMsg(pMsg, msg22); + return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, pMsg, pItem->name, NULL); } SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema; @@ -6136,7 +6122,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (pItem->bytes <= pColSchema->bytes) { - return invalidOperationMsg(pMsg, msg22); + return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_TAG_LENGTH, pMsg, pItem->name, NULL); } SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index cb5e3fc682075a33fd6511d10859a856486be5da..21e67e56e87da03afd400bea6321358541425667 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -4073,6 +4073,31 @@ int32_t tscInvalidOperationMsg(char* msg, const char* additionalInfo, const char return TSDB_CODE_TSC_INVALID_OPERATION; } +int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql) { + const char* msgFormat1 = "%s:%s"; + const char* msgFormat2 = "%s:\'%s\' (%s)"; + const char* msgFormat3 = "%s:\'%s\'"; + + const int32_t BACKWARD_CHAR_STEP = 0; + + if (sql == NULL) { + assert(errMsg != NULL); + sprintf(dstBuffer, msgFormat1, tstrerror(code), errMsg); + return code; + } + + char buf[64] = {0}; // only extract part of sql string + strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); + + if (errMsg != NULL) { + sprintf(dstBuffer, msgFormat2, tstrerror(code), buf, errMsg); + } else { + sprintf(dstBuffer, msgFormat3, tstrerror(code), buf); // no additional information for invalid sql error + } + + return code; +} + bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) { assert(pQueryInfo != NULL && pQueryInfo->clauseLimit != 0); return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit); diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 2214078f5587799ff4daea4f708e920a95e97fcf..000703464cfb8c687e473b7559e1048c42f8a6de 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -103,6 +103,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_FILE_EMPTY TAOS_DEF_ERROR_CODE(0, 0x021A) //"File is empty") #define TSDB_CODE_TSC_LINE_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x021B) //"Syntax error in Line") #define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) //"No table meta cached") +#define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names") +#define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length") +#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid column length") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") @@ -185,6 +188,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x0374) //"Invalid func") #define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0375) //"Invalid func bufSize") +#define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) //"invalid tag length") +#define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) //"invalid column length") + #define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) //"Database not specified or available") #define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) //"Database already exists") #define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) //"Invalid database options") diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index a4ecf5d6f321dee531ed8175bf54aa45d27c1a65..68529ab8a240c2313ae9417bef9f4112759b0c9f 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1518,6 +1518,13 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg) { // update SSchema *schema = (SSchema *) (pStable->schema + col); ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR); + + if (pAlter->schema[0].bytes <= schema->bytes) { + mError("msg:%p, app:%p stable:%s, modify column len. column:%s, len from %d to %d", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, name, schema->bytes, pAlter->schema[0].bytes); + return TSDB_CODE_MND_INVALID_COLUMN_LENGTH; + } + schema->bytes = pAlter->schema[0].bytes; pStable->sversion++; mInfo("msg:%p, app:%p stable %s, start to modify column %s len to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, @@ -1548,6 +1555,12 @@ static int32_t mnodeChangeSuperTableTag(SMnodeMsg *pMsg) { // update SSchema *schema = (SSchema *) (pStable->schema + col + pStable->numOfColumns); ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR); + if (pAlter->schema[0].bytes <= schema->bytes) { + mError("msg:%p, app:%p stable:%s, modify tag len. tag:%s, len from %d to %d", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, name, schema->bytes, pAlter->schema[0].bytes); + return TSDB_CODE_MND_INVALID_TAG_LENGTH; + } + schema->bytes = pAlter->schema[0].bytes; pStable->tversion++; mInfo("msg:%p, app:%p stable %s, start to modify tag len %s to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 42fc76e6c94227b6d8b5fb886239c42e19fc064d..0b9a76004b3e445abe1bbffe593d8061803d1b1b 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -112,6 +112,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, "SQL statement too lon TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FILE_EMPTY, "File is empty") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_LINE_SYNTAX_ERROR, "Syntax error in Line") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_META_CACHED, "No table meta cached") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_COL_NAMES, "duplicated column names") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TAG_LENGTH, "Invalid tag length") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid column length") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") @@ -194,6 +197,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC, "Invalid func") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TAG_LENGTH, "invalid tag length") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COLUMN_LENGTH, "invalid column length") + TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") diff --git a/tests/examples/c/schemaless.c b/tests/examples/c/schemaless.c index 89059501a9d6130d394e58c09466532c37f04cd1..830e59880198b0ef2c21cd78448fd6cb80f98b2f 100644 --- a/tests/examples/c/schemaless.c +++ b/tests/examples/c/schemaless.c @@ -58,11 +58,7 @@ int main(int argc, char* argv[]) { time_t ct = time(0); int64_t ts = ct * 1000; - char* lineFormat = - "sta%d,t0=true,t1=127i8,t2=32767i16,t3=%di32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=" - "\"binaryTagValue\",t12=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32," - "c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" %lldms"; + char* lineFormat = "sta%d,t0=true,t1=127i8,t2=32767i16,t3=%di32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=254u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" %lldms"; char** lines = calloc(numSuperTables * numChildTables * numRowsPerChildTable, sizeof(char*)); int l = 0; @@ -76,7 +72,7 @@ int main(int argc, char* argv[]) { } } } - shuffle(lines, numSuperTables * numChildTables * numRowsPerChildTable); + //shuffle(lines, numSuperTables * numChildTables * numRowsPerChildTable); printf("%s\n", "begin taos_insert_lines"); int64_t begin = getTimeInUs(); @@ -84,140 +80,5 @@ int main(int argc, char* argv[]) { int64_t end = getTimeInUs(); printf("code: %d, %s. time used: %" PRId64 "\n", code, tstrerror(code), end - begin); - char* lines_000_0[] = { - "sta1,id=sta1_1,t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=255u8,t6=32770u16,t7=" - "2147483699u32,t8=9223372036854775899u64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=" - "L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32," - "c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" " - "1626006833639000us"}; - - code = taos_insert_lines(taos, lines_000_0, sizeof(lines_000_0) / sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_000_0 should return error\n"); - return -1; - } - - char* lines_000_1[] = { - "sta2,id=\"sta2_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=255u8,t6=32770u16," - "t7=2147483699u32,t8=9223372036854775899u64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=" - "L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32," - "c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" " - "1626006833639001"}; - - code = taos_insert_lines(taos, lines_000_1, sizeof(lines_000_1) / sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_000_1 should return error\n"); - return -1; - } - - char* lines_000_2[] = { - "sta3,id=\"sta3_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=" - "22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32," - "c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 0"}; - - code = taos_insert_lines(taos, lines_000_2, sizeof(lines_000_2) / sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_000_2 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_001_0[] = { - "sta4,t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64," - "t11=\"binaryTagValue\",t12=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=" - "\"binaryValue\",c12=L\"ncharValue\" 1626006833639000us", - - }; - - code = taos_insert_lines(taos, lines_001_0, sizeof(lines_001_0) / sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_001_0 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_001_1[] = { - "sta5,id=\"sta5_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=" - "22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=" - "\"binaryValue\",c12=L\"ncharValue\" 1626006833639001"}; - - code = taos_insert_lines(taos, lines_001_1, sizeof(lines_001_1) / sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_001_1 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_001_2[] = { - "sta6,id=\"sta6_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=" - "22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" " - "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=" - "\"binaryValue\",c12=L\"ncharValue\" 0"}; - - code = taos_insert_lines(taos, lines_001_2, sizeof(lines_001_2) / sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_001_2 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_002[] = { - "stb,id=\"stb_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345," - "t11=\"binaryTagValue\",t12=L\"ncharTagValue\" " - "c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=" - "\"binaryValue\",c12=L\"ncharValue\" 1626006833639000000ns", - "stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345," - "t11=\"binaryTagValue\",t12=L\"ncharTagValue\" " - "c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=" - "\"binaryValue\",c12=L\"ncharValue\" 1626006833639019us", - "stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345," - "t11=\"binaryTagValue\",t12=L\"ncharTagValue\" " - "c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=" - "\"binaryValue\",c12=L\"ncharValue\" 1626006833640ms", - "stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345," - "t11=\"binaryTagValue\",t12=L\"ncharTagValue\" " - "c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=" - "\"binaryValue\",c12=L\"ncharValue\" 1626006834s"}; - - code = taos_insert_lines(taos, lines_002, sizeof(lines_002) / sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_002 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - // Duplicate key check; - char* lines_003_1[] = {"std,id=\"std_3_1\",t1=4i64,Id=\"std\",t2=true c1=true 1626006834s"}; - - code = taos_insert_lines(taos, lines_003_1, sizeof(lines_003_1) / sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_1 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_003_2[] = {"std,id=\"std_3_2\",tag1=4i64,Tag2=true,tAg3=2,TaG2=\"dup!\" c1=true 1626006834s"}; - - code = taos_insert_lines(taos, lines_003_2, sizeof(lines_003_2) / sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_2 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_003_3[] = {"std,id=\"std_3_3\",tag1=4i64 field1=true,Field2=2,FIElD1=\"dup!\",fIeLd4=true 1626006834s"}; - - code = taos_insert_lines(taos, lines_003_3, sizeof(lines_003_3) / sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_3 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_003_4[] = { - "std,id=\"std_3_4\",tag1=4i64,dupkey=4i16,tag2=T field1=true,dUpkEy=1e3f32,field2=\"1234\" 1626006834s"}; - - code = taos_insert_lines(taos, lines_003_4, sizeof(lines_003_4) / sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_4 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } return 0; } diff --git a/tests/pytest/insert/schemalessInsert.py b/tests/pytest/insert/schemalessInsert.py index 88abea477a9f1b458dc17c93c27d68934f5af03f..fc95f860bcc2d73826d908dd67c0c6f6e62f31cc 100644 --- a/tests/pytest/insert/schemalessInsert.py +++ b/tests/pytest/insert/schemalessInsert.py @@ -1053,7 +1053,7 @@ class TDTestCase: s_stb_d_tb_a_col_m_tag_list = self.genSqlList(stb_name=stb_name)[5] self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_a_col_m_tag_list)) tdSql.query(f"show tables;") - tdSql.checkRows(6) + tdSql.checkRows(3) def sStbDtbDdataAtMcInsertMultiThreadCheckCase(self): """ @@ -1123,7 +1123,7 @@ class TDTestCase: tdSql.checkRows(5) for t in ["t10", "t11"]: tdSql.query(f"select * from {stb_name} where {t} is not NULL;") - tdSql.checkRows(6) + tdSql.checkRows(0) def sStbDtbDdataDtsInsertMultiThreadCheckCase(self): """ @@ -1150,7 +1150,7 @@ class TDTestCase: s_stb_d_tb_d_ts_a_col_m_tag_list = self.genSqlList(stb_name=stb_name)[11] self.multiThreadRun(self.genMultiThreadSeq(s_stb_d_tb_d_ts_a_col_m_tag_list)) tdSql.query(f"show tables;") - tdSql.checkRows(6) + tdSql.checkRows(3) def test(self): input_sql1 = "rfasta,id=\"rfasta_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"ddzhiksj\",t8=L\"ncharTagValue\" c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"bnhwlgvj\",c8=L\"ncharTagValue\",c9=7u64 1626006933640000000ns" @@ -1203,19 +1203,19 @@ class TDTestCase: self.sStbDtbDdataInsertMultiThreadCheckCase() # # ! concurrency conflict - # self.sStbDtbDdataAcMtInsertMultiThreadCheckCase() - # self.sStbDtbDdataAtMcInsertMultiThreadCheckCase() + self.sStbDtbDdataAcMtInsertMultiThreadCheckCase() + self.sStbDtbDdataAtMcInsertMultiThreadCheckCase() self.sStbStbDdataDtsInsertMultiThreadCheckCase() # # ! concurrency conflict - # self.sStbStbDdataDtsAcMtInsertMultiThreadCheckCase() - # self.sStbStbDdataDtsAtMcInsertMultiThreadCheckCase() + self.sStbStbDdataDtsAcMtInsertMultiThreadCheckCase() + self.sStbStbDdataDtsAtMcInsertMultiThreadCheckCase() self.sStbDtbDdataDtsInsertMultiThreadCheckCase() # ! concurrency conflict - # self.sStbDtbDdataDtsAcMtInsertMultiThreadCheckCase() + self.sStbDtbDdataDtsAcMtInsertMultiThreadCheckCase() diff --git a/tests/pytest/query/queryDiffColsOr.py b/tests/pytest/query/queryDiffColsOr.py new file mode 100644 index 0000000000000000000000000000000000000000..feeab84a7ec577c9b48f19416400bcc7b348d543 --- /dev/null +++ b/tests/pytest/query/queryDiffColsOr.py @@ -0,0 +1,545 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from copy import deepcopy +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.common import tdCom + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def insertData(self, tb_name): + insert_sql_list = [f'insert into {tb_name} values ("2021-01-01 12:00:00", 1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1)', + f'insert into {tb_name} values ("2021-01-05 12:00:00", 2, 2, 1, 3, 1.1, 1.1, "binary", "nchar", true, 2)', + f'insert into {tb_name} values ("2021-01-07 12:00:00", 1, 3, 1, 2, 1.1, 1.1, "binary", "nchar", true, 3)', + f'insert into {tb_name} values ("2021-01-09 12:00:00", 1, 2, 4, 3, 1.1, 1.1, "binary", "nchar", true, 4)', + f'insert into {tb_name} values ("2021-01-11 12:00:00", 1, 2, 5, 5, 1.1, 1.1, "binary", "nchar", true, 5)', + f'insert into {tb_name} values ("2021-01-13 12:00:00", 1, 2, 1, 3, 6.6, 1.1, "binary", "nchar", true, 6)', + f'insert into {tb_name} values ("2021-01-15 12:00:00", 1, 2, 1, 3, 1.1, 7.7, "binary", "nchar", true, 7)', + f'insert into {tb_name} values ("2021-01-17 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary8", "nchar", true, 8)', + f'insert into {tb_name} values ("2021-01-19 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary", "nchar9", true, 9)', + f'insert into {tb_name} values ("2021-01-21 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary", "nchar", false, 10)', + f'insert into {tb_name} values ("2021-01-23 12:00:00", 1, 3, 1, 3, 1.1, 1.1, Null, Null, false, 11)' + ] + for sql in insert_sql_list: + tdSql.execute(sql) + + def initTb(self): + tdCom.cleanTb() + tb_name = tdCom.getLongName(8, "letters") + tdSql.execute( + f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 int)") + self.insertData(tb_name) + return tb_name + + def initStb(self): + tdCom.cleanTb() + tb_name = tdCom.getLongName(8, "letters") + tdSql.execute( + f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 int) tags (t1 tinyint, t2 smallint, t3 int, t4 bigint, t5 float, t6 double, t7 binary(100), t8 nchar(200), t9 bool, t10 int)") + tdSql.execute( + f'CREATE TABLE {tb_name}_sub using {tb_name} tags (1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1)') + self.insertData(f'{tb_name}_sub') + return tb_name + + def queryLastC10(self, query_sql, multi=False): + if multi: + res = tdSql.query(query_sql.replace('c10', 'last(*)'), True) + else: + res = tdSql.query(query_sql.replace('*', 'last(*)'), True) + return int(res[0][-1]) + + def queryFullColType(self, tb_name): + ## ts + query_sql = f'select * from {tb_name} where ts > "2021-01-11 12:00:00" or ts < "2021-01-13 12:00:00"' + tdSql.query(query_sql) + tdSql.checkRows(11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## != or + query_sql = f'select * from {tb_name} where c1 != 1 or c2 = 3' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## <> or + query_sql = f'select * from {tb_name} where c1 <> 1 or c3 = 3' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 2) + + ## >= or + query_sql = f'select * from {tb_name} where c1 >= 2 or c3 = 4' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + ## <= or + query_sql = f'select * from {tb_name} where c1 <= 1 or c3 = 4' + tdSql.query(query_sql) + tdSql.checkRows(10) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## <> or is Null + query_sql = f'select * from {tb_name} where c1 <> 1 or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## > or is not Null + query_sql = f'select * from {tb_name} where c2 > 2 or c8 is not Null' + tdSql.query(query_sql) + tdSql.checkRows(11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## > or < or >= or <= or != or <> or = Null + query_sql = f'select * from {tb_name} where c1 > 1 or c2 < 2 or c3 >= 4 or c4 <= 2 or c5 != 1.1 or c6 <> 1.1 or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(8) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## tiny small int big or + query_sql = f'select * from {tb_name} where c1 = 2 or c2 = 3 or c3 = 4 or c4 = 5' + tdSql.query(query_sql) + tdSql.checkRows(5) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## float double binary nchar bool or + query_sql = f'select * from {tb_name} where c5=6.6 or c6=7.7 or c7="binary8" or c8="nchar9" or c9=false' + tdSql.query(query_sql) + tdSql.checkRows(6) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## all types or + query_sql = f'select * from {tb_name} where c1=2 or c2=3 or c3=4 or c4=5 or c5=6.6 or c6=7.7 or c7="binary8" or c8="nchar9" or c9=false' + tdSql.query(query_sql) + tdSql.checkRows(10) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + def queryMultiExpression(self, tb_name): + ## condition_A and condition_B or condition_C (> < >=) + query_sql = f'select * from {tb_name} where c1 > 2 and c2 < 4 or c3 >= 4' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## (condition_A and condition_B) or condition_C (<= != <>) + query_sql = f'select * from {tb_name} where (c1 <= 1 and c2 != 2) or c4 <> 3' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A and (condition_B or condition_C) (Null not Null) + query_sql = f'select * from {tb_name} where c1 is not Null and (c6 = 7.7 or c8 is Null)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A or condition_B and condition_C (> < >=) + query_sql = f'select * from {tb_name} where c1 > 2 or c2 < 4 and c3 >= 4' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## (condition_A or condition_B) and condition_C (<= != <>) + query_sql = f'select * from {tb_name} where (c1 <= 1 or c2 != 2) and c4 <> 3' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## condition_A or (condition_B and condition_C) (Null not Null) + query_sql = f'select * from {tb_name} where c6 >= 7.7 or (c1 is not Null and c3 =5)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 7) + + ## condition_A or (condition_B and condition_C) or condition_D (> != < Null) + query_sql = f'select * from {tb_name} where c1 != 1 or (c2 >2 and c3 < 1) or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A and (condition_B or condition_C) and condition_D (>= = <= not Null) + query_sql = f'select * from {tb_name} where c4 >= 4 and (c1 = 2 or c5 <= 1.1) and c7 is not Null' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## (condition_A and condition_B) or (condition_C or condition_D) (Null >= > =) + query_sql = f'select * from {tb_name} where (c8 is Null and c1 >= 1) or (c3 > 3 or c4 =2)' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## (condition_A or condition_B) or condition_C or (condition_D and condition_E) (>= <= = not Null <>) + query_sql = f'select * from {tb_name} where (c1 >= 2 or c2 <= 1) or c3 = 4 or (c7 is not Null and c6 <> 1.1)' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 7) + + ## condition_A or (condition_B and condition_C) or (condition_D and condition_E) and condition_F + query_sql = f'select * from {tb_name} where c1 != 1 or (c2 <= 1 and c3 <4) or (c3 >= 4 or c7 is not Null) and c9 <> true' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## (condition_A or (condition_B and condition_C) or (condition_D and condition_E)) and condition_F + query_sql = f'select * from {tb_name} where (c1 != 1 or (c2 <= 2 and c3 >= 4) or (c3 >= 4 or c7 is not Null)) and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(9) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## (condition_A or condition_B) or (condition_C or condition_D) and (condition_E or condition_F or condition_G) + query_sql = f'select * from {tb_name} where c1 != 1 or (c2 <= 3 and c3 > 4) and c3 <= 5 and (c7 is not Null and c9 != false)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + def queryMultiIn(self, tb_name): + ## in and in + query_sql = f'select * from {tb_name} where c7 in ("binary") and c8 in ("nchar")' + tdSql.query(query_sql) + tdSql.checkRows(8) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## in or in + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (1, 4)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 2) + + ## in and in or condition_A + query_sql = f'select * from {tb_name} where c7 in ("binary") and c8 in ("nchar") or c10 != 10' + tdSql.query(query_sql) + tdSql.checkRows(11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## in or in and condition_A + query_sql = f'select * from {tb_name} where c7 in ("binary") or c8 in ("nchar") and c10 != 10' + tdSql.query(query_sql) + tdSql.checkRows(10) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## in or in or condition_A + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) or c9 != true' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## in or in or in or in + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) or c9 in (false) or c10 in (5, 6 ,22)' + tdSql.query(query_sql) + tdSql.checkRows(6) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## in or in and in or in + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) and c9 in (false) or c10 in (5, 6 ,22)' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A or in or condition_B and in + query_sql = f'select * from {tb_name} where c1 = 2 or c2 in (2, 4) and c9 = false or c10 in (6 ,22)' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## in and condition_A or in and in and condition_B + query_sql = f'select * from {tb_name} where c1 in (2, 3) and c2 <> 3 or c10 <= 4 and c10 in (4 ,22) and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + ## (in and condition_A or in) and in and condition_B + query_sql = f'select * from {tb_name} where (c1 in (2, 3) and c2 <> 3 or c10 <= 4) and c10 in (4 ,22) and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + def queryMultiLike(self, tb_name): + ## like and like + query_sql = f'select * from {tb_name} where c7 like "bi%" and c8 like ("ncha_")' + tdSql.query(query_sql) + tdSql.checkRows(9) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## like or like + query_sql = f'select * from {tb_name} where c7 like "binar12345" or c8 like "nchar_"' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## like and like or condition_A + query_sql = f'select * from {tb_name} where c7 like "binary_" and c8 like "ncha_" or c1 != 1' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 8) + + ## like or like and condition_A + query_sql = f'select * from {tb_name} where c7 like ("binar_") or c8 like ("nchar_") and c10 != 8' + tdSql.query(query_sql) + tdSql.checkRows(9) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## like or like or condition_A + query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("nchar_") or c10 = 6' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## like or like or like or like + query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("nchar_") or c10 = 6 or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## like or like and like or like + query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("ncha_") and c10 = 6 or c10 = 9' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## condition_A or like or condition_B and like + query_sql = f'select * from {tb_name} where c1 = 2 or c7 like "binary_" or c10 = 3 and c8 like "ncha%"' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 8) + + ## like and condition_A or like and like and condition_B + query_sql = f'select * from {tb_name} where c7 like "bin%" and c2 = 3 or c10 <= 4 and c7 like "binar_" and c8 like "ncha_"' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + ## (like and condition_A or like) and like and condition_B + query_sql = f'select * from {tb_name} where (c7 like "bin%" and c2 = 3 or c8 like "nchar_") and c7 like "binar_" and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + def queryPreCal(self, tb_name): + ## avg sum condition_A or condition_B + query_sql = f'select avg(c3), sum(c3) from {tb_name} where c10 = 5 or c8 is Null' + res = tdSql.query(query_sql, True)[0] + tdSql.checkEqual(int(res[0]), 3) + tdSql.checkEqual(int(res[1]), 6) + + ## avg sum condition_A or condition_B or condition_C + query_sql = f'select avg(c3), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c9 = false ' + res = tdSql.query(query_sql, True)[0] + tdSql.checkEqual(int(res[0]), 2) + tdSql.checkEqual(int(res[1]), 6) + + ## count avg sum condition_A or condition_B or condition_C interval + query_sql = f'select count(*), avg(c3), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c9 = false interval(16d)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(2) + tdSql.checkEqual(int(res[0][1]), 1) + tdSql.checkEqual(int(res[0][2]), 4) + tdSql.checkEqual(int(res[0][3]), 4) + tdSql.checkEqual(int(res[1][1]), 2) + tdSql.checkEqual(int(res[1][2]), 1) + tdSql.checkEqual(int(res[1][3]), 2) + + ## count avg sum condition_A or condition_B or in and like or condition_C interval + query_sql = f'select count(*), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c2 in (1, 2) and c7 like "binary_" or c1 <> 1 interval(16d)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(2) + tdSql.checkEqual(int(res[0][1]), 2) + tdSql.checkEqual(int(res[0][2]), 5) + tdSql.checkEqual(int(res[1][1]), 2) + tdSql.checkEqual(int(res[1][2]), 2) + + def queryMultiTb(self, tb_name): + ## select from (condition_A or condition_B) + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 >=3)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(3) + tdSql.checkEqual(int(res[2][0]), 11) + + ## select from (condition_A or condition_B) where condition_A or condition_B + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 >=3) where c1 =2 or c4 = 2' + res = tdSql.query(query_sql, True) + tdSql.checkRows(2) + tdSql.checkEqual(int(res[1][0]), 3) + + ## select from (condition_A or condition_B and like and in) where condition_A or condition_B or like and in + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) where c1 != 2 or c3 = 1 or c8 like "ncha_" and c9 in (true)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(7) + tdSql.checkEqual(int(res[6][0]), 10) + + ## select count avg sum from (condition_A or condition_B and like and in) where condition_A or condition_B or like and in interval + query_sql = f'select count(*), avg(c6), sum(c3) from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) where c1 != 2 or c3 = 1 or c8 like "ncha_" and c9 in (true) interval(8d)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(3) + tdSql.checkEqual(int(res[0][1]), 3) + tdSql.checkEqual(int(res[0][2]), 1) + tdSql.checkEqual(int(res[0][3]), 10) + tdSql.checkEqual(int(res[1][1]), 3) + tdSql.checkEqual(int(res[1][2]), 3) + tdSql.checkEqual(int(res[1][3]), 3) + tdSql.checkEqual(int(res[2][1]), 1) + tdSql.checkEqual(int(res[2][2]), 1) + tdSql.checkEqual(int(res[2][3]), 1) + + ## cname + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) a where a.c1 != 2 or a.c3 = 1 or a.c8 like "ncha_" and a.c9 in (true)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(7) + tdSql.checkEqual(int(res[6][0]), 10) + + ## multi cname + query_sql = f'select b.c10 from (select * from {tb_name} where c9 = true or c2 = 2) a, (select * from {tb_name} where c7 like "binar_" or c4 in (3, 5)) b where a.ts = b.ts' + res = tdSql.query(query_sql, True) + tdSql.checkRows(10) + tdSql.checkEqual(int(res[9][0]), 10) + + def checkTbColTypeOperator(self): + ''' + Ordinary table full column type and operator + ''' + tb_name = self.initTb() + self.queryFullColType(tb_name) + + def checkStbColTypeOperator(self): + ''' + Super table full column type and operator + ''' + tb_name = self.initStb() + self.queryFullColType(tb_name) + + def checkTbMultiExpression(self): + ''' + Ordinary table multiExpression + ''' + tb_name = self.initTb() + self.queryMultiExpression(tb_name) + + def checkStbMultiExpression(self): + ''' + Super table multiExpression + ''' + tb_name = self.initStb() + self.queryMultiExpression(tb_name) + + def checkTbMultiIn(self): + ''' + Ordinary table multiIn + ''' + tb_name = self.initTb() + self.queryMultiIn(tb_name) + + def checkStbMultiIn(self): + ''' + Super table multiIn + ''' + tb_name = self.initStb() + self.queryMultiIn(tb_name) + + def checkTbMultiLike(self): + ''' + Ordinary table multiLike + ''' + tb_name = self.initTb() + self.queryMultiLike(tb_name) + + def checkStbMultiLike(self): + ''' + Super table multiLike + ''' + tb_name = self.initStb() + self.queryMultiLike(tb_name) + + def checkTbPreCal(self): + ''' + Ordinary table precal + ''' + tb_name = self.initTb() + self.queryPreCal(tb_name) + + def checkStbPreCal(self): + ''' + Super table precal + ''' + tb_name = self.initStb() + self.queryPreCal(tb_name) + + def checkMultiTb(self): + ''' + test "or" in multi ordinary table + ''' + tb_name = self.initTb() + self.queryMultiTb(tb_name) + + def checkMultiStb(self): + ''' + test "or" in multi super table + ''' + tb_name = self.initStb() + self.queryMultiTb(tb_name) + + + # tb_name1 = tdCom.getLongName(8, "letters") + # tb_name2 = tdCom.getLongName(8, "letters") + # tb_name3 = tdCom.getLongName(8, "letters") + # tdSql.execute( + # f"CREATE TABLE {tb_name1} (ts timestamp, c1 tinyint, c2 smallint, c3 int)") + # tdSql.execute( + # f"CREATE TABLE {tb_name2} (ts timestamp, c1 tinyint, c2 smallint, c3 int)") + # tdSql.execute( + # f"CREATE TABLE {tb_name3} (ts timestamp, c1 tinyint, c2 smallint, c3 int)") + # insert_sql_list = [f'insert into {tb_name1} values ("2021-01-01 12:00:00", 1, 5, 1)', + # f'insert into {tb_name1} values ("2021-01-03 12:00:00", 2, 4, 1)', + # f'insert into {tb_name1} values ("2021-01-05 12:00:00", 3, 2, 1)', + # f'insert into {tb_name2} values ("2021-01-01 12:00:00", 4, 2, 1)', + # f'insert into {tb_name2} values ("2021-01-02 12:00:00", 5, 1, 1)', + # f'insert into {tb_name2} values ("2021-01-04 12:00:00", 1, 2, 1)', + # f'insert into {tb_name3} values ("2021-01-02 12:00:00", 4, 2, 1)', + # f'insert into {tb_name3} values ("2021-01-06 12:00:00", 5, 1, 1)', + # f'insert into {tb_name3} values ("2021-01-07 12:00:00", 1, 2, 1)', + # ] + # for sql in insert_sql_list: + # tdSql.execute(sql) + # tdSql.query( + # f'select * from {tb_name1} t1, {tb_name2}, {tb_name3} t3 t2 where (t1.ts=t2.ts or t2.ts=t3.ts)') + # tdSql.checkRows(4) + + + def run(self): + tdSql.prepare() + self.checkTbColTypeOperator() + self.checkStbColTypeOperator() + self.checkTbMultiExpression() + self.checkStbMultiExpression() + self.checkTbMultiIn() + self.checkStbMultiIn() + self.checkTbMultiLike() + self.checkStbMultiLike() + self.checkTbPreCal() + self.checkStbPreCal() + self.checkMultiTb() + self.checkMultiStb() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/schemalessInsertPerformance.py b/tests/pytest/tools/schemalessInsertPerformance.py new file mode 100644 index 0000000000000000000000000000000000000000..acf46b61904e896f7ce7fef3cd24879d40d53029 --- /dev/null +++ b/tests/pytest/tools/schemalessInsertPerformance.py @@ -0,0 +1,264 @@ +################################################################### +# Copyright (c) 2021 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import time +from copy import deepcopy +import numpy as np +from util.log import * +from util.cases import * +from util.sql import * +from util.common import tdCom +import threading +import itertools +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self._conn = conn + self.lock = threading.Lock() + + def genMultiColStr(self, int_count=4, double_count=0, binary_count=0): + ''' + related to self.getPerfSql() + :count = 4 ---> 4 int + :count = 1000 ---> 400 int 400 double 200 binary(128) + :count = 4000 ---> 1900 int 1900 double 200 binary(128) + ''' + col_str = "" + if double_count == 0 and binary_count == 0: + for i in range(0, int_count): + if i < (int_count-1): + col_str += f'c{i}={random.randint(0, 255)}i32,' + else: + col_str += f'c{i}={random.randint(0, 255)}i32 ' + elif double_count > 0 and binary_count == 0: + for i in range(0, int_count): + col_str += f'c{i}={random.randint(0, 255)}i32,' + for i in range(0, double_count): + if i < (double_count-1): + col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64,' + else: + col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64 ' + elif double_count == 0 and binary_count > 0: + for i in range(0, int_count): + col_str += f'c{i}={random.randint(0, 255)}i32,' + for i in range(0, binary_count): + if i < (binary_count-1): + col_str += f'c{i+int_count}=\"{tdCom.getLongName(5, "letters")}\",' + else: + col_str += f'c{i+int_count}=\"{tdCom.getLongName(5, "letters")}\" ' + elif double_count > 0 and binary_count > 0: + for i in range(0, int_count): + col_str += f'c{i}={random.randint(0, 255)}i32,' + for i in range(0, double_count): + col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64,' + for i in range(0, binary_count): + if i < (binary_count-1): + col_str += f'c{i+int_count+double_count}=\"{tdCom.getLongName(5, "letters")}\",' + else: + col_str += f'c{i+int_count+double_count}=\"{tdCom.getLongName(5, "letters")}\" ' + return col_str + + def genLongSql(self, int_count=4, double_count=0, binary_count=0, init=False): + ''' + :init ---> stb insert line + ''' + if init: + tag_str = f'id="init",t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"' + else: + tag_str = f'id="sub_{tdCom.getLongName(5, "letters")}_{tdCom.getLongName(5, "letters")}",t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"' + col_str = self.genMultiColStr(int_count=int_count, double_count=double_count, binary_count=binary_count) + long_sql = 'stb' + ',' + tag_str + ' ' + col_str + '0' + return long_sql + + def getPerfSql(self, count=4, init=False): + ''' + :count = 4 ---> 4 int + :count = 1000 ---> 400 int 400 double 200 binary(128) + :count = 4000 ---> 1900 int 1900 double 200 binary(128) + ''' + if count == 4: + input_sql = self.genLongSql(init=init) + elif count == 1000: + input_sql = self.genLongSql(400, 400, 200, init=init) + elif count == 4000: + input_sql = self.genLongSql(1900, 1900, 200, init=init) + return input_sql + + def replaceLastStr(self, str, new): + ''' + replace last element of str to new element + ''' + list_ori = list(str) + list_ori[-1] = new + return ''.join(list_ori) + + def createStb(self, count=4): + ''' + create 1 stb + ''' + input_sql = self.getPerfSql(count=count, init=True) + self._conn.insertLines([input_sql]) + + def batchCreateTable(self, batch_list): + ''' + schemaless insert api + ''' + print(threading.current_thread().name, "length=", len(batch_list)) + print(threading.current_thread().name, 'firstline', batch_list[0][0:50], '...', batch_list[0][-50:-1]) + print(threading.current_thread().name, 'lastline:', batch_list[-1][0:50], '...', batch_list[-1][-50:-1]) + self._conn.insertLines(batch_list) + print(threading.current_thread().name, 'end') + + def splitGenerator(self, table_list, sub_list_len): + ''' + split a list to n piece of sub_list + [a, b, c, d] ---> [[a, b], [c, d]] + yield type ---> generator + ''' + for i in range(0, len(table_list), sub_list_len): + yield table_list[i:i + sub_list_len] + + def genTbListGenerator(self, table_list, sub_list_len): + ''' + split table_list, after split, every sub_list len is sub_list_len + ''' + table_list_generator = self.splitGenerator(table_list, sub_list_len) + return table_list_generator + + def genTableList(self, count=4, table_count=10000): + ''' + gen len(table_count) table_list + ''' + table_list = list() + for i in range(table_count): + table_list.append(self.getPerfSql(count=count)) + return table_list + + def threadCreateTables(self, table_list_generator, thread_count=10): + ''' + thread create tables + ''' + threads = list() + for i in range(thread_count): + t = threading.Thread(target=self.batchCreateTable, args=(next(table_list_generator),)) + threads.append(t) + return threads + + def batchInsertRows(self, table_list, rows_count): + ''' + add rows in each table ---> count=rows_count + ''' + for input_sql in table_list: + ts = int(time.time()) + input_sql_list = list() + for i in range(rows_count-1): + ts -= 1 + elm_new = self.replaceLastStr(input_sql, str(ts)) + 's' + input_sql_list.append(elm_new) + self.batchCreateTable(input_sql_list) + + def threadsInsertRows(self, rows_generator, rows_count=1000, thread_count=10): + ''' + multi insert rows in each table + ''' + threads = list() + for i in range(thread_count): + self.lock.acquire() + t = threading.Thread(target=self.batchInsertRows, args=(next(rows_generator), rows_count,)) + threads.append(t) + self.lock.release() + return threads + + def multiThreadRun(self, threads): + ''' + multi run threads + ''' + for t in threads: + t.start() + for t in threads: + t.join() + + def createTables(self, count, table_count=10000, sub_list_len=1000, thread_count=10): + ''' + create stb and tb + ''' + table_list = self.genTableList(count=count, table_count=table_count) + create_tables_start_time = time.time() + self.createStb() + table_list_generator = self.genTbListGenerator(table_list, sub_list_len) + create_tables_generator, insert_rows_generator = itertools.tee(table_list_generator, 2) + self.multiThreadRun(self.threadCreateTables(table_list_generator=create_tables_generator, thread_count=thread_count)) + create_tables_end_time = time.time() + create_tables_time = int(create_tables_end_time - create_tables_start_time) + return_str = f'create tables\' time of {count} columns ---> {create_tables_time}s' + return insert_rows_generator, create_tables_time, return_str + + def insertRows(self, count, rows_generator, rows_count=1000, thread_count=10): + ''' + insert rows + ''' + insert_rows_start_time = time.time() + self.multiThreadRun(self.threadsInsertRows(rows_generator=rows_generator, rows_count=rows_count, thread_count=thread_count)) + insert_rows_end_time = time.time() + insert_rows_time = int(insert_rows_end_time - insert_rows_start_time) + return_str = f'insert rows\' time of {count} columns ---> {insert_rows_time}s' + return insert_rows_time, return_str + + def schemalessPerfTest(self, count, table_count=10000, sub_list_len=1000, thread_count=10): + ''' + get performance + ''' + insert_rows_generator = self.createTables(count=count, table_count=table_count, sub_list_len=sub_list_len, thread_count=thread_count)[0] + return self.insertRows(count=count, rows_generator=insert_rows_generator, rows_count=1000, thread_count=10) + + def getPerfResults(self, test_times=3, table_count=10000, sub_list_len=1000, thread_count=10): + col4_time = 0 + col1000_time = 0 + col4000_time = 0 + + # for i in range(test_times): + # time_used = self.schemalessPerfTest(count=4, table_count=table_count, sub_list_len=sub_list_len, thread_count=thread_count)[0] + # col4_time += time_used + # col4_time /= test_times + # print(col4_time) + + tdCom.cleanTb() + for i in range(test_times): + time_used = self.schemalessPerfTest(count=1000, table_count=table_count, sub_list_len=sub_list_len, thread_count=thread_count)[0] + col1000_time += time_used + col1000_time /= test_times + print(col1000_time) + + tdCom.cleanTb() + for i in range(test_times): + time_used = self.schemalessPerfTest(count=4000, table_count=table_count, sub_list_len=sub_list_len, thread_count=thread_count)[0] + col4000_time += time_used + col4000_time /= test_times + print(col4000_time) + + return col4_time, col1000_time, col4000_time + + def run(self): + print("running {}".format(__file__)) + tdSql.prepare() + result = self.getPerfResults(test_times=1, table_count=1000, sub_list_len=100, thread_count=10) + print(result) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 1c7d94a8a4c8cd454d41c6d4d4d89b8377ec88b7..35abc4802f9de2080a6b6a166daf833c9cf04578 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -1,4 +1,4 @@ -################################################################### +################################################################### # Copyright (c) 2016 by TAOS Technologies, Inc. # All rights reserved. # @@ -50,4 +50,4 @@ class TDCom: def close(self): self.cursor.close() -tdCom = TDCom() \ No newline at end of file +tdCom = TDCom()