diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 9ef34c96388907c631d7f9c087e715b0106fa5c0..b8eb0a5286a7b72b3ddd1d34b103e5b6239a496c 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -492,6 +492,7 @@ bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols); char *tscGetErrorMsgPayload(SSqlCmd *pCmd); +int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql); int32_t tscInvalidOperationMsg(char *msg, const char *additionalInfo, const char *sql); int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index eaad78333160a8c7dc0622ec5c670cd28bc517c8..6b12cd0da04c0f791201182c793d647fc54c00b1 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -363,15 +363,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { } if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { // stmt insert - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - code = tscGetTableMeta(pSql, pTableMetaInfo); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); - return; - } else { - assert(code == TSDB_CODE_SUCCESS); - } - (*pSql->fp)(pSql->param, pSql, code); } else if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { // file insert tscImportDataFromFile(pSql); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index b3d6ade31942372dc875ab3fdf7ad938b3ad9aca..1bf27e6cad1d57fdfd4b786d1cdcea981bf3333b 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1777,6 +1777,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow } _error: + pParentSql->res.code = code; tfree(tokenBuf); tfree(line); taos_free_result(pSql); diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 2a16f1aad6525a25da652f643dc7b7f91debd432..6c30c786435e3c81b87632fac0d2b14430c0a2c9 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -32,9 +32,6 @@ typedef struct { uint8_t type; int16_t length; char* value; - - //=================================== - uint32_t fieldSchemaIdx; } TAOS_SML_KV; typedef struct { @@ -47,9 +44,6 @@ typedef struct { // first kv must be timestamp TAOS_SML_KV* fields; int32_t fieldNum; - - //================================ - uint32_t schemaIdx; } TAOS_SML_DATA_POINT; typedef enum { @@ -62,10 +56,23 @@ typedef enum { typedef struct { uint64_t id; - + SHashObj* smlDataToSchema; } SSmlLinesInfo; + //================================================================================================= +static uint64_t linesSmlHandleId = 0; + +uint64_t genLinesSmlId() { + uint64_t id; + + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); + } while (id == 0); + + return id; +} + int compareSmlColKv(const void* p1, const void* p2) { TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; @@ -168,11 +175,46 @@ static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* arra taosHashPut(hash, field.name, tagKeyLen, &fieldIdx, sizeof(fieldIdx)); } - smlKv->fieldSchemaIdx = (uint32_t)fieldIdx; + uintptr_t valPointer = (uintptr_t)smlKv; + taosHashPut(info->smlDataToSchema, &valPointer, sizeof(uintptr_t), &fieldIdx, sizeof(fieldIdx)); return 0; } +static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen, + SSmlLinesInfo* info) { + tscDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id); + qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); + + SStringBuilder sb; memset(&sb, 0, sizeof(sb)); + char sTableName[TSDB_TABLE_NAME_LEN] = {0}; + strtolower(sTableName, point->stableName); + taosStringBuilderAppendString(&sb, sTableName); + for (int j = 0; j < point->tagNum; ++j) { + taosStringBuilderAppendChar(&sb, ','); + TAOS_SML_KV* tagKv = point->tags + j; + char tagName[TSDB_COL_NAME_LEN] = {0}; + strtolower(tagName, tagKv->key); + taosStringBuilderAppendString(&sb, tagName); + taosStringBuilderAppendChar(&sb, '='); + taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); + } + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + MD5_CTX context; + MD5Init(&context); + MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); + MD5Final(&context); + *tableNameLen = snprintf(tableName, *tableNameLen, + "t_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], + context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], + context.digest[12], context.digest[13], context.digest[14], context.digest[15]); + taosStringBuilderDestroy(&sb); + tscDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName); + return 0; +} + static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, SArray* stableSchemas, SSmlLinesInfo* info) { int32_t code = 0; SHashObj* sname2shema = taosHashInit(32, @@ -203,6 +245,15 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, for (int j = 0; j < point->tagNum; ++j) { TAOS_SML_KV* tagKv = point->tags + j; + if (!point->childTableName) { + char childTableName[TSDB_TABLE_NAME_LEN]; + int32_t tableNameLen = TSDB_TABLE_NAME_LEN; + getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info); + point->childTableName = calloc(1, tableNameLen+1); + strncpy(point->childTableName, childTableName, tableNameLen); + point->childTableName[tableNameLen] = '\0'; + } + code = buildSmlKvSchema(tagKv, pStableSchema->tagHash, pStableSchema->tags, info); if (code != 0) { tscError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, tagKv->key); @@ -219,7 +270,8 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, } } - point->schemaIdx = (uint32_t)stableIdx; + uintptr_t valPointer = (uintptr_t)point; + taosHashPut(info->smlDataToSchema, &valPointer, sizeof(uintptr_t), &stableIdx, sizeof(stableIdx)); } size_t numStables = taosArrayGetSize(stableSchemas); @@ -319,7 +371,19 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error: %s", info->id, taos_errstr(res)); + } taos_free_result(res); + + if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || code == TSDB_CODE_TSC_DUP_COL_NAMES) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + } break; } case SCHEMA_ACTION_ADD_TAG: { @@ -328,7 +392,19 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } taos_free_result(res); + + if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST || code == TSDB_CODE_TSC_DUP_COL_NAMES) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + } break; } case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { @@ -337,7 +413,19 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } taos_free_result(res); + + if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + } break; } case SCHEMA_ACTION_CHANGE_TAG_SIZE: { @@ -346,7 +434,19 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } taos_free_result(res); + + if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + } break; } case SCHEMA_ACTION_CREATE_STABLE: { @@ -375,7 +475,19 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf outBytes = snprintf(pos, freeBytes, ")"); TAOS_RES* res = taos_query(taos, result); code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res)); + } taos_free_result(res); + + if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + code = taos_errno(res2); + if (code != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + } break; } @@ -385,7 +497,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf free(result); if (code != 0) { - tscError("SML:0x%"PRIx64 "apply schema action failure. %s", info->id, tstrerror(code)); + tscError("SML:0x%"PRIx64 " apply schema action failure. %s", info->id, tstrerror(code)); } return code; } @@ -398,70 +510,12 @@ static int32_t destroySmlSTableSchema(SSmlSTableSchema* schema) { return 0; } -int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { - int32_t code = 0; - - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - tscDebug("SML:0x%"PRIx64" load table schema. super table name: %s", info->id, tableName); - - char tableNameLowerCase[TSDB_TABLE_NAME_LEN]; - strtolower(tableNameLowerCase, tableName); - - char sql[256]; - snprintf(sql, 256, "describe %s", tableNameLowerCase); - TAOS_RES* res = taos_query(taos, sql); - code = taos_errno(res); - if (code != 0) { - tscError("SML:0x%"PRIx64" describe table failure. %s", info->id, taos_errstr(res)); - taos_free_result(res); - return code; - } - taos_free_result(res); - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL){ - tscError("failed to allocate memory, reason:%s", strerror(errno)); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return code; - } - pSql->pTscObj = taos; - pSql->signature = pSql; - pSql->fp = NULL; - - SStrToken tableToken = {.z=tableNameLowerCase, .n=(uint32_t)strlen(tableNameLowerCase), .type=TK_ID}; - tGetToken(tableNameLowerCase, &tableToken.type); - // Check if the table name available or not - if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pSql->cmd.payload, "table name is invalid"); - tscFreeSqlObj(pSql); - return code; - } - - SName sname = {0}; - if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pSql); - return code; - } - char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; - memset(fullTableName, 0, tListLen(fullTableName)); - tNameExtractFullName(&sname, fullTableName); - tscFreeSqlObj(pSql); - +static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { schema->tags = taosArrayInit(8, sizeof(SSchema)); schema->fields = taosArrayInit(64, sizeof(SSchema)); schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - size_t size = 0; - STableMeta* tableMeta = NULL; - taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void **)&tableMeta, &size); - tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); schema->precision = tableMeta->tableInfo.precision; for (int i=0; itableInfo.numOfColumns; ++i) { @@ -484,9 +538,93 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSm size_t tagIndex = taosArrayGetSize(schema->tags) - 1; taosHashPut(schema->tagHash, field.name, strlen(field.name), &tagIndex, sizeof(tagIndex)); } - tscDebug("SML:0x%"PRIx64 " load table meta succeed. table name: %s, columns number: %d, tag number: %d, precision: %d", + tscDebug("SML:0x%"PRIx64 " load table schema succeed. table name: %s, columns number: %d, tag number: %d, precision: %d", info->id, tableName, tableMeta->tableInfo.numOfColumns, tableMeta->tableInfo.numOfTags, schema->precision); - free(tableMeta); tableMeta = NULL; + return TSDB_CODE_SUCCESS; +} + +static int32_t retrieveTableMeta(TAOS* taos, char* tableName, STableMeta** pTableMeta, SSmlLinesInfo* info) { + int32_t code = 0; + int32_t retries = 0; + STableMeta* tableMeta = NULL; + while (retries++ < TSDB_MAX_REPLICA && tableMeta == NULL) { + STscObj* pObj = (STscObj*)taos; + if (pObj == NULL || pObj->signature != pObj) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + + tscDebug("SML:0x%" PRIx64 " retrieve table meta. super table name: %s", info->id, tableName); + + char tableNameLowerCase[TSDB_TABLE_NAME_LEN]; + strtolower(tableNameLowerCase, tableName); + + char sql[256]; + snprintf(sql, 256, "describe %s", tableNameLowerCase); + TAOS_RES* res = taos_query(taos, sql); + code = taos_errno(res); + if (code != 0) { + tscError("SML:0x%" PRIx64 " describe table failure. %s", info->id, taos_errstr(res)); + taos_free_result(res); + return code; + } + taos_free_result(res); + + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + if (pSql == NULL) { + tscError("SML:0x%" PRIx64 " failed to allocate memory, reason:%s", info->id, strerror(errno)); + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return code; + } + pSql->pTscObj = taos; + pSql->signature = pSql; + pSql->fp = NULL; + + registerSqlObj(pSql); + SStrToken tableToken = {.z = tableNameLowerCase, .n = (uint32_t)strlen(tableNameLowerCase), .type = TK_ID}; + tGetToken(tableNameLowerCase, &tableToken.type); + // Check if the table name available or not + if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + sprintf(pSql->cmd.payload, "table name is invalid"); + tscFreeRegisteredSqlObj(pSql); + return code; + } + + SName sname = {0}; + if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { + tscFreeRegisteredSqlObj(pSql); + return code; + } + char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; + memset(fullTableName, 0, tListLen(fullTableName)); + tNameExtractFullName(&sname, fullTableName); + tscFreeRegisteredSqlObj(pSql); + + size_t size = 0; + taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void**)&tableMeta, &size); + } + + if (tableMeta != NULL) { + *pTableMeta = tableMeta; + return TSDB_CODE_SUCCESS; + } else { + tscError("SML:0x%" PRIx64 " failed to retrieve table meta. super table name: %s", info->id, tableName); + return TSDB_CODE_TSC_NO_META_CACHED; + } +} + +static int32_t loadTableSchemaFromDB(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) { + int32_t code = 0; + STableMeta* tableMeta = NULL; + code = retrieveTableMeta(taos, tableName, &tableMeta, info); + if (code == TSDB_CODE_SUCCESS) { + assert(tableMeta != NULL); + fillDbSchema(tableMeta, tableName, schema, info); + free(tableMeta); + tableMeta = NULL; + } + return code; } @@ -498,7 +636,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* SSmlSTableSchema dbSchema; memset(&dbSchema, 0, sizeof(SSmlSTableSchema)); - code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema, info); + code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; @@ -507,7 +645,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* schemaAction.createSTable.tags = pointSchema->tags; schemaAction.createSTable.fields = pointSchema->fields; applySchemaAction(taos, &schemaAction, info); - code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema, info); + code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info); if (code != 0) { tscError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, pointSchema->sTableName); return code; @@ -567,74 +705,6 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo* return 0; } -static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen, - SSmlLinesInfo* info) { - tscDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id); - qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); - - SStringBuilder sb; memset(&sb, 0, sizeof(sb)); - char sTableName[TSDB_TABLE_NAME_LEN] = {0}; - strtolower(sTableName, point->stableName); - taosStringBuilderAppendString(&sb, sTableName); - for (int j = 0; j < point->tagNum; ++j) { - taosStringBuilderAppendChar(&sb, ','); - TAOS_SML_KV* tagKv = point->tags + j; - char tagName[TSDB_COL_NAME_LEN] = {0}; - strtolower(tagName, tagKv->key); - taosStringBuilderAppendString(&sb, tagName); - taosStringBuilderAppendChar(&sb, '='); - taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); - } - size_t len = 0; - char* keyJoined = taosStringBuilderGetResult(&sb, &len); - MD5_CTX context; - MD5Init(&context); - MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); - MD5Final(&context); - *tableNameLen = snprintf(tableName, *tableNameLen, - "t_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], - context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], - context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], - context.digest[12], context.digest[13], context.digest[14], context.digest[15]); - taosStringBuilderDestroy(&sb); - tscDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName); - return 0; -} - - -static int32_t changeChildTableTagValue(TAOS* taos, const char* cTableName, const char* tagName, TAOS_BIND* bind, SSmlLinesInfo* info) { - char sql[512]; - sprintf(sql, "alter table %s set tag %s=?", cTableName, tagName); - - int32_t code; - TAOS_STMT* stmt = taos_stmt_init(taos); - code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); - - if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); - return code; - } - - code = taos_stmt_bind_param(stmt, bind); - if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); - return code; - } - - code = taos_stmt_execute(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); - return code; - } - - code = taos_stmt_close(stmt); - if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); - return code; - } - return code; -} - static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, const char* sTableName, SArray* tagsSchema, SArray* tagsBind, SSmlLinesInfo* info) { size_t numTags = taosArrayGetSize(tagsSchema); @@ -673,28 +743,28 @@ static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, co free(sql); if (code != 0) { - tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_prepare returns %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_bind_param(stmt, TARRAY_GET_START(tagsBind)); if (code != 0) { - tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_bind_param returns %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_execute(stmt); if (code != 0) { - tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_execute returns %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_close(stmt); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_close return %d:%s", info->id, code, tstrerror(code)); return code; } return code; @@ -726,27 +796,29 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols tscDebug("SML:0x%"PRIx64" insert rows into child table %s. num of rows: %zu", info->id, cTableName, taosArrayGetSize(rowsBind)); int32_t code = 0; - int32_t try = 0; TAOS_STMT* stmt = taos_stmt_init(taos); if (stmt == NULL) { tfree(sql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } + code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); tfree(sql); if (code != 0) { - tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } + bool tryAgain = false; + int32_t try = 0; do { code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { - tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_set_tbname return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } @@ -755,31 +827,52 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols TAOS_BIND* colsBinds = taosArrayGetP(rowsBind, i); code = taos_stmt_bind_param(stmt, colsBinds); if (code != 0) { - tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } code = taos_stmt_add_batch(stmt); if (code != 0) { - tfree(stmt); - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_add_batch return %d:%s", info->id, code, tstrerror(code)); + taos_stmt_close(stmt); return code; } } code = taos_stmt_execute(stmt); if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); + tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s, try:%d", info->id, code, tstrerror(code), try); } - } while (code == TSDB_CODE_TDB_TABLE_RECONFIGURE && try++ < TSDB_MAX_REPLICA); - if (code != 0) { - tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - } else { - taos_stmt_close(stmt); - } + tryAgain = false; + if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID + || code == TSDB_CODE_VND_INVALID_VGROUP_ID + || code == TSDB_CODE_TDB_TABLE_RECONFIGURE + || code == TSDB_CODE_APP_NOT_READY + || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) { + tryAgain = true; + } + if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE"); + int32_t code2 = taos_errno(res2); + if (code2 != TSDB_CODE_SUCCESS) { + tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2)); + } + taos_free_result(res2); + if (tryAgain) { + taosMsleep(50 * (2 << try)); + } + } + if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (tryAgain) { + taosMsleep( 50 * (2 << try)); + } + } + } while (tryAgain); + + + taos_stmt_close(stmt); return code; } @@ -787,16 +880,10 @@ static int32_t arrangePointsByChildTableName(TAOS_SML_DATA_POINT* points, int nu SHashObj* cname2points, SArray* stableSchemas, SSmlLinesInfo* info) { for (int32_t i = 0; i < numPoints; ++i) { TAOS_SML_DATA_POINT * point = points + i; - if (!point->childTableName) { - char childTableName[TSDB_TABLE_NAME_LEN]; - int32_t tableNameLen = TSDB_TABLE_NAME_LEN; - getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info); - point->childTableName = calloc(1, tableNameLen+1); - strncpy(point->childTableName, childTableName, tableNameLen); - point->childTableName[tableNameLen] = '\0'; - } - - SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, point->schemaIdx); + uintptr_t valPointer = (uintptr_t)point; + size_t* pSchemaIndex = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pSchemaIndex != NULL); + SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, *pSchemaIndex); for (int j = 0; j < point->tagNum; ++j) { TAOS_SML_KV* kv = point->tags + j; @@ -840,16 +927,10 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam TAOS_SML_DATA_POINT * pDataPoint = taosArrayGetP(cTablePoints, i); for (int j = 0; j < pDataPoint->tagNum; ++j) { TAOS_SML_KV* kv = pDataPoint->tags + j; - tagKVs[kv->fieldSchemaIdx] = kv; - } - } - - int32_t notNullTagsIndices[TSDB_MAX_TAGS] = {0}; - int32_t numNotNullTags = 0; - for (int32_t i = 0; i < numTags; ++i) { - if (tagKVs[i] != NULL) { - notNullTagsIndices[numNotNullTags] = i; - ++numNotNullTags; + uintptr_t valPointer = (uintptr_t)kv; + size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pFieldSchemaIdx != NULL); + tagKVs[*pFieldSchemaIdx] = kv; } } @@ -863,7 +944,10 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam for (int j = 0; j < numTags; ++j) { if (tagKVs[j] == NULL) continue; TAOS_SML_KV* kv = tagKVs[j]; - TAOS_BIND* bind = taosArrayGet(tagBinds, kv->fieldSchemaIdx); + uintptr_t valPointer = (uintptr_t)kv; + size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pFieldSchemaIdx != NULL); + TAOS_BIND* bind = taosArrayGet(tagBinds, *pFieldSchemaIdx); bind->buffer_type = kv->type; bind->length = malloc(sizeof(uintptr_t*)); *bind->length = kv->length; @@ -871,65 +955,8 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam bind->is_null = NULL; } - // select tag1,tag2,... from stable where tbname in (ctable) - char* sql = malloc(tsMaxSQLStringLen+1); - int freeBytes = tsMaxSQLStringLen + 1; - snprintf(sql, freeBytes, "select tbname, "); - for (int i = 0; i < numNotNullTags ; ++i) { - snprintf(sql + strlen(sql), freeBytes-strlen(sql), "%s,", tagKVs[notNullTagsIndices[i]]->key); - } - snprintf(sql + strlen(sql) - 1, freeBytes - strlen(sql) + 1, - " from %s where tbname in (\'%s\')", sTableName, cTableName); - sql[strlen(sql)] = '\0'; - - TAOS_RES* result = taos_query(taos, sql); - free(sql); - - int32_t code = taos_errno(result); - if (code != 0) { - tscError("SML:0x%"PRIx64" get child table %s tags failed. error string %s", info->id, cTableName, taos_errstr(result)); - goto cleanup; - } + int32_t code = creatChildTableIfNotExists(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, info); - // check tag value and set tag values if different - TAOS_ROW row = taos_fetch_row(result); - if (row != NULL) { - int numFields = taos_field_count(result); - TAOS_FIELD* fields = taos_fetch_fields(result); - int* lengths = taos_fetch_lengths(result); - for (int i = 1; i < numFields; ++i) { - uint8_t dbType = fields[i].type; - int32_t length = lengths[i]; - char* val = row[i]; - - TAOS_SML_KV* tagKV = tagKVs[notNullTagsIndices[i-1]]; - if (tagKV->type != dbType) { - tscError("SML:0x%"PRIx64" child table %s tag %s type mismatch. point type : %d, db type : %d", - info->id, cTableName, tagKV->key, tagKV->type, dbType); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - assert(tagKV->value); - - if (val == NULL || length != tagKV->length || memcmp(tagKV->value, val, length) != 0) { - TAOS_BIND* bind = taosArrayGet(tagBinds, tagKV->fieldSchemaIdx); - code = changeChildTableTagValue(taos, cTableName, tagKV->key, bind, info); - if (code != 0) { - tscError("SML:0x%"PRIx64" change child table tag failed. table name %s, tag %s", info->id, cTableName, tagKV->key); - goto cleanup; - } - } - } - tscDebug("SML:0x%"PRIx64" successfully applied point tags. child table: %s", info->id, cTableName); - } else { - code = creatChildTableIfNotExists(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, info); - if (code != 0) { - goto cleanup; - } - } - -cleanup: - taos_free_result(result); for (int i = 0; i < taosArrayGetSize(tagBinds); ++i) { TAOS_BIND* bind = taosArrayGet(tagBinds, i); free(bind->length); @@ -963,7 +990,10 @@ static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, } for (int j = 0; j < point->fieldNum; ++j) { TAOS_SML_KV* kv = point->fields + j; - TAOS_BIND* bind = colBinds + kv->fieldSchemaIdx; + uintptr_t valPointer = (uintptr_t)kv; + size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pFieldSchemaIdx != NULL); + TAOS_BIND* bind = colBinds + *pFieldSchemaIdx; bind->buffer_type = kv->type; bind->length = malloc(sizeof(uintptr_t*)); *bind->length = kv->length; @@ -1000,9 +1030,11 @@ static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t while (pCTablePoints) { SArray* cTablePoints = *pCTablePoints; - TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, 0); - SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, point->schemaIdx); + uintptr_t valPointer = (uintptr_t)point; + size_t* pSchemaIndex = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t)); + assert(pSchemaIndex != NULL); + SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, *pSchemaIndex); tscDebug("SML:0x%"PRIx64" apply child table tags. child table: %s", info->id, point->childTableName); code = applyChildTableTags(taos, point->childTableName, point->stableName, sTableSchema, cTablePoints, info); @@ -1014,7 +1046,7 @@ static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t tscDebug("SML:0x%"PRIx64" apply child table points. child table: %s", info->id, point->childTableName); code = applyChildTableFields(taos, sTableSchema, point->childTableName, cTablePoints, info); if (code != 0) { - tscError("Apply child table fields failed. child table %s, error %s", point->childTableName, tstrerror(code)); + tscError("SML:0x%"PRIx64" Apply child table fields failed. child table %s, error %s", info->id, point->childTableName, tstrerror(code)); goto cleanup; } @@ -1034,10 +1066,11 @@ cleanup: return code; } -int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info) { +int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info) { tscDebug("SML:0x%"PRIx64" taos_sml_insert. number of points: %d", info->id, numPoint); int32_t code = TSDB_CODE_SUCCESS; + info->smlDataToSchema = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, false); tscDebug("SML:0x%"PRIx64" build data point schemas", info->id); SArray* stableSchemas = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray @@ -1067,6 +1100,15 @@ clean_up: taosArrayDestroy(schema->tags); } taosArrayDestroy(stableSchemas); + taosHashCleanup(info->smlDataToSchema); + return code; +} + +int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { + SSmlLinesInfo* info = calloc(1, sizeof(SSmlLinesInfo)); + info->id = genLinesSmlId(); + int code = tscSmlInsert(taos, points, numPoint, info); + free(info); return code; } @@ -2076,18 +2118,6 @@ int32_t tscParseLine(const char* sql, TAOS_SML_DATA_POINT* smlData, SSmlLinesInf //========================================================================= -static uint64_t linesSmlHandleId = 0; - -uint64_t genLinesSmlId() { - uint64_t id; - - do { - id = atomic_add_fetch_64(&linesSmlHandleId, 1); - } while (id == 0); - - return id; -} - void destroySmlDataPoint(TAOS_SML_DATA_POINT* point) { for (int i=0; itagNum; ++i) { free((point->tags+i)->key); @@ -2157,7 +2187,7 @@ int taos_insert_lines(TAOS* taos, char* lines[], int numLines) { } TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints); - code = taos_sml_insert(taos, points, (int)numPoints, info); + code = tscSmlInsert(taos, points, (int)numPoints, info); if (code != 0) { tscError("SML:0x%"PRIx64" taos_sml_insert error: %s", info->id, tstrerror((code))); } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index e9a47abaad026325995c2d4c6699b741e56a0643..bbddc4bff925de1a7d0b67fd233b6e2e88a618a3 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1540,6 +1540,8 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { pRes->qId = 0; pRes->numOfRows = 1; + registerSqlObj(pSql); + strtolower(pSql->sqlstr, sql); tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr); @@ -1549,8 +1551,6 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { pSql->cmd.insertParam.numOfParams = 0; pSql->cmd.batchSize = 0; - registerSqlObj(pSql); - int32_t ret = stmtParseInsertTbTags(pSql, pStmt); if (ret != TSDB_CODE_SUCCESS) { STMT_RET(ret); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index c6edb6712f7af61c09947ed98345ad6aa7ae0b52..264d8a1a5121e9698b4b59184cb8a78d558ccf3e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -117,7 +117,7 @@ static int32_t validateColumnName(char* name); static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType); static int32_t setCompactVnodeInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); -static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField); +static int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField); static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); static bool hasNormalColumnFilter(SQueryInfo* pQueryInfo); @@ -427,7 +427,6 @@ int32_t readFromFile(char *name, uint32_t *len, void **buf) { return TSDB_CODE_TSC_APP_ERROR; } close(fd); - tfree(*buf); return TSDB_CODE_SUCCESS; } @@ -1545,9 +1544,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC /* * tags name /column name is truncated in sql.y */ -bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { - //const char* msg1 = "timestamp not allowed in tags"; - const char* msg2 = "duplicated column names"; +int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { const char* msg3 = "tag length too long"; const char* msg4 = "invalid tag name"; const char* msg5 = "invalid binary/nchar tag length"; @@ -1562,8 +1559,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { // no more max columns if (numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } // no more than 6 tags @@ -1571,8 +1567,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { char msg[128] = {0}; sprintf(msg, "tags no more than %d", TSDB_MAX_TAGS); - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg); } // no timestamp allowable @@ -1582,8 +1577,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { //} if ((pTagField->type < TSDB_DATA_TYPE_BOOL) || (pTagField->type > TSDB_DATA_TYPE_UBIGINT)) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); @@ -1595,20 +1589,17 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { // length less than TSDB_MAX_TASG_LEN if (nLen + pTagField->bytes > TSDB_MAX_TAGS_LEN) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } // tags name can not be a keyword if (validateColumnName(pTagField->name) != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } // binary(val), val can not be equalled to or less than 0 if ((pTagField->type == TSDB_DATA_TYPE_BINARY || pTagField->type == TSDB_DATA_TYPE_NCHAR) && pTagField->bytes <= 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } // field name must be unique @@ -1616,17 +1607,15 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pTagField->name, pSchema[i].name, sizeof(pTagField->name) - 1) == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - return false; + return tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pTagField->name, NULL); } } - return true; + return TSDB_CODE_SUCCESS; } -bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { +int32_t validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { const char* msg1 = "too many columns"; - const char* msg2 = "duplicated column names"; const char* msg3 = "column length too long"; const char* msg4 = "invalid data type"; const char* msg5 = "invalid column name"; @@ -1641,18 +1630,15 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { // no more max columns if (numOfCols >= TSDB_MAX_COLUMNS || numOfTags + numOfCols >= TSDB_MAX_COLUMNS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pColField->type < TSDB_DATA_TYPE_BOOL || pColField->type > TSDB_DATA_TYPE_UBIGINT) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } if (validateColumnName(pColField->name) != TSDB_CODE_SUCCESS) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } SSchema* pSchema = tscGetTableSchema(pTableMeta); @@ -1663,25 +1649,22 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { } if (pColField->bytes <= 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } // length less than TSDB_MAX_BYTES_PER_ROW if (nLen + pColField->bytes > TSDB_MAX_BYTES_PER_ROW) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); - return false; + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } // field name must be unique for (int32_t i = 0; i < numOfTags + numOfCols; ++i) { if (strncasecmp(pColField->name, pSchema[i].name, sizeof(pColField->name) - 1) == 0) { - invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - return false; + return tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pColField->name, NULL); } } - return true; + return TSDB_CODE_SUCCESS; } /* is contained in pFieldList or not */ @@ -6075,7 +6058,6 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg19 = "invalid new tag name"; const char* msg20 = "table is not super table"; const char* msg21 = "only binary/nchar column length could be modified"; - const char* msg22 = "new column length should be bigger than old one"; const char* msg23 = "only column length coulbe be modified"; const char* msg24 = "invalid binary/nchar column length"; @@ -6127,8 +6109,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } TAOS_FIELD* p = taosArrayGet(pFieldList, 0); - if (!validateOneTags(pCmd, p)) { - return TSDB_CODE_TSC_INVALID_OPERATION; + int32_t ret = validateOneTag(pCmd, p); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); @@ -6305,8 +6288,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } TAOS_FIELD* p = taosArrayGet(pFieldList, 0); - if (!validateOneColumn(pCmd, p)) { - return TSDB_CODE_TSC_INVALID_OPERATION; + int32_t ret = validateOneColumn(pCmd, p); + if (ret != TSDB_CODE_SUCCESS) { + return ret; } tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p); @@ -6369,7 +6353,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (pItem->bytes <= pColSchema->bytes) { - return invalidOperationMsg(pMsg, msg22); + return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, pMsg, pItem->name, NULL); } SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema; @@ -6420,7 +6404,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (pItem->bytes <= pColSchema->bytes) { - return invalidOperationMsg(pMsg, msg22); + return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_TAG_LENGTH, pMsg, pItem->name, NULL); } SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index a6fb3b5dd6bb005e470532defca3dbc5bb271464..a2a891016af91489d3ac58769b5c900eb269c898 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2444,7 +2444,11 @@ static void doSendQueryReqs(SSchedMsg* pSchedMsg) { SSqlObj* pSql = pSchedMsg->ahandle; SPair* p = pSchedMsg->msg; - for(int32_t i = p->first; i < p->second; ++i) { + for (int32_t i = p->first; i < p->second; ++i) { + if (i >= pSql->subState.numOfSub) { + tfree(p); + return; + } SSqlObj* pSub = pSql->pSubs[i]; SRetrieveSupport* pSupport = pSub->param; @@ -2584,7 +2588,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { int32_t numOfTasks = (pState->numOfSub + MAX_REQUEST_PER_TASK - 1)/MAX_REQUEST_PER_TASK; assert(numOfTasks >= 1); - int32_t num = (pState->numOfSub/numOfTasks) + 1; + int32_t num; + if (pState->numOfSub / numOfTasks == MAX_REQUEST_PER_TASK) { + num = MAX_REQUEST_PER_TASK; + } else { + num = pState->numOfSub / numOfTasks + 1; + } tscDebug("0x%"PRIx64 " query will be sent by %d threads", pSql->self, numOfTasks); for(int32_t j = 0; j < numOfTasks; ++j) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 586b929c929201e9864b0ba6843aca28544cc9de..b96fdabd5cc87a57f3e9720ffe74dcf35bd6a310 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -4168,6 +4168,31 @@ int32_t tscInvalidOperationMsg(char* msg, const char* additionalInfo, const char return TSDB_CODE_TSC_INVALID_OPERATION; } +int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql) { + const char* msgFormat1 = "%s:%s"; + const char* msgFormat2 = "%s:\'%s\' (%s)"; + const char* msgFormat3 = "%s:\'%s\'"; + + const int32_t BACKWARD_CHAR_STEP = 0; + + if (sql == NULL) { + assert(errMsg != NULL); + sprintf(dstBuffer, msgFormat1, tstrerror(code), errMsg); + return code; + } + + char buf[64] = {0}; // only extract part of sql string + strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1); + + if (errMsg != NULL) { + sprintf(dstBuffer, msgFormat2, tstrerror(code), buf, errMsg); + } else { + sprintf(dstBuffer, msgFormat3, tstrerror(code), buf); // no additional information for invalid sql error + } + + return code; +} + bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) { assert(pQueryInfo != NULL && pQueryInfo->clauseLimit != 0); return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit); diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 368658377c1a92792e9a03c039a57ddcbc5bd127..c401ab762eb7f17b075ca52ab1e9454eb136a2ab 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -103,6 +103,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_FILE_EMPTY TAOS_DEF_ERROR_CODE(0, 0x021A) //"File is empty") #define TSDB_CODE_TSC_LINE_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x021B) //"Syntax error in Line") #define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) //"No table meta cached") +#define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names") +#define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length") +#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid column length") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") @@ -185,6 +188,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x0374) //"Invalid func") #define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0375) //"Invalid func bufSize") +#define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) //"invalid tag length") +#define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) //"invalid column length") + #define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) //"Database not specified or available") #define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) //"Database already exists") #define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) //"Invalid database options") diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 7bd6842df998f5c99d192fef691b698971e6adb7..df3a741c10d9080d77efe3180cbb497ca4c1093e 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -75,6 +75,8 @@ extern char configDir[]; #define OPT_ABORT 1 /* –abort */ #define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255. +#define DEFAULT_START_TIME 1500000000000 + #define MAX_PREPARED_RAND 1000000 #define INT_BUFF_LEN 11 #define BIGINT_BUFF_LEN 21 @@ -102,6 +104,8 @@ extern char configDir[]; #define NOTE_BUFF_LEN (SMALL_BUFF_LEN*16) #define DEFAULT_TIMESTAMP_STEP 1 +#define DEFAULT_INTERLACE_ROWS 0 +#define DEFAULT_DATATYPE_NUM 3 #define DEFAULT_CHILDTABLES 10000 @@ -202,9 +206,9 @@ enum _describe_table_index { static char *g_dupstr = NULL; typedef struct SArguments_S { - char * metaFile; + char *metaFile; uint32_t test_mode; - char * host; + char *host; uint16_t port; uint16_t iface; char * user; @@ -223,7 +227,7 @@ typedef struct SArguments_S { char * output_file; bool async_mode; char * datatype[MAX_NUM_COLUMNS + 1]; - uint32_t len_of_binary; + uint32_t binwidth; uint32_t num_of_CPR; uint32_t num_of_threads; uint64_t insert_interval; @@ -364,7 +368,7 @@ typedef struct SDbs_S { bool asyncMode; uint32_t threadCount; - uint32_t threadCountByCreateTbl; + uint32_t threadCountForCreateTbl; uint32_t dbCount; SDataBase db[MAX_DB_COUNT]; @@ -583,24 +587,22 @@ char *g_randdouble_buff = NULL; char *g_aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; -#define DEFAULT_DATATYPE_NUM 3 - SArguments g_args = { - NULL, // metaFile - 0, // test_mode - "127.0.0.1", // host - 6030, // port - INTERFACE_BUT, // iface - "root", // user + NULL, // metaFile + 0, // test_mode + "localhost", // host + 6030, // port + INTERFACE_BUT, // iface + "root", // user #ifdef _TD_POWER_ "powerdb", // password #elif (_TD_TQ_ == true) - "tqueue", // password + "tqueue", // password #else - "taosdata", // password + "taosdata", // password #endif - "test", // database - 1, // replica + "test", // database + 1, // replica "d", // tb_prefix NULL, // sqlFile true, // use_metric @@ -617,13 +619,13 @@ SArguments g_args = { "INT", // datatype "FLOAT", // datatype. DEFAULT_DATATYPE_NUM is 3 }, - 64, // len_of_binary + 64, // binwidth 4, // num_of_CPR 10, // num_of_connections/thread 0, // insert_interval DEFAULT_TIMESTAMP_STEP, // timestamp_step 1, // query_times - 0, // interlace_rows; + DEFAULT_INTERLACE_ROWS, // interlace_rows; 30000, // num_of_RPR (1024*1024), // max_sql_len DEFAULT_CHILDTABLES, // num_of_tables @@ -637,8 +639,6 @@ SArguments g_args = { true, // demo_mode; }; - - static SDbs g_Dbs; static int64_t g_totalChildTables = DEFAULT_CHILDTABLES; static int64_t g_actualChildTables = 0; @@ -720,86 +720,93 @@ static void printVersion() { } static void printHelp() { - char indent[10] = " "; - printf("%s%s%s%s\n", indent, "-f", indent, + char indent[10] = " "; + printf("%s\n\n", "Usage: taosdemo [OPTION...]"); + printf("%s%s%s%s\n", indent, "-f, --file=FILE", "\t\t", "The meta file to the execution procedure. Default is './meta.json'."); - printf("%s%s%s%s\n", indent, "-u", indent, - "The TDengine user name to use when connecting to the server. Default is 'root'."); + printf("%s%s%s%s\n", indent, "-u, --user=USER", "\t\t", + "The user name to use when connecting to the server."); #ifdef _TD_POWER_ - printf("%s%s%s%s\n", indent, "-p", indent, - "The password to use when connecting to the server. Default is 'powerdb'."); - printf("%s%s%s%s\n", indent, "-c", indent, + printf("%s%s%s%s\n", indent, "-p, --password", "\t\t", + "The password to use when connecting to the server. Default is 'powerdb'"); + printf("%s%s%s%s\n", indent, "-c, --config-dir=CONFIG_DIR", "\t", "Configuration directory. Default is '/etc/power/'."); #elif (_TD_TQ_ == true) - printf("%s%s%s%s\n", indent, "-p", indent, - "The password to use when connecting to the server. Default is 'tqueue'."); - printf("%s%s%s%s\n", indent, "-c", indent, + printf("%s%s%s%s\n", indent, "-p, --password", "\t\t", + "The password to use when connecting to the server. Default is 'tqueue'"); + printf("%s%s%s%s\n", indent, "-c, --config-dir=CONFIG_DIR", "\t", "Configuration directory. Default is '/etc/tq/'."); #else - printf("%s%s%s%s\n", indent, "-p", indent, - "The password to use when connecting to the server. Default is 'taosdata'."); - printf("%s%s%s%s\n", indent, "-c", indent, - "Configuration directory. Default is '/etc/taos/'."); + printf("%s%s%s%s\n", indent, "-p, --password", "\t\t", + "The password to use when connecting to the server."); + printf("%s%s%s%s\n", indent, "-c, --config-dir=CONFIG_DIR", "\t", + "Configuration directory."); #endif - printf("%s%s%s%s\n", indent, "-h", indent, - "The host to connect to TDengine. Default is localhost."); - printf("%s%s%s%s\n", indent, "-P", indent, - "The TCP/IP port number to use for the connection. Default is 0."); - printf("%s%s%s%s\n", indent, "-I", indent, + printf("%s%s%s%s\n", indent, "-h, --host=HOST", "\t\t", + "TDengine server FQDN to connect. The default host is localhost."); + printf("%s%s%s%s\n", indent, "-P, --port=PORT", "\t\t", + "The TCP/IP port number to use for the connection."); + printf("%s%s%s%s\n", indent, "-I, --interface=INTERFACE", "\t", "The interface (taosc, rest, and stmt) taosdemo uses. Default is 'taosc'."); - printf("%s%s%s%s\n", indent, "-d", indent, + printf("%s%s%s%s\n", indent, "-d, --database=DATABASE", "\t", "Destination database. Default is 'test'."); - printf("%s%s%s%s\n", indent, "-a", indent, + printf("%s%s%s%s\n", indent, "-a, --replica=REPLICA", "\t\t", "Set the replica parameters of the database, Default 1, min: 1, max: 3."); - printf("%s%s%s%s\n", indent, "-m", indent, + printf("%s%s%s%s\n", indent, "-m, --table-prefix=TABLEPREFIX", "\t", "Table prefix name. Default is 'd'."); - printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file."); - printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag."); - printf("%s%s%s%s\n", indent, "-o", indent, + printf("%s%s%s%s\n", indent, "-s, --sql-file=FILE", "\t\t", "The select sql file."); + printf("%s%s%s%s\n", indent, "-N, --normal-table", "\t\t", "Use normal table flag."); + printf("%s%s%s%s\n", indent, "-o, --output=FILE", "\t\t", "Direct output to the named file. Default is './output.txt'."); - printf("%s%s%s%s\n", indent, "-q", indent, + printf("%s%s%s%s\n", indent, "-s, --sql-file=FILE", "\t\t", + "The select sql file."); + printf("%s%s%s%s\n", indent, "-q, --query-mode=MODE", "\t\t", "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC."); - printf("%s%s%s%s\n", indent, "-b", indent, + printf("%s%s%s%s\n", indent, "-b, --data-type=DATATYPE", "\t", "The data_type of columns, default: FLOAT, INT, FLOAT."); - printf("%s%s%s%s%d\n", indent, "-w", indent, - "The length of data_type 'BINARY' or 'NCHAR'. Default is ", - g_args.len_of_binary); - printf("%s%s%s%s%d%s%d\n", indent, "-l", indent, + printf("%s%s%s%s%d\n", indent, "-w, --binwidth=WIDTH", "\t\t", + "The width of data_type 'BINARY' or 'NCHAR'. Default is ", + g_args.binwidth); + printf("%s%s%s%s%d%s%d\n", indent, "-l, --columns=COLUMNS", "\t\t", "The number of columns per record. Demo mode by default is ", DEFAULT_DATATYPE_NUM, " (float, int, float). Max values is ", MAX_NUM_COLUMNS); printf("%s%s%s%s\n", indent, indent, indent, - "All of the new column(s) type is INT. If use -b to specify column type, -l will be ignored."); - printf("%s%s%s%s\n", indent, "-T", indent, + "\t\t\t\tAll of the new column(s) type is INT. If use -b to specify column type, -l will be ignored."); + printf("%s%s%s%s\n", indent, "-T, --threads=NUMBER", "\t\t", "The number of threads. Default is 10."); - printf("%s%s%s%s\n", indent, "-i", indent, + printf("%s%s%s%s\n", indent, "-i, --insert-interval=NUMBER", "\t", "The sleep time (ms) between insertion. Default is 0."); - printf("%s%s%s%s%d.\n", indent, "-S", indent, + printf("%s%s%s%s%d.\n", indent, "-S, --time-step=TIME_STEP", "\t", "The timestamp step between insertion. Default is ", DEFAULT_TIMESTAMP_STEP); - printf("%s%s%s%s\n", indent, "-r", indent, + printf("%s%s%s%s%d.\n", indent, "-B, --interlace-rows=NUMBER", "\t", + "The interlace rows of insertion. Default is ", + DEFAULT_INTERLACE_ROWS); + printf("%s%s%s%s\n", indent, "-r, --rec-per-req=NUMBER", "\t", "The number of records per request. Default is 30000."); - printf("%s%s%s%s\n", indent, "-t", indent, + printf("%s%s%s%s\n", indent, "-t, --tables=NUMBER", "\t\t", "The number of tables. Default is 10000."); - printf("%s%s%s%s\n", indent, "-n", indent, + printf("%s%s%s%s\n", indent, "-n, --records=NUMBER", "\t\t", "The number of records per table. Default is 10000."); - printf("%s%s%s%s\n", indent, "-M", indent, + printf("%s%s%s%s\n", indent, "-M, --random", "\t\t\t", "The value of records generated are totally random."); - printf("%s%s%s%s\n", indent, indent, indent, - " The default is to simulate power equipment senario."); - printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag."); - printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt."); - printf("%s%s%s%s\n", indent, "-O", indent, - "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order."); - printf("%s%s%s%s\n", indent, "-R", indent, + printf("%s\n", "\t\t\t\tThe default is to simulate power equipment senario."); + printf("%s%s%s%s\n", indent, "-x, --no-insert", "\t\t", + "No-insert flag."); + printf("%s%s%s%s\n", indent, "-y, --answer-yes", "\t\t", "Default input yes for prompt."); + printf("%s%s%s%s\n", indent, "-O, --disorder=NUMBER", "\t\t", + "Insert order mode--0: In order, 1 ~ 50: disorder ratio. Default is in order."); + printf("%s%s%s%s\n", indent, "-R, --disorder-range=NUMBER", "\t", "Out of order data's range, ms, default is 1000."); - printf("%s%s%s%s\n", indent, "-g", indent, + printf("%s%s%s%s\n", indent, "-g, --debug", "\t\t\t", "Print debug info."); - printf("%s%s%s\n", indent, "-V, --version\t", - "Print version info."); - printf("%s%s%s%s\n", indent, "--help\t", indent, - "Print command line arguments list info."); + printf("%s%s%s%s\n", indent, "-?, --help\t", "\t\t", + "Give this help list"); + printf("%s%s%s%s\n", indent, " --usage\t", "\t\t", + "Give a short usage message"); + printf("%s%s\n", indent, "-V, --version\t\t\tPrint version info."); /* printf("%s%s%s%s\n", indent, "-D", indent, "Delete database if exists. 0: no, 1: yes, default is 1"); */ @@ -820,66 +827,149 @@ static bool isStringNumber(char *input) return true; } +static void errorPrintReqArg(char *program, char *wrong_arg) +{ + fprintf(stderr, + "%s: option requires an argument -- '%s'\n", + program, wrong_arg); + fprintf(stderr, + "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); +} + +static void errorPrintReqArg2(char *program, char *wrong_arg) +{ + fprintf(stderr, + "%s: option requires a number argument '-%s'\n", + program, wrong_arg); + fprintf(stderr, + "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); +} + +static void errorPrintReqArg3(char *program, char *wrong_arg) +{ + fprintf(stderr, + "%s: option '%s' requires an argument\n", + program, wrong_arg); + fprintf(stderr, + "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); +} + static void parse_args(int argc, char *argv[], SArguments *arguments) { for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "-f") == 0) { - arguments->demo_mode = false; + if ((strcmp(argv[i], "-f") == 0) + || (0 == strncmp(argv[i], "--file", strlen("--file")))) { + if (2 == strlen(argv[i])) { + arguments->demo_mode = false; - if (NULL == argv[i+1]) { - printHelp(); - errorPrint("%s", "\n\t-f need a valid json file following!\n"); - exit(EXIT_FAILURE); - } - arguments->metaFile = argv[++i]; - } else if (strcmp(argv[i], "-c") == 0) { - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-c need a valid path following!\n"); + if (NULL == argv[i+1]) { + errorPrintReqArg3(argv[0], "f"); + exit(EXIT_FAILURE); + } + arguments->metaFile = argv[++i]; + } else if (strlen("--file") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--file"); exit(EXIT_FAILURE); - } - tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN); - } else if (strcmp(argv[i], "-h") == 0) { - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-h need a valid string following!\n"); + } else if (0 == strncmp(argv[i], "--file=", strlen("--file="))) { + arguments->metaFile = (char *)(argv[i] + strlen("--file=")); + } + } else if ((strcmp(argv[i], "-c") == 0) + || (0 == strncmp(argv[i], "--config-dir", strlen("--config-dir")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "c"); + exit(EXIT_FAILURE); + } + tstrncpy(configDir, argv[++i], TSDB_FILENAME_LEN); + } else if (strlen("--config-dir") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--config-dir"); exit(EXIT_FAILURE); - } - arguments->host = argv[++i]; - } else if (strcmp(argv[i], "-P") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-P need a number following!\n"); + } else if (0 == strncmp(argv[i], "--config-dir=", strlen("--config-dir="))) { + tstrncpy(configDir, (char *)(argv[i] + strlen("--config-dir=")), TSDB_FILENAME_LEN); + } + } else if ((strcmp(argv[i], "-h") == 0) + || (0 == strncmp(argv[i], "--host", strlen("--host")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "h"); + exit(EXIT_FAILURE); + } + arguments->host = argv[++i]; + } else if (strlen("--host") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--host"); exit(EXIT_FAILURE); - } - arguments->port = atoi(argv[++i]); - } else if (strcmp(argv[i], "-I") == 0) { - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-I need a valid string following!\n"); + } else if (0 == strncmp(argv[i], "--host=", strlen("--host="))) { + arguments->host = (char *)(argv[i] + strlen("--host=")); + } + } else if ((strcmp(argv[i], "-P") == 0) + || (0 == strncmp(argv[i], "--port", strlen("--port")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "P"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "P"); + exit(EXIT_FAILURE); + } + arguments->port = atoi(argv[++i]); + } else if (strlen("--port") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--port"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--port=", strlen("--port="))) { + if (isStringNumber((char *)(argv[i] + strlen("--port=")))) { + arguments->port = atoi((char *)(argv[i]+strlen("--port="))); + } } - ++i; - if (0 == strcasecmp(argv[i], "taosc")) { - arguments->iface = TAOSC_IFACE; - } else if (0 == strcasecmp(argv[i], "rest")) { - arguments->iface = REST_IFACE; - } else if (0 == strcasecmp(argv[i], "stmt")) { - arguments->iface = STMT_IFACE; - } else { - errorPrint("%s", "\n\t-I need a valid string following!\n"); + } else if ((strcmp(argv[i], "-I") == 0) + || (0 == strncmp(argv[i], "--interface", strlen("--interface")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "I"); + exit(EXIT_FAILURE); + } + if (0 == strcasecmp(argv[i+1], "taosc")) { + arguments->iface = TAOSC_IFACE; + } else if (0 == strcasecmp(argv[i+1], "rest")) { + arguments->iface = REST_IFACE; + } else if (0 == strcasecmp(argv[i+1], "stmt")) { + arguments->iface = STMT_IFACE; + } else { + errorPrintReqArg(argv[0], "I"); + exit(EXIT_FAILURE); + } + i++; + } else if (strlen("--interface") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--interface"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--interface=", strlen("--interface="))) { + if (0 == strcasecmp((char *)(argv[i] + strlen("--interface=")), "taosc")) { + arguments->iface = TAOSC_IFACE; + } else if (0 == strcasecmp((char *)(argv[i] + strlen("--interface=")), "rest")) { + arguments->iface = REST_IFACE; + } else if (0 == strcasecmp((char *)(argv[i] + strlen("--interface=")), "stmt")) { + arguments->iface = STMT_IFACE; + } else { + errorPrintReqArg3(argv[0], "--interface"); + exit(EXIT_FAILURE); + } } - } else if (strcmp(argv[i], "-u") == 0) { - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-u need a valid string following!\n"); + } else if ((strcmp(argv[i], "-u") == 0) + || (0 == strncmp(argv[i], "--user", strlen("--user")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "u"); + exit(EXIT_FAILURE); + } + arguments->user = argv[++i]; + } else if (strlen("--user") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--user"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--user=", strlen("--user="))) { + arguments->user = (char *)(argv[i++] + strlen("--user=")); } - arguments->user = argv[++i]; - } else if (strncmp(argv[i], "-p", 2) == 0) { - if (strlen(argv[i]) == 2) { + } else if ((strncmp(argv[i], "-p", 2) == 0) + || (0 == strcmp(argv[i], "--password"))) { + if ((strlen(argv[i]) == 2) || (0 == strcmp(argv[i], "--password"))) { printf("Enter password: "); taosSetConsoleEcho(false); if (scanf("%s", arguments->password) > 1) { @@ -889,37 +979,100 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } else { tstrncpy(arguments->password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); } - } else if (strcmp(argv[i], "-o") == 0) { - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-o need a valid string following!\n"); + } else if ((strcmp(argv[i], "-o") == 0) + || (0 == strncmp(argv[i], "--output", strlen("--output")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "o"); + exit(EXIT_FAILURE); + } + arguments->output_file = argv[++i]; + } else if (strlen("--output") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--output"); exit(EXIT_FAILURE); - } - arguments->output_file = argv[++i]; - } else if (strcmp(argv[i], "-s") == 0) { - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-s need a valid string following!\n"); + } else if (0 == strncmp(argv[i], "--output=", strlen("--output="))) { + arguments->output_file = (char *)(argv[i++] + strlen("--output=")); + } + } else if ((strcmp(argv[i], "-s") == 0) + || (0 == strncmp(argv[i], "--sql-file", strlen("--sql-file")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "s"); + exit(EXIT_FAILURE); + } + arguments->sqlFile = argv[++i]; + } else if (strlen("--sql-file") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--sql-file"); + exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--sql-file=", strlen("--sql-file="))) { + arguments->host = (char *)(argv[i++] + strlen("--sql-file=")); + } + } else if ((strcmp(argv[i], "-q") == 0) + || (0 == strncmp(argv[i], "--query-mode", strlen("--query-mode")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "q"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "q"); + exit(EXIT_FAILURE); + } + arguments->async_mode = atoi(argv[++i]); + } else if (strlen("--query-mode") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--query-mode"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--query-mode=", strlen("--query-mode="))) { + if (isStringNumber((char *)(argv[i] + strlen("--query-mode=")))) { + arguments->async_mode = atoi((char *)(argv[i]+strlen("--query-mode="))); + } else { + errorPrintReqArg2(argv[0], "--query-mode"); + exit(EXIT_FAILURE); + } } - arguments->sqlFile = argv[++i]; - } else if (strcmp(argv[i], "-q") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, not-0: ASYNC. Default is SYNC.\n"); + } else if ((strcmp(argv[i], "-T") == 0) + || (0 == strncmp(argv[i], "--threads", strlen("--threads")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "T"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "T"); + exit(EXIT_FAILURE); + } + arguments->num_of_threads = atoi(argv[++i]); + } else if (strlen("--threads") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--threads"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--threads=", strlen("--threads="))) { + if (isStringNumber((char *)(argv[i] + strlen("--threads=")))) { + arguments->num_of_threads = atoi((char *)(argv[i]+strlen("--threads="))); + } else { + errorPrintReqArg2(argv[0], "--threads"); + exit(EXIT_FAILURE); + } } - arguments->async_mode = atoi(argv[++i]); - } else if (strcmp(argv[i], "-T") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-T need a number following!\n"); + } else if ((strcmp(argv[i], "-i") == 0) + || (0 == strncmp(argv[i], "--insert-interval", strlen("--insert-interval")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "i"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "i"); + exit(EXIT_FAILURE); + } + arguments->insert_interval = atoi(argv[++i]); + } else if (strlen("--insert-interval")== strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--insert-interval"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--insert-interval=", strlen("--insert-interval="))) { + if (isStringNumber((char *)(argv[i] + 18))) { + arguments->insert_interval = atoi((char *)(argv[i]+strlen("--insert-interval="))); + } else { + errorPrintReqArg3(argv[0], "--insert-innterval"); + exit(EXIT_FAILURE); + } } - arguments->num_of_threads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-i") == 0) { if ((argc == i+1) || (!isStringNumber(argv[i+1]))) { printHelp(); @@ -927,14 +1080,28 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->insert_interval = atoi(argv[++i]); - } else if (strcmp(argv[i], "-S") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("\n\t%s%s", argv[i], " need a number following!\n"); + } else if ((strcmp(argv[i], "-S") == 0) + || (0 == strncmp(argv[i], "--time-step", strlen("--time-step")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "S"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "S"); + exit(EXIT_FAILURE); + } + arguments->async_mode = atoi(argv[++i]); + } else if (strlen("--time-step") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--time-step"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--time-step=", strlen("--time-step="))) { + if (isStringNumber((char *)(argv[i] + strlen("--time-step=")))) { + arguments->async_mode = atoi((char *)(argv[i]+strlen("--time-step="))); + } else { + errorPrintReqArg2(argv[0], "--time-step"); + exit(EXIT_FAILURE); + } } - arguments->timestamp_step = atoi(argv[++i]); } else if (strcmp(argv[i], "-qt") == 0) { if ((argc == i+1) || (!isStringNumber(argv[i+1]))) { @@ -943,56 +1110,134 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { exit(EXIT_FAILURE); } arguments->query_times = atoi(argv[++i]); - } else if (strcmp(argv[i], "-B") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-B need a number following!\n"); + } else if ((strcmp(argv[i], "-B") == 0) + || (0 == strncmp(argv[i], "--interlace-rows", strlen("--interlace-rows")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "B"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "B"); + exit(EXIT_FAILURE); + } + arguments->interlace_rows = atoi(argv[++i]); + } else if (strlen("--interlace-rows")== strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--interlace-rows"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--interlace-rows=", strlen("--interlace-rows="))) { + if (isStringNumber((char *)(argv[i] + strlen("--interlace-rows=")))) { + arguments->interlace_rows = atoi((char *)(argv[i]+strlen("--interlace-rows="))); + } else { + errorPrintReqArg2(argv[0], "--interlace-rows"); + exit(EXIT_FAILURE); + } } - arguments->interlace_rows = atoi(argv[++i]); - } else if (strcmp(argv[i], "-r") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-r need a number following!\n"); + } else if ((strcmp(argv[i], "-r") == 0) + || (0 == strncmp(argv[i], "--rec-per-req", 13))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "r"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "r"); + exit(EXIT_FAILURE); + } + arguments->num_of_RPR = atoi(argv[++i]); + } else if (strlen("--rec-per-req")== strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--rec-per-req"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--rec-per-req=", strlen("--rec-per-req="))) { + if (isStringNumber((char *)(argv[i] + strlen("--rec-per-req=")))) { + arguments->num_of_RPR = atoi((char *)(argv[i]+strlen("--rec-per-req="))); + } else { + errorPrintReqArg2(argv[0], "--rec-per-req"); + exit(EXIT_FAILURE); + } } - arguments->num_of_RPR = atoi(argv[++i]); - } else if (strcmp(argv[i], "-t") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-t need a number following!\n"); + } else if ((strcmp(argv[i], "-t") == 0) + || (0 == strncmp(argv[i], "--tables", strlen("--tables")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "t"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "t"); + exit(EXIT_FAILURE); + } + arguments->num_of_tables = atoi(argv[++i]); + } else if (strlen("--tables") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--tables"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--tables=", strlen("--tables="))) { + if (isStringNumber((char *)(argv[i] + strlen("--tables=")))) { + arguments->num_of_tables = atoi((char *)(argv[i]+strlen("--tables="))); + } else { + errorPrintReqArg2(argv[0], "--tables"); + exit(EXIT_FAILURE); + } } - arguments->num_of_tables = atoi(argv[++i]); - g_totalChildTables = arguments->num_of_tables; - } else if (strcmp(argv[i], "-n") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-n need a number following!\n"); + } else if ((strcmp(argv[i], "-n") == 0) + || (0 == strncmp(argv[i], "--records", strlen("--records")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "n"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "n"); + exit(EXIT_FAILURE); + } + arguments->num_of_DPT = atoi(argv[++i]); + } else if (strlen("--records") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--records"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--records=", strlen("--records="))) { + if (isStringNumber((char *)(argv[i] + strlen("--records=")))) { + arguments->num_of_DPT = atoi((char *)(argv[i]+strlen("--records="))); + } else { + errorPrintReqArg2(argv[0], "--records"); + exit(EXIT_FAILURE); + } } - arguments->num_of_DPT = atoi(argv[++i]); - } else if (strcmp(argv[i], "-d") == 0) { - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-d need a valid string following!\n"); + + g_totalChildTables = arguments->num_of_DPT; + } else if ((strcmp(argv[i], "-d") == 0) + + || (0 == strncmp(argv[i], "--database", strlen("--database")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "d"); + exit(EXIT_FAILURE); + } + arguments->database = argv[++i]; + } else if (strlen("--database") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--database"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--database=", strlen("--database="))) { + arguments->output_file = (char *)(argv[i] + strlen("--database=")); } - arguments->database = argv[++i]; - } else if (strcmp(argv[i], "-l") == 0) { + } else if ((strcmp(argv[i], "-l") == 0) + || (0 == strncmp(argv[i], "--columns", strlen("--columns")))) { arguments->demo_mode = false; - if (argc == i+1) { - if (!isStringNumber(argv[i+1])) { - printHelp(); - errorPrint("%s", "\n\t-l need a number following!\n"); + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "l"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "l"); + exit(EXIT_FAILURE); + } + arguments->num_of_CPR = atoi(argv[++i]); + } else if (strlen("--columns")== strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--columns"); + exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--columns=", strlen("--columns="))) { + if (isStringNumber((char *)(argv[i] + strlen("--columns=")))) { + arguments->num_of_CPR = atoi((char *)(argv[i]+strlen("--columns"))); + } else { + errorPrintReqArg2(argv[0], "--columns"); exit(EXIT_FAILURE); } } - arguments->num_of_CPR = atoi(argv[++i]); if (arguments->num_of_CPR > MAX_NUM_COLUMNS) { printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_COLUMNS); @@ -1006,36 +1251,46 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { for (int col = arguments->num_of_CPR; col < MAX_NUM_COLUMNS; col++) { arguments->datatype[col] = NULL; } - } else if (strcmp(argv[i], "-b") == 0) { + } else if ((strcmp(argv[i], "-b") == 0) + || (0 == strncmp(argv[i], "--data-type", strlen("--data-type")))) { arguments->demo_mode = false; - if (argc == i+1) { - printHelp(); - errorPrint("%s", "\n\t-b need valid string following!\n"); + + char *dataType; + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "b"); + exit(EXIT_FAILURE); + } + dataType = argv[++i]; + } else if (strlen("--data-type") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--data-type"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--data-type=", strlen("--data-type="))) { + dataType = (char *)(argv[i] + strlen("--data-type=")); } - ++i; - if (strstr(argv[i], ",") == NULL) { + + if (strstr(dataType, ",") == NULL) { // only one col - if (strcasecmp(argv[i], "INT") - && strcasecmp(argv[i], "FLOAT") - && strcasecmp(argv[i], "TINYINT") - && strcasecmp(argv[i], "BOOL") - && strcasecmp(argv[i], "SMALLINT") - && strcasecmp(argv[i], "BIGINT") - && strcasecmp(argv[i], "DOUBLE") - && strcasecmp(argv[i], "BINARY") - && strcasecmp(argv[i], "TIMESTAMP") - && strcasecmp(argv[i], "NCHAR")) { + if (strcasecmp(dataType, "INT") + && strcasecmp(dataType, "FLOAT") + && strcasecmp(dataType, "TINYINT") + && strcasecmp(dataType, "BOOL") + && strcasecmp(dataType, "SMALLINT") + && strcasecmp(dataType, "BIGINT") + && strcasecmp(dataType, "DOUBLE") + && strcasecmp(dataType, "BINARY") + && strcasecmp(dataType, "TIMESTAMP") + && strcasecmp(dataType, "NCHAR")) { printHelp(); errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); } - arguments->datatype[0] = argv[i]; + arguments->datatype[0] = dataType; arguments->datatype[1] = NULL; } else { // more than one col int index = 0; - g_dupstr = strdup(argv[i]); + g_dupstr = strdup(dataType); char *running = g_dupstr; char *token = strsep(&running, ","); while(token != NULL) { @@ -1060,75 +1315,155 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->datatype[index] = NULL; } - } else if (strcmp(argv[i], "-w") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-w need a number following!\n"); + } else if ((strcmp(argv[i], "-w") == 0) + || (0 == strncmp(argv[i], "--binwidth", strlen("--binwidth")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "w"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "w"); + exit(EXIT_FAILURE); + } + arguments->binwidth = atoi(argv[++i]); + } else if (strlen("--binwidth") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--binwidth"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--binwidth=", strlen("--binwidth="))) { + if (isStringNumber((char *)(argv[i] + strlen("--binwidth=")))) { + arguments->binwidth = atoi((char *)(argv[i]+strlen("--binwidth="))); + } else { + errorPrintReqArg2(argv[0], "--binwidth"); + exit(EXIT_FAILURE); + } } - arguments->len_of_binary = atoi(argv[++i]); - } else if (strcmp(argv[i], "-m") == 0) { - if ((argc == i+1) || - (isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-m need a letter-initial string following!\n"); + } else if ((strcmp(argv[i], "-m") == 0) + || (0 == strncmp(argv[i], "--table-prefix", strlen("--table-prefix")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg3(argv[0], "m"); + exit(EXIT_FAILURE); + } + arguments->tb_prefix = argv[++i]; + } else if (strlen("--table-prefix") == strlen(argv[i]) + || (strlen("--table-prefix=") == strlen(argv[i]))) { + errorPrintReqArg3(argv[0], "--table-prefix"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--table-prefix=", strlen("--table-prefix="))) { + arguments->tb_prefix = (char *)(argv[i] + strlen("--table-prefix=")); } - arguments->tb_prefix = argv[++i]; - } else if (strcmp(argv[i], "-N") == 0) { + } else if ((strcmp(argv[i], "-N") == 0) + || (0 == strcmp(argv[i], "--normal-table"))) { arguments->use_metric = false; - } else if (strcmp(argv[i], "-M") == 0) { + } else if ((strcmp(argv[i], "-M") == 0) + || (0 == strcmp(argv[i], "--random"))) { arguments->demo_mode = false; - } else if (strcmp(argv[i], "-x") == 0) { + } else if ((strcmp(argv[i], "-x") == 0) + || (0 == strcmp(argv[i], "--no-insert"))) { arguments->insert_only = false; - } else if (strcmp(argv[i], "-y") == 0) { + } else if ((strcmp(argv[i], "-y") == 0) + || (0 == strcmp(argv[i], "--answer-yes"))) { arguments->answer_yes = true; - } else if (strcmp(argv[i], "-g") == 0) { + } else if ((strcmp(argv[i], "-g") == 0) + || (0 == strcmp(argv[i], "--debug"))) { arguments->debug_print = true; } else if (strcmp(argv[i], "-gg") == 0) { arguments->verbose_print = true; } else if (strcmp(argv[i], "-PP") == 0) { arguments->performance_print = true; - } else if (strcmp(argv[i], "-O") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-O need a number following!\n"); + } else if ((strcmp(argv[i], "-O") == 0) + || (0 == strncmp(argv[i], "--disorder", strlen("--disorder")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "O"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "O"); + exit(EXIT_FAILURE); + } + arguments->disorderRatio = atoi(argv[++i]); + } else if (strlen("--disorder") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--disorder"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--disorder=", strlen("--disorder="))) { + if (isStringNumber((char *)(argv[i] + strlen("--disorder=")))) { + arguments->disorderRatio = atoi((char *)(argv[i]+strlen("--disorder="))); + } else { + errorPrintReqArg2(argv[0], "--disorder"); + exit(EXIT_FAILURE); + } } - arguments->disorderRatio = atoi(argv[++i]); - if (arguments->disorderRatio > 50) { + errorPrint("Invalid disorder ratio %d, will be set to %d\n", + arguments->disorderRatio, 50); arguments->disorderRatio = 50; } if (arguments->disorderRatio < 0) { + errorPrint("Invalid disorder ratio %d, will be set to %d\n", + arguments->disorderRatio, 0); arguments->disorderRatio = 0; } - - } else if (strcmp(argv[i], "-R") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-R need a number following!\n"); + } else if ((strcmp(argv[i], "-R") == 0) + || (0 == strncmp(argv[i], "--disorder-range", + strlen("--disorder-range")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "R"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "R"); + exit(EXIT_FAILURE); + } + arguments->disorderRange = atoi(argv[++i]); + } else if (strlen("--disorder-range") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--disorder-range"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--disorder-range=", + strlen("--disorder-range="))) { + if (isStringNumber((char *)(argv[i] + strlen("--disorder-range=")))) { + arguments->disorderRange = + atoi((char *)(argv[i]+strlen("--disorder-rnage="))); + } else { + errorPrintReqArg2(argv[0], "--disorder-range"); + exit(EXIT_FAILURE); + } } - arguments->disorderRange = atoi(argv[++i]); - if (arguments->disorderRange < 0) + if (arguments->disorderRange < 0) { + errorPrint("Invalid disorder range %d, will be set to %d\n", + arguments->disorderRange, 1000); arguments->disorderRange = 1000; - - } else if (strcmp(argv[i], "-a") == 0) { - if ((argc == i+1) || - (!isStringNumber(argv[i+1]))) { - printHelp(); - errorPrint("%s", "\n\t-a need a number following!\n"); + } + } else if ((strcmp(argv[i], "-a") == 0) + || (0 == strncmp(argv[i], "--replica", + strlen("--replica")))) { + if (2 == strlen(argv[i])) { + if (argc == i+1) { + errorPrintReqArg(argv[0], "a"); + exit(EXIT_FAILURE); + } else if (!isStringNumber(argv[i+1])) { + errorPrintReqArg2(argv[0], "a"); + exit(EXIT_FAILURE); + } + arguments->replica = atoi(argv[++i]); + } else if (strlen("--replica") == strlen(argv[i])) { + errorPrintReqArg3(argv[0], "--replica"); exit(EXIT_FAILURE); + } else if (0 == strncmp(argv[i], "--replica=", + strlen("--replica="))) { + if (isStringNumber((char *)(argv[i] + strlen("--replica=")))) { + arguments->replica = + atoi((char *)(argv[i]+strlen("--replica="))); + } else { + errorPrintReqArg2(argv[0], "--replica"); + exit(EXIT_FAILURE); + } } - arguments->replica = atoi(argv[++i]); if (arguments->replica > 3 || arguments->replica < 1) { + errorPrint("Invalid replica value %d, will be set to %d\n", + arguments->replica, 1); arguments->replica = 1; } } else if (strcmp(argv[i], "-D") == 0) { @@ -1137,16 +1472,32 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { errorPrint("%s", "\n\t-D need a valud (0~3) number following!\n"); exit(EXIT_FAILURE); } - } else if ((strcmp(argv[i], "--version") == 0) || - (strcmp(argv[i], "-V") == 0)) { + } else if ((strcmp(argv[i], "--version") == 0) + || (strcmp(argv[i], "-V") == 0)) { printVersion(); exit(0); - } else if (strcmp(argv[i], "--help") == 0) { + } else if ((strcmp(argv[i], "--help") == 0) + || (strcmp(argv[i], "-?") == 0)) { printHelp(); exit(0); + } else if (strcmp(argv[i], "--usage") == 0) { + printf(" Usage: taosdemo [-f JSONFILE] [-u USER] [-p PASSWORD] [-c CONFIG_DIR]\n\ + [-h HOST] [-P PORT] [-I INTERFACE] [-d DATABASE] [-a REPLICA]\n\ + [-m TABLEPREFIX] [-s SQLFILE] [-N] [-o OUTPUTFILE] [-q QUERYMODE]\n\ + [-b DATATYPES] [-w WIDTH_OF_BINARY] [-l COLUNNS] [-T THREADNUMBER]\n\ + [-i SLEEPTIME] [-S TIME_STEP] [-B INTERLACE_ROWS] [-t TABLES]\n\ + [-n RECORDS] [-M] [-x] [-y] [-O ORDERMODE] [-R RANGE] [-a REPLIcA][-g]\n\ + [--help] [--usage] [--version]\n"); + exit(0); } else { - printHelp(); - errorPrint("%s", "ERROR: wrong options\n"); + // to simulate argp_option output + if (strlen(argv[i]) > 2) { + fprintf(stderr, "%s unrecognized options '%s'\n", argv[0], argv[i]); + } else { + fprintf(stderr, "%s invalid options -- '%s'\n", argv[0], + (char *)((char *)argv[i])+1); + } + fprintf(stderr, "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); exit(EXIT_FAILURE); } } @@ -1163,7 +1514,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } g_args.num_of_CPR = columnCount; - if (((arguments->debug_print) && (arguments->metaFile == NULL)) + if (((arguments->debug_print) && (NULL != arguments->metaFile)) || arguments->verbose_print) { printf("###################################################################\n"); printf("# meta file: %s\n", arguments->metaFile); @@ -1189,7 +1540,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->num_of_RPR); printf("# Max SQL length: %"PRIu64"\n", arguments->max_sql_len); - printf("# Length of Binary: %d\n", arguments->len_of_binary); + printf("# Length of Binary: %d\n", arguments->binwidth); printf("# Number of Threads: %d\n", arguments->num_of_threads); printf("# Number of Tables: %"PRId64"\n", arguments->num_of_tables); @@ -1645,7 +1996,7 @@ static int printfInsertMeta() { printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); printf("thread num of insert data: \033[33m%d\033[0m\n", g_Dbs.threadCount); printf("thread num of create table: \033[33m%d\033[0m\n", - g_Dbs.threadCountByCreateTbl); + g_Dbs.threadCountForCreateTbl); printf("top insert interval: \033[33m%"PRIu64"\033[0m\n", g_args.insert_interval); printf("number of records per req: \033[33m%u\033[0m\n", @@ -1853,7 +2204,7 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, "configDir: %s\n", configDir); fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile); fprintf(fp, "thread num of insert data: %d\n", g_Dbs.threadCount); - fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl); + fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountForCreateTbl); fprintf(fp, "number of records per req: %u\n", g_args.num_of_RPR); fprintf(fp, "max sql length: %"PRIu64"\n", g_args.max_sql_len); fprintf(fp, "database count: %d\n", g_Dbs.dbCount); @@ -3453,7 +3804,7 @@ static void createChildTables() { startMultiThreadCreateChildTable( g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, - g_Dbs.threadCountByCreateTbl, + g_Dbs.threadCountForCreateTbl, startFrom, g_Dbs.db[i].superTbls[j].childTblCount, g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); @@ -3467,7 +3818,7 @@ static void createChildTables() { || (strncasecmp(g_args.datatype[j], "NCHAR", strlen("NCHAR")) == 0)) { snprintf(tblColsBuf + len, TSDB_MAX_BYTES_PER_ROW - len, - ",C%d %s(%d)", j, g_args.datatype[j], g_args.len_of_binary); + ",C%d %s(%d)", j, g_args.datatype[j], g_args.binwidth); } else { snprintf(tblColsBuf + len, TSDB_MAX_BYTES_PER_ROW - len, ",C%d %s", j, g_args.datatype[j]); @@ -3482,7 +3833,7 @@ static void createChildTables() { g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf); startMultiThreadCreateChildTable( tblColsBuf, - g_Dbs.threadCountByCreateTbl, + g_Dbs.threadCountForCreateTbl, 0, g_args.num_of_tables, g_Dbs.db[i].dbName, @@ -3839,9 +4190,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl"); if (threads2 && threads2->type == cJSON_Number) { - g_Dbs.threadCountByCreateTbl = threads2->valueint; + g_Dbs.threadCountForCreateTbl = threads2->valueint; } else if (!threads2) { - g_Dbs.threadCountByCreateTbl = 1; + g_Dbs.threadCountForCreateTbl = 1; } else { errorPrint("%s", "failed to read json, threads2 not found\n"); goto PARSE_OVER; @@ -4203,7 +4554,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; } else if (!batchCreateTbl) { - g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000; + g_Dbs.db[i].superTbls[j].batchCreateTableNum = 10; } else { errorPrint("%s", "failed to read json, batch_create_tbl_num not found\n"); goto PARSE_OVER; @@ -5431,7 +5782,7 @@ static int32_t generateDataTailWithoutStb( int64_t retLen = 0; char **data_type = g_args.datatype; - int lenOfBinary = g_args.len_of_binary; + int lenOfBinary = g_args.binwidth; if (g_args.disorderRatio) { retLen = generateData(data, data_type, @@ -6213,7 +6564,7 @@ static int32_t prepareStmtWithoutStb( if ( -1 == prepareStmtBindArrayByType( bind, data_type[i], - g_args.len_of_binary, + g_args.binwidth, pThreadInfo->time_precision, NULL)) { return -1; @@ -6247,7 +6598,7 @@ static int32_t prepareStbStmtBindTag( char *tagsVal, int32_t timePrec) { - char *bindBuffer = calloc(1, DOUBLE_BUFF_LEN); // g_args.len_of_binary); + char *bindBuffer = calloc(1, DOUBLE_BUFF_LEN); // g_args.binwidth); if (bindBuffer == NULL) { errorPrint2("%s() LN%d, Failed to allocate %d bind buffer\n", __func__, __LINE__, DOUBLE_BUFF_LEN); @@ -6279,7 +6630,7 @@ static int32_t prepareStbStmtBindRand( int64_t startTime, int32_t recSeq, int32_t timePrec) { - char *bindBuffer = calloc(1, DOUBLE_BUFF_LEN); // g_args.len_of_binary); + char *bindBuffer = calloc(1, DOUBLE_BUFF_LEN); // g_args.binwidth); if (bindBuffer == NULL) { errorPrint2("%s() LN%d, Failed to allocate %d bind buffer\n", __func__, __LINE__, DOUBLE_BUFF_LEN); @@ -7390,7 +7741,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, } } } else { - start_time = 1500000000000; + start_time = DEFAULT_START_TIME; } debugPrint("%s() LN%d, start_time= %"PRId64"\n", __func__, __LINE__, start_time); @@ -7945,11 +8296,11 @@ static int insertTestProcess() { if (g_totalChildTables > 0) { fprintf(stderr, "creating %"PRId64" table(s) with %d thread(s)\n\n", - g_totalChildTables, g_Dbs.threadCountByCreateTbl); + g_totalChildTables, g_Dbs.threadCountForCreateTbl); if (g_fpOfInsertResult) { fprintf(g_fpOfInsertResult, "creating %"PRId64" table(s) with %d thread(s)\n\n", - g_totalChildTables, g_Dbs.threadCountByCreateTbl); + g_totalChildTables, g_Dbs.threadCountForCreateTbl); } // create child tables @@ -7960,12 +8311,12 @@ static int insertTestProcess() { fprintf(stderr, "\nSpent %.4f seconds to create %"PRId64" table(s) with %d thread(s), actual %"PRId64" table(s) created\n\n", (end - start)/1000.0, g_totalChildTables, - g_Dbs.threadCountByCreateTbl, g_actualChildTables); + g_Dbs.threadCountForCreateTbl, g_actualChildTables); if (g_fpOfInsertResult) { fprintf(g_fpOfInsertResult, "\nSpent %.4f seconds to create %"PRId64" table(s) with %d thread(s), actual %"PRId64" table(s) created\n\n", (end - start)/1000.0, g_totalChildTables, - g_Dbs.threadCountByCreateTbl, g_actualChildTables); + g_Dbs.threadCountForCreateTbl, g_actualChildTables); } } @@ -8880,7 +9231,7 @@ static void setParaFromArg() { } g_Dbs.threadCount = g_args.num_of_threads; - g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + g_Dbs.threadCountForCreateTbl = g_args.num_of_threads; g_Dbs.dbCount = 1; g_Dbs.db[0].drop = true; @@ -8912,7 +9263,7 @@ static void setParaFromArg() { tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", TSDB_TABLE_NAME_LEN); g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; g_Dbs.threadCount = g_args.num_of_threads; - g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + g_Dbs.threadCountForCreateTbl = g_args.num_of_threads; g_Dbs.asyncMode = g_args.async_mode; g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; @@ -8943,7 +9294,7 @@ static void setParaFromArg() { tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, data_type[i], min(DATATYPE_BUFF_LEN, strlen(data_type[i]) + 1)); - g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.binwidth; g_Dbs.db[0].superTbls[0].columnCount++; } @@ -8965,10 +9316,10 @@ static void setParaFromArg() { tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", min(DATATYPE_BUFF_LEN, strlen("BINARY") + 1)); - g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.binwidth; g_Dbs.db[0].superTbls[0].tagCount = 2; } else { - g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + g_Dbs.threadCountForCreateTbl = g_args.num_of_threads; g_Dbs.db[0].superTbls[0].tagCount = 0; } } @@ -9090,7 +9441,7 @@ static void queryResult() { pthread_t read_id; threadInfo *pThreadInfo = calloc(1, sizeof(threadInfo)); assert(pThreadInfo); - pThreadInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 + pThreadInfo->start_time = DEFAULT_START_TIME; // 2017-07-14 10:40:00.000 pThreadInfo->start_table_from = 0; //pThreadInfo->do_aggreFunc = g_Dbs.do_aggreFunc; @@ -9144,7 +9495,6 @@ static void testCmdLine() { } g_args.test_mode = INSERT_TEST; - g_totalChildTables = DEFAULT_CHILDTABLES; insertTestProcess(); if (false == g_Dbs.insert_only) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 30b5d91b10d3c67ad00d6be9716e81ac4ae581af..ae2193a82eb447f0e948abc1757c21cab46ccf34 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -225,16 +225,15 @@ static struct argp_option options[] = { {"password", 'p', 0, 0, "User password to connect to server. Default is taosdata.", 0}, #endif {"port", 'P', "PORT", 0, "Port to connect", 0}, - {"cversion", 'v', "CVERION", 0, "client version", 0}, {"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0}, // input/output file {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, {"inpath", 'i', "INPATH", 0, "Input file path.", 1}, {"resultFile", 'r', "RESULTFILE", 0, "DumpOut/In Result file path and name.", 1}, #ifdef _TD_POWER_ - {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, + {"config-dir", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, #else - {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, + {"config-dir", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, #endif {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, // dump unit options @@ -244,7 +243,7 @@ static struct argp_option options[] = { // dump format options {"schemaonly", 's', 0, 0, "Only dump schema.", 2}, {"without-property", 'N', 0, 0, "Dump schema without properties.", 2}, - {"avro", 'V', 0, 0, "Dump apache avro format data file. By default, dump sql command sequence.", 2}, + {"avro", 'v', 0, 0, "Dump apache avro format data file. By default, dump sql command sequence.", 2}, {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T00:00:00.000+0800 or 2017-10-0100:00:00:000+0800 or '2017-10-01 00:00:00.000+0800'", 4}, {"end-time", 'E', "END_TIME", 0, "End time to dump. Either epoch or ISO8601/RFC3339 format is acceptable. ISO8601 format example: 2017-10-01T00:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 5}, #if TSDB_SUPPORT_NANOSECOND == 1 @@ -267,7 +266,6 @@ typedef struct arguments { char *user; char password[SHELL_MAX_PASSWORD_LEN]; uint16_t port; - char cversion[12]; uint16_t mysqlFlag; // output file char outpath[MAX_FILE_NAME_LEN]; @@ -338,7 +336,6 @@ struct arguments g_args = { "taosdata", #endif 0, - "", 0, // outpath and inpath "", @@ -370,6 +367,24 @@ struct arguments g_args = { false // performance_print }; +static void errorPrintReqArg2(char *program, char *wrong_arg) +{ + fprintf(stderr, + "%s: option requires a number argument '-%s'\n", + program, wrong_arg); + fprintf(stderr, + "Try `taosdump --help' or `taosdump --usage' for more information.\n"); +} + +static void errorPrintReqArg3(char *program, char *wrong_arg) +{ + fprintf(stderr, + "%s: option '%s' requires an argument\n", + program, wrong_arg); + fprintf(stderr, + "Try `taosdump --help' or `taosdump --usage' for more information.\n"); +} + /* Parse a single option. */ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Get the input argument from argp_parse, which we @@ -390,20 +405,15 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'p': break; case 'P': + if (!isStringNumber(arg)) { + errorPrintReqArg2("taosdump", "P"); + exit(EXIT_FAILURE); + } g_args.port = atoi(arg); break; case 'q': g_args.mysqlFlag = atoi(arg); break; - case 'v': - if (wordexp(arg, &full_path, 0) != 0) { - errorPrint("Invalid client vesion %s\n", arg); - return -1; - } - tstrncpy(g_args.cversion, full_path.we_wordv[0], 11); - wordfree(&full_path); - break; - // output file path case 'o': if (wordexp(arg, &full_path, 0) != 0) { errorPrint("Invalid path %s\n", arg); @@ -430,9 +440,13 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { g_args.resultFile = arg; break; case 'c': + if (0 == strlen(arg)) { + errorPrintReqArg3("taosdump", "-c or --config-dir"); + exit(EXIT_FAILURE); + } if (wordexp(arg, &full_path, 0) != 0) { errorPrint("Invalid path %s\n", arg); - return -1; + exit(EXIT_FAILURE); } tstrncpy(configDir, full_path.we_wordv[0], MAX_FILE_NAME_LEN); wordfree(&full_path); @@ -453,7 +467,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'N': g_args.with_property = false; break; - case 'V': + case 'v': g_args.avro = true; break; case 'S': @@ -660,6 +674,9 @@ static void parse_timestamp( } int main(int argc, char *argv[]) { + static char verType[32] = {0}; + sprintf(verType, "version: %s\n", version); + argp_program_version = verType; int ret = 0; /* Parse our arguments; every option seen by parse_opt will be @@ -686,7 +703,6 @@ int main(int argc, char *argv[]) { printf("user: %s\n", g_args.user); printf("password: %s\n", g_args.password); printf("port: %u\n", g_args.port); - printf("cversion: %s\n", g_args.cversion); printf("mysqlFlag: %d\n", g_args.mysqlFlag); printf("outpath: %s\n", g_args.outpath); printf("inpath: %s\n", g_args.inpath); @@ -715,11 +731,6 @@ int main(int argc, char *argv[]) { } } printf("==============================\n"); - - if (g_args.cversion[0] != 0){ - tstrncpy(version, g_args.cversion, 11); - } - if (taosCheckParam(&g_args) < 0) { exit(EXIT_FAILURE); } @@ -737,7 +748,6 @@ int main(int argc, char *argv[]) { fprintf(g_fpOfResult, "user: %s\n", g_args.user); fprintf(g_fpOfResult, "password: %s\n", g_args.password); fprintf(g_fpOfResult, "port: %u\n", g_args.port); - fprintf(g_fpOfResult, "cversion: %s\n", g_args.cversion); fprintf(g_fpOfResult, "mysqlFlag: %d\n", g_args.mysqlFlag); fprintf(g_fpOfResult, "outpath: %s\n", g_args.outpath); fprintf(g_fpOfResult, "inpath: %s\n", g_args.inpath); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index a4ecf5d6f321dee531ed8175bf54aa45d27c1a65..a8a26156726a327bf72b4bc74fc465b2127500b8 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1246,13 +1246,13 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t if (mnodeFindSuperTableColumnIndex(pStable, schema[i].name) > 0) { mError("msg:%p, app:%p stable:%s, add tag, column:%s already exist", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, schema[i].name); - return TSDB_CODE_MND_TAG_ALREAY_EXIST; + return TSDB_CODE_MND_FIELD_ALREAY_EXIST; } if (mnodeFindSuperTableTagIndex(pStable, schema[i].name) > 0) { mError("msg:%p, app:%p stable:%s, add tag, tag:%s already exist", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, schema[i].name); - return TSDB_CODE_MND_FIELD_ALREAY_EXIST; + return TSDB_CODE_MND_TAG_ALREAY_EXIST; } } @@ -1518,6 +1518,13 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg) { // update SSchema *schema = (SSchema *) (pStable->schema + col); ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR); + + if (pAlter->schema[0].bytes <= schema->bytes) { + mError("msg:%p, app:%p stable:%s, modify column len. column:%s, len from %d to %d", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, name, schema->bytes, pAlter->schema[0].bytes); + return TSDB_CODE_MND_INVALID_COLUMN_LENGTH; + } + schema->bytes = pAlter->schema[0].bytes; pStable->sversion++; mInfo("msg:%p, app:%p stable %s, start to modify column %s len to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, @@ -1548,6 +1555,12 @@ static int32_t mnodeChangeSuperTableTag(SMnodeMsg *pMsg) { // update SSchema *schema = (SSchema *) (pStable->schema + col + pStable->numOfColumns); ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR); + if (pAlter->schema[0].bytes <= schema->bytes) { + mError("msg:%p, app:%p stable:%s, modify tag len. tag:%s, len from %d to %d", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, name, schema->bytes, pAlter->schema[0].bytes); + return TSDB_CODE_MND_INVALID_TAG_LENGTH; + } + schema->bytes = pAlter->schema[0].bytes; pStable->tversion++; mInfo("msg:%p, app:%p stable %s, start to modify tag len %s to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 49e46cdde8f7689083b65f747df35a13318c4e46..311abe70df65f6090aa06b4857adb434f4d2f9a0 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -112,6 +112,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, "SQL statement too lon TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FILE_EMPTY, "File is empty") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_LINE_SYNTAX_ERROR, "Syntax error in Line") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_META_CACHED, "No table meta cached") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_COL_NAMES, "duplicated column names") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TAG_LENGTH, "Invalid tag length") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid column length") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") @@ -194,6 +197,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC, "Invalid func") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TAG_LENGTH, "invalid tag length") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COLUMN_LENGTH, "invalid column length") + TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") diff --git a/tests/examples/c/schemaless.c b/tests/examples/c/schemaless.c index 3ea199c9144950526e4bbf59b9356753e2a88da6..1a551cc5f7bd600ccaf87701953f7109743e8302 100644 --- a/tests/examples/c/schemaless.c +++ b/tests/examples/c/schemaless.c @@ -61,7 +61,7 @@ int main(int argc, char* argv[]) { time_t ct = time(0); int64_t ts = ct * 1000; - char* lineFormat = "sta%d,t0=true,t1=127i8,t2=32767i16,t3=%di32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" %lldms"; + char* lineFormat = "sta%d,t0=true,t1=127i8,t2=32767i16,t3=%di32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=254u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" %lldms"; char** lines = calloc(numSuperTables * numChildTables * numRowsPerChildTable, sizeof(char*)); int l = 0; @@ -75,7 +75,7 @@ int main(int argc, char* argv[]) { } } } - shuffle(lines, numSuperTables * numChildTables * numRowsPerChildTable); + //shuffle(lines, numSuperTables * numChildTables * numRowsPerChildTable); printf("%s\n", "begin taos_insert_lines"); int64_t begin = getTimeInUs(); @@ -83,119 +83,5 @@ int main(int argc, char* argv[]) { int64_t end = getTimeInUs(); printf("code: %d, %s. time used: %"PRId64"\n", code, tstrerror(code), end-begin); - char* lines_000_0[] = { - "sta1,id=sta1_1,t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=255u8,t6=32770u16,t7=2147483699u32,t8=9223372036854775899u64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639000us" - }; - - code = taos_insert_lines(taos, lines_000_0 , sizeof(lines_000_0)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_000_0 should return error\n"); - return -1; - } - - char* lines_000_1[] = { - "sta2,id=\"sta2_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=255u8,t6=32770u16,t7=2147483699u32,t8=9223372036854775899u64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639001" - }; - - code = taos_insert_lines(taos, lines_000_1 , sizeof(lines_000_1)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_000_1 should return error\n"); - return -1; - } - - char* lines_000_2[] = { - "sta3,id=\"sta3_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 0" - }; - - code = taos_insert_lines(taos, lines_000_2 , sizeof(lines_000_2)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_000_2 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_001_0[] = { - "sta4,t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639000us", - - }; - - code = taos_insert_lines(taos, lines_001_0 , sizeof(lines_001_0)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_001_0 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_001_1[] = { - "sta5,id=\"sta5_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639001" - }; - - code = taos_insert_lines(taos, lines_001_1 , sizeof(lines_001_1)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_001_1 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_001_2[] = { - "sta6,id=\"sta6_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 0" - }; - - code = taos_insert_lines(taos, lines_001_2 , sizeof(lines_001_2)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_001_2 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_002[] = { - "stb,id=\"stb_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639000000ns", - "stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639019us", - "stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833640ms", - "stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006834s" - }; - - code = taos_insert_lines(taos, lines_002 , sizeof(lines_002)/sizeof(char*)); - if (0 != code) { - printf("taos_insert_lines() lines_002 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - //Duplicate key check; - char* lines_003_1[] = { - "std,id=\"std_3_1\",t1=4i64,Id=\"std\",t2=true c1=true 1626006834s" - }; - - code = taos_insert_lines(taos, lines_003_1 , sizeof(lines_003_1)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_1 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_003_2[] = { - "std,id=\"std_3_2\",tag1=4i64,Tag2=true,tAg3=2,TaG2=\"dup!\" c1=true 1626006834s" - }; - - code = taos_insert_lines(taos, lines_003_2 , sizeof(lines_003_2)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_2 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_003_3[] = { - "std,id=\"std_3_3\",tag1=4i64 field1=true,Field2=2,FIElD1=\"dup!\",fIeLd4=true 1626006834s" - }; - - code = taos_insert_lines(taos, lines_003_3 , sizeof(lines_003_3)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_3 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } - - char* lines_003_4[] = { - "std,id=\"std_3_4\",tag1=4i64,dupkey=4i16,tag2=T field1=true,dUpkEy=1e3f32,field2=\"1234\" 1626006834s" - }; - - code = taos_insert_lines(taos, lines_003_4 , sizeof(lines_003_4)/sizeof(char*)); - if (0 == code) { - printf("taos_insert_lines() lines_003_4 return code:%d (%s)\n", code, (char*)tstrerror(code)); - return -1; - } return 0; } diff --git a/tests/pytest/query/queryDiffColsOr.py b/tests/pytest/query/queryDiffColsOr.py new file mode 100644 index 0000000000000000000000000000000000000000..feeab84a7ec577c9b48f19416400bcc7b348d543 --- /dev/null +++ b/tests/pytest/query/queryDiffColsOr.py @@ -0,0 +1,545 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from copy import deepcopy +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.common import tdCom + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def insertData(self, tb_name): + insert_sql_list = [f'insert into {tb_name} values ("2021-01-01 12:00:00", 1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1)', + f'insert into {tb_name} values ("2021-01-05 12:00:00", 2, 2, 1, 3, 1.1, 1.1, "binary", "nchar", true, 2)', + f'insert into {tb_name} values ("2021-01-07 12:00:00", 1, 3, 1, 2, 1.1, 1.1, "binary", "nchar", true, 3)', + f'insert into {tb_name} values ("2021-01-09 12:00:00", 1, 2, 4, 3, 1.1, 1.1, "binary", "nchar", true, 4)', + f'insert into {tb_name} values ("2021-01-11 12:00:00", 1, 2, 5, 5, 1.1, 1.1, "binary", "nchar", true, 5)', + f'insert into {tb_name} values ("2021-01-13 12:00:00", 1, 2, 1, 3, 6.6, 1.1, "binary", "nchar", true, 6)', + f'insert into {tb_name} values ("2021-01-15 12:00:00", 1, 2, 1, 3, 1.1, 7.7, "binary", "nchar", true, 7)', + f'insert into {tb_name} values ("2021-01-17 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary8", "nchar", true, 8)', + f'insert into {tb_name} values ("2021-01-19 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary", "nchar9", true, 9)', + f'insert into {tb_name} values ("2021-01-21 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary", "nchar", false, 10)', + f'insert into {tb_name} values ("2021-01-23 12:00:00", 1, 3, 1, 3, 1.1, 1.1, Null, Null, false, 11)' + ] + for sql in insert_sql_list: + tdSql.execute(sql) + + def initTb(self): + tdCom.cleanTb() + tb_name = tdCom.getLongName(8, "letters") + tdSql.execute( + f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 int)") + self.insertData(tb_name) + return tb_name + + def initStb(self): + tdCom.cleanTb() + tb_name = tdCom.getLongName(8, "letters") + tdSql.execute( + f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 int) tags (t1 tinyint, t2 smallint, t3 int, t4 bigint, t5 float, t6 double, t7 binary(100), t8 nchar(200), t9 bool, t10 int)") + tdSql.execute( + f'CREATE TABLE {tb_name}_sub using {tb_name} tags (1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1)') + self.insertData(f'{tb_name}_sub') + return tb_name + + def queryLastC10(self, query_sql, multi=False): + if multi: + res = tdSql.query(query_sql.replace('c10', 'last(*)'), True) + else: + res = tdSql.query(query_sql.replace('*', 'last(*)'), True) + return int(res[0][-1]) + + def queryFullColType(self, tb_name): + ## ts + query_sql = f'select * from {tb_name} where ts > "2021-01-11 12:00:00" or ts < "2021-01-13 12:00:00"' + tdSql.query(query_sql) + tdSql.checkRows(11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## != or + query_sql = f'select * from {tb_name} where c1 != 1 or c2 = 3' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## <> or + query_sql = f'select * from {tb_name} where c1 <> 1 or c3 = 3' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 2) + + ## >= or + query_sql = f'select * from {tb_name} where c1 >= 2 or c3 = 4' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + ## <= or + query_sql = f'select * from {tb_name} where c1 <= 1 or c3 = 4' + tdSql.query(query_sql) + tdSql.checkRows(10) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## <> or is Null + query_sql = f'select * from {tb_name} where c1 <> 1 or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## > or is not Null + query_sql = f'select * from {tb_name} where c2 > 2 or c8 is not Null' + tdSql.query(query_sql) + tdSql.checkRows(11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## > or < or >= or <= or != or <> or = Null + query_sql = f'select * from {tb_name} where c1 > 1 or c2 < 2 or c3 >= 4 or c4 <= 2 or c5 != 1.1 or c6 <> 1.1 or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(8) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## tiny small int big or + query_sql = f'select * from {tb_name} where c1 = 2 or c2 = 3 or c3 = 4 or c4 = 5' + tdSql.query(query_sql) + tdSql.checkRows(5) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## float double binary nchar bool or + query_sql = f'select * from {tb_name} where c5=6.6 or c6=7.7 or c7="binary8" or c8="nchar9" or c9=false' + tdSql.query(query_sql) + tdSql.checkRows(6) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## all types or + query_sql = f'select * from {tb_name} where c1=2 or c2=3 or c3=4 or c4=5 or c5=6.6 or c6=7.7 or c7="binary8" or c8="nchar9" or c9=false' + tdSql.query(query_sql) + tdSql.checkRows(10) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + def queryMultiExpression(self, tb_name): + ## condition_A and condition_B or condition_C (> < >=) + query_sql = f'select * from {tb_name} where c1 > 2 and c2 < 4 or c3 >= 4' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## (condition_A and condition_B) or condition_C (<= != <>) + query_sql = f'select * from {tb_name} where (c1 <= 1 and c2 != 2) or c4 <> 3' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A and (condition_B or condition_C) (Null not Null) + query_sql = f'select * from {tb_name} where c1 is not Null and (c6 = 7.7 or c8 is Null)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A or condition_B and condition_C (> < >=) + query_sql = f'select * from {tb_name} where c1 > 2 or c2 < 4 and c3 >= 4' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## (condition_A or condition_B) and condition_C (<= != <>) + query_sql = f'select * from {tb_name} where (c1 <= 1 or c2 != 2) and c4 <> 3' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## condition_A or (condition_B and condition_C) (Null not Null) + query_sql = f'select * from {tb_name} where c6 >= 7.7 or (c1 is not Null and c3 =5)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 7) + + ## condition_A or (condition_B and condition_C) or condition_D (> != < Null) + query_sql = f'select * from {tb_name} where c1 != 1 or (c2 >2 and c3 < 1) or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A and (condition_B or condition_C) and condition_D (>= = <= not Null) + query_sql = f'select * from {tb_name} where c4 >= 4 and (c1 = 2 or c5 <= 1.1) and c7 is not Null' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + ## (condition_A and condition_B) or (condition_C or condition_D) (Null >= > =) + query_sql = f'select * from {tb_name} where (c8 is Null and c1 >= 1) or (c3 > 3 or c4 =2)' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## (condition_A or condition_B) or condition_C or (condition_D and condition_E) (>= <= = not Null <>) + query_sql = f'select * from {tb_name} where (c1 >= 2 or c2 <= 1) or c3 = 4 or (c7 is not Null and c6 <> 1.1)' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 7) + + ## condition_A or (condition_B and condition_C) or (condition_D and condition_E) and condition_F + query_sql = f'select * from {tb_name} where c1 != 1 or (c2 <= 1 and c3 <4) or (c3 >= 4 or c7 is not Null) and c9 <> true' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## (condition_A or (condition_B and condition_C) or (condition_D and condition_E)) and condition_F + query_sql = f'select * from {tb_name} where (c1 != 1 or (c2 <= 2 and c3 >= 4) or (c3 >= 4 or c7 is not Null)) and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(9) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## (condition_A or condition_B) or (condition_C or condition_D) and (condition_E or condition_F or condition_G) + query_sql = f'select * from {tb_name} where c1 != 1 or (c2 <= 3 and c3 > 4) and c3 <= 5 and (c7 is not Null and c9 != false)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 5) + + def queryMultiIn(self, tb_name): + ## in and in + query_sql = f'select * from {tb_name} where c7 in ("binary") and c8 in ("nchar")' + tdSql.query(query_sql) + tdSql.checkRows(8) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## in or in + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (1, 4)' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 2) + + ## in and in or condition_A + query_sql = f'select * from {tb_name} where c7 in ("binary") and c8 in ("nchar") or c10 != 10' + tdSql.query(query_sql) + tdSql.checkRows(11) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## in or in and condition_A + query_sql = f'select * from {tb_name} where c7 in ("binary") or c8 in ("nchar") and c10 != 10' + tdSql.query(query_sql) + tdSql.checkRows(10) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## in or in or condition_A + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) or c9 != true' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## in or in or in or in + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) or c9 in (false) or c10 in (5, 6 ,22)' + tdSql.query(query_sql) + tdSql.checkRows(6) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## in or in and in or in + query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) and c9 in (false) or c10 in (5, 6 ,22)' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## condition_A or in or condition_B and in + query_sql = f'select * from {tb_name} where c1 = 2 or c2 in (2, 4) and c9 = false or c10 in (6 ,22)' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## in and condition_A or in and in and condition_B + query_sql = f'select * from {tb_name} where c1 in (2, 3) and c2 <> 3 or c10 <= 4 and c10 in (4 ,22) and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + ## (in and condition_A or in) and in and condition_B + query_sql = f'select * from {tb_name} where (c1 in (2, 3) and c2 <> 3 or c10 <= 4) and c10 in (4 ,22) and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + def queryMultiLike(self, tb_name): + ## like and like + query_sql = f'select * from {tb_name} where c7 like "bi%" and c8 like ("ncha_")' + tdSql.query(query_sql) + tdSql.checkRows(9) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## like or like + query_sql = f'select * from {tb_name} where c7 like "binar12345" or c8 like "nchar_"' + tdSql.query(query_sql) + tdSql.checkRows(1) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## like and like or condition_A + query_sql = f'select * from {tb_name} where c7 like "binary_" and c8 like "ncha_" or c1 != 1' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 8) + + ## like or like and condition_A + query_sql = f'select * from {tb_name} where c7 like ("binar_") or c8 like ("nchar_") and c10 != 8' + tdSql.query(query_sql) + tdSql.checkRows(9) + tdSql.checkEqual(self.queryLastC10(query_sql), 10) + + ## like or like or condition_A + query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("nchar_") or c10 = 6' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## like or like or like or like + query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("nchar_") or c10 = 6 or c7 is Null' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 11) + + ## like or like and like or like + query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("ncha_") and c10 = 6 or c10 = 9' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + ## condition_A or like or condition_B and like + query_sql = f'select * from {tb_name} where c1 = 2 or c7 like "binary_" or c10 = 3 and c8 like "ncha%"' + tdSql.query(query_sql) + tdSql.checkRows(3) + tdSql.checkEqual(self.queryLastC10(query_sql), 8) + + ## like and condition_A or like and like and condition_B + query_sql = f'select * from {tb_name} where c7 like "bin%" and c2 = 3 or c10 <= 4 and c7 like "binar_" and c8 like "ncha_"' + tdSql.query(query_sql) + tdSql.checkRows(4) + tdSql.checkEqual(self.queryLastC10(query_sql), 4) + + ## (like and condition_A or like) and like and condition_B + query_sql = f'select * from {tb_name} where (c7 like "bin%" and c2 = 3 or c8 like "nchar_") and c7 like "binar_" and c9 != false' + tdSql.query(query_sql) + tdSql.checkRows(2) + tdSql.checkEqual(self.queryLastC10(query_sql), 9) + + def queryPreCal(self, tb_name): + ## avg sum condition_A or condition_B + query_sql = f'select avg(c3), sum(c3) from {tb_name} where c10 = 5 or c8 is Null' + res = tdSql.query(query_sql, True)[0] + tdSql.checkEqual(int(res[0]), 3) + tdSql.checkEqual(int(res[1]), 6) + + ## avg sum condition_A or condition_B or condition_C + query_sql = f'select avg(c3), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c9 = false ' + res = tdSql.query(query_sql, True)[0] + tdSql.checkEqual(int(res[0]), 2) + tdSql.checkEqual(int(res[1]), 6) + + ## count avg sum condition_A or condition_B or condition_C interval + query_sql = f'select count(*), avg(c3), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c9 = false interval(16d)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(2) + tdSql.checkEqual(int(res[0][1]), 1) + tdSql.checkEqual(int(res[0][2]), 4) + tdSql.checkEqual(int(res[0][3]), 4) + tdSql.checkEqual(int(res[1][1]), 2) + tdSql.checkEqual(int(res[1][2]), 1) + tdSql.checkEqual(int(res[1][3]), 2) + + ## count avg sum condition_A or condition_B or in and like or condition_C interval + query_sql = f'select count(*), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c2 in (1, 2) and c7 like "binary_" or c1 <> 1 interval(16d)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(2) + tdSql.checkEqual(int(res[0][1]), 2) + tdSql.checkEqual(int(res[0][2]), 5) + tdSql.checkEqual(int(res[1][1]), 2) + tdSql.checkEqual(int(res[1][2]), 2) + + def queryMultiTb(self, tb_name): + ## select from (condition_A or condition_B) + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 >=3)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(3) + tdSql.checkEqual(int(res[2][0]), 11) + + ## select from (condition_A or condition_B) where condition_A or condition_B + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 >=3) where c1 =2 or c4 = 2' + res = tdSql.query(query_sql, True) + tdSql.checkRows(2) + tdSql.checkEqual(int(res[1][0]), 3) + + ## select from (condition_A or condition_B and like and in) where condition_A or condition_B or like and in + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) where c1 != 2 or c3 = 1 or c8 like "ncha_" and c9 in (true)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(7) + tdSql.checkEqual(int(res[6][0]), 10) + + ## select count avg sum from (condition_A or condition_B and like and in) where condition_A or condition_B or like and in interval + query_sql = f'select count(*), avg(c6), sum(c3) from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) where c1 != 2 or c3 = 1 or c8 like "ncha_" and c9 in (true) interval(8d)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(3) + tdSql.checkEqual(int(res[0][1]), 3) + tdSql.checkEqual(int(res[0][2]), 1) + tdSql.checkEqual(int(res[0][3]), 10) + tdSql.checkEqual(int(res[1][1]), 3) + tdSql.checkEqual(int(res[1][2]), 3) + tdSql.checkEqual(int(res[1][3]), 3) + tdSql.checkEqual(int(res[2][1]), 1) + tdSql.checkEqual(int(res[2][2]), 1) + tdSql.checkEqual(int(res[2][3]), 1) + + ## cname + query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) a where a.c1 != 2 or a.c3 = 1 or a.c8 like "ncha_" and a.c9 in (true)' + res = tdSql.query(query_sql, True) + tdSql.checkRows(7) + tdSql.checkEqual(int(res[6][0]), 10) + + ## multi cname + query_sql = f'select b.c10 from (select * from {tb_name} where c9 = true or c2 = 2) a, (select * from {tb_name} where c7 like "binar_" or c4 in (3, 5)) b where a.ts = b.ts' + res = tdSql.query(query_sql, True) + tdSql.checkRows(10) + tdSql.checkEqual(int(res[9][0]), 10) + + def checkTbColTypeOperator(self): + ''' + Ordinary table full column type and operator + ''' + tb_name = self.initTb() + self.queryFullColType(tb_name) + + def checkStbColTypeOperator(self): + ''' + Super table full column type and operator + ''' + tb_name = self.initStb() + self.queryFullColType(tb_name) + + def checkTbMultiExpression(self): + ''' + Ordinary table multiExpression + ''' + tb_name = self.initTb() + self.queryMultiExpression(tb_name) + + def checkStbMultiExpression(self): + ''' + Super table multiExpression + ''' + tb_name = self.initStb() + self.queryMultiExpression(tb_name) + + def checkTbMultiIn(self): + ''' + Ordinary table multiIn + ''' + tb_name = self.initTb() + self.queryMultiIn(tb_name) + + def checkStbMultiIn(self): + ''' + Super table multiIn + ''' + tb_name = self.initStb() + self.queryMultiIn(tb_name) + + def checkTbMultiLike(self): + ''' + Ordinary table multiLike + ''' + tb_name = self.initTb() + self.queryMultiLike(tb_name) + + def checkStbMultiLike(self): + ''' + Super table multiLike + ''' + tb_name = self.initStb() + self.queryMultiLike(tb_name) + + def checkTbPreCal(self): + ''' + Ordinary table precal + ''' + tb_name = self.initTb() + self.queryPreCal(tb_name) + + def checkStbPreCal(self): + ''' + Super table precal + ''' + tb_name = self.initStb() + self.queryPreCal(tb_name) + + def checkMultiTb(self): + ''' + test "or" in multi ordinary table + ''' + tb_name = self.initTb() + self.queryMultiTb(tb_name) + + def checkMultiStb(self): + ''' + test "or" in multi super table + ''' + tb_name = self.initStb() + self.queryMultiTb(tb_name) + + + # tb_name1 = tdCom.getLongName(8, "letters") + # tb_name2 = tdCom.getLongName(8, "letters") + # tb_name3 = tdCom.getLongName(8, "letters") + # tdSql.execute( + # f"CREATE TABLE {tb_name1} (ts timestamp, c1 tinyint, c2 smallint, c3 int)") + # tdSql.execute( + # f"CREATE TABLE {tb_name2} (ts timestamp, c1 tinyint, c2 smallint, c3 int)") + # tdSql.execute( + # f"CREATE TABLE {tb_name3} (ts timestamp, c1 tinyint, c2 smallint, c3 int)") + # insert_sql_list = [f'insert into {tb_name1} values ("2021-01-01 12:00:00", 1, 5, 1)', + # f'insert into {tb_name1} values ("2021-01-03 12:00:00", 2, 4, 1)', + # f'insert into {tb_name1} values ("2021-01-05 12:00:00", 3, 2, 1)', + # f'insert into {tb_name2} values ("2021-01-01 12:00:00", 4, 2, 1)', + # f'insert into {tb_name2} values ("2021-01-02 12:00:00", 5, 1, 1)', + # f'insert into {tb_name2} values ("2021-01-04 12:00:00", 1, 2, 1)', + # f'insert into {tb_name3} values ("2021-01-02 12:00:00", 4, 2, 1)', + # f'insert into {tb_name3} values ("2021-01-06 12:00:00", 5, 1, 1)', + # f'insert into {tb_name3} values ("2021-01-07 12:00:00", 1, 2, 1)', + # ] + # for sql in insert_sql_list: + # tdSql.execute(sql) + # tdSql.query( + # f'select * from {tb_name1} t1, {tb_name2}, {tb_name3} t3 t2 where (t1.ts=t2.ts or t2.ts=t3.ts)') + # tdSql.checkRows(4) + + + def run(self): + tdSql.prepare() + self.checkTbColTypeOperator() + self.checkStbColTypeOperator() + self.checkTbMultiExpression() + self.checkStbMultiExpression() + self.checkTbMultiIn() + self.checkStbMultiIn() + self.checkTbMultiLike() + self.checkStbMultiLike() + self.checkTbPreCal() + self.checkStbPreCal() + self.checkMultiTb() + self.checkMultiStb() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/schemalessInsertPerformance.py b/tests/pytest/tools/schemalessInsertPerformance.py new file mode 100644 index 0000000000000000000000000000000000000000..5e009a21296bb09c7210a7109aea9dd43bf7a847 --- /dev/null +++ b/tests/pytest/tools/schemalessInsertPerformance.py @@ -0,0 +1,301 @@ +################################################################### +# Copyright (c) 2021 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import traceback +import random +import string +from taos.error import LinesError +import datetime +import time +from copy import deepcopy +import numpy as np +from util.log import * +from util.cases import * +from util.sql import * +from util.common import tdCom +import threading + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self._conn = conn + + def genRandomTs(self): + year = random.randint(2000, 2021) + month = random.randint(10, 12) + day = random.randint(10, 29) + hour = random.randint(10, 24) + minute = random.randint(10, 59) + second = random.randint(10, 59) + m_second = random.randint(101, 199) + date_time = f'{year}-{month}-{day} {hour}:{minute}:{second}' + print(date_time) + timeArray = time.strptime(date_time, "%Y-%m-%d %H:%M:%S") + ts = int(time.mktime(timeArray)) + print("------", ts) + # timestamp = time.mktime(datetime.datetime.strptime(date_time, "%Y-%m-%d %H:%M:%S.%f").timetuple()) + return f'{ts}s' + + def genMultiColStr(self, int_count=4, double_count=0, binary_count=0): + """ + genType must be tag/col + """ + col_str = "" + if double_count == 0 and binary_count == 0: + for i in range(0, int_count): + if i < (int_count-1): + col_str += f'c{i}={random.randint(0, 255)}i32,' + else: + col_str += f'c{i}={random.randint(0, 255)}i32 ' + elif double_count > 0 and binary_count == 0: + for i in range(0, int_count): + col_str += f'c{i}={random.randint(0, 255)}i32,' + for i in range(0, double_count): + if i < (double_count-1): + col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64,' + else: + col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64 ' + elif double_count == 0 and binary_count > 0: + for i in range(0, int_count): + col_str += f'c{i}={random.randint(0, 255)}i32,' + for i in range(0, binary_count): + if i < (binary_count-1): + col_str += f'c{i+int_count}=\"{tdCom.getLongName(5, "letters")}\",' + else: + col_str += f'c{i+int_count}=\"{tdCom.getLongName(5, "letters")}\" ' + elif double_count > 0 and binary_count > 0: + for i in range(0, int_count): + col_str += f'c{i}={random.randint(0, 255)}i32,' + for i in range(0, double_count): + col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64,' + for i in range(0, binary_count): + if i < (binary_count-1): + col_str += f'c{i+int_count+double_count}=\"{tdCom.getLongName(5, "letters")}\",' + else: + col_str += f'c{i+int_count+double_count}=\"{tdCom.getLongName(5, "letters")}\" ' + return col_str + + def genLongSql(self, int_count=4, double_count=0, binary_count=0, init=False): + if init: + tag_str = f'id="init",t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"' + else: + tag_str = f'id="sub_{tdCom.getLongName(5, "letters")}_{tdCom.getLongName(5, "letters")}",t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"' + col_str = self.genMultiColStr(int_count, double_count, binary_count) + long_sql = 'stb' + ',' + tag_str + ' ' + col_str + '0' + return long_sql + + def getPerfSql(self, count=4, init=False): + if count == 4: + input_sql = self.genLongSql(init=init) + elif count == 1000: + input_sql = self.genLongSql(400, 400, 200, init=init) + elif count == 4000: + input_sql = self.genLongSql(1900, 1900, 200, init=init) + return input_sql + + def tableGenerator(self, count=4, table_count=1000): + for i in range(table_count): + yield self.getPerfSql(count) + + + + + + + + def genTableList(self, count=4, table_count=10000): + table_list = list() + for i in range(1, table_count+1): + table_list.append(self.getPerfSql(count)) + return table_list + + def splitTableList(self, count=4, thread_count=10, table_count=1000): + per_list_len = int(table_count/thread_count) + table_list = self.genTableList(count=count) + # ts = int(time.time()) + list_of_group = zip(*(iter(table_list),) *per_list_len) + end_list = [list(i) for i in list_of_group] # i is a tuple + count = len(table_list) % per_list_len + end_list.append(table_list[-count:]) if count !=0 else end_list + return table_list, end_list + + def rowsGenerator(self, end_list): + ts = int(time.time()) + input_sql_list = list() + for elm_list in end_list: + for elm in elm_list: + for i in range(1, 10000): + ts -= 1 + elm_new = self.replaceLastStr(elm, str(ts)) + 's' + input_sql_list.append(elm_new) + yield input_sql_list + + # def insertRows(self, count=4, thread_count=10): + # table_list = self.splitTableList(count=count, thread_count=thread_count)[0] + # for + + + def replaceLastStr(self, str, new): + list_ori = list(str) + list_ori[-1] = new + return ''.join(list_ori) + + def genDataList(self, table_list, row_count=10): + data_list = list() + ts = int(time.time()) + for table_str in table_list: + for i in range(1, row_count+1): + ts -= 1 + table_str_new = self.replaceLastStr(table_str, f'{str(ts)}s') + data_list.append(table_str_new) + print(data_list) + return data_list + + + def insertRows(self, count=4, table_count=1000): + table_generator = self.tableGenerator(count=count, table_count=table_count) + for table_name in table_generator: + pass + + def perfTableInsert(self): + table_generator = self.tableGenerator() + for input_sql in table_generator: + self._conn.insert_lines([input_sql]) + # for i in range(10): + # self._conn.insert_lines([input_sql]) + + def perfDataInsert(self, count=4): + table_generator = self.tableGenerator(count=count) + ts = int(time.time()) + for input_sql in table_generator: + print("input_sql-----------", input_sql) + self._conn.insert_lines([input_sql]) + for i in range(100000): + ts -= 1 + input_sql_new = self.replaceLastStr(input_sql, str(ts)) + 's' + print("input_sql_new---------", input_sql_new) + self._conn.insert_lines([input_sql_new]) + + def batchInsertTable(self, batch_list): + for insert_list in batch_list: + print(threading.current_thread().name, "length=", len(insert_list)) + print(threading.current_thread().name, 'firstline', insert_list[0]) + print(threading.current_thread().name, 'lastline:', insert_list[-1]) + self._conn.insert_lines(insert_list) + print(threading.current_thread().name, 'end') + + def genTableThread(self, thread_count=10): + threads = list() + for i in range(thread_count): + t = threading.Thread(target=self.perfTableInsert) + threads.append(t) + return threads + + def genMultiThread(self, count, thread_count=10): + threads = list() + for i in range(thread_count): + t = threading.Thread(target=self.perfDataInsert,args=(count,)) + threads.append(t) + return threads + + def multiThreadRun(self, threads): + for t in threads: + t.start() + for t in threads: + t.join() + + def createStb(self, count=4): + input_sql = self.getPerfSql(count=count, init=True) + self._conn.insert_lines([input_sql]) + + def threadInsertTable(self, end_list, thread_count=10): + threads = list() + for i in range(thread_count): + t = threading.Thread(target=self.batchInsertTable, args=(end_list,)) + threads.append(t) + return threads + + + def finalRun(self): + self.createStb() + table_list, end_list = self.splitTableList() + batchInsertTableThread = self.threadInsertTable(end_list=end_list) + self.multiThreadRun(batchInsertTableThread) + # print(end_list) + + # def createTb(self, count=4): + # input_sql = self.getPerfSql(count=count) + # for i in range(10000): + # self._conn.insert_lines([input_sql]) + + # def createTb1(self, count=4): + # start_time = time.time() + # self.multiThreadRun(self.genMultiThread(input_sql)) + # end_time = time.time() + # return end_time - start_time + + # def calInsertTableTime(self): + # start_time = time.time() + # self.createStb() + # self.multiThreadRun(self.genMultiThread()) + # end_time = time.time() + # return end_time - start_time + + def calRunTime(self, count=4): + start_time = time.time() + self.createStb() + self.multiThreadRun(self.genMultiThread(count=count)) + end_time = time.time() + return end_time - start_time + + def calRunTime1(self, count=4): + start_time = time.time() + self.createStb() + self.multiThreadRun(self.perfTableInsert()) + # self.perfTableInsert() + + # def schemalessInsertPerfTest(self, count=4): + # input_sql = self.getPerfSql(count) + # self.calRunTime(input_sql) + + # def test(self): + # sql1 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=37i32,c1=217i32,c2=3i32,c3=88i32 1626006833640ms' + # sql2 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=38i32,c1=217i32,c2=3i32,c3=88i32 1626006833641ms' + # self._conn.insert_lines([sql1]) + # self._conn.insert_lines([sql2]) + + def run(self): + print("running {}".format(__file__)) + tdSql.prepare() + self.finalRun() + # print(self.calRunTime1(count=4)) + # print(self.calRunTime(count=4)) + # print(self.genRandomTs()) + # self.calInsertTableTime() + # self.test() + # table_list = self.splitTableList()[0] + # data_list = self.genDataList(table_list) + # print(len(data_list)) + # end_list = [['stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0','stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0'], ['stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0','stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0']] + # rowsGenerator = self.rowsGenerator(end_list) + # for i in rowsGenerator: + # print(i) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 1c7d94a8a4c8cd454d41c6d4d4d89b8377ec88b7..35abc4802f9de2080a6b6a166daf833c9cf04578 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -1,4 +1,4 @@ -################################################################### +################################################################### # Copyright (c) 2016 by TAOS Technologies, Inc. # All rights reserved. # @@ -50,4 +50,4 @@ class TDCom: def close(self): self.cursor.close() -tdCom = TDCom() \ No newline at end of file +tdCom = TDCom() diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 2abb8f5ee730ed187b28f4bff6c28a90b9186411..0208f884b691a20e4b4456fe8165797969305674 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -123,7 +123,7 @@ class TDDnode: "charset":"UTF-8", "asyncLog":"0", "anyIp":"0", - "tsEnableTelemetryReporting":"0", + "telemetryReporting":"0", "dDebugFlag":"135", "tsdbDebugFlag":"135", "mDebugFlag":"135",