diff --git a/include/common/tname.h b/include/common/tname.h index ffa4f8f253a57c5d9897a7cc0f1306998b2857a5..ae2dc32335d8e33692a3c05c773d539e31c8a669 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -17,6 +17,7 @@ #define _TD_COMMON_NAME_H_ #include "tdef.h" +#include "tarray.h" #ifdef __cplusplus extern "C" { @@ -62,6 +63,18 @@ int32_t tNameSetAcctId(SName* dst, int32_t acctId); bool tNameDBNameEqual(SName* left, SName* right); +typedef struct { + // input + SArray *tags; // element is SSmlKV + const char *sTableName; // super table name + uint8_t sTableNameLen; // the length of super table name + + // output + char *childTableName; // must have size of TSDB_TABLE_NAME_LEN; + uint64_t uid; // child table uid, may be useful +} RandTableName; + +void buildChildTableName(RandTableName *rName); #ifdef __cplusplus } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 2a0e85092b522dc8c2945405717db9c9aa6745c4..15d150086033e6044e0ee2a110579dbce26579fc 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -15,6 +15,7 @@ #include "tcommon.h" #include "catalog.h" #include "clientInt.h" +#include "tname.h" //================================================================================================= #define SPACE ' ' @@ -97,6 +98,21 @@ typedef struct { char *buf; } SSmlMsgBuf; +typedef struct { + int32_t code; + int32_t lineNum; + + int32_t numOfSTables; + int32_t numOfCTables; + int32_t numOfCreateSTables; + + int64_t parseTime; + int64_t schemaTime; + int64_t insertBindTime; + int64_t insertRpcTime; + int64_t endTime; +} SSmlCostInfo; + typedef struct { uint64_t id; @@ -114,6 +130,7 @@ typedef struct { SRequestObj *pRequest; SQuery *pQuery; + SSmlCostInfo cost; int32_t affectedRows; SSmlMsgBuf msgBuf; SHashObj *dumplicateKey; // for dumplicate key @@ -147,45 +164,6 @@ static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf* pBuf, const char *msg1, const return TSDB_CODE_SML_INVALID_DATA; } -static int smlCompareKv(const void* p1, const void* p2) { - SSmlKv* kv1 = *(SSmlKv**)p1; - SSmlKv* kv2 = *(SSmlKv**)p2; - int32_t kvLen1 = kv1->keyLen; - int32_t kvLen2 = kv2->keyLen; - int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); - if (res != 0) { - return res; - } else { - return kvLen1-kvLen2; - } -} - -static void smlBuildChildTableName(SSmlTableInfo *tags) { - int32_t size = taosArrayGetSize(tags->tags); - ASSERT(size > 0); - taosArraySort(tags->tags, smlCompareKv); - - SStringBuilder sb = {0}; - taosStringBuilderAppendStringLen(&sb, tags->sTableName, tags->sTableNameLen); - for (int j = 0; j < size; ++j) { - SSmlKv *tagKv = taosArrayGetP(tags->tags, j); - taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); - taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->valueLen); - } - size_t len = 0; - char* keyJoined = taosStringBuilderGetResult(&sb, &len); - T_MD5_CTX context; - tMD5Init(&context); - tMD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); - tMD5Final(&context); - uint64_t digest1 = *(uint64_t*)(context.digest); - //uint64_t digest2 = *(uint64_t*)(context.digest + 8); - //snprintf(tags->childTableName, TSDB_TABLE_NAME_LEN, "t_%016"PRIx64"%016"PRIx64, digest1, digest2); - snprintf(tags->childTableName, TSDB_TABLE_NAME_LEN, "t_%016"PRIx64, digest1); - taosStringBuilderDestroy(&sb); - tags->uid = digest1; -} - static int32_t smlGenerateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, SArray* dbAttrArray, bool isTag, char sTableName[], SSchemaAction* action, bool* actionNeeded, SSmlHandle* info) { // char fieldName[TSDB_COL_NAME_LEN] = {0}; @@ -444,6 +422,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle* info) { uError("SML:0x%"PRIx64" catalogGetSTableMeta failed. super table name %s", info->id, schemaAction.createSTable.sTableName); return code; } + info->cost.numOfCreateSTables++; }else if (code == TSDB_CODE_SUCCESS) { } else { uError("SML:0x%"PRIx64" load table meta error: %s", info->id, tstrerror(code)); @@ -926,20 +905,6 @@ static bool smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { return false; } -static bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlHandle* info) { - char *val = NULL; - val = taosHashGet(pHash, key, strlen(key)); - if (val) { - uError("SML:0x%"PRIx64" Duplicate key detected:%s", info->id, key); - return true; - } - - uint8_t dummy_val = 0; - taosHashPut(pHash, key, strlen(key), &dummy_val, sizeof(uint8_t)); - - return false; -} - static int32_t smlParseString(const char* sql, SSmlLineInfo *elements, SSmlMsgBuf *msg){ if(!sql) return TSDB_CODE_SML_INVALID_DATA; while (*sql != '\0') { // jump the space at the begining @@ -1546,8 +1511,10 @@ static int32_t smlParseLine(SSmlHandle* info, const char* sql) { tinfo->sTableName = elements.measure; tinfo->sTableNameLen = elements.measureLen; - smlBuildChildTableName(tinfo); - uDebug("SML:0x%"PRIx64" child table name: %s", info->id, tinfo->childTableName); + RandTableName rName = {.tags=tinfo->tags, .sTableName=tinfo->sTableName, .sTableNameLen=tinfo->sTableNameLen, + .childTableName=tinfo->childTableName}; + buildChildTableName(&rName); + tinfo->uid = rName.uid; SSmlSTableMeta** tableMeta = taosHashGet(info->superTables, elements.measure, elements.measureLen); if(tableMeta){ // update meta @@ -1604,7 +1571,7 @@ static void smlDestroyInfo(SSmlHandle* info){ static SSmlHandle* smlBuildSmlInfo(TAOS* taos, SRequestObj* request, SMLProtocolType protocol, int8_t precision, bool dataFormat){ int32_t code = TSDB_CODE_SUCCESS; - SSmlHandle* info = taosMemoryMalloc(sizeof(SSmlHandle)); + SSmlHandle* info = taosMemoryCalloc(1, sizeof(SSmlHandle)); if (NULL == info) { return NULL; } @@ -1699,12 +1666,23 @@ static int32_t smlInsertData(SSmlHandle* info) { } smlBuildOutput(info->exec, info->pVgHash); + info->cost.insertRpcTime = taosGetTimestampUs(); + launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true); info->affectedRows = taos_affected_rows(info->pRequest); return info->pRequest->code; } +static void smlPrintStatisticInfo(SSmlHandle *info){ + uError("SML:0x%"PRIx64" smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d \ + parse cost:%"PRId64",schema cost:%"PRId64",bind cost:%"PRId64",rpc cost:%"PRId64",total cost:%"PRId64"", info->id, info->cost.code, + info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->cost.numOfCreateSTables, + info->cost.schemaTime-info->cost.parseTime, info->cost.insertBindTime-info->cost.schemaTime, + info->cost.insertRpcTime-info->cost.insertBindTime, info->cost.endTime-info->cost.insertRpcTime, + info->cost.endTime-info->cost.parseTime); +} + static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) { int32_t code = TSDB_CODE_SUCCESS; @@ -1714,6 +1692,7 @@ static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) { goto cleanup; } + info->cost.parseTime = taosGetTimestampUs(); for (int32_t i = 0; i < numLines; ++i) { code = smlParseLine(info, lines[i]); if (code != TSDB_CODE_SUCCESS) { @@ -1721,24 +1700,29 @@ static int smlInsertLines(SSmlHandle *info, char* lines[], int numLines) { goto cleanup; } } - uDebug("SML:0x%"PRIx64" smlInsertLines parse success. tables %d", info->id, taosHashGetSize(info->childTables)); - uDebug("SML:0x%"PRIx64" smlInsertLines parse success. super tables %d", info->id, taosHashGetSize(info->superTables)); + info->cost.lineNum = numLines; + info->cost.numOfSTables = taosHashGetSize(info->superTables); + info->cost.numOfCTables = taosHashGetSize(info->childTables); + + info->cost.schemaTime = taosGetTimestampUs(); code = smlModifyDBSchemas(info); if (code != 0) { uError("SML:0x%"PRIx64" smlModifyDBSchemas error : %s", info->id, tstrerror(code)); goto cleanup; } + info->cost.insertBindTime = taosGetTimestampUs(); code = smlInsertData(info); if (code != 0) { uError("SML:0x%"PRIx64" smlInsertData error : %s", info->id, tstrerror(code)); goto cleanup; } - - uDebug("SML:0x%"PRIx64" smlInsertLines finish inserting %d lines.", info->id, numLines); + info->cost.endTime = taosGetTimestampUs(); cleanup: + info->cost.code = code; + smlPrintStatisticInfo(info); return code; } @@ -1790,7 +1774,6 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr } smlDestroyInfo(info); -end: return (TAOS_RES*)request; } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index f4755f5b5e4e924ed0c917701430c97c4e08f5ef..62ba4bfb792e34a2c3d45fb300bcf0ead865aa97 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -15,6 +15,8 @@ #define _DEFAULT_SOURCE #include "tname.h" +#include "tcommon.h" +#include "tstrbuild.h" #define VALID_NAME_TYPE(x) ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T) @@ -294,4 +296,43 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { return 0; } +static int compareKv(const void* p1, const void* p2) { + SSmlKv* kv1 = *(SSmlKv**)p1; + SSmlKv* kv2 = *(SSmlKv**)p2; + int32_t kvLen1 = kv1->keyLen; + int32_t kvLen2 = kv2->keyLen; + int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); + if (res != 0) { + return res; + } else { + return kvLen1-kvLen2; + } +} +/* + * use stable name and tags to grearate child table name + */ +void buildChildTableName(RandTableName *rName) { + int32_t size = taosArrayGetSize(rName->tags); + ASSERT(size > 0); + taosArraySort(rName->tags, compareKv); + + SStringBuilder sb = {0}; + taosStringBuilderAppendStringLen(&sb, rName->sTableName, rName->sTableNameLen); + for (int j = 0; j < size; ++j) { + SSmlKv *tagKv = taosArrayGetP(rName->tags, j); + taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); + taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->valueLen); + } + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + T_MD5_CTX context; + tMD5Init(&context); + tMD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); + tMD5Final(&context); + uint64_t digest1 = *(uint64_t*)(context.digest); + uint64_t digest2 = *(uint64_t*)(context.digest + 8); + snprintf(rName->childTableName, TSDB_TABLE_NAME_LEN, "t_%016"PRIx64"%016"PRIx64, digest1, digest2); + taosStringBuilderDestroy(&sb); + rName->uid = digest1; +}