From 806cf56011be0b22f45fd05dbe8d65bf0ad1eb33 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 3 Aug 2021 17:12:44 +0800 Subject: [PATCH] [TD-4199] enhance performance --- src/client/src/tscParseLineProtocol.c | 6 +-- src/client/src/tscSQLParser.c | 23 +++++----- src/client/src/tscServer.c | 35 ++++----------- src/client/src/tscUtil.c | 16 ++++--- src/query/inc/qTableMeta.h | 3 +- src/util/inc/hash.h | 10 +++++ src/util/src/hash.c | 64 +++++++++++++++++++++++++++ 7 files changed, 108 insertions(+), 49 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 7d2823a42e..69e55fd333 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -458,9 +458,9 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSm schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - uint32_t size = tscGetTableMetaMaxSize(); - STableMeta* tableMeta = calloc(1, size); - taosHashGetClone(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, tableMeta); + size_t size = 0; + STableMeta* tableMeta = NULL; + taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void **)&tableMeta, &size); tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); schema->precision = tableMeta->tableInfo.precision; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 27b839169c..6f18ea3753 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -8093,6 +8093,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { SArray* pVgroupList = NULL; SArray* plist = NULL; STableMeta* pTableMeta = NULL; + size_t tableMetaCapacity = 0; SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); pCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); @@ -8119,18 +8120,14 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { } } - uint32_t maxSize = tscGetTableMetaMaxSize(); char name[TSDB_TABLE_FNAME_LEN] = {0}; - assert(maxSize < 80 * TSDB_MAX_COLUMNS); - if (!pSql->pBuf) { - if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _end; - } - } - - pTableMeta = calloc(1, maxSize); + //if (!pSql->pBuf) { + // if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { + // code = TSDB_CODE_TSC_OUT_OF_MEMORY; + // goto _end; + // } + //} plist = taosArrayInit(4, POINTER_BYTES); pVgroupList = taosArrayInit(4, POINTER_BYTES); @@ -8144,10 +8141,10 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { tNameExtractFullName(pname, name); size_t len = strlen(name); - memset(pTableMeta, 0, maxSize); - taosHashGetClone(tscTableMetaMap, name, len, NULL, pTableMeta); + + taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&pTableMeta, &tableMetaCapacity); - if (pTableMeta->id.uid > 0) { + if (pTableMeta && pTableMeta->id.uid > 0) { tscDebug("0x%"PRIx64" retrieve table meta %s from local buf", pSql->self, name); // avoid mem leak, may should update pTableMeta diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index fdb1be9f4e..d1cc0c1fa8 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2840,40 +2840,21 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) { assert(tIsValidName(&pTableMetaInfo->name)); - uint32_t size = tscGetTableMetaMaxSize(); - if (pTableMetaInfo->pTableMeta == NULL) { - pTableMetaInfo->pTableMeta = calloc(1, size); - pTableMetaInfo->tableMetaSize = size; - } else if (pTableMetaInfo->tableMetaSize < size) { - char *tmp = realloc(pTableMetaInfo->pTableMeta, size); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - pTableMetaInfo->pTableMeta = (STableMeta *)tmp; - } - - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; - - pTableMetaInfo->pTableMeta->tableType = -1; - pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; - char name[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&pTableMetaInfo->name, name); size_t len = strlen(name); - taosHashGetClone(tscTableMetaMap, name, len, NULL, pTableMetaInfo->pTableMeta); + taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity); // TODO resize the tableMeta - assert(size < 80 * TSDB_MAX_COLUMNS); - if (!pSql->pBuf) { - if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } - + //assert(size < 80 * TSDB_MAX_COLUMNS); + //if (!pSql->pBuf) { + // if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { + // return TSDB_CODE_TSC_OUT_OF_MEMORY; + // } + //} STableMeta* pMeta = pTableMetaInfo->pTableMeta; - if (pMeta->id.uid > 0) { + if (pMeta && pMeta->id.uid > 0) { // in case of child table, here only get the if (pMeta->tableType == TSDB_CHILD_TABLE) { int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, pSql->pBuf); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index a82e452d0b..b42199ec91 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1657,6 +1657,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { pTableMetaInfo->pTableMeta = tscTableMetaDup(pDataBlock->pTableMeta); pTableMetaInfo->tableMetaSize = tscGetTableMetaSize(pDataBlock->pTableMeta); + pTableMetaInfo->tableMetaCapacity = (size_t)(pTableMetaInfo->tableMetaSize); } /* @@ -3414,6 +3415,8 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableM } else { pTableMetaInfo->tableMetaSize = tscGetTableMetaSize(pTableMeta); } + pTableMetaInfo->tableMetaCapacity = (size_t)(pTableMetaInfo->tableMetaSize); + if (vgroupList != NULL) { pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList); @@ -4446,14 +4449,15 @@ CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { } int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf) { - assert(pChild != NULL && buf != NULL); + assert(pChild != NULL); - STableMeta* p = buf; - taosHashGetClone(tscTableMetaMap, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p); + STableMeta* p = NULL; + size_t sz = 0; + taosHashGetCloneExt(tscTableMetaMap, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, (void **)&p, &sz); // tableMeta exists, build child table meta according to the super table meta // the uid need to be checked in addition to the general name of the super table. - if (p->id.uid > 0 && pChild->suid == p->id.uid) { + if (p && p->id.uid > 0 && pChild->suid == p->id.uid) { pChild->sversion = p->sversion; pChild->tversion = p->tversion; @@ -4461,8 +4465,10 @@ int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, v int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags; memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); + tfree(p); return TSDB_CODE_SUCCESS; } else { // super table has been removed, current tableMeta is also expired. remove it here + tfree(p); taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); return -1; } @@ -4977,4 +4983,4 @@ void tscRemoveTableMetaBuf(STableMetaInfo* pTableMetaInfo, uint64_t id) { taosHashRemove(tscTableMetaMap, fname, len); tscDebug("0x%"PRIx64" remove table meta %s, numOfRemain:%d", id, fname, (int32_t) taosHashGetSize(tscTableMetaMap)); -} \ No newline at end of file +} diff --git a/src/query/inc/qTableMeta.h b/src/query/inc/qTableMeta.h index 4bb5483a10..0dae74ac82 100644 --- a/src/query/inc/qTableMeta.h +++ b/src/query/inc/qTableMeta.h @@ -71,7 +71,8 @@ typedef struct STableMeta { typedef struct STableMetaInfo { STableMeta *pTableMeta; // table meta, cached in client side and acquired by name - uint32_t tableMetaSize; + uint32_t tableMetaSize; + size_t tableMetaCapacity; SVgroupsInfo *vgroupList; SArray *pVgroupTables; // SArray diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index a53aa602c1..6c4145810b 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -127,6 +127,16 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); */ void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d); +/** + * @param pHashObj + * @param key + * @param keyLen + * @param fp + * @param d + * @param sz + * @return + */ +void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void** d, size_t *sz); /** * remove item with the specified key * @param pHashObj diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 2e18f36a17..4398b2d457 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -18,6 +18,8 @@ #include "tulog.h" #include "taosdef.h" +#define EXT_SIZE 512 + #define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) #define DO_FREE_HASH_NODE(_n) \ @@ -296,6 +298,68 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { return taosHashGetClone(pHashObj, key, keyLen, NULL, NULL); } +//TODO(yihaoDeng), merge with taosHashGetClone +void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void** d, size_t *sz) { + if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { + return NULL; + } + + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); + + // only add the read lock to disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[slot]; + + // no data, return directly + if (atomic_load_32(&pe->num) == 0) { + __rd_unlock(&pHashObj->lock, pHashObj->type); + return NULL; + } + + char *data = NULL; + + // lock entry + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosRLockLatch(&pe->latch); + } + + if (pe->num > 0) { + assert(pe->next != NULL); + } else { + assert(pe->next == NULL); + } + + SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal); + if (pNode != NULL) { + if (fp != NULL) { + fp(GET_HASH_NODE_DATA(pNode)); + } + + if (*d == NULL) { + *sz = pNode->dataLen + EXT_SIZE; + *d = calloc(1, *sz); + } else if (*sz < pNode->dataLen){ + *sz = pNode->dataLen + EXT_SIZE; + *d = realloc(*d, *sz); + } + memcpy(*d, GET_HASH_NODE_DATA(pNode), pNode->dataLen); + // just make runtime happy + if ((*sz) - pNode->dataLen > 0) { + memset((*d) + pNode->dataLen, 0, (*sz) - pNode->dataLen); + } + + data = GET_HASH_NODE_DATA(pNode); + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosRUnLockLatch(&pe->latch); + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return data; +} void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d) { if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { -- GitLab