diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 687d182bb696dd745eb3795ccc23b6284cec0a30..b3674a7bf5bf94806dfd576b4e81af95413bb5bd 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -340,7 +340,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild); uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); uint32_t tscGetTableMetaMaxSize(); -int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf); +int32_t tscCreateTableMetaFromSTableMeta(STableMeta** pChild, const char* name, size_t *tableMetaCapacity); STableMeta* tscTableMetaDup(STableMeta* pTableMeta); SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo); diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 7d2823a42edc3ea7616e822d47cfdbf8706f4a56..69e55fd3339c7dac3de80872dbb3a0f09fc38a6e 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -458,9 +458,9 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSm schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - uint32_t size = tscGetTableMetaMaxSize(); - STableMeta* tableMeta = calloc(1, size); - taosHashGetClone(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, tableMeta); + size_t size = 0; + STableMeta* tableMeta = NULL; + taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void **)&tableMeta, &size); tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); schema->precision = tableMeta->tableInfo.precision; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e6720c0f4e327413de9cd4a80e67aa0bd12b72a9..7adc0812ae12bd8711cba5988806a4c7ba740cf5 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -8089,6 +8089,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { SArray* pVgroupList = NULL; SArray* plist = NULL; STableMeta* pTableMeta = NULL; + size_t tableMetaCapacity = 0; SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); pCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); @@ -8115,18 +8116,14 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { } } - uint32_t maxSize = tscGetTableMetaMaxSize(); char name[TSDB_TABLE_FNAME_LEN] = {0}; - assert(maxSize < 80 * TSDB_MAX_COLUMNS); - if (!pSql->pBuf) { - if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _end; - } - } - - pTableMeta = calloc(1, maxSize); + //if (!pSql->pBuf) { + // if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { + // code = TSDB_CODE_TSC_OUT_OF_MEMORY; + // goto _end; + // } + //} plist = taosArrayInit(4, POINTER_BYTES); pVgroupList = taosArrayInit(4, POINTER_BYTES); @@ -8140,16 +8137,16 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { tNameExtractFullName(pname, name); size_t len = strlen(name); - memset(pTableMeta, 0, maxSize); - taosHashGetClone(tscTableMetaMap, name, len, NULL, pTableMeta); + + taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&pTableMeta, &tableMetaCapacity); - if (pTableMeta->id.uid > 0) { + if (pTableMeta && pTableMeta->id.uid > 0) { tscDebug("0x%"PRIx64" retrieve table meta %s from local buf", pSql->self, name); // avoid mem leak, may should update pTableMeta void* pVgroupIdList = NULL; if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - code = tscCreateTableMetaFromSTableMeta(pTableMeta, name, pSql->pBuf); + code = tscCreateTableMetaFromSTableMeta((STableMeta **)(&pTableMeta), name, &tableMetaCapacity); // create the child table meta from super table failed, try load it from mnode if (code != TSDB_CODE_SUCCESS) { @@ -8304,7 +8301,8 @@ static int32_t doLoadAllTableMeta(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNod tNameExtractFullName(&pTableMetaInfo->name, fname); STableMetaVgroupInfo* p = taosHashGet(pCmd->pTableMetaMap, fname, strnlen(fname, TSDB_TABLE_FNAME_LEN)); - pTableMetaInfo->pTableMeta = tscTableMetaDup(p->pTableMeta); + pTableMetaInfo->pTableMeta = tscTableMetaDup(p->pTableMeta); + pTableMetaInfo->tableMetaCapacity = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); assert(pTableMetaInfo->pTableMeta != NULL); if (p->vgroupIdList != NULL) { @@ -8404,7 +8402,8 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pS if (pTableMetaInfo1 == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pTableMetaInfo1->pTableMeta = extractTempTableMetaFromSubquery(pSub); + pTableMetaInfo1->pTableMeta = extractTempTableMetaFromSubquery(pSub); + pTableMetaInfo1->tableMetaCapacity = tscGetTableMetaSize(pTableMetaInfo1->pTableMeta); if (subInfo->aliasName.n > 0) { if (subInfo->aliasName.n >= TSDB_TABLE_FNAME_LEN) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 53fd05863b994d86c1e06b24fc35b2a1d8e088a6..3e8dfac1da0cb192282989b9ab1137427365b6d0 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2840,43 +2840,22 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) { assert(tIsValidName(&pTableMetaInfo->name)); - uint32_t size = tscGetTableMetaMaxSize(); - if (pTableMetaInfo->pTableMeta == NULL) { - pTableMetaInfo->pTableMeta = calloc(1, size); - pTableMetaInfo->tableMetaSize = size; - } else if (pTableMetaInfo->tableMetaSize < size) { - char *tmp = realloc(pTableMetaInfo->pTableMeta, size); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - pTableMetaInfo->pTableMeta = (STableMeta *)tmp; - } - - memset(pTableMetaInfo->pTableMeta, 0, size); - pTableMetaInfo->tableMetaSize = size; - - pTableMetaInfo->pTableMeta->tableType = -1; - pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; - char name[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&pTableMetaInfo->name, name); size_t len = strlen(name); - taosHashGetClone(tscTableMetaMap, name, len, NULL, pTableMetaInfo->pTableMeta); - - // TODO resize the tableMeta - assert(size < 80 * TSDB_MAX_COLUMNS); - if (!pSql->pBuf) { - if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } + if (pTableMetaInfo->tableMetaCapacity != 0) { + if (pTableMetaInfo->pTableMeta != NULL) { + memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity); + } + } + taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity); STableMeta* pMeta = pTableMetaInfo->pTableMeta; - if (pMeta->id.uid > 0) { + if (pMeta && pMeta->id.uid > 0) { // in case of child table, here only get the if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, pSql->pBuf); + int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity); if (code != TSDB_CODE_SUCCESS) { return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index b5f965627715f75b0d9e8d8b8148e27942e573ac..5a724af7bf17e4aa09eac880f6c6176db4bdd96d 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1657,6 +1657,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { pTableMetaInfo->pTableMeta = tscTableMetaDup(pDataBlock->pTableMeta); pTableMetaInfo->tableMetaSize = tscGetTableMetaSize(pDataBlock->pTableMeta); + pTableMetaInfo->tableMetaCapacity = (size_t)(pTableMetaInfo->tableMetaSize); } /* @@ -3414,6 +3415,8 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableM } else { pTableMetaInfo->tableMetaSize = tscGetTableMetaSize(pTableMeta); } + pTableMetaInfo->tableMetaCapacity = (size_t)(pTableMetaInfo->tableMetaSize); + if (vgroupList != NULL) { pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList); @@ -4445,24 +4448,36 @@ CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) { return cMeta; } -int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf) { - assert(pChild != NULL && buf != NULL); +int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity) { + assert(*ppChild != NULL); - STableMeta* p = buf; - taosHashGetClone(tscTableMetaMap, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p); + STableMeta* p = NULL; + size_t sz = 0; + STableMeta* pChild = *ppChild; + + taosHashGetCloneExt(tscTableMetaMap, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, (void **)&p, &sz); // tableMeta exists, build child table meta according to the super table meta // the uid need to be checked in addition to the general name of the super table. - if (p->id.uid > 0 && pChild->suid == p->id.uid) { + if (p && p->id.uid > 0 && pChild->suid == p->id.uid) { + + int32_t totalBytes = (p->tableInfo.numOfColumns + p->tableInfo.numOfTags) * sizeof(SSchema); + int32_t tableMetaSize = sizeof(STableMeta) + totalBytes; + if (*tableMetaCapacity < tableMetaSize) { + pChild = realloc(pChild, tableMetaSize); + *tableMetaCapacity = (size_t)tableMetaSize; + } + pChild->sversion = p->sversion; pChild->tversion = p->tversion; - memcpy(&pChild->tableInfo, &p->tableInfo, sizeof(STableComInfo)); - int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags; + memcpy(pChild->schema, p->schema, totalBytes); - memcpy(pChild->schema, p->schema, sizeof(SSchema) *total); + *ppChild = pChild; + tfree(p); return TSDB_CODE_SUCCESS; } else { // super table has been removed, current tableMeta is also expired. remove it here + tfree(p); taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); return -1; } @@ -4992,4 +5007,4 @@ void tscRemoveTableMetaBuf(STableMetaInfo* pTableMetaInfo, uint64_t id) { taosHashRemove(tscTableMetaMap, fname, len); tscDebug("0x%"PRIx64" remove table meta %s, numOfRemain:%d", id, fname, (int32_t) taosHashGetSize(tscTableMetaMap)); -} \ No newline at end of file +} diff --git a/src/query/inc/qTableMeta.h b/src/query/inc/qTableMeta.h index 4bb5483a10dc8f1b0904c17b5b3c81973115470d..0dae74ac8224329810064c773ecd478d4a89ad15 100644 --- a/src/query/inc/qTableMeta.h +++ b/src/query/inc/qTableMeta.h @@ -71,7 +71,8 @@ typedef struct STableMeta { typedef struct STableMetaInfo { STableMeta *pTableMeta; // table meta, cached in client side and acquired by name - uint32_t tableMetaSize; + uint32_t tableMetaSize; + size_t tableMetaCapacity; SVgroupsInfo *vgroupList; SArray *pVgroupTables; // SArray diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index a53aa602c1e309ef0c25e370ae084b5a33e4144c..6c4145810b9819e60a9cb979a8ab42795df3cf00 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -127,6 +127,16 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); */ void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d); +/** + * @param pHashObj + * @param key + * @param keyLen + * @param fp + * @param d + * @param sz + * @return + */ +void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void** d, size_t *sz); /** * remove item with the specified key * @param pHashObj diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 2e18f36a17f9b3112c4d1747afa37944a1e93d28..6118aa7bef8b426a909cee19b6ee2aba2ed6dfc4 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -18,6 +18,8 @@ #include "tulog.h" #include "taosdef.h" +#define EXT_SIZE 1024 + #define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) #define DO_FREE_HASH_NODE(_n) \ @@ -296,6 +298,68 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { return taosHashGetClone(pHashObj, key, keyLen, NULL, NULL); } +//TODO(yihaoDeng), merge with taosHashGetClone +void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void** d, size_t *sz) { + if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { + return NULL; + } + + uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); + + // only add the read lock to disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[slot]; + + // no data, return directly + if (atomic_load_32(&pe->num) == 0) { + __rd_unlock(&pHashObj->lock, pHashObj->type); + return NULL; + } + + char *data = NULL; + + // lock entry + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosRLockLatch(&pe->latch); + } + + if (pe->num > 0) { + assert(pe->next != NULL); + } else { + assert(pe->next == NULL); + } + + SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal); + if (pNode != NULL) { + if (fp != NULL) { + fp(GET_HASH_NODE_DATA(pNode)); + } + + if (*d == NULL) { + *sz = pNode->dataLen + EXT_SIZE; + *d = calloc(1, *sz); + } else if (*sz < pNode->dataLen){ + *sz = pNode->dataLen + EXT_SIZE; + *d = realloc(*d, *sz); + } + memcpy((char *)(*d), GET_HASH_NODE_DATA(pNode), pNode->dataLen); + // just make runtime happy + if ((*sz) - pNode->dataLen > 0) { + memset((char *)(*d) + pNode->dataLen, 0, (*sz) - pNode->dataLen); + } + + data = GET_HASH_NODE_DATA(pNode); + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosRUnLockLatch(&pe->latch); + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return data; +} void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d) { if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) {