diff --git a/include/util/thash.h b/include/util/thash.h index 781c22a56aaba0d449d1f711b32fe4bd75a39003..f4d09eb0906b04bfd40d97c39ec66feb3b1967a1 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -210,6 +210,8 @@ void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp); */ void taosHashSetFreeFp(SHashObj *pHashObj, _hash_free_fn_t fp); +int64_t taosHashGetCompTimes(SHashObj *pHashObj); + #ifdef __cplusplus } #endif diff --git a/source/util/src/thash.c b/source/util/src/thash.c index aee84a0d55336c63840d1a5df887da7752592841..0029f1ab1ec83278de5336cdedf1666bec839df0 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -21,7 +21,7 @@ // the add ref count operation may trigger the warning if the reference count is greater than the MAX_WARNING_REF_COUNT #define MAX_WARNING_REF_COUNT 10000 -#define HASH_MAX_CAPACITY (1024 * 1024 * 16) +#define HASH_MAX_CAPACITY (1024 * 1024 * 1024) #define HASH_DEFAULT_LOAD_FACTOR (0.75) #define HASH_INDEX(v, c) ((v) & ((c)-1)) @@ -67,6 +67,7 @@ struct SHashObj { bool enableUpdate; // enable update SArray *pMemBlock; // memory block allocated for SHashEntry _hash_before_fn_t callbackFp; // function invoked before return the value to caller + int64_t compTimes; }; /* @@ -146,6 +147,7 @@ static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntr uint32_t hashVal) { SHashNode *pNode = pe->next; while (pNode) { + atomic_add_fetch_64(&pHashObj->compTimes, 1); if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { assert(pNode->hashVal == hashVal); @@ -882,3 +884,7 @@ void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) { } void taosHashRelease(SHashObj *pHashObj, void *p) { taosHashCancelIterate(pHashObj, p); } + +int64_t taosHashGetCompTimes(SHashObj *pHashObj) { return atomic_load_64(&pHashObj->compTimes); } + + diff --git a/source/util/test/hashTest.cpp b/source/util/test/hashTest.cpp index 99f5a761c5d0d3a489176749883da981c847011d..5a426f931711852554b3c6b5bd318d18d0ed756a 100644 --- a/source/util/test/hashTest.cpp +++ b/source/util/test/hashTest.cpp @@ -197,6 +197,172 @@ void acquireRleaseTest() { taosMemoryFreeClear(data.p); } +void perfTest() { + SHashObj* hash1h = (SHashObj*) taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SHashObj* hash1s = (SHashObj*) taosHashInit(1000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SHashObj* hash10s = (SHashObj*) taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SHashObj* hash100s = (SHashObj*) taosHashInit(100000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SHashObj* hash1m = (SHashObj*) taosHashInit(1000000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SHashObj* hash10m = (SHashObj*) taosHashInit(10000000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SHashObj* hash100m = (SHashObj*) taosHashInit(100000000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + + char *name = (char*)taosMemoryCalloc(50000000, 9); + for (int64_t i = 0; i < 50000000; ++i) { + sprintf(name + i * 9, "t%08d", i); + } + + for (int64_t i = 0; i < 50; ++i) { + taosHashPut(hash1h, name + i * 9, 9, &i, sizeof(i)); + } + + for (int64_t i = 0; i < 500; ++i) { + taosHashPut(hash1s, name + i * 9, 9, &i, sizeof(i)); + } + + for (int64_t i = 0; i < 5000; ++i) { + taosHashPut(hash10s, name + i * 9, 9, &i, sizeof(i)); + } + + for (int64_t i = 0; i < 50000; ++i) { + taosHashPut(hash100s, name + i * 9, 9, &i, sizeof(i)); + } + + for (int64_t i = 0; i < 500000; ++i) { + taosHashPut(hash1m, name + i * 9, 9, &i, sizeof(i)); + } + + for (int64_t i = 0; i < 5000000; ++i) { + taosHashPut(hash10m, name + i * 9, 9, &i, sizeof(i)); + } + + for (int64_t i = 0; i < 50000000; ++i) { + taosHashPut(hash100m, name + i * 9, 9, &i, sizeof(i)); + } + + int64_t start1h = taosGetTimestampMs(); + int64_t start1hCt = taosHashGetCompTimes(hash1h); + for (int64_t i = 0; i < 10000000; ++i) { + ASSERT(taosHashGet(hash1h, name + (i % 50) * 9, 9)); + } + int64_t end1h = taosGetTimestampMs(); + int64_t end1hCt = taosHashGetCompTimes(hash1h); + + int64_t start1s = taosGetTimestampMs(); + int64_t start1sCt = taosHashGetCompTimes(hash1s); + for (int64_t i = 0; i < 10000000; ++i) { + ASSERT(taosHashGet(hash1s, name + (i % 500) * 9, 9)); + } + int64_t end1s = taosGetTimestampMs(); + int64_t end1sCt = taosHashGetCompTimes(hash1s); + + int64_t start10s = taosGetTimestampMs(); + int64_t start10sCt = taosHashGetCompTimes(hash10s); + for (int64_t i = 0; i < 10000000; ++i) { + ASSERT(taosHashGet(hash10s, name + (i % 5000) * 9, 9)); + } + int64_t end10s = taosGetTimestampMs(); + int64_t end10sCt = taosHashGetCompTimes(hash10s); + + int64_t start100s = taosGetTimestampMs(); + int64_t start100sCt = taosHashGetCompTimes(hash100s); + for (int64_t i = 0; i < 10000000; ++i) { + ASSERT(taosHashGet(hash100s, name + (i % 50000) * 9, 9)); + } + int64_t end100s = taosGetTimestampMs(); + int64_t end100sCt = taosHashGetCompTimes(hash100s); + + int64_t start1m = taosGetTimestampMs(); + int64_t start1mCt = taosHashGetCompTimes(hash1m); + for (int64_t i = 0; i < 10000000; ++i) { + ASSERT(taosHashGet(hash1m, name + (i % 500000) * 9, 9)); + } + int64_t end1m = taosGetTimestampMs(); + int64_t end1mCt = taosHashGetCompTimes(hash1m); + + int64_t start10m = taosGetTimestampMs(); + int64_t start10mCt = taosHashGetCompTimes(hash10m); + for (int64_t i = 0; i < 10000000; ++i) { + ASSERT(taosHashGet(hash10m, name + (i % 5000000) * 9, 9)); + } + int64_t end10m = taosGetTimestampMs(); + int64_t end10mCt = taosHashGetCompTimes(hash10m); + + int64_t start100m = taosGetTimestampMs(); + int64_t start100mCt = taosHashGetCompTimes(hash100m); + for (int64_t i = 0; i < 10000000; ++i) { + ASSERT(taosHashGet(hash100m, name + (i % 50000000) * 9, 9)); + } + int64_t end100m = taosGetTimestampMs(); + int64_t end100mCt = taosHashGetCompTimes(hash100m); + + int64_t start100mS = taosGetTimestampMs(); + int64_t start100mSCt = taosHashGetCompTimes(hash100m); + _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + for (int64_t i = 0; i < 10000000; ++i) { + ASSERT(taosHashGet(hash100m, name + (i % 50000000) * 9, 9)); + } + int64_t end100mS = taosGetTimestampMs(); + int64_t end100mSCt = taosHashGetCompTimes(hash100m); + + + printf("1h \t %" PRId64 "ms,%" PRId64 "\n", end1h - start1h, end1hCt - start1hCt); + printf("1s \t %" PRId64 "ms,%" PRId64 "\n", end1s - start1s, end1sCt - start1sCt); + printf("10s \t %" PRId64 "ms,%" PRId64 "\n", end10s - start10s, end10sCt - start10sCt); + printf("100s \t %" PRId64 "ms,%" PRId64 "\n", end100s - start100s, end100sCt - start100sCt); + printf("1m \t %" PRId64 "ms,%" PRId64 "\n", end1m - start1m, end1mCt - start1mCt); + printf("10m \t %" PRId64 "ms,%" PRId64 "\n", end10m - start10m, end10mCt - start10mCt); + printf("100m \t %" PRId64 "ms,%" PRId64 "\n", end100m - start100m, end100mCt - start100mCt); + + taosHashCleanup(hash1h); + taosHashCleanup(hash1s); + taosHashCleanup(hash10s); + taosHashCleanup(hash100s); + taosHashCleanup(hash1m); + taosHashCleanup(hash10m); + taosHashCleanup(hash100m); + + SHashObj *mhash[1000] = {0}; + for (int64_t i = 0; i < 1000; ++i) { + mhash[i] = (SHashObj*) taosHashInit(100000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + } + + for (int64_t i = 0; i < 50000000; ++i) { +#if 0 + taosHashPut(mhash[i%1000], name + i * 9, 9, &i, sizeof(i)); +#else + taosHashPut(mhash[i/50000], name + i * 9, 9, &i, sizeof(i)); +#endif + } + + int64_t startMhashCt = 0; + for (int64_t i = 0; i < 1000; ++i) { + startMhashCt += taosHashGetCompTimes(mhash[i]); + } + + int64_t startMhash = taosGetTimestampMs(); +#if 0 + for (int32_t i = 0; i < 10000000; ++i) { + ASSERT(taosHashGet(mhash[i%1000], name + i * 9, 9)); + } +#else + for (int64_t i = 0; i < 10000000; ++i) { + ASSERT(taosHashGet(mhash[i/50000], name + i * 9, 9)); + } +#endif + int64_t endMhash = taosGetTimestampMs(); + int64_t endMhashCt = 0; + for (int64_t i = 0; i < 1000; ++i) { + printf(" %" PRId64 , taosHashGetCompTimes(mhash[i])); + endMhashCt += taosHashGetCompTimes(mhash[i]); + } + printf("\n100m \t %" PRId64 "ms,%" PRId64 "\n", endMhash - startMhash, endMhashCt - startMhashCt); + + for (int64_t i = 0; i < 1000; ++i) { + taosHashCleanup(mhash[i]); + } +} + + } int main(int argc, char** argv) { @@ -210,4 +376,5 @@ TEST(testCase, hashTest) { noLockPerformanceTest(); multithreadsTest(); acquireRleaseTest(); + //perfTest(); }