#include #include #include #include "os.h" #include "taos.h" #include "taosdef.h" #include "thash.h" namespace { typedef struct TESTSTRUCT { char* p; } TESTSTRUCT; // the simple test code for basic operations void simpleTest() { SHashObj* hashTable = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); ASSERT_EQ(taosHashGetSize(hashTable), 0); // put 400 elements in the hash table for (int32_t i = -200; i < 200; ++i) { taosHashPut(hashTable, (const char*)&i, sizeof(int32_t), (char*)&i, sizeof(int32_t)); } ASSERT_EQ(taosHashGetSize(hashTable), 400); for (int32_t i = 0; i < 200; ++i) { char* p = (char*)taosHashGet(hashTable, (const char*)&i, sizeof(int32_t)); ASSERT_TRUE(p != nullptr); ASSERT_EQ(*reinterpret_cast(p), i); } for (int32_t i = 1000; i < 2000; ++i) { taosHashRemove(hashTable, (const char*)&i, sizeof(int32_t)); } ASSERT_EQ(taosHashGetSize(hashTable), 400); for (int32_t i = 0; i < 100; ++i) { taosHashRemove(hashTable, (const char*)&i, sizeof(int32_t)); } ASSERT_EQ(taosHashGetSize(hashTable), 300); for (int32_t i = 100; i < 150; ++i) { taosHashRemove(hashTable, (const char*)&i, sizeof(int32_t)); } ASSERT_EQ(taosHashGetSize(hashTable), 250); taosHashCleanup(hashTable); } void stringKeyTest() { auto* hashTable = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); ASSERT_EQ(taosHashGetSize(hashTable), 0); char key[128] = {0}; // put 200 elements in the hash table for (int32_t i = 0; i < 1000; ++i) { int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); taosHashPut(hashTable, key, len, (char*)&i, sizeof(int32_t)); } ASSERT_EQ(taosHashGetSize(hashTable), 1000); for (int32_t i = 0; i < 1000; ++i) { int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); char* p = (char*)taosHashGet(hashTable, key, len); ASSERT_TRUE(p != nullptr); ASSERT_EQ(*reinterpret_cast(p), i); } for (int32_t i = 500; i < 1000; ++i) { int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); taosHashRemove(hashTable, key, len); } ASSERT_EQ(taosHashGetSize(hashTable), 500); for (int32_t i = 0; i < 499; ++i) { int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); taosHashRemove(hashTable, key, len); } ASSERT_EQ(taosHashGetSize(hashTable), 1); taosHashCleanup(hashTable); } void functionTest() {} /** * evaluate the performance issue, by add 10million elements in to hash table in * a single threads situation */ void noLockPerformanceTest() { auto* hashTable = (SHashObj*)taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); ASSERT_EQ(taosHashGetSize(hashTable), 0); char key[128] = {0}; int32_t num = 5000; int64_t st = taosGetTimestampUs(); // put 10M elements in the hash table for (int32_t i = 0; i < num; ++i) { int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); taosHashPut(hashTable, key, len, (char*)&i, sizeof(int32_t)); } ASSERT_EQ(taosHashGetSize(hashTable), num); int64_t et = taosGetTimestampUs(); printf("Elpased time:%" PRId64 " us to add %d elements, avg cost:%lf us\n", et - st, num, (et - st) / (double)num); st = taosGetTimestampUs(); for (int32_t i = 0; i < num; ++i) { int32_t len = sprintf(key, "%d_1_%dabcefg_", i, i + 10); char* p = (char*)taosHashGet(hashTable, key, len); ASSERT_TRUE(p != nullptr); ASSERT_EQ(*reinterpret_cast(p), i); } et = taosGetTimestampUs(); printf("Elpased time:%" PRId64 " us to fetch all %d elements, avg cost:%lf us\n", et - st, num, (et - st) / (double)num); printf("The maximum length of overflow linklist in hash table is:%d\n", taosHashGetMaxOverflowLinkLength(hashTable)); taosHashCleanup(hashTable); } void multithreadsTest() { // todo } // check the function robustness void invalidOperationTest() {} void acquireRleaseTest() { SHashObj* hashTable = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); ASSERT_EQ(taosHashGetSize(hashTable), 0); int32_t key = 2; int32_t code = 0; int32_t num = 0; TESTSTRUCT data = {0}; const char* str1 = "abcdefg"; const char* str2 = "aaaaaaa"; const char* str3 = "123456789"; data.p = (char*)taosMemoryMalloc(10); strcpy(data.p, str1); code = taosHashPut(hashTable, &key, sizeof(key), &data, sizeof(data)); ASSERT_EQ(code, 0); TESTSTRUCT* pdata = (TESTSTRUCT*)taosHashAcquire(hashTable, &key, sizeof(key)); ASSERT_TRUE(pdata != nullptr); ASSERT_TRUE(strcmp(pdata->p, str1) == 0); code = taosHashRemove(hashTable, &key, sizeof(key)); ASSERT_EQ(code, 0); ASSERT_TRUE(strcmp(pdata->p, str1) == 0); num = taosHashGetSize(hashTable); ASSERT_EQ(num, 1); strcpy(pdata->p, str3); data.p = (char*)taosMemoryMalloc(10); strcpy(data.p, str2); code = taosHashPut(hashTable, &key, sizeof(key), &data, sizeof(data)); ASSERT_EQ(code, 0); num = taosHashGetSize(hashTable); ASSERT_EQ(num, 2); printf("%s,expect:%s", pdata->p, str3); ASSERT_TRUE(strcmp(pdata->p, str3) == 0); taosMemoryFreeClear(pdata->p); taosHashRelease(hashTable, pdata); num = taosHashGetSize(hashTable); ASSERT_EQ(num, 1); taosHashCleanup(hashTable); taosMemoryFreeClear(data.p); } void perfTest() { SHashObj* hash1h = (SHashObj*)taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SHashObj* hash1s = (SHashObj*)taosHashInit(1000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SHashObj* hash10s = (SHashObj*)taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SHashObj* hash100s = (SHashObj*)taosHashInit(100000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SHashObj* hash1m = (SHashObj*)taosHashInit(1000000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SHashObj* hash10m = (SHashObj*)taosHashInit(10000000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SHashObj* hash100m = (SHashObj*)taosHashInit(100000000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); char* name = (char*)taosMemoryCalloc(50000000, 9); for (int64_t i = 0; i < 50000000; ++i) { sprintf(name + i * 9, "t%08" PRId64, i); } for (int64_t i = 0; i < 50; ++i) { taosHashPut(hash1h, name + i * 9, 9, &i, sizeof(i)); } for (int64_t i = 0; i < 500; ++i) { taosHashPut(hash1s, name + i * 9, 9, &i, sizeof(i)); } for (int64_t i = 0; i < 5000; ++i) { taosHashPut(hash10s, name + i * 9, 9, &i, sizeof(i)); } for (int64_t i = 0; i < 50000; ++i) { taosHashPut(hash100s, name + i * 9, 9, &i, sizeof(i)); } for (int64_t i = 0; i < 500000; ++i) { taosHashPut(hash1m, name + i * 9, 9, &i, sizeof(i)); } for (int64_t i = 0; i < 5000000; ++i) { taosHashPut(hash10m, name + i * 9, 9, &i, sizeof(i)); } for (int64_t i = 0; i < 50000000; ++i) { taosHashPut(hash100m, name + i * 9, 9, &i, sizeof(i)); } int64_t start1h = taosGetTimestampMs(); int64_t start1hCt = taosHashGetCompTimes(hash1h); for (int64_t i = 0; i < 10000000; ++i) { ASSERT(taosHashGet(hash1h, name + (i % 50) * 9, 9)); } int64_t end1h = taosGetTimestampMs(); int64_t end1hCt = taosHashGetCompTimes(hash1h); int64_t start1s = taosGetTimestampMs(); int64_t start1sCt = taosHashGetCompTimes(hash1s); for (int64_t i = 0; i < 10000000; ++i) { ASSERT(taosHashGet(hash1s, name + (i % 500) * 9, 9)); } int64_t end1s = taosGetTimestampMs(); int64_t end1sCt = taosHashGetCompTimes(hash1s); int64_t start10s = taosGetTimestampMs(); int64_t start10sCt = taosHashGetCompTimes(hash10s); for (int64_t i = 0; i < 10000000; ++i) { ASSERT(taosHashGet(hash10s, name + (i % 5000) * 9, 9)); } int64_t end10s = taosGetTimestampMs(); int64_t end10sCt = taosHashGetCompTimes(hash10s); int64_t start100s = taosGetTimestampMs(); int64_t start100sCt = taosHashGetCompTimes(hash100s); for (int64_t i = 0; i < 10000000; ++i) { ASSERT(taosHashGet(hash100s, name + (i % 50000) * 9, 9)); } int64_t end100s = taosGetTimestampMs(); int64_t end100sCt = taosHashGetCompTimes(hash100s); int64_t start1m = taosGetTimestampMs(); int64_t start1mCt = taosHashGetCompTimes(hash1m); for (int64_t i = 0; i < 10000000; ++i) { ASSERT(taosHashGet(hash1m, name + (i % 500000) * 9, 9)); } int64_t end1m = taosGetTimestampMs(); int64_t end1mCt = taosHashGetCompTimes(hash1m); int64_t start10m = taosGetTimestampMs(); int64_t start10mCt = taosHashGetCompTimes(hash10m); for (int64_t i = 0; i < 10000000; ++i) { ASSERT(taosHashGet(hash10m, name + (i % 5000000) * 9, 9)); } int64_t end10m = taosGetTimestampMs(); int64_t end10mCt = taosHashGetCompTimes(hash10m); int64_t start100m = taosGetTimestampMs(); int64_t start100mCt = taosHashGetCompTimes(hash100m); for (int64_t i = 0; i < 10000000; ++i) { ASSERT(taosHashGet(hash100m, name + (i % 50000000) * 9, 9)); } int64_t end100m = taosGetTimestampMs(); int64_t end100mCt = taosHashGetCompTimes(hash100m); SArray* sArray[1000] = {0}; for (int64_t i = 0; i < 1000; ++i) { sArray[i] = taosArrayInit(100000, 9); } int64_t cap = 4; while (cap < 100000000) cap = (cap << 1u); _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); int32_t slotR = cap / 1000 + 1; for (int64_t i = 0; i < 10000000; ++i) { char* p = name + (i % 50000000) * 9; uint32_t v = (*hashFp)(p, 9); taosArrayPush(sArray[(v % cap) / slotR], p); } SArray* slArray = taosArrayInit(100000000, 9); for (int64_t i = 0; i < 1000; ++i) { int32_t num = taosArrayGetSize(sArray[i]); SArray* pArray = sArray[i]; for (int64_t m = 0; m < num; ++m) { char* p = (char*)taosArrayGet(pArray, m); ASSERT(taosArrayPush(slArray, p)); } } int64_t start100mS = taosGetTimestampMs(); int64_t start100mSCt = taosHashGetCompTimes(hash100m); int32_t num = taosArrayGetSize(slArray); for (int64_t i = 0; i < num; ++i) { ASSERT(taosHashGet(hash100m, (char*)TARRAY_GET_ELEM(slArray, i), 9)); } int64_t end100mS = taosGetTimestampMs(); int64_t end100mSCt = taosHashGetCompTimes(hash100m); for (int64_t i = 0; i < 1000; ++i) { taosArrayDestroy(sArray[i]); } taosArrayDestroy(slArray); printf("1h \t %" PRId64 "ms,%" PRId64 "\n", end1h - start1h, end1hCt - start1hCt); printf("1s \t %" PRId64 "ms,%" PRId64 "\n", end1s - start1s, end1sCt - start1sCt); printf("10s \t %" PRId64 "ms,%" PRId64 "\n", end10s - start10s, end10sCt - start10sCt); printf("100s \t %" PRId64 "ms,%" PRId64 "\n", end100s - start100s, end100sCt - start100sCt); printf("1m \t %" PRId64 "ms,%" PRId64 "\n", end1m - start1m, end1mCt - start1mCt); printf("10m \t %" PRId64 "ms,%" PRId64 "\n", end10m - start10m, end10mCt - start10mCt); printf("100m \t %" PRId64 "ms,%" PRId64 "\n", end100m - start100m, end100mCt - start100mCt); printf("100mS \t %" PRId64 "ms,%" PRId64 "\n", end100mS - start100mS, end100mSCt - start100mSCt); taosHashCleanup(hash1h); taosHashCleanup(hash1s); taosHashCleanup(hash10s); taosHashCleanup(hash100s); taosHashCleanup(hash1m); taosHashCleanup(hash10m); taosHashCleanup(hash100m); SHashObj* mhash[1000] = {0}; for (int64_t i = 0; i < 1000; ++i) { mhash[i] = (SHashObj*)taosHashInit(100000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } for (int64_t i = 0; i < 50000000; ++i) { #if 0 taosHashPut(mhash[i%1000], name + i * 9, 9, &i, sizeof(i)); #else taosHashPut(mhash[i / 50000], name + i * 9, 9, &i, sizeof(i)); #endif } int64_t startMhashCt = 0; for (int64_t i = 0; i < 1000; ++i) { startMhashCt += taosHashGetCompTimes(mhash[i]); } int64_t startMhash = taosGetTimestampMs(); #if 0 for (int32_t i = 0; i < 10000000; ++i) { ASSERT(taosHashGet(mhash[i%1000], name + i * 9, 9)); } #else // for (int64_t i = 0; i < 10000000; ++i) { for (int64_t i = 0; i < 50000000; i += 5) { ASSERT(taosHashGet(mhash[i / 50000], name + i * 9, 9)); } #endif int64_t endMhash = taosGetTimestampMs(); int64_t endMhashCt = 0; for (int64_t i = 0; i < 1000; ++i) { printf(" %" PRId64, taosHashGetCompTimes(mhash[i])); endMhashCt += taosHashGetCompTimes(mhash[i]); } printf("\n100m \t %" PRId64 "ms,%" PRId64 "\n", endMhash - startMhash, endMhashCt - startMhashCt); for (int64_t i = 0; i < 1000; ++i) { taosHashCleanup(mhash[i]); } } } // namespace int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } TEST(testCase, hashTest) { simpleTest(); stringKeyTest(); noLockPerformanceTest(); multithreadsTest(); acquireRleaseTest(); // perfTest(); }