enh: add hash performance test

include/util/thash.h

@@ -210,6 +210,8 @@ void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp);
  */
 void taosHashSetFreeFp(SHashObj *pHashObj, _hash_free_fn_t fp);
 
+int64_t taosHashGetCompTimes(SHashObj *pHashObj);
+
 #ifdef __cplusplus
 }
 #endif

source/util/src/thash.c

@@ -21,7 +21,7 @@
 
 // the add ref count operation may trigger the warning if the reference count is greater than the MAX_WARNING_REF_COUNT
 #define MAX_WARNING_REF_COUNT    10000
-#define HASH_MAX_CAPACITY        (1024 * 1024 * 16)
+#define HASH_MAX_CAPACITY        (1024 * 1024 * 1024)
 #define HASH_DEFAULT_LOAD_FACTOR (0.75)
 #define HASH_INDEX(v, c)         ((v) & ((c)-1))
 
@@ -67,6 +67,7 @@ struct SHashObj {
   bool              enableUpdate;  // enable update
   SArray           *pMemBlock;     // memory block allocated for SHashEntry
   _hash_before_fn_t callbackFp;    // function invoked before return the value to caller
+  int64_t           compTimes;
 };
 
 /*
@@ -146,6 +147,7 @@ static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntr
                                                    uint32_t hashVal) {
   SHashNode *pNode = pe->next;
   while (pNode) {
+    atomic_add_fetch_64(&pHashObj->compTimes, 1);
     if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) &&
         pNode->removed == 0) {
       assert(pNode->hashVal == hashVal);
@@ -882,3 +884,7 @@ void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) {
 }
 
 void taosHashRelease(SHashObj *pHashObj, void *p) { taosHashCancelIterate(pHashObj, p); }
+
+int64_t taosHashGetCompTimes(SHashObj *pHashObj) { return atomic_load_64(&pHashObj->compTimes); }
+
+

source/util/test/hashTest.cpp

@@ -197,6 +197,172 @@ void acquireRleaseTest() {
   taosMemoryFreeClear(data.p);
 }
 
+void perfTest() {
+  SHashObj* hash1h = (SHashObj*) taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  SHashObj* hash1s = (SHashObj*) taosHashInit(1000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  SHashObj* hash10s = (SHashObj*) taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  SHashObj* hash100s = (SHashObj*) taosHashInit(100000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  SHashObj* hash1m = (SHashObj*) taosHashInit(1000000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  SHashObj* hash10m = (SHashObj*) taosHashInit(10000000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  SHashObj* hash100m = (SHashObj*) taosHashInit(100000000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+
+  char *name = (char*)taosMemoryCalloc(50000000, 9);
+  for (int64_t i = 0; i < 50000000; ++i) {
+    sprintf(name + i * 9, "t%08d", i);
+  }
+  
+  for (int64_t i = 0; i < 50; ++i) {
+    taosHashPut(hash1h, name + i * 9, 9, &i, sizeof(i));
+  }
+
+  for (int64_t i = 0; i < 500; ++i) {
+    taosHashPut(hash1s, name + i * 9, 9, &i, sizeof(i));
+  }
+
+  for (int64_t i = 0; i < 5000; ++i) {
+    taosHashPut(hash10s, name + i * 9, 9, &i, sizeof(i));
+  }
+
+  for (int64_t i = 0; i < 50000; ++i) {
+    taosHashPut(hash100s, name + i * 9, 9, &i, sizeof(i));
+  }
+
+  for (int64_t i = 0; i < 500000; ++i) {
+    taosHashPut(hash1m, name + i * 9, 9, &i, sizeof(i));
+  }
+
+  for (int64_t i = 0; i < 5000000; ++i) {
+    taosHashPut(hash10m, name + i * 9, 9, &i, sizeof(i));
+  }
+
+  for (int64_t i = 0; i < 50000000; ++i) {
+    taosHashPut(hash100m, name + i * 9, 9, &i, sizeof(i));
+  }
+
+  int64_t start1h = taosGetTimestampMs();
+  int64_t start1hCt = taosHashGetCompTimes(hash1h);
+  for (int64_t i = 0; i < 10000000; ++i) {
+    ASSERT(taosHashGet(hash1h, name + (i % 50) * 9, 9));
+  }
+  int64_t end1h = taosGetTimestampMs();
+  int64_t end1hCt = taosHashGetCompTimes(hash1h);
+
+  int64_t start1s = taosGetTimestampMs();
+  int64_t start1sCt = taosHashGetCompTimes(hash1s);
+  for (int64_t i = 0; i < 10000000; ++i) {
+    ASSERT(taosHashGet(hash1s, name + (i % 500) * 9, 9));
+  }
+  int64_t end1s = taosGetTimestampMs();
+  int64_t end1sCt = taosHashGetCompTimes(hash1s);
+
+  int64_t start10s = taosGetTimestampMs();
+  int64_t start10sCt = taosHashGetCompTimes(hash10s);
+  for (int64_t i = 0; i < 10000000; ++i) {
+    ASSERT(taosHashGet(hash10s, name + (i % 5000) * 9, 9));
+  }
+  int64_t end10s = taosGetTimestampMs();
+  int64_t end10sCt = taosHashGetCompTimes(hash10s);
+
+  int64_t start100s = taosGetTimestampMs();
+  int64_t start100sCt = taosHashGetCompTimes(hash100s);
+  for (int64_t i = 0; i < 10000000; ++i) {
+    ASSERT(taosHashGet(hash100s, name + (i % 50000) * 9, 9));
+  }
+  int64_t end100s = taosGetTimestampMs();
+  int64_t end100sCt = taosHashGetCompTimes(hash100s);
+
+  int64_t start1m = taosGetTimestampMs();
+  int64_t start1mCt = taosHashGetCompTimes(hash1m);
+  for (int64_t i = 0; i < 10000000; ++i) {
+    ASSERT(taosHashGet(hash1m, name + (i % 500000) * 9, 9));
+  }
+  int64_t end1m = taosGetTimestampMs();
+  int64_t end1mCt = taosHashGetCompTimes(hash1m);
+
+  int64_t start10m = taosGetTimestampMs();
+  int64_t start10mCt = taosHashGetCompTimes(hash10m);
+  for (int64_t i = 0; i < 10000000; ++i) {
+    ASSERT(taosHashGet(hash10m, name + (i % 5000000) * 9, 9));
+  }
+  int64_t end10m = taosGetTimestampMs();
+  int64_t end10mCt = taosHashGetCompTimes(hash10m);
+
+  int64_t start100m = taosGetTimestampMs();
+  int64_t start100mCt = taosHashGetCompTimes(hash100m);
+  for (int64_t i = 0; i < 10000000; ++i) {
+    ASSERT(taosHashGet(hash100m, name + (i % 50000000) * 9, 9));
+  }
+  int64_t end100m = taosGetTimestampMs();
+  int64_t end100mCt = taosHashGetCompTimes(hash100m);
+
+  int64_t start100mS = taosGetTimestampMs();
+  int64_t start100mSCt = taosHashGetCompTimes(hash100m);
+  _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
+  for (int64_t i = 0; i < 10000000; ++i) {
+    ASSERT(taosHashGet(hash100m, name + (i % 50000000) * 9, 9));
+  }
+  int64_t end100mS = taosGetTimestampMs();
+  int64_t end100mSCt = taosHashGetCompTimes(hash100m);
+
+
+  printf("1h \t %" PRId64 "ms,%" PRId64 "\n", end1h - start1h, end1hCt - start1hCt);
+  printf("1s \t %" PRId64 "ms,%" PRId64 "\n", end1s - start1s, end1sCt - start1sCt);
+  printf("10s \t %" PRId64 "ms,%" PRId64 "\n", end10s - start10s, end10sCt - start10sCt);
+  printf("100s \t %" PRId64 "ms,%" PRId64 "\n", end100s - start100s, end100sCt - start100sCt);
+  printf("1m \t %" PRId64 "ms,%" PRId64 "\n", end1m - start1m, end1mCt - start1mCt);
+  printf("10m \t %" PRId64 "ms,%" PRId64 "\n", end10m - start10m, end10mCt - start10mCt);
+  printf("100m \t %" PRId64 "ms,%" PRId64 "\n", end100m - start100m, end100mCt - start100mCt);
+
+  taosHashCleanup(hash1h);
+  taosHashCleanup(hash1s);
+  taosHashCleanup(hash10s);
+  taosHashCleanup(hash100s);
+  taosHashCleanup(hash1m);
+  taosHashCleanup(hash10m);
+  taosHashCleanup(hash100m);
+
+  SHashObj *mhash[1000] = {0};
+  for (int64_t i = 0; i < 1000; ++i) {
+    mhash[i] = (SHashObj*) taosHashInit(100000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  }
+
+  for (int64_t i = 0; i < 50000000; ++i) {
+#if 0
+    taosHashPut(mhash[i%1000], name + i * 9, 9, &i, sizeof(i));
+#else
+    taosHashPut(mhash[i/50000], name + i * 9, 9, &i, sizeof(i));
+#endif    
+  }
+
+  int64_t startMhashCt = 0;
+  for (int64_t i = 0; i < 1000; ++i) {
+    startMhashCt += taosHashGetCompTimes(mhash[i]);
+  }
+
+  int64_t startMhash = taosGetTimestampMs();
+#if 0
+  for (int32_t i = 0; i < 10000000; ++i) {
+    ASSERT(taosHashGet(mhash[i%1000], name + i * 9, 9));
+  }
+#else
+  for (int64_t i = 0; i < 10000000; ++i) {
+    ASSERT(taosHashGet(mhash[i/50000], name + i * 9, 9));
+  }
+#endif  
+  int64_t endMhash = taosGetTimestampMs();
+  int64_t endMhashCt = 0;
+  for (int64_t i = 0; i < 1000; ++i) {
+    printf(" %" PRId64 , taosHashGetCompTimes(mhash[i]));
+    endMhashCt += taosHashGetCompTimes(mhash[i]);
+  }
+  printf("\n100m \t %" PRId64 "ms,%" PRId64 "\n", endMhash - startMhash, endMhashCt - startMhashCt);
+
+  for (int64_t i = 0; i < 1000; ++i) {
+    taosHashCleanup(mhash[i]);
+  }
+}
+
+
 }
 
 int main(int argc, char** argv) {
@@ -210,4 +376,5 @@ TEST(testCase, hashTest) {
   noLockPerformanceTest();
   multithreadsTest();
   acquireRleaseTest();
+  //perfTest();
 }