Merge pull request #8760 from haoyifan/hashtable_cleanup

Hashtable cleanup

src/query/src/qExecutor.c

@@ -2235,6 +2235,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf
         if (pRuntimeEnv->proot == NULL) {
           goto _clean;
         }
+
         int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType;
         if (opType != OP_DummyInput) {
           setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot);
@@ -10041,6 +10042,9 @@ static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type
 
 static int64_t getQuerySupportBufSize(size_t numOfTables) {
   size_t s1 = sizeof(STableQueryInfo);
+
+  // TODO: struct SHashNode is an internal implementation of
+  // hash table. The implementation should not leak here.
   size_t s2 = sizeof(SHashNode);
 
 //  size_t s3 = sizeof(STableCheckInfo);  buffer consumption in tsdb

src/util/inc/hash.h

@@ -24,24 +24,18 @@ extern "C" {
 #include "hashfunc.h"
 #include "tlockfree.h"
 
-#define HASH_MAX_CAPACITY (1024 * 1024 * 16)
-#define HASH_DEFAULT_LOAD_FACTOR (0.75)
-#define HASH_INDEX(v, c) ((v) & ((c)-1))
-
-typedef void (*_hash_free_fn_t)(void *param);
-
+// TODO: SHashNode is an internal implementation and should not
+// be in the public header file.
 typedef struct SHashNode {
   struct SHashNode *next;
   uint32_t          hashVal;     // the hash value of key
   uint32_t          dataLen;     // length of data
   uint32_t          keyLen;      // length of the key
   int8_t            removed;     // flag to indicate removed
-  int32_t           count;       // reference count
+  int32_t           refCount;    // reference count
   char              data[];
 } SHashNode;
   
-#define GET_HASH_NODE_KEY(_n)  ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen)
-#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode))
 #define GET_HASH_PNODE(_n) ((SHashNode *)((char*)(_n) - sizeof(SHashNode)))
 
 typedef enum SHashLockTypeE {
@@ -49,39 +43,22 @@ typedef enum SHashLockTypeE {
   HASH_ENTRY_LOCK  = 1,
 } SHashLockTypeE;
 
-typedef struct SHashEntry {
-  int32_t    num;      // number of elements in current entry
-  SRWLatch   latch;    // entry latch
-  SHashNode *next;
-} SHashEntry;
-
-typedef struct SHashObj {
-  SHashEntry    **hashList;
-  size_t          capacity;     // number of slots
-  size_t          size;         // number of elements in hash table
-  _hash_fn_t      hashFp;       // hash function
-  _hash_free_fn_t freeFp;       // hash node free callback function
-  _equal_fn_t     equalFp;       // equal function
-
-  SRWLatch        lock;         // read-write spin lock
-  SHashLockTypeE  type;         // lock type
-  bool            enableUpdate; // enable update
-  SArray         *pMemBlock;    // memory block allocated for SHashEntry
-} SHashObj;
+typedef struct SHashObj SHashObj;
 
 /**
- * init the hash table
+ * initialize a hash table
  *
  * @param capacity   initial capacity of the hash table
- * @param fn          hash function to generate the hash value
- * @param threadsafe  thread safe or not
- * @return
+ * @param fn         hash function
+ * @param update     whether the hash table allows in place update
+ * @param type       whether the hash table has per entry lock
+ * @return           hash table object
  */
 SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type);
 
-
 /**
  * set equal func of the hash table
+ *
  * @param pHashObj
  * @param equalFp
  * @return
@@ -92,6 +69,7 @@ void taosHashSetFreeFp(SHashObj *pHashObj, _hash_free_fn_t fp);
 
 /**
  * return the size of hash table
+ *
  * @param pHashObj
  * @return
  */
@@ -99,73 +77,114 @@ int32_t taosHashGetSize(const SHashObj *pHashObj);
 
 /**
  * put element into hash table, if the element with the same key exists, update it
- * @param pHashObj
- * @param key
- * @param keyLen
- * @param data
- * @param size
- * @return
+ *
+ * @param pHashObj   hash table object
+ * @param key        key
+ * @param keyLen     length of key
+ * @param data       data
+ * @param size       size of data
+ * @return           0 if success, -1 otherwise
  */
 int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size);
 
 /**
  * return the payload data with the specified key
  *
- * @param pHashObj
- * @param key
- * @param keyLen
- * @return
+ * @param pHashObj   hash table object
+ * @param key        key
+ * @param keyLen     length of key
+ * @return           pointer to data
  */
 void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen);
 
 /**
- * apply the udf before return the result
- * @param pHashObj
- * @param key
- * @param keyLen
- * @param fp
- * @param d
- * @return
+ * Get the data associated with "key". Note that caller needs to make sure
+ * "d" has enough capacity to accomodate the data.
+ *
+ * @param pHashObj   hash table object
+ * @param key        key
+ * @param keyLen     length of key
+ * @param fp         function to be called on hash node when the data is found
+ * @param d          buffer
+ * @return           pointer to data
  */
 void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d);
 
 /**
- * @param pHashObj
- * @param key
- * @param keyLen
- * @param fp
- * @param d
- * @param sz 
- * @return
+ * Get the data associated with "key". Note that caller needs to take ownership
+ * of the data "d" and make sure it is deallocated.
+ *
+ * @param pHashObj   hash table object
+ * @param key        key
+ * @param keyLen     length of key
+ * @param fp         function to be called on hash node when the data is found
+ * @param d          buffer
+ * @param sz         size of the data buffer
+ * @return           pointer to data
  */
 void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void** d, size_t *sz);
+
 /**
  * remove item with the specified key
- * @param pHashObj
- * @param key
- * @param keyLen
+ *
+ * @param pHashObj   hash table object
+ * @param key        key
+ * @param keyLen     length of key
+ * @return           0 if success, -1 otherwise
  */
 int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen);
 
+/**
+ * remove item with the specified key
+ *
+ * @param pHashObj   hash table object
+ * @param key        key
+ * @param keyLen     length of key
+ * @param data       buffer for data
+ * @param dsize      size of data buffer
+ * @return           0 if success, -1 otherwise
+ */
 int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void* data, size_t dsize);
 
-int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param);
+/**
+ * traverse through all objects in the hash table and apply "fp" on each node.
+ * If "fp" returns false when applied on top of a node, the node will also be
+ * removed from table.
+ *
+ * @param pHashObj   hash table object
+ * @param fp         function pointer applied on each node
+ * @param param      parameter fed into "fp"
+ */
+void taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param);
 
+/**
+ * clear the contents of the hash table
+ *
+ * @param pHashObj   hash table object
+ */
 void taosHashClear(SHashObj *pHashObj);
 
 /**
  * clean up hash table
- * @param handle
+ *
+ * @param pHashObj   hash table object
  */
 void taosHashCleanup(SHashObj *pHashObj);
 
 /**
+ * return the number of collisions in the hash table
  *
- * @param pHashObj
- * @return
+ * @param pHashObj   hash table object
+ * @return           maximum number of collisions
  */
-int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj);
+int32_t taosHashGetMaxOverflowLinkLength(SHashObj *pHashObj);
 
+/**
+ * return the consumed memory of the hash table
+ *
+ * @param pHashObj   hash table object
+ * @return           consumed memory of the hash table
+ */
 size_t taosHashGetMemSize(const SHashObj *pHashObj);
 
 void *taosHashIterate(SHashObj *pHashObj, void *p);

src/util/inc/hashfunc.h

@@ -22,6 +22,8 @@ typedef uint32_t (*_hash_fn_t)(const char *, uint32_t);
 
 typedef int32_t (*_equal_fn_t)(const void *a, const void *b, size_t sz); 
 
+typedef void (*_hash_free_fn_t)(void *param);
+
 /**
  * murmur hash algorithm
  * @key  usually string