Merge pull request #6454 from taosdata/feature/TD-4614

[TD-4614]: add equal func to hashTable

src/client/src/tscSQLParser.c

@@ -150,12 +150,9 @@ int16_t getNewResColId(SSqlCmd* pCmd) {
 // formate  "type | size | value"
 bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType) {
   bool ret = false;
-  if (!pList || pList->size <= 0) {
+  if (!pList || pList->size <= 0 || colType < 0) {
     return ret;
   }  
-  if (colType == TSDB_DATA_TYPE_DOUBLE || colType == TSDB_DATA_TYPE_FLOAT) {
-    return ret;
-  }
   
   tSqlExprItem* item = (tSqlExprItem *)taosArrayGet(pList, 0); 
   int32_t firstTokenType = item->pNode->token.type; 
@@ -4562,11 +4559,7 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) {
       free(tmp);
     } else {
       double tmp;
-      if (p->_node.optr == TSDB_RELATION_IN) {
-        retVal = validateParamOfRelationIn(vVariant, schemaType);
-      } else {
-        retVal = tVariantDump(vVariant, (char*)&tmp, schemaType, false);
-      }
+      retVal = tVariantDump(vVariant, (char*)&tmp, schemaType, false);
     }
     
     if (retVal != TSDB_CODE_SUCCESS) {
@@ -8065,18 +8058,23 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS
       
       return TSDB_CODE_SUCCESS;
     } else if (pSqlExpr->tokenId == TK_SET) {
-      int32_t type = -1;
+      int32_t colType = -1;
       STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta;
       if (pCols != NULL) {
         SColIndex* idx = taosArrayGet(pCols, 0);
         SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, idx->colIndex);
         if (pSchema != NULL) {
-          type = pSchema->type; 
+          colType = pSchema->type; 
         }
       }
 
       tVariant *pVal;
-      if (serializeExprListToVariant(pSqlExpr->pParam, &pVal, type) == false) {
+      if (colType >= TSDB_DATA_TYPE_TINYINT && colType <= TSDB_DATA_TYPE_BIGINT) {
+        colType = TSDB_DATA_TYPE_BIGINT;
+      } else if (colType == TSDB_DATA_TYPE_FLOAT || colType == TSDB_DATA_TYPE_DOUBLE) {
+        colType = TSDB_DATA_TYPE_DOUBLE;
+      }
+      if (serializeExprListToVariant(pSqlExpr->pParam, &pVal, colType) == false) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression");
       }
       *pExpr = calloc(1, sizeof(tExprNode));

src/common/src/texpr.c

@@ -470,6 +470,9 @@ void buildFilterSetFromBinary(void **q, const char *buf, int32_t len) {
   SBufferReader br = tbufInitReader(buf, len, false); 
   uint32_t type  = tbufReadUint32(&br);     
   SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false);
+  
+  taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); 
+  
   int dummy = -1;
   int32_t sz = tbufReadInt32(&br);
   for (int32_t i = 0; i < sz; i++) {

src/inc/taosdef.h

@@ -166,7 +166,7 @@ do { \
 #define TSDB_BINARY_OP_REMAINDER  34
 
 
-#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) <= TSDB_RELATION_IN))
+#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN))
 #define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER))
 
 #define TS_PATH_DELIMITER_LEN     1

src/query/src/qExecutor.c

@@ -2282,7 +2282,7 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis
   if (pDataStatis == NULL || pQueryAttr->numOfFilterCols == 0) {
     return true;
   }
-
+  bool ret = true;
   for (int32_t k = 0; k < pQueryAttr->numOfFilterCols; ++k) {
     SSingleColumnFilterInfo *pFilterInfo = &pQueryAttr->pFilterInfo[k];
 
@@ -2319,26 +2319,34 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis
     }
 
     SDataStatis* pDataBlockst = &pDataStatis[index];
-
+    
     if (pFilterInfo->info.type == TSDB_DATA_TYPE_FLOAT) {
       float minval = (float)(*(double *)(&pDataBlockst->min));
       float maxval = (float)(*(double *)(&pDataBlockst->max));
-
+       
       for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) {
-        if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT)) {
-          return true;
+        if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) {
+           continue;   
+        }
+        ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT);
+        if (ret == false) {
+          return false;
         }
       }
     } else {
       for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) {
-        if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type)) {
-          return true;
+        if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) {
+           continue; 
+        }
+        ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type); 
+        if (ret == false) {
+          return false;
         }
       }
     }
   }
 
-  return false;
+  return ret;
 }
 
 static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) {

src/query/src/qFilterfunc.c

@@ -262,7 +262,7 @@ bool inOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxv
       return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&minv, sizeof(minv));     
     }
     return true; 
-  } else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_DOUBLE) {
+  } else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_FLOAT) {
     double v;
     GET_TYPED_DATA(v, double, type, minval);
     return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&v, sizeof(v));     

src/util/inc/hash.h

@@ -61,6 +61,7 @@ typedef struct SHashObj {
   size_t          size;         // number of elements in hash table
   _hash_fn_t      hashFp;       // hash function
   _hash_free_fn_t freeFp;       // hash node free callback function
+  _equal_fn_t     equalFp;       // equal function
 
   SRWLatch        lock;         // read-write spin lock
   SHashLockTypeE  type;         // lock type
@@ -78,6 +79,15 @@ typedef struct SHashObj {
  */
 SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type);
 
+
+/**
+ * set equal func of the hash table  
+ * @param pHashObj    
+ * @param equalFp       
+ * @return
+ */
+void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp);
+
 /**
  * return the size of hash table
  * @param pHashObj

src/util/inc/hashfunc.h

@@ -20,6 +20,8 @@
 
 typedef uint32_t (*_hash_fn_t)(const char *, uint32_t);
 
+typedef int32_t (*_equal_fn_t)(const void *a, const void *b, size_t sz); 
+
 /**
  * murmur hash algorithm
  * @key  usually string
@@ -36,9 +38,14 @@ uint32_t MurmurHash3_32(const char *key, uint32_t len);
  * @return
  */
 uint32_t taosIntHash_32(const char *key, uint32_t len);
-
 uint32_t taosIntHash_64(const char *key, uint32_t len);
 
+
+int32_t  taosFloatEqual(const void *a, const void *b, size_t sz);
+int32_t  taosDoubleEqual(const void *a,const void *b, size_t sz); 
+
 _hash_fn_t taosGetDefaultHashFunction(int32_t type);
 
+_equal_fn_t taosGetDefaultEqualFunction(int32_t type);
+
 #endif //TDENGINE_HASHUTIL_H

src/util/inc/tcompare.h

@@ -27,6 +27,13 @@ extern "C" {
 #define TSDB_PATTERN_NOWILDCARDMATCH 2
 #define TSDB_PATTERN_STRING_MAX_LEN 20
 
+#define FLT_COMPAR_TOL_FACTOR    4
+#define FLT_EQUAL(_x, _y)        (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON))
+#define FLT_GREATER(_x, _y)      (!FLT_EQUAL((_x), (_y)) && ((_x) > (_y)))
+#define FLT_LESS(_x, _y)         (!FLT_EQUAL((_x), (_y)) && ((_x) < (_y)))
+#define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y)))
+#define FLT_LESSEQUAL(_x, _y)    (FLT_EQUAL((_x), (_y)) || ((_x) < (_y)))
+
 #define PATTERN_COMPARE_INFO_INITIALIZER { '%', '_' }
 
 typedef struct SPatternCompareInfo {

src/util/src/hash.c

@@ -73,10 +73,10 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
   return i;
 }
 
-static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) {
+static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) {
   SHashNode *pNode = pe->next;
   while (pNode) {
-    if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) {
+    if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) {
       assert(pNode->hashVal == hashVal);
       break;
     }
@@ -165,8 +165,8 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
   // the max slots is not defined by user
   pHashObj->capacity = taosHashCapacity((int32_t)capacity);
   assert((pHashObj->capacity & (pHashObj->capacity - 1)) == 0);
-
-  pHashObj->hashFp = fn;
+  pHashObj->equalFp = memcmp;
+  pHashObj->hashFp  = fn;
   pHashObj->type = type;
   pHashObj->enableUpdate = update;
 
@@ -189,6 +189,12 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
   return pHashObj;
 }
 
+void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp) {
+  if (pHashObj != NULL && fp != NULL) {
+    pHashObj->equalFp = fp;
+  } 
+} 
+
 int32_t taosHashGetSize(const SHashObj *pHashObj) { return (int32_t)((pHashObj == NULL) ? 0 : pHashObj->size); }
 
 int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) {
@@ -223,7 +229,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
 
   SHashNode* prev = NULL;
   while (pNode) {
-    if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) {
+    if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) {
       assert(pNode->hashVal == hashVal);
       break;
     }
@@ -306,7 +312,7 @@ void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void
     assert(pe->next == NULL);
   }
 
-  SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal);
+  SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal);
   if (pNode != NULL) {
     if (fp != NULL) {
       fp(GET_HASH_NODE_DATA(pNode));
@@ -362,7 +368,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
   SHashNode *prevNode = NULL;
 
   while (pNode) {
-    if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) 
+    if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) 
       break;
 
     prevNode = pNode;

src/util/src/tcompare.c

@@ -90,13 +90,10 @@ int32_t compareFloatVal(const void *pLeft, const void *pRight) {
   if (isnan(p2)) {
     return 1;
   }
-
-  float ret = p1 - p2;
-  if (fabs(ret) < FLT_EPSILON) {
+  if (FLT_EQUAL(p1, p2)) {
     return 0;
-  } else {
-    return ret > 0? 1 : -1;
-  }
+  } 
+  return FLT_GREATER(p1, p2) ? 1: -1; 
 }
 
 int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
@@ -114,13 +111,10 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
   if (isnan(p2)) {
     return 1;
   }
-
-  double ret = p1 - p2;
-  if (fabs(ret) < FLT_EPSILON) {
+  if (FLT_EQUAL(p1, p2)) {
     return 0;
-  } else {
-    return ret > 0? 1 : -1;
-  }
+  } 
+  return FLT_GREATER(p1, p2) ? 1: -1; 
 }
 
 int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {

src/util/src/thashutil.c

@@ -16,17 +16,20 @@
 #include "os.h"
 #include "hashfunc.h"
 #include "tutil.h"
+#include "tcompare.h"
 
 #define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r)))
 
+#define DLT (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON)
+#define BASE 1000
+
 #define FMIX32(h)      \
   do {                 \
     (h) ^= (h) >> 16;  \
     (h) *= 0x85ebca6b; \
     (h) ^= (h) >> 13;  \
     (h) *= 0xc2b2ae35; \
-    (h) ^= (h) >> 16;  \
-  } while (0)
+    (h) ^= (h) >> 16; } while (0)
   
 uint32_t MurmurHash3_32(const char *key, uint32_t len) {
   const uint8_t *data = (const uint8_t *)key;
@@ -78,7 +81,38 @@ uint32_t MurmurHash3_32(const char *key, uint32_t len) {
 uint32_t taosIntHash_32(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint32_t *)key; }
 uint32_t taosIntHash_16(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint16_t *)key; }
 uint32_t taosIntHash_8(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint8_t *)key; }
+uint32_t taosFloatHash(const char *key, uint32_t UNUSED_PARAM(len)) {
+  float f = GET_FLOAT_VAL(key); 
+  if (isnan(f)) {
+    return 0x7fc00000;
+  }
+   
+  if (FLT_EQUAL(f, 0.0)) {
+    return 0;
+  } 
+  if (fabs(f) < FLT_MAX/BASE - DLT) {
+   int t = (int)(round(BASE * (f + DLT)));
+   return (uint32_t)t;
+  } else {
+   return 0x7fc00000;
+  }
+}
+uint32_t taosDoubleHash(const char *key, uint32_t UNUSED_PARAM(len)) {
+  double f = GET_DOUBLE_VAL(key);  
+  if (isnan(f)) {
+    return 0x7fc00000;
+  }
 
+  if (FLT_EQUAL(f, 0.0)) {
+    return 0;
+  } 
+  if (fabs(f) < DBL_MAX/BASE - DLT) {
+   int t = (int)(round(BASE * (f + DLT)));
+   return (uint32_t)t;
+  } else {
+   return 0x7fc00000;
+  } 
+}
 uint32_t taosIntHash_64(const char *key, uint32_t UNUSED_PARAM(len)) {
   uint64_t val = *(uint64_t *)key;
 
@@ -92,13 +126,35 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) {
   _hash_fn_t fn = NULL;
   switch(type) {
     case TSDB_DATA_TYPE_TIMESTAMP:
-    case TSDB_DATA_TYPE_BIGINT: fn = taosIntHash_64;break;
-    case TSDB_DATA_TYPE_BINARY: fn = MurmurHash3_32;break;
-    case TSDB_DATA_TYPE_INT: fn = taosIntHash_32; break;
+    case TSDB_DATA_TYPE_BIGINT:   fn = taosIntHash_64;break;
+    case TSDB_DATA_TYPE_BINARY:   fn = MurmurHash3_32;break;
+    case TSDB_DATA_TYPE_NCHAR:    fn = MurmurHash3_32;break;
+    case TSDB_DATA_TYPE_INT:      fn = taosIntHash_32; break;
     case TSDB_DATA_TYPE_SMALLINT: fn = taosIntHash_16; break;
-    case TSDB_DATA_TYPE_TINYINT: fn = taosIntHash_8; break;
+    case TSDB_DATA_TYPE_TINYINT:  fn = taosIntHash_8; break;
+    case TSDB_DATA_TYPE_FLOAT:    fn = taosFloatHash; break;                             
+    case TSDB_DATA_TYPE_DOUBLE:   fn = taosDoubleHash; break;                             
     default: fn = taosIntHash_32;break;
   }
   
   return fn;
-}
\ No newline at end of file
+}
+
+int32_t taosFloatEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) {
+  return getComparFunc(TSDB_DATA_TYPE_FLOAT, -1)(a, b);  
+}
+
+int32_t taosDoubleEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) {
+  return getComparFunc(TSDB_DATA_TYPE_DOUBLE, -1)(a, b);  
+}
+
+_equal_fn_t taosGetDefaultEqualFunction(int32_t type) {
+  _equal_fn_t fn = NULL;
+  switch (type) {
+    case TSDB_DATA_TYPE_FLOAT:  fn = taosFloatEqual;  break; 
+    case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleEqual; break;
+    default: fn = memcmp; break;
+  }
+  return fn;
+  
+}