diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index b7c582ed85139dd4ca8818d0f99281de65fc2d65..d8524c3e03a615a250eac3f72f75c72b8f209c84 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -150,12 +150,9 @@ int16_t getNewResColId(SSqlCmd* pCmd) { // formate "type | size | value" bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType) { bool ret = false; - if (!pList || pList->size <= 0) { + if (!pList || pList->size <= 0 || colType < 0) { return ret; } - if (colType == TSDB_DATA_TYPE_DOUBLE || colType == TSDB_DATA_TYPE_FLOAT) { - return ret; - } tSqlExprItem* item = (tSqlExprItem *)taosArrayGet(pList, 0); int32_t firstTokenType = item->pNode->token.type; @@ -4562,11 +4559,7 @@ static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { free(tmp); } else { double tmp; - if (p->_node.optr == TSDB_RELATION_IN) { - retVal = validateParamOfRelationIn(vVariant, schemaType); - } else { - retVal = tVariantDump(vVariant, (char*)&tmp, schemaType, false); - } + retVal = tVariantDump(vVariant, (char*)&tmp, schemaType, false); } if (retVal != TSDB_CODE_SUCCESS) { @@ -8065,18 +8058,23 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pS return TSDB_CODE_SUCCESS; } else if (pSqlExpr->tokenId == TK_SET) { - int32_t type = -1; + int32_t colType = -1; STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; if (pCols != NULL) { SColIndex* idx = taosArrayGet(pCols, 0); SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, idx->colIndex); if (pSchema != NULL) { - type = pSchema->type; + colType = pSchema->type; } } tVariant *pVal; - if (serializeExprListToVariant(pSqlExpr->pParam, &pVal, type) == false) { + if (colType >= TSDB_DATA_TYPE_TINYINT && colType <= TSDB_DATA_TYPE_BIGINT) { + colType = TSDB_DATA_TYPE_BIGINT; + } else if (colType == TSDB_DATA_TYPE_FLOAT || colType == TSDB_DATA_TYPE_DOUBLE) { + colType = TSDB_DATA_TYPE_DOUBLE; + } + if (serializeExprListToVariant(pSqlExpr->pParam, &pVal, colType) == false) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression"); } *pExpr = calloc(1, sizeof(tExprNode)); diff --git a/src/common/src/texpr.c b/src/common/src/texpr.c index 6fa4d68734df6fde94725026bd65b219cc6564ab..0e9ec053cc5fb0f402fe9ffd42f3435fb4ca3cad 100644 --- a/src/common/src/texpr.c +++ b/src/common/src/texpr.c @@ -470,6 +470,9 @@ void buildFilterSetFromBinary(void **q, const char *buf, int32_t len) { SBufferReader br = tbufInitReader(buf, len, false); uint32_t type = tbufReadUint32(&br); SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); + + taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); + int dummy = -1; int32_t sz = tbufReadInt32(&br); for (int32_t i = 0; i < sz; i++) { diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index e6eec73681dbde6c4c5b18f7b7c0351d40f7d134..dac2dc84b6065d27514bbbbd2a836a81422408ad 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -166,7 +166,7 @@ do { \ #define TSDB_BINARY_OP_REMAINDER 34 -#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) <= TSDB_RELATION_IN)) +#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) #define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) #define TS_PATH_DELIMITER_LEN 1 diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 77ceabbd635c56b0c95ad481982ec0a211531482..395903f75ef4a0cb4626baf9416c17a9b8221052 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2282,7 +2282,7 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis if (pDataStatis == NULL || pQueryAttr->numOfFilterCols == 0) { return true; } - + bool ret = true; for (int32_t k = 0; k < pQueryAttr->numOfFilterCols; ++k) { SSingleColumnFilterInfo *pFilterInfo = &pQueryAttr->pFilterInfo[k]; @@ -2319,26 +2319,34 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis } SDataStatis* pDataBlockst = &pDataStatis[index]; - + if (pFilterInfo->info.type == TSDB_DATA_TYPE_FLOAT) { float minval = (float)(*(double *)(&pDataBlockst->min)); float maxval = (float)(*(double *)(&pDataBlockst->max)); - + for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { - if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT)) { - return true; + if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) { + continue; + } + ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT); + if (ret == false) { + return false; } } } else { for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) { - if (pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type)) { - return true; + if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) { + continue; + } + ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type); + if (ret == false) { + return false; } } } } - return false; + return ret; } static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { diff --git a/src/query/src/qFilterfunc.c b/src/query/src/qFilterfunc.c index 663f8814a0d57eb8cecad7f2d62944425510f029..7e8dd66ed12ee38d9dba5c982a5b78fda9a872ac 100644 --- a/src/query/src/qFilterfunc.c +++ b/src/query/src/qFilterfunc.c @@ -262,7 +262,7 @@ bool inOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxv return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&minv, sizeof(minv)); } return true; - } else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_DOUBLE) { + } else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_FLOAT) { double v; GET_TYPED_DATA(v, double, type, minval); return NULL != taosHashGet((SHashObj *)pFilter->q, (char *)&v, sizeof(v)); diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index cd4850e47e855cde4c1a7c281edb0afc23404370..c37426069cf64bb35ac5e1100d49e8103c851625 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -61,6 +61,7 @@ typedef struct SHashObj { size_t size; // number of elements in hash table _hash_fn_t hashFp; // hash function _hash_free_fn_t freeFp; // hash node free callback function + _equal_fn_t equalFp; // equal function SRWLatch lock; // read-write spin lock SHashLockTypeE type; // lock type @@ -78,6 +79,15 @@ typedef struct SHashObj { */ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type); + +/** + * set equal func of the hash table + * @param pHashObj + * @param equalFp + * @return + */ +void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp); + /** * return the size of hash table * @param pHashObj diff --git a/src/util/inc/hashfunc.h b/src/util/inc/hashfunc.h index 047f1889d78d6f8559bd0e320a0e9bae2beaa681..a9563d03941496a42e71a91fc7b7aab014ff4f50 100644 --- a/src/util/inc/hashfunc.h +++ b/src/util/inc/hashfunc.h @@ -20,6 +20,8 @@ typedef uint32_t (*_hash_fn_t)(const char *, uint32_t); +typedef int32_t (*_equal_fn_t)(const void *a, const void *b, size_t sz); + /** * murmur hash algorithm * @key usually string @@ -36,9 +38,14 @@ uint32_t MurmurHash3_32(const char *key, uint32_t len); * @return */ uint32_t taosIntHash_32(const char *key, uint32_t len); - uint32_t taosIntHash_64(const char *key, uint32_t len); + +int32_t taosFloatEqual(const void *a, const void *b, size_t sz); +int32_t taosDoubleEqual(const void *a,const void *b, size_t sz); + _hash_fn_t taosGetDefaultHashFunction(int32_t type); +_equal_fn_t taosGetDefaultEqualFunction(int32_t type); + #endif //TDENGINE_HASHUTIL_H diff --git a/src/util/inc/tcompare.h b/src/util/inc/tcompare.h index 5e951923c1513a4e90b8d9cccea15cbeb5664ee5..612ce7ede0976764843a16002c7770cfa2a6bbb7 100644 --- a/src/util/inc/tcompare.h +++ b/src/util/inc/tcompare.h @@ -27,6 +27,13 @@ extern "C" { #define TSDB_PATTERN_NOWILDCARDMATCH 2 #define TSDB_PATTERN_STRING_MAX_LEN 20 +#define FLT_COMPAR_TOL_FACTOR 4 +#define FLT_EQUAL(_x, _y) (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON)) +#define FLT_GREATER(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) > (_y))) +#define FLT_LESS(_x, _y) (!FLT_EQUAL((_x), (_y)) && ((_x) < (_y))) +#define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y))) +#define FLT_LESSEQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) < (_y))) + #define PATTERN_COMPARE_INFO_INITIALIZER { '%', '_' } typedef struct SPatternCompareInfo { diff --git a/src/util/src/hash.c b/src/util/src/hash.c index ede3e1e4ce3d1c9e5cadbd6bc21c009f0c5f5ca3..5b3218382fff38382a536c8ece077fd2f350adf2 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -73,10 +73,10 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { return i; } -static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { +static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { SHashNode *pNode = pe->next; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; } @@ -165,8 +165,8 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp // the max slots is not defined by user pHashObj->capacity = taosHashCapacity((int32_t)capacity); assert((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); - - pHashObj->hashFp = fn; + pHashObj->equalFp = memcmp; + pHashObj->hashFp = fn; pHashObj->type = type; pHashObj->enableUpdate = update; @@ -189,6 +189,12 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp return pHashObj; } +void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp) { + if (pHashObj != NULL && fp != NULL) { + pHashObj->equalFp = fp; + } +} + int32_t taosHashGetSize(const SHashObj *pHashObj) { return (int32_t)((pHashObj == NULL) ? 0 : pHashObj->size); } int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { @@ -223,7 +229,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da SHashNode* prev = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; } @@ -306,7 +312,7 @@ void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void assert(pe->next == NULL); } - SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal); + SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal); if (pNode != NULL) { if (fp != NULL) { fp(GET_HASH_NODE_DATA(pNode)); @@ -362,7 +368,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe SHashNode *prevNode = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) break; prevNode = pNode; diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index f91d13e6d0a4310181594b840631249bd1bcade8..e953f4c4649b6f906b8517a7165600d246da2176 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -90,13 +90,10 @@ int32_t compareFloatVal(const void *pLeft, const void *pRight) { if (isnan(p2)) { return 1; } - - float ret = p1 - p2; - if (fabs(ret) < FLT_EPSILON) { + if (FLT_EQUAL(p1, p2)) { return 0; - } else { - return ret > 0? 1 : -1; - } + } + return FLT_GREATER(p1, p2) ? 1: -1; } int32_t compareDoubleVal(const void *pLeft, const void *pRight) { @@ -114,13 +111,10 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) { if (isnan(p2)) { return 1; } - - double ret = p1 - p2; - if (fabs(ret) < FLT_EPSILON) { + if (FLT_EQUAL(p1, p2)) { return 0; - } else { - return ret > 0? 1 : -1; - } + } + return FLT_GREATER(p1, p2) ? 1: -1; } int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) { diff --git a/src/util/src/thashutil.c b/src/util/src/thashutil.c index 3634dca4c0f635b61b9a26e3f96c6f00bc9bb34d..ffe167977ba3fd08ac75f0b142c4fcb7db1d14a2 100644 --- a/src/util/src/thashutil.c +++ b/src/util/src/thashutil.c @@ -16,17 +16,20 @@ #include "os.h" #include "hashfunc.h" #include "tutil.h" +#include "tcompare.h" #define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r))) +#define DLT (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON) +#define BASE 1000 + #define FMIX32(h) \ do { \ (h) ^= (h) >> 16; \ (h) *= 0x85ebca6b; \ (h) ^= (h) >> 13; \ (h) *= 0xc2b2ae35; \ - (h) ^= (h) >> 16; \ - } while (0) + (h) ^= (h) >> 16; } while (0) uint32_t MurmurHash3_32(const char *key, uint32_t len) { const uint8_t *data = (const uint8_t *)key; @@ -78,7 +81,38 @@ uint32_t MurmurHash3_32(const char *key, uint32_t len) { uint32_t taosIntHash_32(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint32_t *)key; } uint32_t taosIntHash_16(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint16_t *)key; } uint32_t taosIntHash_8(const char *key, uint32_t UNUSED_PARAM(len)) { return *(uint8_t *)key; } +uint32_t taosFloatHash(const char *key, uint32_t UNUSED_PARAM(len)) { + float f = GET_FLOAT_VAL(key); + if (isnan(f)) { + return 0x7fc00000; + } + + if (FLT_EQUAL(f, 0.0)) { + return 0; + } + if (fabs(f) < FLT_MAX/BASE - DLT) { + int t = (int)(round(BASE * (f + DLT))); + return (uint32_t)t; + } else { + return 0x7fc00000; + } +} +uint32_t taosDoubleHash(const char *key, uint32_t UNUSED_PARAM(len)) { + double f = GET_DOUBLE_VAL(key); + if (isnan(f)) { + return 0x7fc00000; + } + if (FLT_EQUAL(f, 0.0)) { + return 0; + } + if (fabs(f) < DBL_MAX/BASE - DLT) { + int t = (int)(round(BASE * (f + DLT))); + return (uint32_t)t; + } else { + return 0x7fc00000; + } +} uint32_t taosIntHash_64(const char *key, uint32_t UNUSED_PARAM(len)) { uint64_t val = *(uint64_t *)key; @@ -92,13 +126,35 @@ _hash_fn_t taosGetDefaultHashFunction(int32_t type) { _hash_fn_t fn = NULL; switch(type) { case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: fn = taosIntHash_64;break; - case TSDB_DATA_TYPE_BINARY: fn = MurmurHash3_32;break; - case TSDB_DATA_TYPE_INT: fn = taosIntHash_32; break; + case TSDB_DATA_TYPE_BIGINT: fn = taosIntHash_64;break; + case TSDB_DATA_TYPE_BINARY: fn = MurmurHash3_32;break; + case TSDB_DATA_TYPE_NCHAR: fn = MurmurHash3_32;break; + case TSDB_DATA_TYPE_INT: fn = taosIntHash_32; break; case TSDB_DATA_TYPE_SMALLINT: fn = taosIntHash_16; break; - case TSDB_DATA_TYPE_TINYINT: fn = taosIntHash_8; break; + case TSDB_DATA_TYPE_TINYINT: fn = taosIntHash_8; break; + case TSDB_DATA_TYPE_FLOAT: fn = taosFloatHash; break; + case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleHash; break; default: fn = taosIntHash_32;break; } return fn; -} \ No newline at end of file +} + +int32_t taosFloatEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { + return getComparFunc(TSDB_DATA_TYPE_FLOAT, -1)(a, b); +} + +int32_t taosDoubleEqual(const void *a, const void *b, size_t UNUSED_PARAM(sz)) { + return getComparFunc(TSDB_DATA_TYPE_DOUBLE, -1)(a, b); +} + +_equal_fn_t taosGetDefaultEqualFunction(int32_t type) { + _equal_fn_t fn = NULL; + switch (type) { + case TSDB_DATA_TYPE_FLOAT: fn = taosFloatEqual; break; + case TSDB_DATA_TYPE_DOUBLE: fn = taosDoubleEqual; break; + default: fn = memcmp; break; + } + return fn; + +}