Merge pull request #12613 from taosdata/fix/filterFloat

Fix/filter float

source/libs/index/src/indexComm.c

@@ -20,12 +20,13 @@
 #include "tcompare.h"
 #include "tdataformat.h"
 #include "ttypes.h"
+#include "tvariant.h"
 
 char JSON_COLUMN[] = "JSON";
 char JSON_VALUE_DELIM = '&';
 
 char* indexInt2str(int64_t val, char* dst, int radix) {
-  char     buffer[65];
+  char     buffer[65] = {0};
   char*    p;
   int64_t  new_val;
   uint64_t uval = (uint64_t)val;
@@ -74,28 +75,70 @@ static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) {
   return tCompare(func, QUERY_GREATER_EQUAL, a, b, type);
 }
 TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) {
-  if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR) {
+  if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY) {
     return tDoCompare(func, cmptype, a, b);
   }
 #if 1
-  int8_t bytes = tDataTypes[dtype].bytes;
-  if (bytes == 1) {
+  if (dtype == TSDB_DATA_TYPE_TIMESTAMP) {
+    int64_t va = taosStr2int64(a);
+    int64_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (dtype == TSDB_DATA_TYPE_BOOL || dtype == TSDB_DATA_TYPE_UTINYINT) {
+    uint8_t va = taosStr2int64(a);
+    uint8_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (dtype == TSDB_DATA_TYPE_TINYINT) {
     int8_t va = taosStr2int64(a);
     int8_t vb = taosStr2int64(b);
     return tDoCompare(func, cmptype, &va, &vb);
-  } else if (bytes == 2) {
+  } else if (dtype == TSDB_DATA_TYPE_SMALLINT) {
     int16_t va = taosStr2int64(a);
     int16_t vb = taosStr2int64(b);
     return tDoCompare(func, cmptype, &va, &vb);
-  } else if (bytes == 4) {
+  } else if (dtype == TSDB_DATA_TYPE_USMALLINT) {
+    uint16_t va = taosStr2int64(a);
+    uint16_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (dtype == TSDB_DATA_TYPE_INT) {
     int32_t va = taosStr2int64(a);
     int32_t vb = taosStr2int64(b);
     return tDoCompare(func, cmptype, &va, &vb);
-  } else {
+  } else if (dtype == TSDB_DATA_TYPE_UINT) {
+    uint32_t va = taosStr2int64(a);
+    uint32_t vb = taosStr2int64(b);
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (dtype == TSDB_DATA_TYPE_BIGINT) {
     int64_t va = taosStr2int64(a);
     int64_t vb = taosStr2int64(b);
     return tDoCompare(func, cmptype, &va, &vb);
+  } else if (dtype == TSDB_DATA_TYPE_UBIGINT) {
+    uint64_t va, vb;
+    if (0 != toUInteger(a, strlen(a), 10, &va) || 0 != toUInteger(b, strlen(b), 10, &vb)) {
+      return CONTINUE;
+    }
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (dtype == TSDB_DATA_TYPE_FLOAT) {
+    float va = strtod(a, NULL);
+    if (errno == ERANGE && va == -1) {
+      return CONTINUE;
+    }
+    float vb = strtod(b, NULL);
+    if (errno == ERANGE && va == -1) {
+      return CONTINUE;
+    }
+    return tDoCompare(func, cmptype, &va, &vb);
+  } else if (dtype == TSDB_DATA_TYPE_DOUBLE) {
+    double va = strtod(a, NULL);
+    if (errno == ERANGE && va == -1) {
+      return CONTINUE;
+    }
+    double vb = strtod(b, NULL);
+    if (errno == ERANGE && va == -1) {
+      return CONTINUE;
+    }
+    return tDoCompare(func, cmptype, &va, &vb);
   }
+  assert(0);
 #endif
 }
 TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) {
@@ -248,20 +291,16 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
       break;
     }
     case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
-#if 1
       tlen = taosEncodeBinary(NULL, src, strlen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
       tlen = taosEncodeBinary(dst, src, strlen(src));
       break;
-#endif
     }
     case TSDB_DATA_TYPE_VARBINARY:
-#if 1
       tlen = taosEncodeBinary(NULL, src, strlen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
       tlen = taosEncodeBinary(dst, src, strlen(src));
       break;
-#endif
     default:
       TASSERT(0);
       break;
@@ -271,87 +310,73 @@ int32_t indexConvertData(void* src, int8_t type, void** dst) {
   return tlen;
 }
 int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) {
-  int tlen = tDataTypes[type].bytes;
-
+  int     tlen = tDataTypes[type].bytes;
+  int32_t bufSize = 64;
   switch (type) {
     case TSDB_DATA_TYPE_TIMESTAMP:
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(int64_t*)src, *dst, -1);
       break;
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_UTINYINT:
-      // tlen = taosEncodeFixedU8(NULL, *(uint8_t*)src);
-      //*dst = taosMemoryCalloc(1, tlen + 1);
-      // tlen = taosEncodeFixedU8(dst, *(uint8_t*)src);
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(uint8_t*)src, *dst, 1);
       break;
     case TSDB_DATA_TYPE_TINYINT:
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(int8_t*)src, *dst, 1);
       break;
     case TSDB_DATA_TYPE_SMALLINT:
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(int16_t*)src, *dst, -1);
       break;
     case TSDB_DATA_TYPE_USMALLINT:
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(uint16_t*)src, *dst, -1);
       break;
     case TSDB_DATA_TYPE_INT:
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(int32_t*)src, *dst, -1);
       break;
-    case TSDB_DATA_TYPE_FLOAT:
-      tlen = taosEncodeBinary(NULL, src, sizeof(float));
-      *dst = taosMemoryCalloc(1, tlen + 1);
-      tlen = taosEncodeBinary(dst, src, sizeof(float));
-      *dst = (char*) * dst - tlen;
-      break;
     case TSDB_DATA_TYPE_UINT:
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(uint32_t*)src, *dst, 1);
       break;
     case TSDB_DATA_TYPE_BIGINT:
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
-      indexInt2str(*(int64_t*)src, *dst, 1);
-      break;
-    case TSDB_DATA_TYPE_DOUBLE:
-      tlen = taosEncodeBinary(NULL, src, sizeof(double));
-      *dst = taosMemoryCalloc(1, tlen + 1);
-      tlen = taosEncodeBinary(dst, src, sizeof(double));
-      *dst = (char*) * dst - tlen;
+      *dst = taosMemoryCalloc(1, bufSize + 1);
+      sprintf(*dst, "%" PRIu64, *(uint64_t*)src);
       break;
     case TSDB_DATA_TYPE_UBIGINT:
-      assert(0);
-      *dst = taosMemoryCalloc(1, sizeof(int64_t) + 1);
+      *dst = taosMemoryCalloc(1, bufSize + 1);
       indexInt2str(*(uint64_t*)src, *dst, 1);
+    case TSDB_DATA_TYPE_FLOAT:
+      *dst = taosMemoryCalloc(1, bufSize + 1);
+      sprintf(*dst, "%.9lf", *(float*)src);
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      *dst = taosMemoryCalloc(1, bufSize + 1);
+      sprintf(*dst, "%.9lf", *(double*)src);
       break;
     case TSDB_DATA_TYPE_NCHAR: {
       tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
       tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src));
-      *dst = (char*) * dst - tlen;
-
+      *dst = *dst - tlen;
       break;
     }
     case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
-#if 1
       tlen = taosEncodeBinary(NULL, src, strlen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
       tlen = taosEncodeBinary(dst, src, strlen(src));
-      *dst = (char*) * dst - tlen;
+      *dst = (char*)*dst - tlen;
       break;
-#endif
     }
     case TSDB_DATA_TYPE_VARBINARY:
-#if 1
       tlen = taosEncodeBinary(NULL, src, strlen(src));
       *dst = taosMemoryCalloc(1, tlen + 1);
       tlen = taosEncodeBinary(dst, src, strlen(src));
-      *dst = (char*) * dst - tlen;
+      *dst = (char*)*dst - tlen;
       break;
-#endif
     default:
       TASSERT(0);
       break;

source/libs/index/src/indexTfile.c

@@ -20,6 +20,7 @@ p *
 #include "indexFstCountingWriter.h"
 #include "indexUtil.h"
 #include "taosdef.h"
+#include "taoserror.h"
 #include "tcoding.h"
 #include "tcompare.h"
 
@@ -472,16 +473,16 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
 
     int32_t sz = 0;
     char*   ch = (char*)fstSliceData(s, &sz);
-    char*   tmp = taosMemoryCalloc(1, sz + 1);
-    memcpy(tmp, ch, sz);
+    // char*   tmp = taosMemoryCalloc(1, sz + 1);
+    // memcpy(tmp, ch, sz);
 
-    if (0 != strncmp(tmp, p, skip)) {
+    if (0 != strncmp(ch, p, skip)) {
       swsResultDestroy(rt);
-      taosMemoryFree(tmp);
+      // taosMemoryFree(tmp);
       break;
     }
 
-    TExeCond cond = cmpFn(tmp + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType));
+    TExeCond cond = cmpFn(ch + skip, tem->colVal, INDEX_TYPE_GET_TYPE(tem->colType));
 
     if (MATCH == cond) {
       tfileReaderLoadTableIds((TFileReader*)reader, rt->out.out, tr->total);
@@ -490,7 +491,7 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTempR
       swsResultDestroy(rt);
       break;
     }
-    taosMemoryFree(tmp);
+    // taosMemoryFree(tmp);
     swsResultDestroy(rt);
   }
   streamWithStateDestroy(st);
@@ -533,10 +534,12 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c
   tfileGenFileFullName(fullname, path, suid, colName, version);
 
   WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024);
-  indexInfo("open read file name:%s, file size: %d", wc->file.buf, wc->file.size);
   if (wc == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    indexError("failed to open readonly file: %s, reason: %s", fullname, terrstr());
     return NULL;
   }
+  indexInfo("open read file name:%s, file size: %d", wc->file.buf, wc->file.size);
 
   TFileReader* reader = tfileReaderCreate(wc);
   return reader;
@@ -613,9 +616,7 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
     if (tfileWriteData(tw, v) != 0) {
       indexError("failed to write data: %s, offset: %d len: %d", v->colVal, v->offset,
                  (int)taosArrayGetSize(v->tableId));
-      // printf("write faile\n");
     } else {
-      // printf("write sucee\n");
       // indexInfo("success to write data: %s, offset: %d len: %d", v->colVal, v->offset,
       //          (int)taosArrayGetSize(v->tableId));
 

source/libs/index/test/jsonUT.cc

@@ -553,7 +553,7 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_FLOAT) {
     float       val = 2.0;
     std::string colName("test1");
     for (int i = 0; i < 1000; i++) {
-      WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i);
+      WriteData(index, colName, TSDB_DATA_TYPE_FLOAT, &val, sizeof(val), i + 1000);
     }
   }
   {