[td-225] fix bugs for nchar handling

src/client/src/tscSQLParser.c

@@ -5563,8 +5563,8 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
       }
       
-      *(VarDataLenT*)tagVal = pList->a[i].pVar.nLen;
-      ret = tVariantDump(&(pList->a[i].pVar), tagVal + VARSTR_HEADER_SIZE, pTagSchema[i].type);
+      ret = tVariantDump(&(pList->a[i].pVar), varDataVal(tagVal), pTagSchema[i].type);
+      varDataSetLen(tagVal, pList->a[i].pVar.nLen);
     } else {
       ret = tVariantDump(&(pList->a[i].pVar), tagVal, pTagSchema[i].type);
     }

src/query/src/qast.c

@@ -540,8 +540,11 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) {
 
 static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) {
   SSkipListIterator* iter = NULL;
+  
   SQueryCond cond = {0};
-  setQueryCond(pQueryInfo, &cond);
+  if (setQueryCond(pQueryInfo, &cond) != TSDB_CODE_SUCCESS) {
+    //todo handle error
+  }
 
   if (cond.start != NULL) {
     iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC);

src/query/src/tvariant.c

@@ -389,6 +389,7 @@ static int32_t toBinary(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
   return 0;
 }
 
+// todo handle the error
 static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
   char tmpBuf[40] = {0};
   
@@ -424,7 +425,12 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
     
     pVariant->wpz = (wchar_t *)tmp;
   } else {
-    taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, NULL);
+    size_t output = -1;
+    taosMbsToUcs4(pDst, nLen, *pDest, (nLen + 1) * TSDB_NCHAR_SIZE, &output);
+    
+    if (pDestSize != NULL) {
+      *pDestSize = output;
+    }
   }
   
   return 0;
@@ -779,7 +785,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, char type) {
     }
     case TSDB_DATA_TYPE_NCHAR: {
       if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
-        *(uint32_t *)payload = TSDB_DATA_NCHAR_NULL;
+        *(uint32_t *) payload = TSDB_DATA_NCHAR_NULL;
       } else {
         if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) {
           toNchar(pVariant, &payload, &pVariant->nLen);

src/tsdb/src/tsdbMeta.c

@@ -8,7 +8,7 @@
 #define TSDB_SUPER_TABLE_SL_LEVEL 5 // TODO: may change here
 #define TSDB_META_FILE_NAME "META"
 
-const int32_t DEFAULT_TAG_INDEX_COLUMN = 0;
+const int32_t DEFAULT_TAG_INDEX_COLUMN = 0;   // skip list built based on the first column of tags
 
 static int     tsdbFreeTable(STable *pTable);
 static int32_t tsdbCheckTableCfg(STableCfg *pCfg);
@@ -548,7 +548,10 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
   STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid);
   assert(pSTable != NULL);
   
-  char* key = dataRowTuple(pTable->tagVal); // key
+  STSchema* pSchema = tsdbGetTableTagSchema(pMeta, pTable);
+  STColumn* pCol = &pSchema->columns[DEFAULT_TAG_INDEX_COLUMN];
+  
+  char* key = tdGetRowDataOfCol(pTable->tagVal, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
   bool ret = tSkipListRemove(pSTable->pIndex, key);
   
   assert(ret);

src/tsdb/src/tsdbRead.c

@@ -626,7 +626,7 @@ static void filterDataInDataBlock(STsdbQueryHandle* pQueryHandle, STableCheckInf
           for(int32_t k = start; k < pQueryHandle->realNumOfRows + start; ++k) {
             char* p = tdGetColDataOfRow(src, k);
             memcpy(dst, p, varDataTLen(p));
-            dst += varDataTLen(p);
+            dst += bytes;
           }
         }
         
@@ -1311,8 +1311,8 @@ void filterPrepare(void* expr, void* param) {
   } else {
     pInfo->q = calloc(1, pSchema->bytes);
     if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
-      varDataSetLen(pInfo->q, pCond->nLen);
       tVariantDump(pCond, varDataVal(pInfo->q), pSchema->type);
+      varDataSetLen(pInfo->q, pCond->nLen);  // the length may be changed after dump, so assign its value after dump
     } else {
       tVariantDump(pCond, pInfo->q, pSchema->type);
     }

src/util/src/tcompare.c

@@ -67,33 +67,29 @@ int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {
   if (len1 != len2) {
     return len1 > len2? 1:-1;
   } else {
-    return (int32_t) strncmp(varDataVal(pLeft), varDataVal(pRight), len1);
+    int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1);
+    if (ret == 0) {
+      return 0;
+    } else {
+      return ret > 0 ? 1:-1;
+    }
   }
 }
 
-int32_t compareWStrVal(const void *pLeft, const void *pRight) {
-  //  SSkipListKey *pL = (SSkipListKey *)pLeft;
-  //  SSkipListKey *pR = (SSkipListKey *)pRight;
-  //
-  //  if (pL->nLen == 0 && pR->nLen == 0) {
-  //    return 0;
-  //  }
-  //
-  //  // handle only one-side bound compare situation, there is only lower bound or only upper bound
-  //  if (pL->nLen == -1) {
-  //    return 1;  // no lower bound, lower bound is minimum, always return -1;
-  //  } else if (pR->nLen == -1) {
-  //    return -1;  // no upper bound, upper bound is maximum situation, always return 1;
-  //  }
-  //
-  //  int32_t ret = wcscmp(((SSkipListKey *)pLeft)->wpz, ((SSkipListKey *)pRight)->wpz);
-  //
-  //  if (ret == 0) {
-  //    return 0;
-  //  } else {
-  //    return ret > 0 ? 1 : -1;
-  //  }
-  return 0;
+int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
+  int32_t len1 = varDataLen(pLeft);
+  int32_t len2 = varDataLen(pRight);
+  
+  if (len1 != len2) {
+    return len1 > len2? 1:-1;
+  } else {
+    int32_t ret = wcsncmp(varDataVal(pLeft), varDataVal(pRight), len1);
+    if (ret == 0) {
+      return 0;
+    } else {
+      return ret > 0 ? 1 : -1;
+    }
+  }
 }
 
 /*
@@ -284,7 +280,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
       if (optr == TSDB_RELATION_LIKE) {
         comparFn = compareWStrPatternComp;
       } else {
-        comparFn = compareWStrVal;
+        comparFn = compareLenPrefixedWStr;
       }
     
       break;
@@ -326,7 +322,7 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) {
       break;
     
     case TSDB_DATA_TYPE_NCHAR:
-      comparFn = compareWStrVal;
+      comparFn = compareLenPrefixedWStr;
       break;
     
     default:

tests/script/general/parser/nchar.sim

@@ -194,8 +194,10 @@ sql reset query cache
 sql select * from $mt where tgcol = '1'
 step2:
 if $rows != 100  then
+  print expect 100, actual: $rows
   return -1
 endi
+
 sql select * from $mt where tgcol > '0'
 #print rows = $rows
 if $rows != 100 then

tests/script/general/parser/testSuite.sim

@@ -35,6 +35,12 @@ run general/parser/first_last.sim
 sleep 2000
 run general/parser/import_file.sim
 sleep 2000
+run general/parser/lastrow.sim
+sleep 2000
+run general/parser/nchar.sim
+sleep 2000
+run general/parser/null_char.sim
+sleep 2000
 run general/parser/fill.sim
 sleep 2000
 run general/parser/fill_stb.sim
@@ -42,7 +48,6 @@ sleep 2000
 run general/parser/tags_dynamically_specifiy.sim
 sleep 2000
 run general/parser/interp.sim
-run general/parser/lastrow.sim
 sleep 2000
 run general/parser/limit.sim
 sleep 2000
@@ -54,10 +59,6 @@ run general/parser/limit2.sim
 sleep 2000
 run general/parser/mixed_blocks.sim
 sleep 2000
-run general/parser/nchar.sim
-sleep 2000
-run general/parser/null_char.sim
-sleep 2000
 run general/parser/selectResNum.sim
 sleep 2000
 run general/parser/select_across_vnodes.sim