Merge branch 'develop' into feature/query

src/client/src/tscSQLParser.c

@@ -6374,13 +6374,13 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
   }
   
-  SArray* pSrcMeterName = pInfo->pCreateTableInfo->pSelect->from;
-  if (pSrcMeterName == NULL || taosArrayGetSize(pSrcMeterName) == 0) {
+  SFromInfo* pFromInfo = pInfo->pCreateTableInfo->pSelect->from;
+  if (pFromInfo == NULL || taosArrayGetSize(pFromInfo->tableList) == 0) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
   }
   
-  tVariantListItem* p1 = taosArrayGet(pSrcMeterName, 0);
-  SStrToken srcToken = {.z = p1->pVar.pz, .n = p1->pVar.nLen, .type = TK_STRING};
+  STableNamePair* p1 = taosArrayGet(pFromInfo->tableList, 0);
+  SStrToken srcToken = {.z = p1->name.z, .n = p1->name.n, .type = TK_STRING};
   if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
   }
@@ -6498,7 +6498,7 @@ static int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) {
 }
 
 int32_t doValidateSqlNode(SSqlObj* pSql, SQuerySqlNode* pQuerySqlNode, int32_t index) {
-  assert(pQuerySqlNode != NULL && (pQuerySqlNode->from == NULL || taosArrayGetSize(pQuerySqlNode->from) > 0));
+  assert(pQuerySqlNode != NULL && (pQuerySqlNode->from == NULL || taosArrayGetSize(pQuerySqlNode->from->tableList) > 0));
 
   const char* msg0 = "invalid table name";
   const char* msg1 = "point interpolation query needs timestamp";
@@ -6508,6 +6508,7 @@ int32_t doValidateSqlNode(SSqlObj* pSql, SQuerySqlNode* pQuerySqlNode, int32_t i
   const char* msg5 = "too many columns in selection clause";
   const char* msg6 = "too many tables in from clause";
   const char* msg7 = "invalid table alias name";
+  const char* msg8 = "alias name too long";
 
   int32_t code = TSDB_CODE_SUCCESS;
 
@@ -6539,71 +6540,71 @@ int32_t doValidateSqlNode(SSqlObj* pSql, SQuerySqlNode* pQuerySqlNode, int32_t i
     return doLocalQueryProcess(pCmd, pQueryInfo, pQuerySqlNode);
   }
 
-  size_t fromSize = taosArrayGetSize(pQuerySqlNode->from);
-  if (fromSize > TSDB_MAX_JOIN_TABLE_NUM * 2) {
+  size_t fromSize = taosArrayGetSize(pQuerySqlNode->from->tableList);
+  if (fromSize > TSDB_MAX_JOIN_TABLE_NUM) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
   }
 
   pQueryInfo->command = TSDB_SQL_SELECT;
-
-  if (fromSize > 4) {
+  if (fromSize > 2) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
   }
 
   // set all query tables, which are maybe more than one.
-  for (int32_t i = 0; i < fromSize; ) {
-    tVariantListItem* item = taosArrayGet(pQuerySqlNode->from, i);
-    tVariant* pTableItem = &item->pVar;
+  for (int32_t i = 0; i < fromSize; ++i) {
+    STableNamePair* item = taosArrayGet(pQuerySqlNode->from->tableList, i);
+    SStrToken* pTableItem = &item->name;
 
-    if (pTableItem->nType != TSDB_DATA_TYPE_BINARY) {
+    if (pTableItem->type != TSDB_DATA_TYPE_BINARY) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0);
     }
 
-    pTableItem->nLen = strdequote(pTableItem->pz);
+    tscDequoteAndTrimToken(pTableItem);
 
-    SStrToken tableName = {.z = pTableItem->pz, .n = pTableItem->nLen, .type = TK_STRING};
+    SStrToken tableName = {.z = pTableItem->z, .n = pTableItem->n, .type = TK_STRING};
     if (tscValidateName(&tableName) != TSDB_CODE_SUCCESS) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0);
     }
 
-    if (pQueryInfo->numOfTables <= i/2) {  // more than one table
+    if (pQueryInfo->numOfTables <= i) {  // more than one table
       tscAddEmptyMetaInfo(pQueryInfo);
     }
 
-    STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, i/2);
-
-    SStrToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz};
-    code = tscSetTableFullName(pTableMetaInfo1, &t, pSql);
+    STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, i);
+    code = tscSetTableFullName(pTableMetaInfo1, pTableItem, pSql);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
 
-    tVariantListItem* p1 = taosArrayGet(pQuerySqlNode->from, i + 1);
-    if (p1->pVar.nType != TSDB_DATA_TYPE_BINARY) {
+    SStrToken* aliasName = &item->aliasName;
+    if (TPARSER_HAS_TOKEN(*aliasName)) {
+      if (aliasName->type != TSDB_DATA_TYPE_BINARY) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
       }
 
-    SStrToken aliasName = {.z = p1->pVar.pz, .n = p1->pVar.nLen, .type = TK_STRING};
-    if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) {
+      tscDequoteAndTrimToken(aliasName);
+
+      SStrToken aliasName1 = {.z = aliasName->z, .n = aliasName->n, .type = TK_STRING};
+      if (tscValidateName(&aliasName1) != TSDB_CODE_SUCCESS) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
       }
 
-    // has no table alias name
-    if (memcmp(pTableItem->pz, p1->pVar.pz, p1->pVar.nLen) == 0) {
-      strncpy(pTableMetaInfo1->aliasName, tNameGetTableName(&pTableMetaInfo1->name), tListLen(pTableMetaInfo->aliasName));
+      if (aliasName1.n >= TSDB_TABLE_NAME_LEN) {
+        return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8);
+      }
+
+      strncpy(pTableMetaInfo1->aliasName, aliasName1.z, aliasName1.n);
     } else {
-      tstrncpy(pTableMetaInfo1->aliasName, p1->pVar.pz, sizeof(pTableMetaInfo1->aliasName));
+      strncpy(pTableMetaInfo1->aliasName, tNameGetTableName(&pTableMetaInfo1->name), tListLen(pTableMetaInfo1->aliasName));
     }
 
     code = tscGetTableMeta(pSql, pTableMetaInfo1);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
-
-    i += 2;
   }
 
-  assert(pQueryInfo->numOfTables == taosArrayGetSize(pQuerySqlNode->from) / 2);
+  assert(pQueryInfo->numOfTables == taosArrayGetSize(pQuerySqlNode->from->tableList));
   bool isSTable = false;
   
   if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
@@ -6637,12 +6638,12 @@ int32_t doValidateSqlNode(SSqlObj* pSql, SQuerySqlNode* pQuerySqlNode, int32_t i
       pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000;
     }
   } else {  // set the time rang
-    if (taosArrayGetSize(pQuerySqlNode->from) > 2) { // it is a join query, no wher clause is not allowed.
+    if (taosArrayGetSize(pQuerySqlNode->from->tableList) > 1) { // it is a join query, no where clause is not allowed.
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query ");
     }
   }
 
-  int32_t joinQuery = (pQuerySqlNode->from != NULL && taosArrayGetSize(pQuerySqlNode->from) > 2);
+  int32_t joinQuery = (pQuerySqlNode->from != NULL && taosArrayGetSize(pQuerySqlNode->from->tableList) > 1);
   int32_t timeWindowQuery =
       (TPARSER_HAS_TOKEN(pQuerySqlNode->interval.interval) || TPARSER_HAS_TOKEN(pQuerySqlNode->sessionVal.gap));
 

src/query/inc/qSqlparser.h

@@ -39,6 +39,11 @@ enum SQL_NODE_TYPE {
   SQL_NODE_EXPR        = 4,
 };
 
+enum SQL_NODE_FROM_TYPE {
+  SQL_NODE_FROM_SUBQUERY  = 1,
+  SQL_NODE_FROM_NAMELIST  = 2,
+};
+
 extern char tTokenTypeSwitcher[13];
 
 #define toTSDBType(x)                          \
@@ -78,9 +83,11 @@ typedef struct SSessionWindowVal {
   SStrToken          gap;
 } SSessionWindowVal;
 
+struct SFromInfo;
+
 typedef struct SQuerySqlNode {
   struct SArray     *pSelectList;  // select clause
-  SArray            *from;         // from clause  SArray<SQuerySqlNode>
+  struct SFromInfo  *from;         // from clause SArray<SQuerySqlNode>
   struct tSqlExpr   *pWhere;       // where clause [optional]
   SArray            *pGroupby;     // groupby clause, only for tags[optional], SArray<tVariantListItem>
   SArray            *pSortOrder;   // orderby [optional],  SArray<tVariantListItem>
@@ -93,6 +100,24 @@ typedef struct SQuerySqlNode {
   SStrToken          sqlstr;       // sql string in select clause
 } SQuerySqlNode;
 
+typedef struct STableNamePair {
+  SStrToken name;
+  SStrToken aliasName;
+} STableNamePair;
+
+typedef struct SSubclauseInfo {  // "UNION" multiple select sub-clause
+  SQuerySqlNode    **pClause;
+  int32_t            numOfClause;
+} SSubclauseInfo;
+
+typedef struct SFromInfo {
+  int32_t            type;        // nested query|table name list
+  union {
+    SSubclauseInfo  *pNode;
+    SArray          *tableList;   // SArray<STableNamePair>
+  };
+} SFromInfo;
+
 typedef struct SCreatedTableInfo {
   SStrToken          name;        // table name token
   SStrToken          stableName;  // super table name token , for using clause
@@ -188,11 +213,6 @@ typedef struct SMiscInfo {
   };
 } SMiscInfo;
 
-typedef struct SSubclauseInfo {  // "UNION" multiple select sub-clause
-  SQuerySqlNode    **pClause;
-  int32_t            numOfClause;
-} SSubclauseInfo;
-
 typedef struct SSqlInfo {
   int32_t            type;
   bool               valid;
@@ -233,6 +253,10 @@ SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder);
 SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index);
 SArray *tVariantListAppendToken(SArray *pList, SStrToken *pAliasToken, uint8_t sortOrder);
 
+SFromInfo *setTableNameList(SFromInfo* pFromInfo, SStrToken *pName, SStrToken* pAlias);
+SFromInfo *setSubquery(SFromInfo* pFromInfo, SQuerySqlNode *pSqlNode);
+void      *destroyFromInfo(SFromInfo* pFromInfo);
+
 // sql expr leaf node
 tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType);
 tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType);
@@ -246,7 +270,7 @@ void      tSqlExprDestroy(tSqlExpr *pExpr);
 SArray   *tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SStrToken *pDistinct, SStrToken *pToken);
 void      tSqlExprListDestroy(SArray *pList);
 
-SQuerySqlNode *tSetQuerySqlNode(SStrToken *pSelectToken, SArray *pSelectList, SArray *pFrom, tSqlExpr *pWhere,
+SQuerySqlNode *tSetQuerySqlNode(SStrToken *pSelectToken, SArray *pSelectList, SFromInfo *pFrom, tSqlExpr *pWhere,
                                 SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *ps,
                                 SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pgLimit);
 

src/query/src/qExecutor.c

@@ -2366,13 +2366,13 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key, bool asc
 }
 
 void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols,
-                        SSDataBlock* pBlock, STSBuf* pTsBuf, bool ascQuery) {
+                        SSDataBlock* pBlock, bool ascQuery) {
   int32_t numOfRows = pBlock->info.rows;
 
   int8_t *p = calloc(numOfRows, sizeof(int8_t));
   bool    all = true;
 
-  if (pTsBuf != NULL) {
+  if (pRuntimeEnv->pTsBuf != NULL) {
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);
 
     TSKEY* k = (TSKEY*) pColInfoData->pData;
@@ -2393,6 +2393,9 @@ void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInf
         break;
       }
     }
+
+    // save the cursor status
+    pRuntimeEnv->pQuery->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf);
   } else {
     for (int32_t i = 0; i < numOfRows; ++i) {
       bool qualified = false;
@@ -2653,8 +2656,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa
 
     doSetFilterColumnInfo(pQuery, pBlock);
     if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL) {
-      filterRowsInDataBlock(pRuntimeEnv, pQuery->pFilterInfo, pQuery->numOfFilterCols, pBlock, pRuntimeEnv->pTsBuf,
-                            ascQuery);
+      filterRowsInDataBlock(pRuntimeEnv, pQuery->pFilterInfo, pQuery->numOfFilterCols, pBlock, ascQuery);
     }
   }
 

src/query/src/qSqlParser.c

@@ -443,6 +443,52 @@ SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int
   return pList;
 }
 
+SFromInfo *setTableNameList(SFromInfo* pFromInfo, SStrToken *pName, SStrToken* pAlias) {
+  if (pFromInfo == NULL) {
+    pFromInfo = calloc(1, sizeof(SFromInfo));
+    pFromInfo->tableList = taosArrayInit(4, sizeof(STableNamePair));
+  }
+
+  pFromInfo->type = SQL_NODE_FROM_NAMELIST;
+  STableNamePair p = {.name = *pName};
+  if (pAlias != NULL) {
+    p.aliasName = *pAlias;
+  } else {
+    TPARSER_SET_NONE_TOKEN(p.aliasName);
+  }
+
+  taosArrayPush(pFromInfo->tableList, &p);
+
+  return pFromInfo;
+}
+
+SFromInfo *setSubquery(SFromInfo* pFromInfo, SQuerySqlNode* pSqlNode) {
+  if (pFromInfo == NULL) {
+    pFromInfo = calloc(1, sizeof(SFromInfo));
+  }
+
+  pFromInfo->type = SQL_NODE_FROM_SUBQUERY;
+  pFromInfo->pNode->pClause[pFromInfo->pNode->numOfClause - 1] = pSqlNode;
+
+  return pFromInfo;
+}
+
+void* destroyFromInfo(SFromInfo* pFromInfo) {
+  if (pFromInfo == NULL) {
+    return NULL;
+  }
+
+  if (pFromInfo->type == SQL_NODE_FROM_NAMELIST) {
+    taosArrayDestroy(pFromInfo->tableList);
+  } else {
+    destroyAllSelectClause(pFromInfo->pNode);
+  }
+
+  tfree(pFromInfo);
+  return NULL;
+}
+
+
 void tSetDbName(SStrToken *pCpxName, SStrToken *pDb) {
   pCpxName->type = pDb->type;
   pCpxName->z = pDb->z;
@@ -582,9 +628,10 @@ void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
 /*
  * extract the select info out of sql string
  */
-SQuerySqlNode *tSetQuerySqlNode(SStrToken *pSelectToken, SArray *pSelectList, SArray *pFrom, tSqlExpr *pWhere,
-                             SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *pSession,
-                             SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *psLimit) {
+SQuerySqlNode *tSetQuerySqlNode(SStrToken *pSelectToken, SArray *pSelectList, SFromInfo *pFrom, tSqlExpr *pWhere,
+                                SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval,
+                                SSessionWindowVal *pSession, SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit,
+                                SLimitVal *psLimit) {
   assert(pSelectList != NULL);
 
   SQuerySqlNode *pSqlNode = calloc(1, sizeof(SQuerySqlNode));
@@ -668,8 +715,7 @@ void destroyQuerySqlNode(SQuerySqlNode *pQuerySql) {
   taosArrayDestroyEx(pQuerySql->pGroupby, freeVariant);
   pQuerySql->pGroupby = NULL;
 
-  taosArrayDestroyEx(pQuerySql->from, freeVariant);
-  pQuerySql->from = NULL;
+  pQuerySql->from = destroyFromInfo(pQuerySql->from);
 
   taosArrayDestroyEx(pQuerySql->fillType, freeVariant);
   pQuerySql->fillType = NULL;

tests/pytest/query/queryNormal.py

@@ -94,8 +94,9 @@ class TDTestCase:
         tdSql.query("select * from stb1 limit 2 offset 3")
         tdSql.checkRows(1)
 
-        # query ... alias for table ---- bug
-        tdSql.error("select t.ts from tb1 t")        
+        # query ... alias for table
+        tdSql.query("select t.ts from tb1 t")
+        tdSql.checkRows(2)
 
         # query ... tbname
         tdSql.query("select tbname from stb1")

tests/script/general/parser/join.sim

@@ -448,7 +448,7 @@ endi
 sql select first(join_tb0.c8),first(join_tb0.c9) from join_tb1 , join_tb0 where join_tb1.ts = join_tb0.ts and join_tb1.ts <= 100002 and join_tb0.c7 = true
 
 #====================group by=========================================
-
+print =================>"group by not supported"
 
 
 

tests/script/general/parser/join_manyblocks.sim

+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
+system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4
+
+system sh/exec.sh -n dnode1 -s start
+sql connect
+sleep 100
+
+$dbPrefix = join_m_db
+$tbPrefix = join_tb
+$mtPrefix = join_mt
+$tbNum = 3
+$rowNum = 20000
+$totalNum = $tbNum * $rowNum
+
+print =============== join_manyBlocks.sim
+$i = 0
+$db = $dbPrefix . $i
+$mt = $mtPrefix . $i
+
+$tstart = 100000
+
+sql drop database if exists $db -x step1
+step1:
+sql create database if not exists $db keep 36500
+
+sql use $db
+sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))
+
+$mt1 = $mtPrefix . 1 . $i
+sql create table $mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12), t3 int)
+
+$i = 0
+$tbPrefix1 = join_1_tb
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+  $tg2 = ' . abc
+  $tg2 = $tg2 . '
+  sql create table $tb using $mt tags( $i , $tg2 )
+
+  $tb1 = $tbPrefix1 . $i
+  $c = $i
+  $t3 = $i + 1
+
+  $binary = ' . abc
+  $binary = $binary . $i
+  $binary = $binary . '
+
+  print $binary
+  sql create table $tb1 using $mt1 tags( $i , $binary , $t3 )
+
+  $x = 0
+  while $x < $rowNum
+    $ms = $x . m
+    $c = $x / 100
+    $c = $c * 100
+    $c = $x - $c
+
+    $binary = ' . binary
+    $binary = $binary . $c
+    $binary = $binary . '
+
+    $nchar = ' . nchar
+    $nchar = $nchar . $c
+    $nchar = $nchar . '
+
+    sql insert into $tb values ($tstart , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) $tb1 values ($tstart , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )
+
+    $tstart = $tstart + 1
+    $x = $x + 1
+  endw
+
+  $i = $i + 1
+  $tstart = 100000
+endw
+
+sleep 100
+
+print ===============join_manyblocks.sim
+print ==============> td-3313
+sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1;
+
+print $row
+if $row != 60000 then
+  print expect 60000, actual: $row
+  return -1
+endi
+
+print ======= second tags join

tests/script/general/table/delete_writing.sim

@@ -41,7 +41,7 @@ while $x < 15
 
 	sql create table db.tb (ts timestamp, i int)
 
-	sleep 2000
+	sleep 1000
 	
 	$x = $x + 1
 endw