Merge pull request #4981 from taosdata/feature/distinct_tag

Feature/distinct tag

src/client/inc/tsclient.h

@@ -223,6 +223,8 @@ typedef struct SQueryInfo {
 
   int32_t          udColumnId;    // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
   int16_t          resColumnId;   // result column id
+  bool             distinctTag;   // distinct tag or not
+  
 } SQueryInfo;
 
 typedef struct {

src/client/src/tscLocalMerge.c

@@ -1101,7 +1101,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx)
      * the number of output result is decided by main output
      */
     int32_t functionId = pCtx[j].functionId;
-    if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) {
+    if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG) {
       continue;
     }
 
@@ -1183,7 +1183,7 @@ bool needToMerge(SQueryInfo *pQueryInfo, SLocalMerger *pLocalMerge, tFilePage *t
   int16_t functionId = pLocalMerge->pCtx[0].functionId;
 
   // todo opt performance
-  if ((/*functionId == TSDB_FUNC_PRJ || */functionId == TSDB_FUNC_ARITHM) || (tscIsProjectionQueryOnSTable(pQueryInfo, 0))) {  // column projection query
+  if ((/*functionId == TSDB_FUNC_PRJ || */functionId == TSDB_FUNC_ARITHM) || (tscIsProjectionQueryOnSTable(pQueryInfo, 0) && pQueryInfo->distinctTag == false)) {  // column projection query
     ret = 1;                                                            // disable merge procedure
   } else {
     tOrderDescriptor *pDesc = pLocalMerge->pDesc;

src/client/src/tscSQLParser.c

@@ -1505,23 +1505,39 @@ static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo) {
   pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY;
 }
 
+bool isValidDistinctSql(SQueryInfo* pQueryInfo) {
+  if (pQueryInfo == NULL) {
+    return false;
+  }
+  if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_QUERY)  != TSDB_QUERY_TYPE_STABLE_QUERY) {
+    return false;
+  }
+  if (tscQueryTags(pQueryInfo) && tscSqlExprNumOfExprs(pQueryInfo) == 1){
+    return true;
+  }
+  return false;
+}
 int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery, bool intervalQuery) {
   assert(pSelection != NULL && pCmd != NULL);
 
   const char* msg2 = "functions can not be mixed up";
   const char* msg3 = "not support query expression";
   const char* msg5 = "invalid function name";
+  const char* msg6 = "only support distinct one tag";
 
   SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);
 
   if (pQueryInfo->colList == NULL) {
     pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES);
   }
-
+  bool hasDistinct = false;
   for (int32_t i = 0; i < pSelection->nExpr; ++i) {
     int32_t outputIndex = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
     tSqlExprItem* pItem = &pSelection->a[i];
-
+     
+    if (hasDistinct == false) {
+       hasDistinct = (pItem->distinct == true); 
+    }
     // project on all fields
     int32_t optr = pItem->pNode->nSQLOptr;
 
@@ -1555,6 +1571,13 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
     }
   }
 
+  if (hasDistinct == true) {
+    if (!isValidDistinctSql(pQueryInfo)) {
+      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
+    }
+    pQueryInfo->distinctTag = true;
+  }
+  
   // there is only one user-defined column in the final result field, add the timestamp column.
   size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList);
   if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) {
@@ -4657,6 +4680,12 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
   setDefaultOrderInfo(pQueryInfo);
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
 
+
+  if (pQueryInfo->distinctTag == true) {
+    pQueryInfo->order.order = TSDB_ORDER_ASC;
+    pQueryInfo->order.orderColId = 0; 
+    return TSDB_CODE_SUCCESS;
+  }
   if (pQuerySql->pSortOrder == NULL) {
     return TSDB_CODE_SUCCESS;
   }

src/inc/ttokendef.h

@@ -16,6 +16,7 @@
 #ifndef TDENGINE_TTOKENDEF_H
 #define TDENGINE_TTOKENDEF_H
 
+
 #define TK_ID                               1
 #define TK_BOOL                             2
 #define TK_TINYINT                          3
@@ -127,104 +128,104 @@
 #define TK_SELECT                         109
 #define TK_UNION                          110
 #define TK_ALL                            111
-#define TK_FROM                           112
-#define TK_VARIABLE                       113
-#define TK_INTERVAL                       114
-#define TK_FILL                           115
-#define TK_SLIDING                        116
-#define TK_ORDER                          117
-#define TK_BY                             118
-#define TK_ASC                            119
-#define TK_DESC                           120
-#define TK_GROUP                          121
-#define TK_HAVING                         122
-#define TK_LIMIT                          123
-#define TK_OFFSET                         124
-#define TK_SLIMIT                         125
-#define TK_SOFFSET                        126
-#define TK_WHERE                          127
-#define TK_NOW                            128
-#define TK_RESET                          129
-#define TK_QUERY                          130
-#define TK_ADD                            131
-#define TK_COLUMN                         132
-#define TK_TAG                            133
-#define TK_CHANGE                         134
-#define TK_SET                            135
-#define TK_KILL                           136
-#define TK_CONNECTION                     137
-#define TK_STREAM                         138
-#define TK_COLON                          139
-#define TK_ABORT                          140
-#define TK_AFTER                          141
-#define TK_ATTACH                         142
-#define TK_BEFORE                         143
-#define TK_BEGIN                          144
-#define TK_CASCADE                        145
-#define TK_CLUSTER                        146
-#define TK_CONFLICT                       147
-#define TK_COPY                           148
-#define TK_DEFERRED                       149
-#define TK_DELIMITERS                     150
-#define TK_DETACH                         151
-#define TK_EACH                           152
-#define TK_END                            153
-#define TK_EXPLAIN                        154
-#define TK_FAIL                           155
-#define TK_FOR                            156
-#define TK_IGNORE                         157
-#define TK_IMMEDIATE                      158
-#define TK_INITIALLY                      159
-#define TK_INSTEAD                        160
-#define TK_MATCH                          161
-#define TK_KEY                            162
-#define TK_OF                             163
-#define TK_RAISE                          164
-#define TK_REPLACE                        165
-#define TK_RESTRICT                       166
-#define TK_ROW                            167
-#define TK_STATEMENT                      168
-#define TK_TRIGGER                        169
-#define TK_VIEW                           170
-#define TK_COUNT                          171
-#define TK_SUM                            172
-#define TK_AVG                            173
-#define TK_MIN                            174
-#define TK_MAX                            175
-#define TK_FIRST                          176
-#define TK_LAST                           177
-#define TK_TOP                            178
-#define TK_BOTTOM                         179
-#define TK_STDDEV                         180
-#define TK_PERCENTILE                     181
-#define TK_APERCENTILE                    182
-#define TK_LEASTSQUARES                   183
-#define TK_HISTOGRAM                      184
-#define TK_DIFF                           185
-#define TK_SPREAD                         186
-#define TK_TWA                            187
-#define TK_INTERP                         188
-#define TK_LAST_ROW                       189
-#define TK_RATE                           190
-#define TK_IRATE                          191
-#define TK_SUM_RATE                       192
-#define TK_SUM_IRATE                      193
-#define TK_AVG_RATE                       194
-#define TK_AVG_IRATE                      195
-#define TK_TBID                           196
-#define TK_SEMI                           197
-#define TK_NONE                           198
-#define TK_PREV                           199
-#define TK_LINEAR                         200
-#define TK_IMPORT                         201
-#define TK_METRIC                         202
-#define TK_TBNAME                         203
-#define TK_JOIN                           204
-#define TK_METRICS                        205
-#define TK_INSERT                         206
-#define TK_INTO                           207
-#define TK_VALUES                         208
-
+#define TK_DISTINCT                       112
+#define TK_FROM                           113
+#define TK_VARIABLE                       114
+#define TK_INTERVAL                       115
+#define TK_FILL                           116
+#define TK_SLIDING                        117
+#define TK_ORDER                          118
+#define TK_BY                             119
+#define TK_ASC                            120
+#define TK_DESC                           121
+#define TK_GROUP                          122
+#define TK_HAVING                         123
+#define TK_LIMIT                          124
+#define TK_OFFSET                         125
+#define TK_SLIMIT                         126
+#define TK_SOFFSET                        127
+#define TK_WHERE                          128
+#define TK_NOW                            129
+#define TK_RESET                          130
+#define TK_QUERY                          131
+#define TK_ADD                            132
+#define TK_COLUMN                         133
+#define TK_TAG                            134
+#define TK_CHANGE                         135
+#define TK_SET                            136
+#define TK_KILL                           137
+#define TK_CONNECTION                     138
+#define TK_STREAM                         139
+#define TK_COLON                          140
+#define TK_ABORT                          141
+#define TK_AFTER                          142
+#define TK_ATTACH                         143
+#define TK_BEFORE                         144
+#define TK_BEGIN                          145
+#define TK_CASCADE                        146
+#define TK_CLUSTER                        147
+#define TK_CONFLICT                       148
+#define TK_COPY                           149
+#define TK_DEFERRED                       150
+#define TK_DELIMITERS                     151
+#define TK_DETACH                         152
+#define TK_EACH                           153
+#define TK_END                            154
+#define TK_EXPLAIN                        155
+#define TK_FAIL                           156
+#define TK_FOR                            157
+#define TK_IGNORE                         158
+#define TK_IMMEDIATE                      159
+#define TK_INITIALLY                      160
+#define TK_INSTEAD                        161
+#define TK_MATCH                          162
+#define TK_KEY                            163
+#define TK_OF                             164
+#define TK_RAISE                          165
+#define TK_REPLACE                        166
+#define TK_RESTRICT                       167
+#define TK_ROW                            168
+#define TK_STATEMENT                      169
+#define TK_TRIGGER                        170
+#define TK_VIEW                           171
+#define TK_COUNT                          172
+#define TK_SUM                            173
+#define TK_AVG                            174
+#define TK_MIN                            175
+#define TK_MAX                            176
+#define TK_FIRST                          177
+#define TK_LAST                           178
+#define TK_TOP                            179
+#define TK_BOTTOM                         180
+#define TK_STDDEV                         181
+#define TK_PERCENTILE                     182
+#define TK_APERCENTILE                    183
+#define TK_LEASTSQUARES                   184
+#define TK_HISTOGRAM                      185
+#define TK_DIFF                           186
+#define TK_SPREAD                         187
+#define TK_TWA                            188
+#define TK_INTERP                         189
+#define TK_LAST_ROW                       190
+#define TK_RATE                           191
+#define TK_IRATE                          192
+#define TK_SUM_RATE                       193
+#define TK_SUM_IRATE                      194
+#define TK_AVG_RATE                       195
+#define TK_AVG_IRATE                      196
+#define TK_TBID                           197
+#define TK_SEMI                           198
+#define TK_NONE                           199
+#define TK_PREV                           200
+#define TK_LINEAR                         201
+#define TK_IMPORT                         202
+#define TK_METRIC                         203
+#define TK_TBNAME                         204
+#define TK_JOIN                           205
+#define TK_METRICS                        206
+#define TK_INSERT                         207
+#define TK_INTO                           208
+#define TK_VALUES                         209
 
 
 #define TK_SPACE                          300

src/query/inc/qSqlparser.h

@@ -200,6 +200,7 @@ typedef struct tSQLExpr {
 typedef struct tSqlExprItem {
   tSQLExpr *pNode;      // The list of expressions
   char *    aliasName;  // alias name, null-terminated string
+  bool      distinct;
 } tSqlExprItem;
 
 // todo refactor by using SArray
@@ -232,7 +233,7 @@ tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType);
 
 void tSqlExprDestroy(tSQLExpr *pExpr);
 
-tSQLExprList *tSqlExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken);
+tSQLExprList *tSqlExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pDistinct, SStrToken *pToken);
 
 void tSqlExprListDestroy(tSQLExprList *pList);
 

src/query/inc/sql.y

@@ -161,7 +161,7 @@ cmd ::= ALTER DNODE ids(X) ids(Y).              { setDCLSQLElems(pInfo, TSDB_SQL
 cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z).       { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z);      }
 cmd ::= ALTER LOCAL ids(X).                     { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X);              }
 cmd ::= ALTER LOCAL ids(X) ids(Y).              { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y);          }
-cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0};  setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);}
+cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0};  setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);}
 
 cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z).      { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);}
 cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z).      { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);}
@@ -186,7 +186,7 @@ ifnotexists(X) ::= .                { X.n = 0;}
 cmd ::= CREATE DNODE   ids(X).     { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &X);}
 cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z).
                                 { setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);}
-cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y).  { setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
+cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y).  { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
 cmd ::= CREATE USER ids(X) PASS ids(Y).     { setCreateUserSql(pInfo, &X, &Y);}
 
 pps(Y) ::= .                                { Y.n = 0;   }
@@ -457,13 +457,13 @@ select(A) ::= SELECT(T) selcollist(W). {
 %destructor sclp {tSqlExprListDestroy($$);}
 sclp(A) ::= selcollist(X) COMMA.             {A = X;}
 sclp(A) ::= .                                {A = 0;}
-selcollist(A) ::= sclp(P) expr(X) as(Y).     {
-   A = tSqlExprListAppend(P, X, Y.n?&Y:0);
+selcollist(A) ::= sclp(P) distinct(Z) expr(X) as(Y).     {
+   A = tSqlExprListAppend(P, X,  Z.n? &Z:0, Y.n?&Y:0);
 }
 
 selcollist(A) ::= sclp(P) STAR. {
    tSQLExpr *pNode = tSqlExprIdValueCreate(NULL, TK_ALL);
-   A = tSqlExprListAppend(P, pNode, 0);
+   A = tSqlExprListAppend(P, pNode, 0, 0);
 }
 
 // An option "AS <id>" phrase that can follow one of the expressions that
@@ -474,6 +474,10 @@ as(X) ::= AS ids(Y).    { X = Y;    }
 as(X) ::= ids(Y).       { X = Y;    }
 as(X) ::= .             { X.n = 0;  }
 
+%type distinct {SStrToken}
+distinct(X) ::= DISTINCT(Y). { X = Y;  }
+distinct(X) ::= .            { X.n = 0;}
+
 // A complete FROM clause.
 %type from {SArray*}
 // current not support query from no-table
@@ -681,8 +685,8 @@ expr(A) ::= expr(X) IN LP exprlist(Y) RP.   {A = tSqlExprCreate(X, (tSQLExpr*)Y,
 %type expritem {tSQLExpr*}
 %destructor expritem {tSqlExprDestroy($$);}
 
-exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSqlExprListAppend(X,Y,0);}
-exprlist(A) ::= expritem(X).                   {A = tSqlExprListAppend(0,X,0);}
+exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSqlExprListAppend(X,Y,0, 0);}
+exprlist(A) ::= expritem(X).                   {A = tSqlExprListAppend(0,X,0, 0);}
 expritem(A) ::= expr(X).                       {A = X;}
 expritem(A) ::= .                              {A = 0;}
 

src/query/src/qParserImpl.c

@@ -71,7 +71,7 @@ abort_parse:
   return sqlInfo;
 }
 
-tSQLExprList *tSqlExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken) {
+tSQLExprList *tSqlExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pDistinct, SStrToken *pToken) {
   if (pList == NULL) {
     pList = calloc(1, sizeof(tSQLExprList));
   }
@@ -97,6 +97,7 @@ tSQLExprList *tSqlExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken
 
       strdequote(pItem->aliasName);
     }
+    pItem->distinct = (pDistinct != NULL);
   }
   return pList;
 }

src/query/src/qTokenizer.c

@@ -240,6 +240,7 @@ static SKeyword keywordTable[] = {
     {"AVG_RATE",     TK_AVG_RATE},
     {"AVG_IRATE",    TK_AVG_IRATE},
     {"CACHELAST",    TK_CACHELAST},
+    {"DISTINCT",     TK_DISTINCT},
 };
 
 static const char isIdChar[] = {

tests/script/general/parser/select_distinct_tag.sim

+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5
+system sh/exec.sh -n dnode1 -s start
+sleep 100
+sql connect
+
+$dbPrefix = sav_db
+$tbPrefix = sav_tb
+$stbPrefix = sav_stb
+$tbNum = 20
+$rowNum = 10
+$totalNum = $tbNum * $rowNum
+$ts0 = 1537146000000
+$delta = 600000
+print ========== alter.sim
+$i = 0
+$db = $dbPrefix
+$stb = $stbPrefix
+
+sql drop database if exists $db
+sql create database $db
+sql use $db
+print ====== create tables
+sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int, t2 int)
+
+$i = 0
+$ts = $ts0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+  sql create table $tb using $stb tags( $i , 0 )
+  $i = $i + 1
+endw 
+
+print ====== table created  
+
+#### select distinct tag
+sql select distinct t1 from $stb
+if $rows != $tbNum then
+  return -1
+endi
+
+#### select distinct tag
+sql select distinct t2 from $stb
+if $rows != 1 then
+  print $rows
+  return -1
+endi
+
+#### unsupport sql
+sql_error select distinct t1, t2 from &stb
+
+sql drop database $db
+sql show databases
+if $rows != 0 then 
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/jenkins/basic.txt

@@ -159,6 +159,7 @@ cd ../../../debug; make
 ./test.sh -f general/parser/union.sim
 ./test.sh -f general/parser/topbot.sim
 ./test.sh -f general/parser/function.sim
+./test.sh -f general/parser/select_distinct_tag.sim
 
 ./test.sh -f general/stable/disk.sim
 ./test.sh -f general/stable/dnode3.sim