Merge pull request #7523 from taosdata/feature/szhou/tagregex

[TD-6145]<feature>:tagfilter-add parser support

src/client/src/tscSQLParser.c

@@ -22,6 +22,7 @@
 
 #include <qSqlparser.h>
 #include "os.h"
+#include "regex.h"
 #include "qPlan.h"
 #include "qSqlparser.h"
 #include "qTableMeta.h"
@@ -278,6 +279,8 @@ static uint8_t convertRelationalOperator(SStrToken *pToken) {
       return TSDB_BINARY_OP_REMAINDER;
     case TK_LIKE:
       return TSDB_RELATION_LIKE;
+    case TK_MATCH:
+      return TSDB_RELATION_MATCH;
     case TK_ISNULL:
       return TSDB_RELATION_ISNULL;
     case TK_NOTNULL:
@@ -3776,6 +3779,9 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
     case TK_LIKE:
       pColumnFilter->lowerRelOptr = TSDB_RELATION_LIKE;
       break;
+    case TK_MATCH:
+      pColumnFilter->lowerRelOptr = TSDB_RELATION_MATCH;
+      break;
     case TK_ISNULL:
       pColumnFilter->lowerRelOptr = TSDB_RELATION_ISNULL;
       break;
@@ -3839,9 +3845,15 @@ static int32_t tablenameListToString(tSqlExpr* pExpr, SStringBuilder* sb) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t tablenameCondToString(tSqlExpr* pExpr, SStringBuilder* sb) {
-  taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN);
-  taosStringBuilderAppendString(sb, pExpr->value.pz);
+static int32_t tablenameCondToString(tSqlExpr* pExpr, uint32_t opToken, SStringBuilder* sb) {
+  assert(opToken == TK_LIKE || opToken == TK_MATCH);
+  if (opToken == TK_LIKE) {
+    taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN);
+    taosStringBuilderAppendString(sb, pExpr->value.pz);
+  } else if (opToken == TK_MATCH) {
+    taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN);
+    taosStringBuilderAppendString(sb, pExpr->value.pz);
+  }
 
   return TSDB_CODE_SUCCESS;
 }
@@ -3862,7 +3874,7 @@ static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCol
   STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
   SSchema*    pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
   int32_t     ret = 0;
-  const char* msg1 = "non binary column not support like operator";
+  const char* msg1 = "non binary column not support like/match operator";
   const char* msg2 = "binary column not support this operator";  
   const char* msg3 = "bool column not support this operator";
   const char* msg4 = "primary key not support this operator";
@@ -3890,12 +3902,13 @@ static int32_t checkColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCol
       && pExpr->tokenId != TK_ISNULL
       && pExpr->tokenId != TK_NOTNULL
       && pExpr->tokenId != TK_LIKE
+      && pExpr->tokenId != TK_MATCH
       && pExpr->tokenId != TK_IN) {
       ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
       goto _err_ret;
     }
   } else {
-    if (pExpr->tokenId == TK_LIKE) {
+    if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH) {
       ret = invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
       goto _err_ret;
     }
@@ -3943,12 +3956,12 @@ static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr*
 
   if (pTableCond->tokenId == TK_IN) {
     ret = tablenameListToString(pRight, sb);
-  } else if (pTableCond->tokenId == TK_LIKE) {
+  } else if (pTableCond->tokenId == TK_LIKE || pTableCond->tokenId == TK_MATCH) {
     if (pRight->tokenId != TK_STRING) {
       return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
     }
     
-    ret = tablenameCondToString(pRight, sb);
+    ret = tablenameCondToString(pRight, pTableCond->tokenId, sb);
   }
 
   if (ret != TSDB_CODE_SUCCESS) {
@@ -4397,7 +4410,7 @@ static bool validateJoinExprNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr
 }
 
 static bool validTableNameOptr(tSqlExpr* pExpr) {
-  const char nameFilterOptr[] = {TK_IN, TK_LIKE};
+  const char nameFilterOptr[] = {TK_IN, TK_LIKE, TK_MATCH};
 
   for (int32_t i = 0; i < tListLen(nameFilterOptr); ++i) {
     if (pExpr->tokenId == nameFilterOptr[i]) {
@@ -4489,6 +4502,45 @@ static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t
   return TSDB_CODE_SUCCESS;
 }
 
+// check for match expression
+static int32_t validateMatchExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) {
+  const char* msg1 = "regular expression string should be less than %d characters";
+  const char* msg2 = "illegal column type for match";
+  const char* msg3 = "invalid regular expression";
+
+  tSqlExpr* pLeft  = pExpr->pLeft;
+  tSqlExpr* pRight = pExpr->pRight;
+
+  if (pExpr->tokenId == TK_MATCH) {
+    if (pRight->value.nLen > tsMaxRegexStringLen) {
+      char tmp[64] = {0};
+      sprintf(tmp, msg1, tsMaxRegexStringLen);
+      return invalidOperationMsg(msgBuf, tmp);
+    }
+
+    SSchema* pSchema = tscGetTableSchema(pTableMeta);
+    if ((!isTablenameToken(&pLeft->columnName)) && !IS_VAR_DATA_TYPE(pSchema[index].type)) {
+      return invalidOperationMsg(msgBuf, msg2);
+    }
+
+    int errCode = 0;
+    regex_t regex;
+    char    regErrBuf[256] = {0};
+
+    const char* pattern = pRight->value.pz;
+    int cflags = REG_EXTENDED;
+    if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
+      regerror(errCode, &regex, regErrBuf, sizeof(regErrBuf));
+      tscError("Failed to compile regex pattern %s. reason %s", pattern, regErrBuf);
+      return invalidOperationMsg(msgBuf, msg3);
+    }
+    regfree(&regex);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 int32_t handleNeOptr(tSqlExpr** rexpr, tSqlExpr* expr) {
   tSqlExpr* left = tSqlExprClone(expr);
   tSqlExpr* right = expr;
@@ -4540,6 +4592,12 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
     return code;
   }
 
+  // validate the match expression
+  code = validateMatchExpr(*pExpr, pTableMeta, index.columnIndex, tscGetErrorMsgPayload(pCmd));
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+
   SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex);
   if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP && index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) {  // query on time range
     if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) {
@@ -4867,65 +4925,66 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
   STagCond* pTagCond = &pQueryInfo->tagCond;
   pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->id.uid;
 
-  assert(pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_IN);
+  assert(pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH || pExpr->tokenId == TK_IN);
 
-  if (pExpr->tokenId == TK_LIKE) {
+  if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH) {
     char* str = taosStringBuilderGetResult(sb, NULL);
     pQueryInfo->tagCond.tbnameCond.cond = strdup(str);
     pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str);
     return TSDB_CODE_SUCCESS;
-  }
-
-  SStringBuilder sb1; memset(&sb1, 0, sizeof(sb1));
-  taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN);
+  } else {
+    SStringBuilder sb1;
+    memset(&sb1, 0, sizeof(sb1));
+    taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN);
 
-  // remove the duplicated input table names
-  int32_t num = 0;
-  char*   tableNameString = taosStringBuilderGetResult(sb, NULL);
+    // remove the duplicated input table names
+    int32_t num = 0;
+    char*   tableNameString = taosStringBuilderGetResult(sb, NULL);
 
-  char** segments = strsplit(tableNameString + QUERY_COND_REL_PREFIX_IN_LEN, TBNAME_LIST_SEP, &num);
-  qsort(segments, num, POINTER_BYTES, tableNameCompar);
+    char** segments = strsplit(tableNameString + QUERY_COND_REL_PREFIX_IN_LEN, TBNAME_LIST_SEP, &num);
+    qsort(segments, num, POINTER_BYTES, tableNameCompar);
 
-  int32_t j = 1;
-  for (int32_t i = 1; i < num; ++i) {
-    if (strcmp(segments[i], segments[i - 1]) != 0) {
-      segments[j++] = segments[i];
+    int32_t j = 1;
+    for (int32_t i = 1; i < num; ++i) {
+      if (strcmp(segments[i], segments[i - 1]) != 0) {
+        segments[j++] = segments[i];
+      }
     }
-  }
-  num = j;
+    num = j;
 
-  char name[TSDB_DB_NAME_LEN] = {0};
-  tNameGetDbName(&pTableMetaInfo->name, name);
-  SStrToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) };
-  
-  for (int32_t i = 0; i < num; ++i) {
-    if (i >= 1) {
-      taosStringBuilderAppendStringLen(&sb1, TBNAME_LIST_SEP, 1);
-    }
+    char name[TSDB_DB_NAME_LEN] = {0};
+    tNameGetDbName(&pTableMetaInfo->name, name);
+    SStrToken dbToken = {.type = TK_STRING, .z = name, .n = (uint32_t)strlen(name)};
+
+    for (int32_t i = 0; i < num; ++i) {
+      if (i >= 1) {
+        taosStringBuilderAppendStringLen(&sb1, TBNAME_LIST_SEP, 1);
+      }
 
-    char      idBuf[TSDB_TABLE_FNAME_LEN] = {0};
-    int32_t   xlen = (int32_t)strlen(segments[i]);
-    SStrToken t = {.z = segments[i], .n = xlen, .type = TK_STRING};
+      char      idBuf[TSDB_TABLE_FNAME_LEN] = {0};
+      int32_t   xlen = (int32_t)strlen(segments[i]);
+      SStrToken t = {.z = segments[i], .n = xlen, .type = TK_STRING};
 
-    int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen);
-    if (ret != TSDB_CODE_SUCCESS) {
-      taosStringBuilderDestroy(&sb1);
-      tfree(segments);
+      int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen);
+      if (ret != TSDB_CODE_SUCCESS) {
+        taosStringBuilderDestroy(&sb1);
+        tfree(segments);
 
-      invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg);
-      return ret;
-    }
+        invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg);
+        return ret;
+      }
 
-    taosStringBuilderAppendString(&sb1, idBuf);
-  }
+      taosStringBuilderAppendString(&sb1, idBuf);
+    }
 
-  char* str = taosStringBuilderGetResult(&sb1, NULL);
-  pQueryInfo->tagCond.tbnameCond.cond = strdup(str);
-  pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str);
+    char* str = taosStringBuilderGetResult(&sb1, NULL);
+    pQueryInfo->tagCond.tbnameCond.cond = strdup(str);
+    pQueryInfo->tagCond.tbnameCond.len = (int32_t)strlen(str);
 
-  taosStringBuilderDestroy(&sb1);
-  tfree(segments);
-  return TSDB_CODE_SUCCESS;
+    taosStringBuilderDestroy(&sb1);
+    tfree(segments);
+    return TSDB_CODE_SUCCESS;
+  }
 }
 
 int32_t mergeTimeRange(SSqlCmd* pCmd, STimeWindow* res, STimeWindow* win, int32_t optr) {
@@ -8112,7 +8171,7 @@ int32_t tscGetExprFilters(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelect
 }
 
 static int32_t handleExprInHavingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelectNodeList, tSqlExpr* pExpr, int32_t sqlOptr) {
-  const char* msg1 = "non binary column not support like operator";
+  const char* msg1 = "non binary column not support like/match operator";
   const char* msg2 = "invalid operator for binary column in having clause";
   const char* msg3 = "invalid operator for bool column in having clause";
 
@@ -8164,11 +8223,12 @@ static int32_t handleExprInHavingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, S
         && pExpr->tokenId != TK_ISNULL
         && pExpr->tokenId != TK_NOTNULL
         && pExpr->tokenId != TK_LIKE
+        && pExpr->tokenId != TK_MATCH
         ) {
       return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
     }
   } else {
-    if (pExpr->tokenId == TK_LIKE) {
+    if (pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_MATCH) {
       return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
     }
 

src/common/inc/texpr.h

@@ -33,9 +33,11 @@ struct SSchema;
 
 #define QUERY_COND_REL_PREFIX_IN "IN|"
 #define QUERY_COND_REL_PREFIX_LIKE "LIKE|"
+#define QUERY_COND_REL_PREFIX_MATCH "MATCH|"
 
 #define QUERY_COND_REL_PREFIX_IN_LEN   3
 #define QUERY_COND_REL_PREFIX_LIKE_LEN 5
+#define QUERY_COND_REL_PREFIX_MATCH_LEN 6
 
 typedef bool (*__result_filter_fn_t)(const void *, void *);
 typedef void (*__do_filter_suppl_fn_t)(void *, void *);

src/common/inc/tglobal.h

@@ -74,6 +74,7 @@ extern int8_t   tsKeepOriginalColumnName;
 // client
 extern int32_t tsMaxSQLStringLen;
 extern int32_t tsMaxWildCardsLen;
+extern int32_t tsMaxRegexStringLen;
 extern int8_t  tsTscEnableRecordSql;
 extern int32_t tsMaxNumOfOrderedResults;
 extern int32_t tsMinSlidingTime;

src/common/src/texpr.c

@@ -430,6 +430,17 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
     pVal->nType = TSDB_DATA_TYPE_BINARY;
     pVal->nLen = (int32_t)len;
 
+  } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN) == 0) {
+    right->nodeType = TSQL_NODE_VALUE;
+    expr->_node.optr = TSDB_RELATION_MATCH;
+    tVariant* pVal = exception_calloc(1, sizeof(tVariant));
+    right->pVal = pVal;
+    size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_MATCH_LEN) + 1;
+    pVal->pz = exception_malloc(len);
+    memcpy(pVal->pz, tbnameCond + QUERY_COND_REL_PREFIX_MATCH_LEN, len);
+    pVal->nType = TSDB_DATA_TYPE_BINARY;
+    pVal->nLen = (int32_t)len;
+
   } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) {
     right->nodeType = TSQL_NODE_VALUE;
     expr->_node.optr = TSDB_RELATION_IN;

src/common/src/tglobal.c

@@ -85,6 +85,8 @@ int32_t tsCompressColData = -1;
 // client
 int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN;
 int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN;
+int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN;
+
 int8_t  tsTscEnableRecordSql = 0;
 
 // the maximum number of results for projection query on super table that are returned from

src/inc/taosdef.h

@@ -164,6 +164,8 @@ do { \
 #define TSDB_RELATION_OR          12
 #define TSDB_RELATION_NOT         13
 
+#define TSDB_RELATION_MATCH       14
+
 #define TSDB_BINARY_OP_ADD        30
 #define TSDB_BINARY_OP_SUBTRACT   31
 #define TSDB_BINARY_OP_MULTIPLY   32

src/inc/ttokendef.h

@@ -37,160 +37,160 @@
 #define TK_NOTNULL                         19
 #define TK_IS                              20
 #define TK_LIKE                            21
-#define TK_GLOB                            22
-#define TK_BETWEEN                         23
-#define TK_IN                              24
-#define TK_GT                              25
-#define TK_GE                              26
-#define TK_LT                              27
-#define TK_LE                              28
-#define TK_BITAND                          29
-#define TK_BITOR                           30
-#define TK_LSHIFT                          31
-#define TK_RSHIFT                          32
-#define TK_PLUS                            33
-#define TK_MINUS                           34
-#define TK_DIVIDE                          35
-#define TK_TIMES                           36
-#define TK_STAR                            37
-#define TK_SLASH                           38
-#define TK_REM                             39
-#define TK_CONCAT                          40
-#define TK_UMINUS                          41
-#define TK_UPLUS                           42
-#define TK_BITNOT                          43
-#define TK_SHOW                            44
-#define TK_DATABASES                       45
-#define TK_TOPICS                          46
-#define TK_FUNCTIONS                       47
-#define TK_MNODES                          48
-#define TK_DNODES                          49
-#define TK_ACCOUNTS                        50
-#define TK_USERS                           51
-#define TK_MODULES                         52
-#define TK_QUERIES                         53
-#define TK_CONNECTIONS                     54
-#define TK_STREAMS                         55
-#define TK_VARIABLES                       56
-#define TK_SCORES                          57
-#define TK_GRANTS                          58
-#define TK_VNODES                          59
-#define TK_DOT                             60
-#define TK_CREATE                          61
-#define TK_TABLE                           62
-#define TK_STABLE                          63
-#define TK_DATABASE                        64
-#define TK_TABLES                          65
-#define TK_STABLES                         66
-#define TK_VGROUPS                         67
-#define TK_DROP                            68
-#define TK_TOPIC                           69
-#define TK_FUNCTION                        70
-#define TK_DNODE                           71
-#define TK_USER                            72
-#define TK_ACCOUNT                         73
-#define TK_USE                             74
-#define TK_DESCRIBE                        75
-#define TK_DESC                            76
-#define TK_ALTER                           77
-#define TK_PASS                            78
-#define TK_PRIVILEGE                       79
-#define TK_LOCAL                           80
-#define TK_COMPACT                         81
-#define TK_LP                              82
-#define TK_RP                              83
-#define TK_IF                              84
-#define TK_EXISTS                          85
-#define TK_AS                              86
-#define TK_OUTPUTTYPE                      87
-#define TK_AGGREGATE                       88
-#define TK_BUFSIZE                         89
-#define TK_PPS                             90
-#define TK_TSERIES                         91
-#define TK_DBS                             92
-#define TK_STORAGE                         93
-#define TK_QTIME                           94
-#define TK_CONNS                           95
-#define TK_STATE                           96
-#define TK_COMMA                           97
-#define TK_KEEP                            98
-#define TK_CACHE                           99
-#define TK_REPLICA                        100
-#define TK_QUORUM                         101
-#define TK_DAYS                           102
-#define TK_MINROWS                        103
-#define TK_MAXROWS                        104
-#define TK_BLOCKS                         105
-#define TK_CTIME                          106
-#define TK_WAL                            107
-#define TK_FSYNC                          108
-#define TK_COMP                           109
-#define TK_PRECISION                      110
-#define TK_UPDATE                         111
-#define TK_CACHELAST                      112
-#define TK_PARTITIONS                     113
-#define TK_UNSIGNED                       114
-#define TK_TAGS                           115
-#define TK_USING                          116
-#define TK_NULL                           117
-#define TK_NOW                            118
-#define TK_SELECT                         119
-#define TK_UNION                          120
-#define TK_ALL                            121
-#define TK_DISTINCT                       122
-#define TK_FROM                           123
-#define TK_VARIABLE                       124
-#define TK_INTERVAL                       125
-#define TK_EVERY                          126
-#define TK_SESSION                        127
-#define TK_STATE_WINDOW                   128
-#define TK_FILL                           129
-#define TK_SLIDING                        130
-#define TK_ORDER                          131
-#define TK_BY                             132
-#define TK_ASC                            133
-#define TK_GROUP                          134
-#define TK_HAVING                         135
-#define TK_LIMIT                          136
-#define TK_OFFSET                         137
-#define TK_SLIMIT                         138
-#define TK_SOFFSET                        139
-#define TK_WHERE                          140
-#define TK_RESET                          141
-#define TK_QUERY                          142
-#define TK_SYNCDB                         143
-#define TK_ADD                            144
-#define TK_COLUMN                         145
-#define TK_MODIFY                         146
-#define TK_TAG                            147
-#define TK_CHANGE                         148
-#define TK_SET                            149
-#define TK_KILL                           150
-#define TK_CONNECTION                     151
-#define TK_STREAM                         152
-#define TK_COLON                          153
-#define TK_ABORT                          154
-#define TK_AFTER                          155
-#define TK_ATTACH                         156
-#define TK_BEFORE                         157
-#define TK_BEGIN                          158
-#define TK_CASCADE                        159
-#define TK_CLUSTER                        160
-#define TK_CONFLICT                       161
-#define TK_COPY                           162
-#define TK_DEFERRED                       163
-#define TK_DELIMITERS                     164
-#define TK_DETACH                         165
-#define TK_EACH                           166
-#define TK_END                            167
-#define TK_EXPLAIN                        168
-#define TK_FAIL                           169
-#define TK_FOR                            170
-#define TK_IGNORE                         171
-#define TK_IMMEDIATE                      172
-#define TK_INITIALLY                      173
-#define TK_INSTEAD                        174
-#define TK_MATCH                          175
+#define TK_MATCH                           22
+#define TK_GLOB                            23
+#define TK_BETWEEN                         24
+#define TK_IN                              25
+#define TK_GT                              26
+#define TK_GE                              27
+#define TK_LT                              28
+#define TK_LE                              29
+#define TK_BITAND                          30
+#define TK_BITOR                           31
+#define TK_LSHIFT                          32
+#define TK_RSHIFT                          33
+#define TK_PLUS                            34
+#define TK_MINUS                           35
+#define TK_DIVIDE                          36
+#define TK_TIMES                           37
+#define TK_STAR                            38
+#define TK_SLASH                           39
+#define TK_REM                             40
+#define TK_CONCAT                          41
+#define TK_UMINUS                          42
+#define TK_UPLUS                           43
+#define TK_BITNOT                          44
+#define TK_SHOW                            45
+#define TK_DATABASES                       46
+#define TK_TOPICS                          47
+#define TK_FUNCTIONS                       48
+#define TK_MNODES                          49
+#define TK_DNODES                          50
+#define TK_ACCOUNTS                        51
+#define TK_USERS                           52
+#define TK_MODULES                         53
+#define TK_QUERIES                         54
+#define TK_CONNECTIONS                     55
+#define TK_STREAMS                         56
+#define TK_VARIABLES                       57
+#define TK_SCORES                          58
+#define TK_GRANTS                          59
+#define TK_VNODES                          60
+#define TK_DOT                             61
+#define TK_CREATE                          62
+#define TK_TABLE                           63
+#define TK_STABLE                          64
+#define TK_DATABASE                        65
+#define TK_TABLES                          66
+#define TK_STABLES                         67
+#define TK_VGROUPS                         68
+#define TK_DROP                            69
+#define TK_TOPIC                           70
+#define TK_FUNCTION                        71
+#define TK_DNODE                           72
+#define TK_USER                            73
+#define TK_ACCOUNT                         74
+#define TK_USE                             75
+#define TK_DESCRIBE                        76
+#define TK_DESC                            77
+#define TK_ALTER                           78
+#define TK_PASS                            79
+#define TK_PRIVILEGE                       80
+#define TK_LOCAL                           81
+#define TK_COMPACT                         82
+#define TK_LP                              83
+#define TK_RP                              84
+#define TK_IF                              85
+#define TK_EXISTS                          86
+#define TK_AS                              87
+#define TK_OUTPUTTYPE                      88
+#define TK_AGGREGATE                       89
+#define TK_BUFSIZE                         90
+#define TK_PPS                             91
+#define TK_TSERIES                         92
+#define TK_DBS                             93
+#define TK_STORAGE                         94
+#define TK_QTIME                           95
+#define TK_CONNS                           96
+#define TK_STATE                           97
+#define TK_COMMA                           98
+#define TK_KEEP                            99
+#define TK_CACHE                          100
+#define TK_REPLICA                        101
+#define TK_QUORUM                         102
+#define TK_DAYS                           103
+#define TK_MINROWS                        104
+#define TK_MAXROWS                        105
+#define TK_BLOCKS                         106
+#define TK_CTIME                          107
+#define TK_WAL                            108
+#define TK_FSYNC                          109
+#define TK_COMP                           110
+#define TK_PRECISION                      111
+#define TK_UPDATE                         112
+#define TK_CACHELAST                      113
+#define TK_PARTITIONS                     114
+#define TK_UNSIGNED                       115
+#define TK_TAGS                           116
+#define TK_USING                          117
+#define TK_NULL                           118
+#define TK_NOW                            119
+#define TK_SELECT                         120
+#define TK_UNION                          121
+#define TK_ALL                            122
+#define TK_DISTINCT                       123
+#define TK_FROM                           124
+#define TK_VARIABLE                       125
+#define TK_INTERVAL                       126
+#define TK_EVERY                          127
+#define TK_SESSION                        128
+#define TK_STATE_WINDOW                   129
+#define TK_FILL                           130
+#define TK_SLIDING                        131
+#define TK_ORDER                          132
+#define TK_BY                             133
+#define TK_ASC                            134
+#define TK_GROUP                          135
+#define TK_HAVING                         136
+#define TK_LIMIT                          137
+#define TK_OFFSET                         138
+#define TK_SLIMIT                         139
+#define TK_SOFFSET                        140
+#define TK_WHERE                          141
+#define TK_RESET                          142
+#define TK_QUERY                          143
+#define TK_SYNCDB                         144
+#define TK_ADD                            145
+#define TK_COLUMN                         146
+#define TK_MODIFY                         147
+#define TK_TAG                            148
+#define TK_CHANGE                         149
+#define TK_SET                            150
+#define TK_KILL                           151
+#define TK_CONNECTION                     152
+#define TK_STREAM                         153
+#define TK_COLON                          154
+#define TK_ABORT                          155
+#define TK_AFTER                          156
+#define TK_ATTACH                         157
+#define TK_BEFORE                         158
+#define TK_BEGIN                          159
+#define TK_CASCADE                        160
+#define TK_CLUSTER                        161
+#define TK_CONFLICT                       162
+#define TK_COPY                           163
+#define TK_DEFERRED                       164
+#define TK_DELIMITERS                     165
+#define TK_DETACH                         166
+#define TK_EACH                           167
+#define TK_END                            168
+#define TK_EXPLAIN                        169
+#define TK_FAIL                           170
+#define TK_FOR                            171
+#define TK_IGNORE                         172
+#define TK_IMMEDIATE                      173
+#define TK_INITIALLY                      174
+#define TK_INSTEAD                        175
 #define TK_KEY                            176
 #define TK_OF                             177
 #define TK_RAISE                          178

src/query/inc/sql.y

@@ -11,7 +11,7 @@
 %left OR.
 %left AND.
 %right NOT.
-%left EQ NE ISNULL NOTNULL IS LIKE GLOB BETWEEN IN.
+%left EQ NE ISNULL NOTNULL IS LIKE MATCH GLOB BETWEEN IN.
 %left GT GE LT LE.
 %left BITAND BITOR LSHIFT RSHIFT.
 %left PLUS MINUS.
@@ -751,6 +751,9 @@ expr(A) ::= expr(X) REM   expr(Y).   {A = tSqlExprCreate(X, Y, TK_REM);   }
 // like expression
 expr(A) ::= expr(X) LIKE expr(Y).    {A = tSqlExprCreate(X, Y, TK_LIKE);  }
 
+// match expression
+expr(A) ::= expr(X) MATCH expr(Y).    {A = tSqlExprCreate(X, Y, TK_MATCH);  }
+
 //in expression
 expr(A) ::= expr(X) IN LP exprlist(Y) RP.   {A = tSqlExprCreate(X, (tSqlExpr*)Y, TK_IN); }
 

src/query/src/qFilter.c

@@ -28,6 +28,7 @@ OptrStr gOptrStr[] = {
   {TSDB_RELATION_GREATER_EQUAL,            ">="},
   {TSDB_RELATION_NOT_EQUAL,                "!="},
   {TSDB_RELATION_LIKE,                     "like"},
+  {TSDB_RELATION_MATCH,                    "match"},
   {TSDB_RELATION_ISNULL,                   "is null"},
   {TSDB_RELATION_NOTNULL,                  "not null"},
   {TSDB_RELATION_IN,                       "in"},
@@ -156,7 +157,7 @@ int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) {
 __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, compareFloatVal,
   compareDoubleVal, compareLenPrefixedStr, compareStrPatternComp, compareFindItemInSet, compareWStrPatternComp, 
   compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val,
-  setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8
+  setCompareBytes1, setCompareBytes2, setCompareBytes4, setCompareBytes8, compareStrRegexComp,
 };
 
 int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
@@ -195,7 +196,9 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
     case TSDB_DATA_TYPE_FLOAT:     comparFn = 4;  break;
     case TSDB_DATA_TYPE_DOUBLE:    comparFn = 5; break;
     case TSDB_DATA_TYPE_BINARY: {
-      if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
+      if (optr == TSDB_RELATION_MATCH) {
+        comparFn = 19;
+      } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
         comparFn = 7;
       } else if (optr == TSDB_RELATION_IN) {
         comparFn = 8;
@@ -207,7 +210,9 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
     }
   
     case TSDB_DATA_TYPE_NCHAR: {
-      if (optr == TSDB_RELATION_LIKE) {
+      if (optr == TSDB_RELATION_MATCH) {
+        comparFn = 19;
+      } else if (optr == TSDB_RELATION_LIKE) {
         comparFn = 9;
       } else if (optr == TSDB_RELATION_IN) {
         comparFn = 8;
@@ -1871,6 +1876,9 @@ bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right)
     case TSDB_RELATION_LIKE: {
       return ret == 0;
     }
+    case TSDB_RELATION_MATCH: {
+      return ret == 0;
+    }
     case TSDB_RELATION_IN: {
       return ret == 1;
     }
@@ -2641,7 +2649,7 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SDataStatis *pDataStatis, int32_t
     }
 
     if (cunit->optr == TSDB_RELATION_ISNULL || cunit->optr == TSDB_RELATION_NOTNULL 
-     || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE
+     || cunit->optr == TSDB_RELATION_IN || cunit->optr == TSDB_RELATION_LIKE || cunit->optr == TSDB_RELATION_MATCH
      || cunit->optr == TSDB_RELATION_NOT_EQUAL) {
       continue;
     }

src/tsdb/src/tsdbRead.c

@@ -3709,6 +3709,9 @@ static bool tableFilterFp(const void* pNode, void* param) {
     case TSDB_RELATION_LIKE: {
       return ret == 0;
     }
+    case TSDB_RELATION_MATCH: {
+      return ret == 0;
+    }
     case TSDB_RELATION_IN: {
       return ret == 1;
     }
@@ -4042,6 +4045,8 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) {
     pCond->start->v    = queryColInfo->q; 
   } else if (optr == TSDB_RELATION_LIKE) {
     assert(0);
+  } else if (optr == TSDB_RELATION_MATCH) {
+    assert(0);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -4199,7 +4204,7 @@ static void queryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, S
     if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) {
       if (pQueryInfo->optr == TSDB_RELATION_IN) {
         addToResult = pQueryInfo->compare(name, pQueryInfo->q);
-      } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) {
+      } else if (pQueryInfo->optr == TSDB_RELATION_LIKE || pQueryInfo->optr == TSDB_RELATION_MATCH) {
         addToResult = !pQueryInfo->compare(name, pQueryInfo->q);
       }
     } else {
@@ -4231,7 +4236,8 @@ void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *re
     param->setupInfoFn(pExpr, param->pExtInfo);
 
     tQueryInfo *pQueryInfo = pExpr->_node.info;
-    if (pQueryInfo->indexed && (pQueryInfo->optr != TSDB_RELATION_LIKE && pQueryInfo->optr != TSDB_RELATION_IN)) {
+    if (pQueryInfo->indexed && (pQueryInfo->optr != TSDB_RELATION_LIKE && pQueryInfo->optr != TSDB_RELATION_MATCH
+                                && pQueryInfo->optr != TSDB_RELATION_IN)) {
       queryIndexedColumn(pSkipList, pQueryInfo, result);
     } else {
       queryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn);

src/util/inc/tcompare.h

@@ -22,10 +22,11 @@ extern "C" {
 
 #include "os.h"
 
-#define TSDB_PATTERN_MATCH               0
-#define TSDB_PATTERN_NOMATCH             1
-#define TSDB_PATTERN_NOWILDCARDMATCH     2
-#define TSDB_PATTERN_STRING_DEFAULT_LEN  100
+#define TSDB_PATTERN_MATCH 0
+#define TSDB_PATTERN_NOMATCH 1
+#define TSDB_PATTERN_NOWILDCARDMATCH 2
+#define TSDB_PATTERN_STRING_DEFAULT_LEN 100
+#define TSDB_REGEX_STRING_DEFAULT_LEN 128
 
 #define FLT_COMPAR_TOL_FACTOR    4
 #define FLT_EQUAL(_x, _y)        (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON))
@@ -82,6 +83,7 @@ int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight);
 
 int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight);
 int32_t compareStrPatternComp(const void* pLeft, const void* pRight);
+int32_t compareStrRegexComp(const void* pLeft, const void* pRight);
 int32_t compareFindItemInSet(const void *pLeft, const void* pRight);
 int32_t compareWStrPatternComp(const void* pLeft, const void* pRight);
 

src/util/src/tcompare.c

@@ -12,11 +12,17 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+#define _BSD_SOURCE
+#define _GNU_SOURCE
+#define _XOPEN_SOURCE
+#define _DEFAULT_SOURCE
 
-#include "os.h"
-#include "ttype.h"
 #include "tcompare.h"
+#include "tulog.h"
 #include "hash.h"
+#include "regex.h"
+#include "os.h"
+#include "ttype.h"
 
 int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
   return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
@@ -344,6 +350,43 @@ int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
   return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
 }
 
+int32_t compareStrRegexComp(const void* pLeft, const void* pRight) {
+  size_t sz = varDataLen(pRight);
+  char *pattern = malloc(sz + 1);
+  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
+  pattern[sz] = 0;
+
+  sz = varDataLen(pLeft);
+  char *str = malloc(sz + 1);
+  memcpy(str, varDataVal(pLeft), sz);
+  str[sz] = 0;
+
+  int errCode = 0;
+  regex_t regex;
+  char    msgbuf[256] = {0};
+
+  int cflags = REG_EXTENDED;
+  if ((errCode = regcomp(&regex, pattern, cflags)) != 0) {
+    regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
+    uError("Failed to compile regex pattern %s. reason %s", pattern, msgbuf);
+    regfree(&regex);
+    free(str);
+    free(pattern);
+    return 1;
+  }
+
+  errCode = regexec(&regex, str, 0, NULL, 0);
+  if (errCode != 0 && errCode != REG_NOMATCH) {
+    regerror(errCode, &regex, msgbuf, sizeof(msgbuf));
+    uDebug("Failed to match %s with pattern %s, reason %s", str, pattern, msgbuf)
+  }
+  int32_t result = (errCode == 0) ? 0 : 1;
+  regfree(&regex);
+  free(str);
+  free(pattern);
+  return result;
+}
+
 int32_t taosArrayCompareString(const void* a, const void* b) {
   const char* x = *(const char**)a;
   const char* y = *(const char**)b;
@@ -405,7 +448,9 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
     case TSDB_DATA_TYPE_FLOAT:     comparFn = compareFloatVal;  break;
     case TSDB_DATA_TYPE_DOUBLE:    comparFn = compareDoubleVal; break;
     case TSDB_DATA_TYPE_BINARY: {
-      if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
+      if (optr == TSDB_RELATION_MATCH) {
+        comparFn = compareStrRegexComp;
+      } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
         comparFn = compareStrPatternComp;
       } else if (optr == TSDB_RELATION_IN) {
         comparFn = compareFindItemInSet;
@@ -417,7 +462,9 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
     }
 
     case TSDB_DATA_TYPE_NCHAR: {
-      if (optr == TSDB_RELATION_LIKE) {
+      if (optr == TSDB_RELATION_MATCH) {
+        comparFn = compareStrRegexComp;
+      } else if (optr == TSDB_RELATION_LIKE) {
         comparFn = compareWStrPatternComp;
       } else if (optr == TSDB_RELATION_IN) {
         comparFn = compareFindItemInSet;

src/util/src/ttokenizer.c

@@ -53,6 +53,7 @@ static SKeyword keywordTable[] = {
     {"NOTNULL",      TK_NOTNULL},
     {"IS",           TK_IS},
     {"LIKE",         TK_LIKE},
+    {"MATCH",        TK_MATCH},
     {"GLOB",         TK_GLOB},
     {"BETWEEN",      TK_BETWEEN},
     {"IN",           TK_IN},

tests/script/fullGeneralSuite.sim

@@ -222,3 +222,4 @@ run general/stream/metrics_replica1_vnoden.sim
 run general/db/show_create_db.sim
 run general/db/show_create_table.sim
 run general/parser/like.sim
+run general/parser/regex.sim

tests/script/general/parser/regex.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/exec.sh -n dnode1 -s start
+
+sleep 100
+sql connect
+
+$db = testdb
+sql drop database if exists $db
+sql create database $db
+sql use $db
+
+print ======================== regular expression match test
+$st_name = st
+$ct1_name = ct1
+$ct2_name = ct2
+
+sql create table $st_name (ts timestamp, c1b binary(20)) tags(t1b binary(20));
+sql create table $ct1_name using $st_name tags('taosdata1')
+sql create table $ct2_name using $st_name tags('taosdata2')
+sql create table not_match using $st_name tags('NOTMATCH')
+
+sql select tbname from $st_name where tbname match '.*'
+if $rows != 3 then
+  return -1
+endi
+
+
+sql select tbname from $st_name where tbname match '^ct[[:digit:]]'
+
+if $rows != 2 then
+  return -1
+endi
+
+sql select tbname from $st_name where tbname match '.*'
+if $rows !=3 then
+  return -1
+endi
+
+sql select tbname from $st_name where t1b match '[[:lower:]]+'
+if $rows != 2 then
+  return -1
+endi
+
+sql insert into $ct1_name values(now, 'this is engine')
+sql insert into $ct2_name values(now, 'this is app egnine')
+
+sql select c1b from $st_name where c1b match 'engine'
+if $data00 != @this is engine@ then
+  return -1
+endi
+
+if $rows != 1 then
+  return -1
+endi  
+
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+
+