[TD-3178]<feature> fix generate file

src/query/inc/sql.h

+#define TK_ID                              1
+#define TK_BOOL                            2
+#define TK_TINYINT                         3
+#define TK_SMALLINT                        4
+#define TK_INTEGER                         5
+#define TK_BIGINT                          6
+#define TK_FLOAT                           7
+#define TK_DOUBLE                          8
+#define TK_STRING                          9
+#define TK_TIMESTAMP                      10
+#define TK_BINARY                         11
+#define TK_NCHAR                          12
+#define TK_OR                             13
+#define TK_AND                            14
+#define TK_NOT                            15
+#define TK_EQ                             16
+#define TK_NE                             17
+#define TK_ISNULL                         18
+#define TK_NOTNULL                        19
+#define TK_IS                             20
+#define TK_LIKE                           21
+#define TK_GLOB                           22
+#define TK_BETWEEN                        23
+#define TK_IN                             24
+#define TK_GT                             25
+#define TK_GE                             26
+#define TK_LT                             27
+#define TK_LE                             28
+#define TK_BITAND                         29
+#define TK_BITOR                          30
+#define TK_LSHIFT                         31
+#define TK_RSHIFT                         32
+#define TK_PLUS                           33
+#define TK_MINUS                          34
+#define TK_DIVIDE                         35
+#define TK_TIMES                          36
+#define TK_STAR                           37
+#define TK_SLASH                          38
+#define TK_REM                            39
+#define TK_CONCAT                         40
+#define TK_UMINUS                         41
+#define TK_UPLUS                          42
+#define TK_BITNOT                         43
+#define TK_SHOW                           44
+#define TK_DATABASES                      45
+#define TK_TOPICS                         46
+#define TK_MNODES                         47
+#define TK_DNODES                         48
+#define TK_ACCOUNTS                       49
+#define TK_USERS                          50
+#define TK_MODULES                        51
+#define TK_QUERIES                        52
+#define TK_CONNECTIONS                    53
+#define TK_STREAMS                        54
+#define TK_VARIABLES                      55
+#define TK_SCORES                         56
+#define TK_GRANTS                         57
+#define TK_VNODES                         58
+#define TK_IPTOKEN                        59
+#define TK_DOT                            60
+#define TK_CREATE                         61
+#define TK_TABLE                          62
+#define TK_STABLE                         63
+#define TK_DATABASE                       64
+#define TK_TABLES                         65
+#define TK_STABLES                        66
+#define TK_VGROUPS                        67
+#define TK_DROP                           68
+#define TK_TOPIC                          69
+#define TK_DNODE                          70
+#define TK_USER                           71
+#define TK_ACCOUNT                        72
+#define TK_USE                            73
+#define TK_DESCRIBE                       74
+#define TK_ALTER                          75
+#define TK_PASS                           76
+#define TK_PRIVILEGE                      77
+#define TK_LOCAL                          78
+#define TK_COMPACT                        79
+#define TK_LP                             80
+#define TK_RP                             81
+#define TK_IF                             82
+#define TK_EXISTS                         83
+#define TK_PPS                            84
+#define TK_TSERIES                        85
+#define TK_DBS                            86
+#define TK_STORAGE                        87
+#define TK_QTIME                          88
+#define TK_CONNS                          89
+#define TK_STATE                          90
+#define TK_KEEP                           91
+#define TK_CACHE                          92
+#define TK_REPLICA                        93
+#define TK_QUORUM                         94
+#define TK_DAYS                           95
+#define TK_MINROWS                        96
+#define TK_MAXROWS                        97
+#define TK_BLOCKS                         98
+#define TK_CTIME                          99
+#define TK_WAL                            100
+#define TK_FSYNC                          101
+#define TK_COMP                           102
+#define TK_PRECISION                      103
+#define TK_UPDATE                         104
+#define TK_CACHELAST                      105
+#define TK_PARTITIONS                     106
+#define TK_UNSIGNED                       107
+#define TK_TAGS                           108
+#define TK_USING                          109
+#define TK_COMMA                          110
+#define TK_AS                             111
+#define TK_NULL                           112
+#define TK_SELECT                         113
+#define TK_UNION                          114
+#define TK_ALL                            115
+#define TK_DISTINCT                       116
+#define TK_FROM                           117
+#define TK_VARIABLE                       118
+#define TK_INTERVAL                       119
+#define TK_SESSION                        120
+#define TK_FILL                           121
+#define TK_SLIDING                        122
+#define TK_ORDER                          123
+#define TK_BY                             124
+#define TK_ASC                            125
+#define TK_DESC                           126
+#define TK_GROUP                          127
+#define TK_HAVING                         128
+#define TK_LIMIT                          129
+#define TK_OFFSET                         130
+#define TK_SLIMIT                         131
+#define TK_SOFFSET                        132
+#define TK_WHERE                          133
+#define TK_NOW                            134
+#define TK_RESET                          135
+#define TK_QUERY                          136
+#define TK_SYNCDB                         137
+#define TK_ADD                            138
+#define TK_COLUMN                         139
+#define TK_TAG                            140
+#define TK_CHANGE                         141
+#define TK_SET                            142
+#define TK_KILL                           143
+#define TK_CONNECTION                     144
+#define TK_STREAM                         145
+#define TK_COLON                          146
+#define TK_ABORT                          147
+#define TK_AFTER                          148
+#define TK_ATTACH                         149
+#define TK_BEFORE                         150
+#define TK_BEGIN                          151
+#define TK_CASCADE                        152
+#define TK_CLUSTER                        153
+#define TK_CONFLICT                       154
+#define TK_COPY                           155
+#define TK_DEFERRED                       156
+#define TK_DELIMITERS                     157
+#define TK_DETACH                         158
+#define TK_EACH                           159
+#define TK_END                            160
+#define TK_EXPLAIN                        161
+#define TK_FAIL                           162
+#define TK_FOR                            163
+#define TK_IGNORE                         164
+#define TK_IMMEDIATE                      165
+#define TK_INITIALLY                      166
+#define TK_INSTEAD                        167
+#define TK_MATCH                          168
+#define TK_KEY                            169
+#define TK_OF                             170
+#define TK_RAISE                          171
+#define TK_REPLACE                        172
+#define TK_RESTRICT                       173
+#define TK_ROW                            174
+#define TK_STATEMENT                      175
+#define TK_TRIGGER                        176
+#define TK_VIEW                           177
+#define TK_SEMI                           178
+#define TK_NONE                           179
+#define TK_PREV                           180
+#define TK_LINEAR                         181
+#define TK_IMPORT                         182
+#define TK_TBNAME                         183
+#define TK_JOIN                           184
+#define TK_INSERT                         185
+#define TK_INTO                           186
+#define TK_VALUES                         187

src/util/inc/tstoken.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TTOKEN_H
+#define TDENGINE_TTOKEN_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "os.h"
+#include "tutil.h"
+#include "ttokendef.h"
+
+#define TSQL_TBNAME   "TBNAME"
+#define TSQL_TBNAME_L "tbname"
+
+#define TSQL_BLOCK_DIST   "_BLOCK_DIST"
+#define TSQL_BLOCK_DIST_L "_block_dist"
+
+// used to denote the minimum unite in sql parsing
+typedef struct SStrToken {
+  uint32_t n;
+  uint32_t type;
+  char    *z;
+} SStrToken;
+
+/**
+ * tokenizer for sql string
+ * @param z
+ * @param tokenType
+ * @return
+ */
+uint32_t tSQLGetToken(char *z, uint32_t *tokenType);
+
+/**
+ * enhanced tokenizer for sql string.
+ *
+ * @param str
+ * @param i
+ * @param isPrevOptr
+ * @return
+ */
+SStrToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr);
+
+/**
+ * check if it is a keyword or not
+ * @param z
+ * @param len
+ * @return
+ */
+bool isKeyWord(const char *z, int32_t len);
+
+/**
+ * check if it is a number or not
+ * @param pToken
+ * @return
+ */
+#define isNumber(tk) \
+((tk)->type == TK_INTEGER || (tk)->type == TK_FLOAT || (tk)->type == TK_HEX || (tk)->type == TK_BIN)
+
+
+/**
+ * check if it is a token or not
+ * @param pToken
+ * @return        token type, if it is not a number, TK_ILLEGAL will return
+ */
+static FORCE_INLINE int32_t tGetNumericStringType(const SStrToken* pToken) {
+  const char* z = pToken->z;
+  int32_t type = TK_ILLEGAL;
+
+  uint32_t i = 0;
+  for(; i < pToken->n; ++i) {
+    switch (z[i]) {
+      case '+':
+      case '-': {
+        break;
+      }
+
+      case '.': {
+        /*
+         * handle the the float number with out integer part
+         * .123
+         * .123e4
+         */
+        if (!isdigit(z[i+1])) {
+          return TK_ILLEGAL;
+        }
+
+        for (i += 2; isdigit(z[i]); i++) {
+        }
+
+        if ((z[i] == 'e' || z[i] == 'E') &&
+            (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) {
+          i += 2;
+          while (isdigit(z[i])) {
+            i++;
+          }
+        }
+
+        type = TK_FLOAT;
+        goto _end;
+      }
+
+      case '0': {
+        char next = z[i + 1];
+        if (next == 'b') { // bin number
+          type = TK_BIN;
+          for (i += 2; (z[i] == '0' || z[i] == '1'); ++i) {
+          }
+
+          goto _end;
+        } else if (next == 'x') {  //hex number
+          type = TK_HEX;
+          for (i += 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) {
+          }
+
+          goto _end;
+        }
+      }
+      case '1':
+      case '2':
+      case '3':
+      case '4':
+      case '5':
+      case '6':
+      case '7':
+      case '8':
+      case '9': {
+        type = TK_INTEGER;
+        for (; isdigit(z[i]); i++) {
+        }
+
+        int32_t seg = 0;
+        while (z[i] == '.' && isdigit(z[i + 1])) {
+          i += 2;
+
+          while (isdigit(z[i])) {
+            i++;
+          }
+
+          seg++;
+          type = TK_FLOAT;
+        }
+
+        if (seg > 1) {
+          return TK_ILLEGAL;
+        }
+
+        if ((z[i] == 'e' || z[i] == 'E') &&
+            (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) {
+          i += 2;
+          while (isdigit(z[i])) {
+            i++;
+          }
+
+          type = TK_FLOAT;
+        }
+
+        goto _end;
+      }
+      default:
+        return TK_ILLEGAL;
+    }
+  }
+
+  _end:
+  return (i < pToken->n)? TK_ILLEGAL:type;
+}
+
+void taosCleanupKeywordsTable();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TTOKEN_H