From 2f5eb2ab2dc560498d5446b4f494ef1081207d19 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 8 Oct 2021 10:25:09 +0800 Subject: [PATCH] [TD-10529]add new files. --- include/common/common.h | 37 ++ include/common/taosdef.h | 448 +++++++++++++++++++++++ include/common/tname.h | 59 +++ include/common/ttypes.h | 215 +++++++++++ include/util/thash.h | 206 +++++++++++ source/libs/parser/inc/astGenerator.h | 360 ++++++++++++++++++ source/libs/parser/inc/insertParser.h | 19 + source/libs/parser/inc/parserUtil.h | 28 ++ source/libs/parser/inc/ttoken.h | 185 ++++++++++ source/libs/parser/inc/ttokendef.h | 228 ++++++++++++ source/libs/parser/inc/tvariant.h | 66 ++++ source/libs/parser/src/astGenerator.c | 507 ++++++++++++++++++++++++++ source/libs/parser/src/astValidate.c | 26 ++ 13 files changed, 2384 insertions(+) create mode 100644 include/common/common.h create mode 100644 include/common/taosdef.h create mode 100644 include/common/tname.h create mode 100644 include/common/ttypes.h create mode 100644 include/util/thash.h create mode 100644 source/libs/parser/inc/astGenerator.h create mode 100644 source/libs/parser/inc/insertParser.h create mode 100644 source/libs/parser/inc/parserUtil.h create mode 100644 source/libs/parser/inc/ttoken.h create mode 100644 source/libs/parser/inc/ttokendef.h create mode 100644 source/libs/parser/inc/tvariant.h create mode 100644 source/libs/parser/src/astGenerator.c create mode 100644 source/libs/parser/src/astValidate.c diff --git a/include/common/common.h b/include/common/common.h new file mode 100644 index 0000000000..c03b978c98 --- /dev/null +++ b/include/common/common.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_COMMON_H +#define TDENGINE_COMMON_H + +typedef struct STimeWindow { + TSKEY skey; + TSKEY ekey; +} STimeWindow; + +typedef struct { + int32_t dataLen; + char name[TSDB_TABLE_FNAME_LEN]; + char *data; +} STagData; + +typedef struct SSchema { + uint8_t type; + char name[TSDB_COL_NAME_LEN]; + int16_t colId; + int16_t bytes; +} SSchema; + +#endif // TDENGINE_COMMON_H diff --git a/include/common/taosdef.h b/include/common/taosdef.h new file mode 100644 index 0000000000..dd5bbace8c --- /dev/null +++ b/include/common/taosdef.h @@ -0,0 +1,448 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TAOS_DEF_H +#define TDENGINE_TAOS_DEF_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" + +#define TSDB__packed + +#define TSKEY int64_t + +#define TSWINDOW_INITIALIZER ((STimeWindow) {INT64_MIN, INT64_MAX}) +#define TSWINDOW_DESC_INITIALIZER ((STimeWindow) {INT64_MAX, INT64_MIN}) +#define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) + +#define TSKEY_INITIAL_VAL INT64_MIN + +// Bytes for each type. +extern const int32_t TYPE_BYTES[15]; + +// TODO: replace and remove code below +#define CHAR_BYTES sizeof(int8_t) +#define SHORT_BYTES sizeof(int16_t) +#define INT_BYTES sizeof(int32_t) +#define LONG_BYTES sizeof(int64_t) +#define FLOAT_BYTES sizeof(float) +#define DOUBLE_BYTES sizeof(double) +#define POINTER_BYTES sizeof(ptrdiff_t) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) +#define TSDB_KEYSIZE sizeof(TSKEY) +#define TSDB_NCHAR_SIZE sizeof(int32_t) + +// NULL definition +#define TSDB_DATA_BOOL_NULL 0x02 +#define TSDB_DATA_TINYINT_NULL 0x80 +#define TSDB_DATA_SMALLINT_NULL 0x8000 +#define TSDB_DATA_INT_NULL 0x80000000L +#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L +#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL + +#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF +#define TSDB_DATA_BINARY_NULL 0xFF + +#define TSDB_DATA_UTINYINT_NULL 0xFF +#define TSDB_DATA_USMALLINT_NULL 0xFFFF +#define TSDB_DATA_UINT_NULL 0xFFFFFFFF +#define TSDB_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL + +#define TSDB_DATA_NULL_STR "NULL" +#define TSDB_DATA_NULL_STR_L "null" +#define TSDB_DEFAULT_USER "root" + +#ifdef _TD_POWER_ +#define TSDB_DEFAULT_PASS "powerdb" +#elif (_TD_TQ_ == true) +#define TSDB_DEFAULT_PASS "tqueue" +#else +#define TSDB_DEFAULT_PASS "taosdata" +#endif + +#define SHELL_MAX_PASSWORD_LEN 20 + +#define TSDB_TRUE 1 +#define TSDB_FALSE 0 +#define TSDB_OK 0 +#define TSDB_ERR -1 + +#define TS_PATH_DELIMITER "." +#define TSQL_TBNAME "TBNAME" +#define TSQL_TBNAME_L "tbname" +#define TSQL_BLOCK_DIST "_BLOCK_DIST" +#define TSQL_BLOCK_DIST_L "_block_dist" + +#define TSDB_TIME_PRECISION_MILLI 0 +#define TSDB_TIME_PRECISION_MICRO 1 +#define TSDB_TIME_PRECISION_NANO 2 + +#define TSDB_TIME_PRECISION_MILLI_STR "ms" +#define TSDB_TIME_PRECISION_MICRO_STR "us" +#define TSDB_TIME_PRECISION_NANO_STR "ns" + +#define TSDB_TICK_PER_SECOND(precision) ((int64_t)((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) + +#define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#define T_APPEND_MEMBER(dst, ptr, type, member) \ +do {\ + memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member));\ + dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member));\ +} while(0) +#define T_READ_MEMBER(src, type, target) \ +do { \ + (target) = *(type *)(src); \ + (src) = (void *)((char *)src + sizeof(type));\ +} while(0) + +#define GET_INT8_VAL(x) (*(int8_t *)(x)) +#define GET_INT16_VAL(x) (*(int16_t *)(x)) +#define GET_INT32_VAL(x) (*(int32_t *)(x)) +#define GET_INT64_VAL(x) (*(int64_t *)(x)) +#define GET_UINT8_VAL(x) (*(uint8_t*) (x)) +#define GET_UINT16_VAL(x) (*(uint16_t *)(x)) +#define GET_UINT32_VAL(x) (*(uint32_t *)(x)) +#define GET_UINT64_VAL(x) (*(uint64_t *)(x)) + +#ifdef _TD_ARM_32 + float taos_align_get_float(const char* pBuf); + double taos_align_get_double(const char* pBuf); + + #define GET_FLOAT_VAL(x) taos_align_get_float(x) + #define GET_DOUBLE_VAL(x) taos_align_get_double(x) + #define SET_FLOAT_VAL(x, y) { float z = (float)(y); (*(int32_t*) x = *(int32_t*)(&z)); } + #define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); } + #define SET_FLOAT_PTR(x, y) { (*(int32_t*) x = *(int32_t*)y); } + #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); } +#else + #define GET_FLOAT_VAL(x) (*(float *)(x)) + #define GET_DOUBLE_VAL(x) (*(double *)(x)) + #define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); } + #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); } + #define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); } + #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); } +#endif + +// TODO: check if below is necessary +#define TSDB_RELATION_INVALID 0 +#define TSDB_RELATION_LESS 1 +#define TSDB_RELATION_GREATER 2 +#define TSDB_RELATION_EQUAL 3 +#define TSDB_RELATION_LESS_EQUAL 4 +#define TSDB_RELATION_GREATER_EQUAL 5 +#define TSDB_RELATION_NOT_EQUAL 6 +#define TSDB_RELATION_LIKE 7 +#define TSDB_RELATION_ISNULL 8 +#define TSDB_RELATION_NOTNULL 9 +#define TSDB_RELATION_IN 10 + +#define TSDB_RELATION_AND 11 +#define TSDB_RELATION_OR 12 +#define TSDB_RELATION_NOT 13 + +#define TSDB_RELATION_MATCH 14 +#define TSDB_RELATION_NMATCH 15 + +#define TSDB_BINARY_OP_ADD 30 +#define TSDB_BINARY_OP_SUBTRACT 31 +#define TSDB_BINARY_OP_MULTIPLY 32 +#define TSDB_BINARY_OP_DIVIDE 33 +#define TSDB_BINARY_OP_REMAINDER 34 + + +#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) +#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) + +#define TS_PATH_DELIMITER_LEN 1 + +#define TSDB_UNI_LEN 24 +#define TSDB_USER_LEN TSDB_UNI_LEN + +// ACCOUNT is a 32 bit positive integer +// this is the length of its string representation, including the terminator zero +#define TSDB_ACCT_ID_LEN 11 + +#define TSDB_MAX_COLUMNS 4096 +#define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns + +#define TSDB_NODE_NAME_LEN 64 +#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_DB_NAME_LEN 33 +#define TSDB_FUNC_NAME_LEN 65 +#define TSDB_FUNC_CODE_LEN (65535 - 512) +#define TSDB_FUNC_BUF_SIZE 512 +#define TSDB_TYPE_STR_MAX_LEN 32 +#define TSDB_TABLE_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN) +#define TSDB_COL_NAME_LEN 65 +#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 +#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE +#define TSDB_MAX_SQL_SHOW_LEN 512 +#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb + +#define TSDB_APPNAME_LEN TSDB_UNI_LEN + +/** + * In some scenarios uint16_t (0~65535) is used to store the row len. + * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. + * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus + * the final value is 65531-(4096-1)*4 = 49151. + */ +#define TSDB_MAX_BYTES_PER_ROW 49151 +#define TSDB_MAX_TAGS_LEN 16384 +#define TSDB_MAX_TAGS 128 +#define TSDB_MAX_TAG_CONDITIONS 1024 + +#define TSDB_AUTH_LEN 16 +#define TSDB_KEY_LEN 16 +#define TSDB_VERSION_LEN 12 +#define TSDB_LOCALE_LEN 64 +#define TSDB_TIMEZONE_LEN 96 +#define TSDB_LABEL_LEN 8 + +#define TSDB_CLUSTER_ID_LEN 40 +#define TSDB_FQDN_LEN 128 +#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) +#define TSDB_IPv4ADDR_LEN 16 +#define TSDB_FILENAME_LEN 128 +#define TSDB_SHOW_SQL_LEN 512 +#define TSDB_SHOW_SUBQUERY_LEN 1000 +#define TSDB_SLOW_QUERY_SQL_LEN 512 + +#define TSDB_STEP_NAME_LEN 32 +#define TSDB_STEP_DESC_LEN 128 + +#define TSDB_MQTT_HOSTNAME_LEN 64 +#define TSDB_MQTT_PORT_LEN 8 +#define TSDB_MQTT_USER_LEN 24 +#define TSDB_MQTT_PASS_LEN 24 +#define TSDB_MQTT_TOPIC_LEN 64 +#define TSDB_MQTT_CLIENT_ID_LEN 32 + +#define TSDB_DB_TYPE_DEFAULT 0 +#define TSDB_DB_TYPE_TOPIC 1 + +#define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE + +#define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE +#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value +#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth +#define TSDB_CQ_SQL_SIZE 1024 +#define TSDB_MIN_VNODES 64 +#define TSDB_MAX_VNODES 2048 +#define TSDB_MIN_VNODES_PER_DB 2 +#define TSDB_MAX_VNODES_PER_DB 64 + +#define TSDB_DNODE_ROLE_ANY 0 +#define TSDB_DNODE_ROLE_MGMT 1 +#define TSDB_DNODE_ROLE_VNODE 2 + +#define TSDB_MAX_REPLICA 5 + +#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_UD_COLUMN_INDEX (-1000) +#define TSDB_RES_COL_ID (-5000) + +#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta + +#define TSDB_MIN_CACHE_BLOCK_SIZE 1 +#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode +#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 + +#define TSDB_MIN_TOTAL_BLOCKS 3 +#define TSDB_MAX_TOTAL_BLOCKS 10000 +#define TSDB_DEFAULT_TOTAL_BLOCKS 6 + +#define TSDB_MIN_TABLES 4 +#define TSDB_MAX_TABLES 10000000 +#define TSDB_DEFAULT_TABLES 1000000 +#define TSDB_TABLES_STEP 1000 +#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default + +#define TSDB_MIN_DAYS_PER_FILE 1 +#define TSDB_MAX_DAYS_PER_FILE 3650 +#define TSDB_DEFAULT_DAYS_PER_FILE 10 + +#define TSDB_MIN_KEEP 1 // data in db to be reserved. +#define TSDB_MAX_KEEP 365000 // data in db to be reserved. +#define TSDB_DEFAULT_KEEP 3650 // ten years + +#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 +#define TSDB_MIN_MIN_ROW_FBLOCK 10 +#define TSDB_MAX_MIN_ROW_FBLOCK 1000 + +#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 +#define TSDB_MIN_MAX_ROW_FBLOCK 200 +#define TSDB_MAX_MAX_ROW_FBLOCK 10000 + +#define TSDB_MIN_COMMIT_TIME 30 +#define TSDB_MAX_COMMIT_TIME 40960 +#define TSDB_DEFAULT_COMMIT_TIME 3600 + +#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI +#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO +#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI + +#define TSDB_MIN_COMP_LEVEL 0 +#define TSDB_MAX_COMP_LEVEL 2 +#define TSDB_DEFAULT_COMP_LEVEL 2 + +#define TSDB_MIN_WAL_LEVEL 0 +#define TSDB_MAX_WAL_LEVEL 2 +#define TSDB_DEFAULT_WAL_LEVEL 1 + +#define TSDB_MIN_DB_UPDATE 0 +#define TSDB_MAX_DB_UPDATE 2 +#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 + +#define TSDB_MIN_DB_CACHE_LAST_ROW 0 +#define TSDB_MAX_DB_CACHE_LAST_ROW 3 +#define TSDB_DEFAULT_CACHE_LAST_ROW 0 + +#define TSDB_MIN_FSYNC_PERIOD 0 +#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond +#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second + +#define TSDB_MIN_DB_REPLICA_OPTION 1 +#define TSDB_MAX_DB_REPLICA_OPTION 3 +#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 + +#define TSDB_MIN_DB_PARTITON_OPTION 0 +#define TSDB_MAX_DB_PARTITON_OPTION 1000 +#define TSDB_DEFAULT_DB_PARTITON_OPTION 4 + +#define TSDB_MIN_DB_QUORUM_OPTION 1 +#define TSDB_MAX_DB_QUORUM_OPTION 2 +#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 + +#define TSDB_MAX_JOIN_TABLE_NUM 10 +#define TSDB_MAX_UNION_CLAUSE 5 + +#define TSDB_MAX_FIELD_LEN 16384 +#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 +#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 +#define PRIMARYKEY_TIMESTAMP_COL_INDEX 0 + +#define TSDB_MAX_RPC_THREADS 5 + +#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type +#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode + +#define TSDB_UDF_TYPE_SCALAR 1 +#define TSDB_UDF_TYPE_AGGREGATE 2 + +/* + * TODO remove the following definitions + * 1. ordinary sub query for select * from super_table + * 2. all sqlobj generated by createSubqueryObj with this flag + */ +#define TSDB_QUERY_TYPE_SUBQUERY 0x02u +#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table + +#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side +#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table +#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query +#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query +#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage + +#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u +#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type +#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u +#define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file +#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type +#define TSDB_QUERY_TYPE_NEST_SUBQUERY 0x1000u // nested sub query + +#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) +#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) +#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) +#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) + +#define TSDB_ORDER_ASC 1 +#define TSDB_ORDER_DESC 2 + +#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 +#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 +#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 +#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 +#define TSDB_DEFAULT_USERS_HASH_SIZE 20 +#define TSDB_DEFAULT_DBS_HASH_SIZE 100 +#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 +#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 +#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 + +#define TSDB_PORT_DNODESHELL 0 +#define TSDB_PORT_DNODEDNODE 5 +#define TSDB_PORT_SYNC 10 +#define TSDB_PORT_HTTP 11 +#define TSDB_PORT_ARBITRATOR 12 + +#define TSDB_MAX_WAL_SIZE (1024*1024*3) +#define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P + +typedef enum { + TAOS_QTYPE_RPC = 1, + TAOS_QTYPE_FWD = 2, + TAOS_QTYPE_WAL = 3, + TAOS_QTYPE_CQ = 4, + TAOS_QTYPE_QUERY = 5 +} EQType; + +#define TSDB_MAX_TIERS 3 +#define TSDB_MAX_DISKS_PER_TIER 16 +#define TSDB_MAX_DISKS (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER) + +typedef enum { + TSDB_SUPER_TABLE = 1, // super table + TSDB_CHILD_TABLE = 2, // table created from super table + TSDB_NORMAL_TABLE = 3, // ordinary table + TSDB_STREAM_TABLE = 4, // table created from stream computing + TSDB_TEMP_TABLE = 5, // temp table created by nest query + TSDB_TABLE_MAX = 6 +} ETableType; + +typedef enum { + TSDB_MOD_MNODE = 1, + TSDB_MOD_HTTP = 2, + TSDB_MOD_MONITOR = 3, + TSDB_MOD_MQTT = 4, + TSDB_MOD_MAX = 5 +} EModuleType; + +typedef enum { + TSDB_CHECK_ITEM_NETWORK, + TSDB_CHECK_ITEM_MEM, + TSDB_CHECK_ITEM_CPU, + TSDB_CHECK_ITEM_DISK, + TSDB_CHECK_ITEM_OS, + TSDB_CHECK_ITEM_ACCESS, + TSDB_CHECK_ITEM_VERSION, + TSDB_CHECK_ITEM_DATAFILE, + TSDB_CHECK_ITEM_MAX +} ECheckItemType; + +typedef enum { + TD_ROW_DISCARD_UPDATE = 0, + TD_ROW_OVERWRITE_UPDATE = 1, + TD_ROW_PARTIAL_UPDATE = 2 +} TDUpdateConfig; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/common/tname.h b/include/common/tname.h new file mode 100644 index 0000000000..18526f54d4 --- /dev/null +++ b/include/common/tname.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TNAME_H +#define TDENGINE_TNAME_H + +#define TSDB_DB_NAME_T 1 +#define TSDB_TABLE_NAME_T 2 + +#define T_NAME_ACCT 0x1u +#define T_NAME_DB 0x2u +#define T_NAME_TABLE 0x4u + +typedef struct SName { + uint8_t type; //db_name_t, table_name_t + char acctId[TSDB_ACCT_ID_LEN]; + char dbname[TSDB_DB_NAME_LEN]; + char tname[TSDB_TABLE_NAME_LEN]; +} SName; + +int32_t tNameExtractFullName(const SName* name, char* dst); + +int32_t tNameLen(const SName* name); + +SName* tNameDup(const SName* name); + +bool tIsValidName(const SName* name); + +const char* tNameGetTableName(const SName* name); + +int32_t tNameGetDbName(const SName* name, char* dst); + +int32_t tNameGetFullDbName(const SName* name, char* dst); + +bool tNameIsEmpty(const SName* name); + +void tNameAssign(SName* dst, const SName* src); + +int32_t tNameFromString(SName* dst, const char* str, uint32_t type); + +int32_t tNameSetAcctId(SName* dst, const char* acct); + +#if 0 +int32_t tNameSetDbName(SName* dst, const char* acct, SToken* dbToken); +#endif + +#endif // TDENGINE_TNAME_H diff --git a/include/common/ttypes.h b/include/common/ttypes.h new file mode 100644 index 0000000000..6fe6e11d05 --- /dev/null +++ b/include/common/ttypes.h @@ -0,0 +1,215 @@ +#ifndef TDENGINE_TTYPE_H +#define TDENGINE_TTYPE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taosdef.h" + +// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR +typedef int32_t VarDataOffsetT; +typedef int16_t VarDataLenT; // maxVarDataLen: 32767 +typedef uint16_t TDRowLenT; // not including overhead: 0 ~ 65535 +typedef uint32_t TDRowTLenT; // total length, including overhead + +typedef struct tstr { + VarDataLenT len; + char data[]; +} tstr; + +#pragma pack(push, 1) +typedef struct { + VarDataLenT len; + uint8_t data; +} SBinaryNullT; + +typedef struct { + VarDataLenT len; + uint32_t data; +} SNCharNullT; +#pragma pack(pop) + +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + +#define varDataLen(v) ((VarDataLenT *)(v))[0] +#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) +#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) +#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) +#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) +#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len)) +#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) + +#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) +#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) + + +// this data type is internally used only in 'in' query to hold the values +#define TSDB_DATA_TYPE_POINTER_ARRAY (1000) +#define TSDB_DATA_TYPE_VALUE_ARRAY (1001) + +#define GET_TYPED_DATA(_v, _finalType, _type, _data) \ + do { \ + switch (_type) { \ + case TSDB_DATA_TYPE_BOOL: \ + case TSDB_DATA_TYPE_TINYINT: \ + (_v) = (_finalType)GET_INT8_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_UTINYINT: \ + (_v) = (_finalType)GET_UINT8_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_SMALLINT: \ + (_v) = (_finalType)GET_INT16_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_USMALLINT: \ + (_v) = (_finalType)GET_UINT16_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_TIMESTAMP:\ + case TSDB_DATA_TYPE_BIGINT: \ + (_v) = (_finalType)(GET_INT64_VAL(_data)); \ + break; \ + case TSDB_DATA_TYPE_UBIGINT: \ + (_v) = (_finalType)(GET_UINT64_VAL(_data)); \ + break; \ + case TSDB_DATA_TYPE_FLOAT: \ + (_v) = (_finalType)GET_FLOAT_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_DOUBLE: \ + (_v) = (_finalType)GET_DOUBLE_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_UINT: \ + (_v) = (_finalType)GET_UINT32_VAL(_data); \ + break; \ + default: \ + (_v) = (_finalType)GET_INT32_VAL(_data); \ + break; \ + } \ + } while (0) + +#define SET_TYPED_DATA(_v, _type, _data) \ + do { \ + switch (_type) { \ + case TSDB_DATA_TYPE_BOOL: \ + case TSDB_DATA_TYPE_TINYINT: \ + *(int8_t *)(_v) = (int8_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_UTINYINT: \ + *(uint8_t *)(_v) = (uint8_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_SMALLINT: \ + *(int16_t *)(_v) = (int16_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_USMALLINT: \ + *(uint16_t *)(_v) = (uint16_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_BIGINT: \ + *(int64_t *)(_v) = (int64_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_UBIGINT: \ + *(uint64_t *)(_v) = (uint64_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_FLOAT: \ + *(float *)(_v) = (float)(_data); \ + break; \ + case TSDB_DATA_TYPE_DOUBLE: \ + *(double *)(_v) = (double)(_data); \ + break; \ + case TSDB_DATA_TYPE_UINT: \ + *(uint32_t *)(_v) = (uint32_t)(_data); \ + break; \ + default: \ + *(int32_t *)(_v) = (int32_t)(_data); \ + break; \ + } \ + } while (0) + +#define IS_SIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT) +#define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT) +#define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) + +#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) + +#define IS_VALID_TINYINT(_t) ((_t) > INT8_MIN && (_t) <= INT8_MAX) +#define IS_VALID_SMALLINT(_t) ((_t) > INT16_MIN && (_t) <= INT16_MAX) +#define IS_VALID_INT(_t) ((_t) > INT32_MIN && (_t) <= INT32_MAX) +#define IS_VALID_BIGINT(_t) ((_t) > INT64_MIN && (_t) <= INT64_MAX) +#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) < UINT8_MAX) +#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) < UINT16_MAX) +#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) < UINT32_MAX) +#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) < UINT64_MAX) +#define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) +#define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) + +static FORCE_INLINE bool isNull(const void *val, int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + return *(uint8_t *)val == TSDB_DATA_BOOL_NULL; + case TSDB_DATA_TYPE_TINYINT: + return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL; + case TSDB_DATA_TYPE_SMALLINT: + return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL; + case TSDB_DATA_TYPE_INT: + return *(uint32_t *)val == TSDB_DATA_INT_NULL; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL; + case TSDB_DATA_TYPE_FLOAT: + return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL; + case TSDB_DATA_TYPE_DOUBLE: + return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL; + case TSDB_DATA_TYPE_NCHAR: + return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL; + case TSDB_DATA_TYPE_BINARY: + return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL; + case TSDB_DATA_TYPE_UTINYINT: + return *(uint8_t*) val == TSDB_DATA_UTINYINT_NULL; + case TSDB_DATA_TYPE_USMALLINT: + return *(uint16_t*) val == TSDB_DATA_USMALLINT_NULL; + case TSDB_DATA_TYPE_UINT: + return *(uint32_t*) val == TSDB_DATA_UINT_NULL; + case TSDB_DATA_TYPE_UBIGINT: + return *(uint64_t*) val == TSDB_DATA_UBIGINT_NULL; + + default: + return false; + }; +} + +typedef struct tDataTypeDescriptor { + int16_t type; + int16_t nameLen; + int32_t bytes; + char * name; + int64_t minValue; + int64_t maxValue; + int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, + char algorithm, char *const buffer, int bufferSize); + int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize); + void (*statisFunc)(const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minindex, int16_t *maxindex, int16_t *numofnull); +} tDataTypeDescriptor; + +extern tDataTypeDescriptor tDataTypes[15]; + +bool isValidDataType(int32_t type); + +void setVardataNull(void* val, int32_t type); +void setNull(void *val, int32_t type, int32_t bytes); +void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems); +const void *getNullValue(int32_t type); + +void assignVal(char *val, const char *src, int32_t len, int32_t type); +void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); +void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); +void* getDataMin(int32_t type); +void* getDataMax(int32_t type); + + +#define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL) + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TTYPE_H diff --git a/include/util/thash.h b/include/util/thash.h new file mode 100644 index 0000000000..e863396f38 --- /dev/null +++ b/include/util/thash.h @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_HASH_H +#define TDENGINE_HASH_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tarray.h" +#include "tlockfree.h" + +typedef uint32_t (*_hash_fn_t)(const char *, uint32_t); +typedef int32_t (*_equal_fn_t)(const void*, const void*); +typedef void (*_hash_before_fn_t)(void *); +typedef void (*_hash_free_fn_t)(void *); + +#define HASH_MAX_CAPACITY (1024 * 1024 * 16) +#define HASH_DEFAULT_LOAD_FACTOR (0.75) + +#define HASH_INDEX(v, c) ((v) & ((c)-1)) + +/** + * murmur hash algorithm + * @key usually string + * @len key length + * @seed hash seed + * @out an int32 value + */ +uint32_t MurmurHash3_32(const char *key, uint32_t len); + +/** + * + * @param key + * @param len + * @return + */ +uint32_t taosIntHash_32(const char *key, uint32_t len); +uint32_t taosIntHash_64(const char *key, uint32_t len); + +_hash_fn_t taosGetDefaultHashFunction(int32_t type); + +typedef struct SHashNode { + struct SHashNode *next; + uint32_t hashVal; // the hash value of key + uint32_t dataLen; // length of data + uint32_t keyLen; // length of the key + uint16_t count; // reference count + int8_t removed; // flag to indicate removed + char data[]; +} SHashNode; + +#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) +#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) +#define GET_HASH_PNODE(_n) ((SHashNode *)((char*)(_n) - sizeof(SHashNode))) + +typedef enum SHashLockTypeE { + HASH_NO_LOCK = 0, + HASH_ENTRY_LOCK = 1, +} SHashLockTypeE; + +typedef struct SHashEntry { + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch + SHashNode *next; +} SHashEntry; + +typedef struct SHashObj { + SHashEntry **hashList; + uint32_t capacity; // number of slots + uint32_t size; // number of elements in hash table + + _hash_fn_t hashFp; // hash function + _hash_free_fn_t freeFp; // hash node free callback function + _equal_fn_t equalFp; // equal function + _hash_before_fn_t callbackFp; // function invoked before return the value to caller + + SRWLatch lock; // read-write spin lock + SHashLockTypeE type; // lock type + bool enableUpdate; // enable update + SArray *pMemBlock; // memory block allocated for SHashEntry +} SHashObj; + +/** + * init the hash table + * + * @param capacity initial capacity of the hash table + * @param fn hash function to generate the hash value + * @param threadsafe thread safe or not + * @return + */ +SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type); + +/** + * return the size of hash table + * @param pHashObj + * @return + */ +int32_t taosHashGetSize(const SHashObj *pHashObj); + +/** + * put element into hash table, if the element with the same key exists, update it + * @param pHashObj + * @param key + * @param keyLen + * @param data + * @param size + * @return + */ +int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size); + +/** + * return the payload data with the specified key + * + * @param pHashObj + * @param key + * @param keyLen + * @return + */ +void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); + +/** + * Clone the result to destination buffer + * @param pHashObj + * @param key + * @param keyLen + * @param destBuf + * @return + */ +void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void* destBuf); + +/** + * remove item with the specified key + * @param pHashObj + * @param key + * @param keyLen + */ +int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen); + +/** + * Clear the hash table. + * @param pHashObj + */ +void taosHashClear(SHashObj *pHashObj); + +/** + * Clean up hash table and release all allocated resources. + * @param handle + */ +void taosHashCleanup(SHashObj *pHashObj); + +/** + * Get the max overflow link list length + * @param pHashObj + * @return + */ +int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj); + +/** + * Get the hash table size + * @param pHashObj + * @return + */ +size_t taosHashGetMemSize(const SHashObj *pHashObj); + +/** + * Create the hash table iterator + * @param pHashObj + * @param p + * @return + */ +void *taosHashIterate(SHashObj *pHashObj, void *p); + +/** + * Cancel the hash table iterator + * @param pHashObj + * @param p + */ +void taosHashCancelIterate(SHashObj *pHashObj, void *p); + +/** + * Get the corresponding key information for a given data in hash table + * @param pHashObj + * @param data + * @return + */ +int32_t taosHashGetKey(SHashObj *pHashObj, void *data, void** key, size_t* keyLen); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_HASH_H diff --git a/source/libs/parser/inc/astGenerator.h b/source/libs/parser/inc/astGenerator.h new file mode 100644 index 0000000000..b6d7545f3b --- /dev/null +++ b/source/libs/parser/inc/astGenerator.h @@ -0,0 +1,360 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_ASTGENERATOR_H +#define TDENGINE_ASTGENERATOR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "ttoken.h" +#include "tvariant.h" +#include "parser.h" + +#define ParseTOKENTYPE SToken + +#define NON_ARITHMEIC_EXPR 0 +#define NORMAL_ARITHMETIC 1 +#define AGG_ARIGHTMEIC 2 + +enum SQL_NODE_TYPE { + SQL_NODE_TABLE_COLUMN= 1, + SQL_NODE_SQLFUNCTION = 2, + SQL_NODE_VALUE = 3, + SQL_NODE_EXPR = 4, +}; + +enum SQL_NODE_FROM_TYPE { + SQL_NODE_FROM_SUBQUERY = 1, + SQL_NODE_FROM_TABLELIST = 2, +}; + +//enum SQL_EXPR_FLAG { +// EXPR_FLAG_TS_ERROR = 1, +// EXPR_FLAG_NS_TIMESTAMP = 2, +// EXPR_FLAG_TIMESTAMP_VAR = 3, +//}; + +extern char tTokenTypeSwitcher[13]; + +#define toTSDBType(x) \ + do { \ + if ((x) >= tListLen(tTokenTypeSwitcher)) { \ + (x) = TSDB_DATA_TYPE_BINARY; \ + } else { \ + (x) = tTokenTypeSwitcher[(x)]; \ + } \ + } while (0) + +#define TPARSER_HAS_TOKEN(_t) ((_t).n > 0) +#define TPARSER_SET_NONE_TOKEN(_t) ((_t).n = 0) + +typedef struct SListItem { + SVariant pVar; + uint8_t sortOrder; +} SListItem; + +typedef struct SIntervalVal { + int32_t token; + SToken interval; + SToken offset; +} SIntervalVal; + +typedef struct SSessionWindowVal { + SToken col; + SToken gap; +} SSessionWindowVal; + +typedef struct SWindowStateVal { + SToken col; +} SWindowStateVal; + +struct SRelationInfo; + +typedef struct SSqlNode { + struct SArray *pSelNodeList; // select clause + struct SRelationInfo *from; // from clause SArray + struct tSqlExpr *pWhere; // where clause [optional] + SArray *pGroupby; // groupby clause, only for tags[optional], SArray + SArray *pSortOrder; // orderby [optional], SArray + SArray *fillType; // fill type[optional], SArray + SIntervalVal interval; // (interval, interval_offset) [optional] + SSessionWindowVal sessionVal; // session window [optional] + SWindowStateVal windowstateVal; // window_state(col) [optional] + SToken sliding; // sliding window [optional] + SLimit limit; // limit offset [optional] + SLimit slimit; // group limit offset [optional] + SToken sqlstr; // sql string in select clause + struct tSqlExpr *pHaving; // having clause [optional] +} SSqlNode; + +typedef struct SRelElementPair { + union { + SToken tableName; + SArray *pSubquery; + }; + + SToken aliasName; +} SRelElementPair; + +typedef struct SRelationInfo { + int32_t type; // nested query|table name list + SArray *list; // SArray +} SRelationInfo; + +typedef struct SCreatedTableInfo { + SToken name; // table name token + SToken stableName; // super table name token , for using clause + SArray *pTagNames; // create by using super table, tag name + SArray *pTagVals; // create by using super table, tag value + char *fullname; // table full name + STagData tagdata; // true tag data, super table full name is in STagData + int8_t igExist; // ignore if exists +} SCreatedTableInfo; + +typedef struct SCreateTableSql { + SToken name; // table name, create table [name] xxx + int8_t type; // create normal table/from super table/ stream + bool existCheck; + + struct { + SArray *pTagColumns; // SArray + SArray *pColumns; // SArray + } colInfo; + + SArray *childTableInfo; // SArray + SSqlNode *pSelect; +} SCreateTableSql; + +typedef struct SAlterTableInfo { + SToken name; + int16_t tableType; + int16_t type; + STagData tagData; + SArray *pAddColumns; // SArray + SArray *varList; // set t=val or: change src dst, SArray +} SAlterTableInfo; + +typedef struct SCreateDbInfo { + SToken dbname; + int32_t replica; + int32_t cacheBlockSize; + int32_t maxTablesPerVnode; + int32_t numOfBlocks; + int32_t daysPerFile; + int32_t minRowsPerBlock; + int32_t maxRowsPerBlock; + int32_t fsyncPeriod; + int64_t commitTime; + int32_t walLevel; + int32_t quorum; + int32_t compressionLevel; + SToken precision; + bool ignoreExists; + int8_t update; + int8_t cachelast; + SArray *keep; + int8_t dbType; + int16_t partitions; +} SCreateDbInfo; + +typedef struct SCreateFuncInfo { + SToken name; + SToken path; + int32_t type; + int32_t bufSize; + SField output; +} SCreateFuncInfo; + +typedef struct SCreateAcctInfo { + int32_t maxUsers; + int32_t maxDbs; + int32_t maxTimeSeries; + int32_t maxStreams; + int32_t maxPointsPerSecond; + int64_t maxStorage; + int64_t maxQueryTime; + int32_t maxConnections; + SToken stat; +} SCreateAcctInfo; + +typedef struct SShowInfo { + uint8_t showType; + SToken prefix; + SToken pattern; +} SShowInfo; + +typedef struct SUserInfo { + SToken user; + SToken passwd; + SToken privilege; + int16_t type; +} SUserInfo; + +typedef struct SMiscInfo { + SArray *a; // SArray + bool existsCheck; + int16_t dbType; + int16_t tableType; + SUserInfo user; + union { + SCreateDbInfo dbOpt; + SCreateAcctInfo acctOpt; + SCreateFuncInfo funcOpt; + SShowInfo showOpt; + SToken id; + }; +} SMiscInfo; + +typedef struct SSqlInfo { + int32_t type; + bool valid; + SArray *list; // todo refactor + char msg[256]; + SArray *funcs; + union { + SCreateTableSql *pCreateTableInfo; + SAlterTableInfo *pAlterInfo; + SMiscInfo *pMiscInfo; + }; +} SSqlInfo; + +typedef struct tSqlExpr { + uint16_t type; // sql node type + uint32_t tokenId; // TK_LE: less than(binary expr) + + // The complete string of the function(col, param), and the function name is kept in exprToken + struct { + SToken operand; + struct SArray *paramList; // function parameters list + } Expr; + + SToken columnName; // table column info + SVariant value; // the use input value + SToken exprToken; // original sql expr string or function name of sql function + struct tSqlExpr *pLeft; // the left child + struct tSqlExpr *pRight; // the right child +} tSqlExpr; + +// used in select clause. select from xxx +typedef struct tSqlExprItem { + tSqlExpr *pNode; // The list of expressions + char *aliasName; // alias name, null-terminated string + bool distinct; +} tSqlExprItem; + +SArray *tListItemAppend(SArray *pList, SVariant *pVar, uint8_t sortOrder); +SArray *tListItemInsert(SArray *pList, SVariant *pVar, uint8_t sortOrder, int32_t index); +SArray *tListItemAppendToken(SArray *pList, SToken *pAliasToken, uint8_t sortOrder); + +SRelationInfo *setTableNameList(SRelationInfo *pRelationInfo, SToken *pName, SToken *pAlias); +void * destroyRelationInfo(SRelationInfo *pFromInfo); +SRelationInfo *addSubquery(SRelationInfo *pRelationInfo, SArray *pSub, SToken *pAlias); + +// sql expr leaf node +tSqlExpr *tSqlExprCreateIdValue(SToken *pToken, int32_t optrType); +tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SToken *pFuncToken, SToken *endToken, int32_t optType); +SArray * tAppendFuncName(SArray *pList, SToken *pToken); + +tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType); +tSqlExpr *tSqlExprClone(tSqlExpr *pSrc); +void tSqlExprCompact(tSqlExpr **pExpr); +bool tSqlExprIsLeaf(tSqlExpr *pExpr); +bool tSqlExprIsParentOfLeaf(tSqlExpr *pExpr); +void tSqlExprDestroy(tSqlExpr *pExpr); +SArray * tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SToken *pDistinct, SToken *pToken); +void tSqlExprListDestroy(SArray *pList); + +SSqlNode *tSetQuerySqlNode(SToken *pSelectToken, SArray *pSelNodeList, SRelationInfo *pFrom, tSqlExpr *pWhere, + SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *ps, + SWindowStateVal *pw, SToken *pSliding, SArray *pFill, SLimit *pLimit, SLimit *pgLimit, tSqlExpr *pHaving); +int32_t tSqlExprCompare(tSqlExpr *left, tSqlExpr *right); + +SCreateTableSql *tSetCreateTableInfo(SArray *pCols, SArray *pTags, SSqlNode *pSelect, int32_t type); + +SAlterTableInfo * tSetAlterTableInfo(SToken *pTableName, SArray *pCols, SArray *pVals, int32_t type, + int16_t tableTable); +SCreatedTableInfo createNewChildTableInfo(SToken *pTableName, SArray *pTagNames, SArray *pTagVals, SToken *pToken, + SToken *igExists); + +void destroyAllSqlNode(SArray *pSqlNode); +void destroySqlNode(SSqlNode *pSql); +void freeCreateTableInfo(void* p); + +SSqlInfo *setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SToken *pTableName, int32_t type); +SArray *setSubclause(SArray *pList, void *pSqlNode); +SArray *appendSelectClause(SArray *pList, void *pSubclause); + +void setCreatedTableName(SSqlInfo *pInfo, SToken *pTableNameToken, SToken *pIfNotExists); + +void SqlInfoDestroy(SSqlInfo *pInfo); + +void setDCLSqlElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); +void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SToken* pToken, SToken* existsCheck,int16_t dbType,int16_t tableType); +void setShowOptions(SSqlInfo *pInfo, int32_t type, SToken* prefix, SToken* pPatterns); + +void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SToken *pToken, SCreateDbInfo *pDB, SToken *pIgExists); + +void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SToken *pName, SToken *pPwd, SCreateAcctInfo *pAcctInfo); +void setCreateUserSql(SSqlInfo *pInfo, SToken *pName, SToken *pPasswd); +void setKillSql(SSqlInfo *pInfo, int32_t type, SToken *ip); +void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SToken *pName, SToken* pPwd, SToken *pPrivilege); + +void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam); + +void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo); +void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo); + +// prefix show db.tables; +void tSetDbName(SToken *pCpxName, SToken *pDb); + +void tSetColumnInfo(struct SField *pField, SToken *pName, struct SField *pType); +void tSetColumnType(struct SField *pField, SToken *type); + +/** + * The main parse function. + * @param yyp The parser + * @param yymajor The major token code number + * @param yyminor The value for the token + */ +void Parse(void *yyp, int yymajor, ParseTOKENTYPE yyminor, SSqlInfo *); + +/** + * Free the allocated resources in case of failure. + * @param p The parser to be deleted + * @param freeProc Function used to reclaim memory + */ +void ParseFree(void *p, void (*freeProc)(void *)); + +/** + * Allocated callback function. + * @param mallocProc The parser allocator + * @return + */ +void *ParseAlloc(void *(*mallocProc)(size_t)); + +/** + * + * @param str sql string + * @return sql ast + */ +SSqlInfo genAST(const char *str); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_ASTGENERATOR_H diff --git a/source/libs/parser/inc/insertParser.h b/source/libs/parser/inc/insertParser.h new file mode 100644 index 0000000000..49e678cd54 --- /dev/null +++ b/source/libs/parser/inc/insertParser.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_INSERTPARSER_H +#define TDENGINE_INSERTPARSER_H + +#endif // TDENGINE_INSERTPARSER_H diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h new file mode 100644 index 0000000000..4e6af0a901 --- /dev/null +++ b/source/libs/parser/inc/parserUtil.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_PARSERUTIL_H +#define TDENGINE_PARSERUTIL_H + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_PARSERUTIL_H diff --git a/source/libs/parser/inc/ttoken.h b/source/libs/parser/inc/ttoken.h new file mode 100644 index 0000000000..bacabe299e --- /dev/null +++ b/source/libs/parser/inc/ttoken.h @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TTOKEN_H +#define TDENGINE_TTOKEN_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" +#include "ttokendef.h" + +// used to denote the minimum unite in sql parsing +typedef struct SToken { + uint32_t n; + uint32_t type; + char *z; +} SToken; + +/** + * check if it is a number or not + * @param pToken + * @return + */ +#define isNumber(tk) \ +((tk)->type == TK_INTEGER || (tk)->type == TK_FLOAT || (tk)->type == TK_HEX || (tk)->type == TK_BIN) + +/** + * tokenizer for sql string + * @param z + * @param tokenType + * @return + */ +uint32_t tGetToken(char *z, uint32_t *tokenType); + +/** + * enhanced tokenizer for sql string. + * + * @param str + * @param i + * @param isPrevOptr + * @return + */ +SToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr); + +/** + * check if it is a keyword or not + * @param z + * @param len + * @return + */ +bool taosIsKeyWordToken(const char *z, int32_t len); + +/** + * check if it is a token or not + * @param pToken + * @return token type, if it is not a number, TK_ILLEGAL will return + */ +static FORCE_INLINE int32_t tGetNumericStringType(const SToken* pToken) { + const char* z = pToken->z; + int32_t type = TK_ILLEGAL; + + uint32_t i = 0; + for(; i < pToken->n; ++i) { + switch (z[i]) { + case '+': + case '-': { + break; + } + + case '.': { + /* + * handle the the float number with out integer part + * .123 + * .123e4 + */ + if (!isdigit(z[i+1])) { + return TK_ILLEGAL; + } + + for (i += 2; isdigit(z[i]); i++) { + } + + if ((z[i] == 'e' || z[i] == 'E') && + (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) { + i += 2; + while (isdigit(z[i])) { + i++; + } + } + + type = TK_FLOAT; + goto _end; + } + + case '0': { + char next = z[i + 1]; + if (next == 'b') { // bin number + type = TK_BIN; + for (i += 2; (z[i] == '0' || z[i] == '1'); ++i) { + } + + goto _end; + } else if (next == 'x') { //hex number + type = TK_HEX; + for (i += 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) { + } + + goto _end; + } + } + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + type = TK_INTEGER; + for (; isdigit(z[i]); i++) { + } + + int32_t seg = 0; + while (z[i] == '.' && isdigit(z[i + 1])) { + i += 2; + + while (isdigit(z[i])) { + i++; + } + + seg++; + type = TK_FLOAT; + } + + if (seg > 1) { + return TK_ILLEGAL; + } + + if ((z[i] == 'e' || z[i] == 'E') && + (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) { + i += 2; + while (isdigit(z[i])) { + i++; + } + + type = TK_FLOAT; + } + + goto _end; + } + default: + return TK_ILLEGAL; + } + } + + _end: + return (i < pToken->n)? TK_ILLEGAL:type; +} + +void taosCleanupKeywordsTable(); + +SToken tscReplaceStrToken(char **str, SToken *token, const char* newToken); + +SToken taosTokenDup(SToken* pToken, char* buf, int32_t len); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TTOKEN_H diff --git a/source/libs/parser/inc/ttokendef.h b/source/libs/parser/inc/ttokendef.h new file mode 100644 index 0000000000..9b0ad2cf13 --- /dev/null +++ b/source/libs/parser/inc/ttokendef.h @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TTOKENDEF_H +#define TDENGINE_TTOKENDEF_H + +#define TK_ID 1 +#define TK_BOOL 2 +#define TK_TINYINT 3 +#define TK_SMALLINT 4 +#define TK_INTEGER 5 +#define TK_BIGINT 6 +#define TK_FLOAT 7 +#define TK_DOUBLE 8 +#define TK_STRING 9 +#define TK_TIMESTAMP 10 +#define TK_BINARY 11 +#define TK_NCHAR 12 +#define TK_OR 13 +#define TK_AND 14 +#define TK_NOT 15 +#define TK_EQ 16 +#define TK_NE 17 +#define TK_ISNULL 18 +#define TK_NOTNULL 19 +#define TK_IS 20 +#define TK_LIKE 21 +#define TK_MATCH 22 +#define TK_NMATCH 23 +#define TK_GLOB 24 +#define TK_BETWEEN 25 +#define TK_IN 26 +#define TK_GT 27 +#define TK_GE 28 +#define TK_LT 29 +#define TK_LE 30 +#define TK_BITAND 31 +#define TK_BITOR 32 +#define TK_LSHIFT 33 +#define TK_RSHIFT 34 +#define TK_PLUS 35 +#define TK_MINUS 36 +#define TK_DIVIDE 37 +#define TK_TIMES 38 +#define TK_STAR 39 +#define TK_SLASH 40 +#define TK_REM 41 +#define TK_CONCAT 42 +#define TK_UMINUS 43 +#define TK_UPLUS 44 +#define TK_BITNOT 45 +#define TK_SHOW 46 +#define TK_DATABASES 47 +#define TK_TOPICS 48 +#define TK_FUNCTIONS 49 +#define TK_MNODES 50 +#define TK_DNODES 51 +#define TK_ACCOUNTS 52 +#define TK_USERS 53 +#define TK_MODULES 54 +#define TK_QUERIES 55 +#define TK_CONNECTIONS 56 +#define TK_STREAMS 57 +#define TK_VARIABLES 58 +#define TK_SCORES 59 +#define TK_GRANTS 60 +#define TK_VNODES 61 +#define TK_DOT 62 +#define TK_CREATE 63 +#define TK_TABLE 64 +#define TK_STABLE 65 +#define TK_DATABASE 66 +#define TK_TABLES 67 +#define TK_STABLES 68 +#define TK_VGROUPS 69 +#define TK_DROP 70 +#define TK_TOPIC 71 +#define TK_FUNCTION 72 +#define TK_DNODE 73 +#define TK_USER 74 +#define TK_ACCOUNT 75 +#define TK_USE 76 +#define TK_DESCRIBE 77 +#define TK_DESC 78 +#define TK_ALTER 79 +#define TK_PASS 80 +#define TK_PRIVILEGE 81 +#define TK_LOCAL 82 +#define TK_COMPACT 83 +#define TK_LP 84 +#define TK_RP 85 +#define TK_IF 86 +#define TK_EXISTS 87 +#define TK_AS 88 +#define TK_OUTPUTTYPE 89 +#define TK_AGGREGATE 90 +#define TK_BUFSIZE 91 +#define TK_PPS 92 +#define TK_TSERIES 93 +#define TK_DBS 94 +#define TK_STORAGE 95 +#define TK_QTIME 96 +#define TK_CONNS 97 +#define TK_STATE 98 +#define TK_COMMA 99 +#define TK_KEEP 100 +#define TK_CACHE 101 +#define TK_REPLICA 102 +#define TK_QUORUM 103 +#define TK_DAYS 104 +#define TK_MINROWS 105 +#define TK_MAXROWS 106 +#define TK_BLOCKS 107 +#define TK_CTIME 108 +#define TK_WAL 109 +#define TK_FSYNC 110 +#define TK_COMP 111 +#define TK_PRECISION 112 +#define TK_UPDATE 113 +#define TK_CACHELAST 114 +#define TK_PARTITIONS 115 +#define TK_UNSIGNED 116 +#define TK_TAGS 117 +#define TK_USING 118 +#define TK_NULL 119 +#define TK_NOW 120 +#define TK_SELECT 121 +#define TK_UNION 122 +#define TK_ALL 123 +#define TK_DISTINCT 124 +#define TK_FROM 125 +#define TK_VARIABLE 126 +#define TK_INTERVAL 127 +#define TK_EVERY 128 +#define TK_SESSION 129 +#define TK_STATE_WINDOW 130 +#define TK_FILL 131 +#define TK_SLIDING 132 +#define TK_ORDER 133 +#define TK_BY 134 +#define TK_ASC 135 +#define TK_GROUP 136 +#define TK_HAVING 137 +#define TK_LIMIT 138 +#define TK_OFFSET 139 +#define TK_SLIMIT 140 +#define TK_SOFFSET 141 +#define TK_WHERE 142 +#define TK_RESET 143 +#define TK_QUERY 144 +#define TK_SYNCDB 145 +#define TK_ADD 146 +#define TK_COLUMN 147 +#define TK_MODIFY 148 +#define TK_TAG 149 +#define TK_CHANGE 150 +#define TK_SET 151 +#define TK_KILL 152 +#define TK_CONNECTION 153 +#define TK_STREAM 154 +#define TK_COLON 155 +#define TK_ABORT 156 +#define TK_AFTER 157 +#define TK_ATTACH 158 +#define TK_BEFORE 159 +#define TK_BEGIN 160 +#define TK_CASCADE 161 +#define TK_CLUSTER 162 +#define TK_CONFLICT 163 +#define TK_COPY 164 +#define TK_DEFERRED 165 +#define TK_DELIMITERS 166 +#define TK_DETACH 167 +#define TK_EACH 168 +#define TK_END 169 +#define TK_EXPLAIN 170 +#define TK_FAIL 171 +#define TK_FOR 172 +#define TK_IGNORE 173 +#define TK_IMMEDIATE 174 +#define TK_INITIALLY 175 +#define TK_INSTEAD 176 +#define TK_KEY 177 +#define TK_OF 178 +#define TK_RAISE 179 +#define TK_REPLACE 180 +#define TK_RESTRICT 181 +#define TK_ROW 182 +#define TK_STATEMENT 183 +#define TK_TRIGGER 184 +#define TK_VIEW 185 +#define TK_IPTOKEN 186 +#define TK_SEMI 187 +#define TK_NONE 188 +#define TK_PREV 189 +#define TK_LINEAR 190 +#define TK_IMPORT 191 +#define TK_TBNAME 192 +#define TK_JOIN 193 +#define TK_INSERT 194 +#define TK_INTO 195 +#define TK_VALUES 196 + + +#define TK_SPACE 300 +#define TK_COMMENT 301 +#define TK_ILLEGAL 302 +#define TK_HEX 303 // hex number 0x123 +#define TK_OCT 304 // oct number +#define TK_BIN 305 // bin format data 0b111 +#define TK_FILE 306 +#define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query + +#endif + + diff --git a/source/libs/parser/inc/tvariant.h b/source/libs/parser/inc/tvariant.h new file mode 100644 index 0000000000..941d1920cb --- /dev/null +++ b/source/libs/parser/inc/tvariant.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TVARIANT_H +#define TDENGINE_TVARIANT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tarray.h" +#include "ttoken.h" + +// variant, each number/string/field_id has a corresponding struct during parsing sql +typedef struct SVariant { + uint32_t nType; + int32_t nLen; // only used for string, for number, it is useless + union { + int64_t i64; + uint64_t u64; + double d; + char *pz; + wchar_t *wpz; + SArray *arr; // only for 'in' query to hold value list, not value for a field + }; +} SVariant; + +bool taosVariantIsValid(SVariant *pVar); + +void taosVariantCreate(SVariant *pVar, SStrToken *token); + +void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type); + +void taosVariantDestroy(SVariant *pV); + +void taosVariantAssign(SVariant *pDst, const SVariant *pSrc); + +int32_t taosVariantCompare(const SVariant* p1, const SVariant* p2); + +int32_t taosVariantToString(SVariant *pVar, char *dst); + +int32_t taosVariantDump(SVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix); + +#if 0 +int32_t taosVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, char *extInfo); +#endif + +int32_t taosVariantTypeSetType(SVariant *pVariant, char type); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TVARIANT_H diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c new file mode 100644 index 0000000000..74642ea811 --- /dev/null +++ b/source/libs/parser/src/astGenerator.c @@ -0,0 +1,507 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taos.h" +#include "astGenerator.h" + +int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned) { + errno = 0; + int32_t ret = 0; + + char* endPtr = NULL; + if (type == TK_FLOAT) { + double v = strtod(z, &endPtr); + if ((errno == ERANGE && v == HUGE_VALF) || isinf(v) || isnan(v)) { + ret = -1; + } else if ((issigned && (v < INT64_MIN || v > INT64_MAX)) || ((!issigned) && (v < 0 || v > UINT64_MAX))) { + ret = -1; + } else { + *value = (int64_t) round(v); + } + + errno = 0; + return ret; + } + + int32_t radix = 10; + if (type == TK_HEX) { + radix = 16; + } else if (type == TK_BIN) { + radix = 2; + } + + // the string may be overflow according to errno + if (!issigned) { + const char *p = z; + while(*p != 0 && *p == ' ') p++; + if (*p != 0 && *p == '-') { return -1;} + + *value = strtoull(z, &endPtr, radix); + } else { + *value = strtoll(z, &endPtr, radix); + } + + // not a valid integer number, return error + if (endPtr - z != n || errno == ERANGE) { + ret = -1; + } + + errno = 0; + return ret; +} + +SArray *tListItemAppend(SArray *pList, SVariant *pVar, uint8_t sortOrder) { + if (pList == NULL) { + pList = taosArrayInit(4, sizeof(SListItem)); + } + + if (pVar == NULL) { + return pList; + } + + /* + * Here we do not employ the assign function, since we need the pz attribute of structure , which is the point to char string. + * Otherwise, the original pointer may be lost, which causes memory leak. + */ + SListItem item; + item.pVar = *pVar; + item.sortOrder = sortOrder; + + taosArrayPush(pList, &item); + return pList; +} + +SArray *tListItemInsert(SArray *pList, SVariant *pVar, uint8_t sortOrder, int32_t index) { + if (pList == NULL || pVar == NULL || index >= taosArrayGetSize(pList)) { + return tListItemAppend(pList, pVar, sortOrder); + } + + SListItem item; + item.pVar = *pVar; + item.sortOrder = sortOrder; + + taosArrayInsert(pList, index, &item); + return pList; +} + +SArray *tListItemAppendToken(SArray *pList, SToken *pAliasToken, uint8_t sortOrder) { + if (pList == NULL) { + pList = taosArrayInit(4, sizeof(SListItem)); + } + + if (pAliasToken) { + SListItem item; + taosVariantCreate(&item.pVar, pAliasToken); + item.sortOrder = sortOrder; + + taosArrayPush(pList, &item); + } + + return pList; +} + +SRelationInfo *setTableNameList(SRelationInfo *pRelationInfo, SToken *pName, SToken *pAlias) { + if (pRelationInfo == NULL) { + pRelationInfo = calloc(1, sizeof(SRelationInfo)); + pRelationInfo->list = taosArrayInit(4, sizeof(SRelElementPair)); + } + + pRelationInfo->type = SQL_NODE_FROM_TABLELIST; + SRelElementPair p = {.tableName = *pName}; + if (pAlias != NULL) { + p.aliasName = *pAlias; + } else { + TPARSER_SET_NONE_TOKEN(p.aliasName); + } + + taosArrayPush(pRelationInfo->list, &p); + return pRelationInfo; +} + +void *destroyRelationInfo(SRelationInfo *pRelationInfo) { + if (pRelationInfo == NULL) { + return NULL; + } + + if (pRelationInfo->type == SQL_NODE_FROM_TABLELIST) { + taosArrayDestroy(pRelationInfo->list); + } else { + size_t size = taosArrayGetSize(pRelationInfo->list); + for(int32_t i = 0; i < size; ++i) { + SArray* pa = taosArrayGetP(pRelationInfo->list, i); + destroyAllSqlNode(pa); + } + taosArrayDestroy(pRelationInfo->list); + } + + tfree(pRelationInfo); + return NULL; +} + +SRelationInfo *addSubquery(SRelationInfo *pRelationInfo, SArray *pSub, SToken *pAlias) { + if (pRelationInfo == NULL) { + pRelationInfo = calloc(1, sizeof(SRelationInfo)); + pRelationInfo->list = taosArrayInit(4, sizeof(SRelElementPair)); + } + + pRelationInfo->type = SQL_NODE_FROM_SUBQUERY; + + SRelElementPair p = {.pSubquery = pSub}; + if (pAlias != NULL) { + p.aliasName = *pAlias; + } else { + TPARSER_SET_NONE_TOKEN(p.aliasName); + } + + taosArrayPush(pRelationInfo->list, &p); + return pRelationInfo; +} + +// sql expr leaf node +// todo Evalute the value during the validation process of AST. +tSqlExpr *tSqlExprCreateIdValue(SToken *pToken, int32_t optrType) { + tSqlExpr *pSqlExpr = calloc(1, sizeof(tSqlExpr)); + + if (pToken != NULL) { + pSqlExpr->exprToken = *pToken; + } + + if (optrType == TK_NULL) { +// if (pToken) { +// pToken->type = TSDB_DATA_TYPE_NULL; +// tVariantCreate(&pSqlExpr->value, pToken); +// } + pSqlExpr->tokenId = optrType; + pSqlExpr->type = SQL_NODE_VALUE; + } else if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { +// if (pToken) { +// toTSDBType(pToken->type); +// tVariantCreate(&pSqlExpr->value, pToken); +// } + pSqlExpr->tokenId = optrType; + pSqlExpr->type = SQL_NODE_VALUE; + } else if (optrType == TK_NOW) { + // use nanosecond by default TODO set value after getting database precision +// pSqlExpr->value.i64 = taosGetTimestamp(TSDB_TIME_PRECISION_NANO); +// pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT; + pSqlExpr->tokenId = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond + pSqlExpr->type = SQL_NODE_VALUE; +// pSqlExpr->flags |= 1 << EXPR_FLAG_NS_TIMESTAMP; + } else if (optrType == TK_VARIABLE) { + // use nanosecond by default + // TODO set value after getting database precision +// if (pToken) { +// char unit = 0; +// int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->value.i64, &unit, TSDB_TIME_PRECISION_NANO); +// if (ret != TSDB_CODE_SUCCESS) { +// terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; +// } +// } + +// pSqlExpr->flags |= 1 << EXPR_FLAG_NS_TIMESTAMP; +// pSqlExpr->flags |= 1 << EXPR_FLAG_TIMESTAMP_VAR; +// pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT; + pSqlExpr->tokenId = TK_TIMESTAMP; + pSqlExpr->type = SQL_NODE_VALUE; + } else { + // Here it must be the column name (tk_id) if it is not a number or string. + assert(optrType == TK_ID || optrType == TK_ALL); + if (pToken != NULL) { + pSqlExpr->columnName = *pToken; + } + + pSqlExpr->tokenId = optrType; + pSqlExpr->type = SQL_NODE_TABLE_COLUMN; + } + + return pSqlExpr; +} + +tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SToken *pFuncToken, SToken *endToken, int32_t optType) { + if (pFuncToken == NULL) { + return NULL; + } + + tSqlExpr *pExpr = calloc(1, sizeof(tSqlExpr)); + pExpr->tokenId = optType; + pExpr->type = SQL_NODE_SQLFUNCTION; + pExpr->Expr.paramList = pParam; + + int32_t len = (int32_t)((endToken->z + endToken->n) - pFuncToken->z); + pExpr->Expr.operand = (*pFuncToken); + + pExpr->exprToken.n = len; + pExpr->exprToken.z = pFuncToken->z; + pExpr->exprToken.type = pFuncToken->type; + + return pExpr; +} + +SArray *tAppendFuncName(SArray *pList, SToken *pToken) { + assert(pList != NULL && pToken != NULL); + taosArrayPush(pList, pToken); + return pList; +} + +tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) { + tSqlExpr *pExpr = calloc(1, sizeof(tSqlExpr)); + pExpr->type = SQL_NODE_EXPR; + + if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) { + char* endPos = pRight->exprToken.z + pRight->exprToken.n; + pExpr->exprToken.z = pLeft->exprToken.z; + pExpr->exprToken.n = (uint32_t)(endPos - pExpr->exprToken.z); + pExpr->exprToken.type = pLeft->exprToken.type; + } + + if ((pLeft != NULL && pRight != NULL) && + (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM)) { + /* + * if a exprToken is noted as the TK_TIMESTAMP, the time precision is microsecond + * Otherwise, the time precision is adaptive, determined by the time precision from databases. + */ + if ((pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_INTEGER) || + (pLeft->tokenId == TK_TIMESTAMP && pRight->tokenId == TK_TIMESTAMP)) { + pExpr->value.nType = TSDB_DATA_TYPE_BIGINT; + pExpr->tokenId = pLeft->tokenId; + pExpr->type = SQL_NODE_VALUE; + + switch (optrType) { + case TK_PLUS: { + pExpr->value.i64 = pLeft->value.i64 + pRight->value.i64; + break; + } + + case TK_MINUS: { + pExpr->value.i64 = pLeft->value.i64 - pRight->value.i64; + break; + } + case TK_STAR: { + pExpr->value.i64 = pLeft->value.i64 * pRight->value.i64; + break; + } + case TK_DIVIDE: { + pExpr->tokenId = TK_FLOAT; + pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE; + pExpr->value.d = (double)pLeft->value.i64 / pRight->value.i64; + break; + } + case TK_REM: { + pExpr->value.i64 = pLeft->value.i64 % pRight->value.i64; + break; + } + } + + tSqlExprDestroy(pLeft); + tSqlExprDestroy(pRight); + } else if ((pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_INTEGER) || + (pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_FLOAT) || + (pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_FLOAT)) { + pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE; + pExpr->tokenId = TK_FLOAT; + pExpr->type = SQL_NODE_VALUE; + + double left = (pLeft->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->value.d : pLeft->value.i64; + double right = (pRight->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->value.d : pRight->value.i64; + + switch (optrType) { + case TK_PLUS: { + pExpr->value.d = left + right; + break; + } + case TK_MINUS: { + pExpr->value.d = left - right; + break; + } + case TK_STAR: { + pExpr->value.d = left * right; + break; + } + case TK_DIVIDE: { + pExpr->value.d = left / right; + break; + } + case TK_REM: { + pExpr->value.d = left - ((int64_t)(left / right)) * right; + break; + } + } + + tSqlExprDestroy(pLeft); + tSqlExprDestroy(pRight); + + } else { + pExpr->tokenId = optrType; + pExpr->pLeft = pLeft; + pExpr->pRight = pRight; + } + } else if (optrType == TK_IN) { + pExpr->tokenId = optrType; + pExpr->pLeft = pLeft; + + tSqlExpr *pRSub = calloc(1, sizeof(tSqlExpr)); + pRSub->tokenId = TK_SET; // TODO refactor ..... + pRSub->Expr.paramList = (SArray *)pRight; + + pExpr->pRight = pRSub; + } else { + pExpr->tokenId = optrType; + pExpr->pLeft = pLeft; + + if (pLeft != NULL && pRight == NULL) { + pRight = calloc(1, sizeof(tSqlExpr)); + } + + pExpr->pRight = pRight; + } + + return pExpr; +} + +tSqlExpr *tSqlExprClone(tSqlExpr *pSrc); +void tSqlExprCompact(tSqlExpr **pExpr); +bool tSqlExprIsLeaf(tSqlExpr *pExpr); +bool tSqlExprIsParentOfLeaf(tSqlExpr *pExpr); +void tSqlExprDestroy(tSqlExpr *pExpr); +SArray * tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SToken *pDistinct, SToken *pToken); +void tSqlExprListDestroy(SArray *pList); + +SSqlNode *tSetQuerySqlNode(SToken *pSelectToken, SArray *pSelNodeList, SRelationInfo *pFrom, tSqlExpr *pWhere, + SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *ps, + SWindowStateVal *pw, SToken *pSliding, SArray *pFill, SLimit *pLimit, SLimit *pgLimit, tSqlExpr *pHaving); +int32_t tSqlExprCompare(tSqlExpr *left, tSqlExpr *right); + +SCreateTableSql *tSetCreateTableInfo(SArray *pCols, SArray *pTags, SSqlNode *pSelect, int32_t type); + +SAlterTableInfo * tSetAlterTableInfo(SToken *pTableName, SArray *pCols, SArray *pVals, int32_t type, + int16_t tableTable); +SCreatedTableInfo createNewChildTableInfo(SToken *pTableName, SArray *pTagNames, SArray *pTagVals, SToken *pToken, + SToken *igExists); + +void destroyAllSqlNode(SArray *pSqlNode); +void destroySqlNode(SSqlNode *pSql); +void freeCreateTableInfo(void* p); + +SSqlInfo *setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SToken *pTableName, int32_t type); +SArray *setSubclause(SArray *pList, void *pSqlNode); +SArray *appendSelectClause(SArray *pList, void *pSubclause); + +void setCreatedTableName(SSqlInfo *pInfo, SToken *pTableNameToken, SToken *pIfNotExists); + +void SqlInfoDestroy(SSqlInfo *pInfo); + +void setDCLSqlElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); +void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SToken* pToken, SToken* existsCheck,int16_t dbType,int16_t tableType); +void setShowOptions(SSqlInfo *pInfo, int32_t type, SToken* prefix, SToken* pPatterns); + +void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SToken *pToken, SCreateDbInfo *pDB, SToken *pIgExists); + +void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SToken *pName, SToken *pPwd, SCreateAcctInfo *pAcctInfo); +void setCreateUserSql(SSqlInfo *pInfo, SToken *pName, SToken *pPasswd); +void setKillSql(SSqlInfo *pInfo, int32_t type, SToken *ip); +void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SToken *pName, SToken* pPwd, SToken *pPrivilege); + +void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam); + +void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo); +void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo); + +// prefix show db.tables; +void tSetDbName(SToken *pCpxName, SToken *pDb); + +void tSetColumnInfo(struct SField *pField, SToken *pName, struct SField *pType); +void tSetColumnType(struct SField *pField, SToken *type); + +/** + * + * @param yyp The parser + * @param yymajor The major token code number + * @param yyminor The value for the token + */ +void Parse(void *yyp, int yymajor, ParseTOKENTYPE yyminor, SSqlInfo *); + +/** + * + * @param p The parser to be deleted + * @param freeProc Function used to reclaim memory + */ +void ParseFree(void *p, void (*freeProc)(void *)); + +/** + * + * @param mallocProc The parser allocator + * @return + */ +void *ParseAlloc(void *(*mallocProc)(size_t)); + +SSqlInfo genAST(const char *pStr) { + void *pParser = ParseAlloc(malloc); + + SSqlInfo sqlInfo = {0}; + + sqlInfo.valid = true; + sqlInfo.funcs = taosArrayInit(4, sizeof(SToken)); + + int32_t i = 0; + while (1) { + SToken t0 = {0}; + + if (pStr[i] == 0) { + Parse(pParser, 0, t0, &sqlInfo); + goto abort_parse; + } + + t0.n = tGetToken((char *)&pStr[i], &t0.type); + t0.z = (char *)(pStr + i); + i += t0.n; + + switch (t0.type) { + case TK_SPACE: + case TK_COMMENT: { + break; + } + case TK_SEMI: { + Parse(pParser, 0, t0, &sqlInfo); + goto abort_parse; + } + + case TK_QUESTION: + case TK_ILLEGAL: { + snprintf(sqlInfo.msg, tListLen(sqlInfo.msg), "unrecognized token: \"%s\"", t0.z); + sqlInfo.valid = false; + goto abort_parse; + } + + case TK_HEX: + case TK_OCT: + case TK_BIN: { + snprintf(sqlInfo.msg, tListLen(sqlInfo.msg), "unsupported token: \"%s\"", t0.z); + sqlInfo.valid = false; + goto abort_parse; + } + + default: + Parse(pParser, t0.type, t0, &sqlInfo); + if (sqlInfo.valid == false) { + goto abort_parse; + } + } + } + +abort_parse: + ParseFree(pParser, free); + return sqlInfo; +} diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c new file mode 100644 index 0000000000..7718dc0758 --- /dev/null +++ b/source/libs/parser/src/astValidate.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +#include "parserInt.h" + +int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msg) { + //1. if it is a query, get the meta info and continue. + + + +// qParserExtractRequestedMetaInfo(pInfo->) + return 0; +} -- GitLab