tscUtil.h 13.5 KB
Newer Older
H
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef TDENGINE_TSCUTIL_H
#define TDENGINE_TSCUTIL_H

#ifdef __cplusplus
extern "C" {
#endif

23
#include "exception.h"
H
Haojun Liao 已提交
24 25
#include "os.h"
#include "qExtbuffer.h"
26
#include "taosdef.h"
H
Haojun Liao 已提交
27
#include "tbuffer.h"
28
#include "tscLocalMerge.h"
S
slguan 已提交
29
#include "tsclient.h"
H
hzcheng 已提交
30

weixin_48148422's avatar
weixin_48148422 已提交
31
#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo)  \
H
hjxilinx 已提交
32
  (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE))
33
#define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \
H
hjxilinx 已提交
34
  (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE))
H
hjxilinx 已提交
35
  
weixin_48148422's avatar
weixin_48148422 已提交
36 37
#define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo)\
  (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo)))
H
hzcheng 已提交
38

39 40 41 42
#pragma pack(push,1)
// this struct is transfered as binary, padding two bytes to avoid
// an 'uid' whose low bytes is 0xff being recoginized as NULL,
// and set 'pack' to 1 to avoid break existing code.
43
typedef struct STidTags {
44
  int16_t  padding;
45 46 47 48 49
  int64_t  uid;
  int32_t  tid;
  int32_t  vgId;
  char     tag[];
} STidTags;
50
#pragma pack(pop)
51 52

typedef struct SJoinSupporter {
S
slguan 已提交
53 54
  SSqlObj*        pObj;           // parent SqlObj
  int32_t         subqueryIndex;  // index of sub query
55
  SInterval       interval;
S
slguan 已提交
56
  SLimitVal       limit;          // limit info
H
Haojun Liao 已提交
57
  uint64_t        uid;            // query table uid
H
hjxilinx 已提交
58 59
  SArray*         colList;        // previous query information, no need to use this attribute, and the corresponding attribution
  SArray*         exprList;
S
slguan 已提交
60 61
  SFieldInfo      fieldsInfo;
  STagCond        tagCond;
H
Haojun Liao 已提交
62
  SSqlGroupbyExpr groupInfo;       // group by info
H
hjxilinx 已提交
63 64
  struct STSBuf*  pTSBuf;          // the TSBuf struct that holds the compressed timestamp array
  FILE*           f;               // temporary file in order to create TSBuf
65 66 67 68 69
  char            path[PATH_MAX];  // temporary file path, todo dynamic allocate memory
  int32_t         tagSize;         // the length of each in the first filter stage
  char*           pIdTagList;      // result of first stage tags
  int32_t         totalLen;
  int32_t         num;
H
Haojun Liao 已提交
70
  SArray*         pVgroupTables;
71 72
} SJoinSupporter;

D
dapan1121 已提交
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87

typedef struct SMergeCtx {
  SJoinSupporter* p;
  int32_t         idx;
  SArray*         res;
  int8_t          compared;
}SMergeCtx;

typedef struct SMergeTsCtx {
  SJoinSupporter* p;
  STSBuf*         res;
  int64_t         numOfInput;
  int8_t          compared;
}SMergeTsCtx;

88
typedef struct SVgroupTableInfo {
S
TD-1732  
Shengliang Guan 已提交
89
  SVgroupInfo vgInfo;
H
Haojun Liao 已提交
90
  SArray     *itemList;   // SArray<STableIdInfo>
91
} SVgroupTableInfo;
S
slguan 已提交
92

H
Haojun Liao 已提交
93
static FORCE_INLINE SQueryInfo* tscGetQueryInfo(SSqlCmd* pCmd, int32_t subClauseIndex) {
94
  assert(pCmd != NULL && subClauseIndex >= 0);
H
Haojun Liao 已提交
95 96 97 98 99 100 101
  if (pCmd->pQueryInfo == NULL || subClauseIndex >= pCmd->numOfClause) {
    return NULL;
  }

  return pCmd->pQueryInfo[subClauseIndex];
}

H
Haojun Liao 已提交
102
SQueryInfo* tscGetActiveQueryInfo(SSqlCmd* pCmd);
H
Haojun Liao 已提交
103

104
int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks);
105
void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta);
H
hjxilinx 已提交
106
void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf);
H
hjxilinx 已提交
107

H
Haojun Liao 已提交
108 109
void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo);

110
SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes,
H
hjxilinx 已提交
111 112
                                   uint32_t offset);

113
void*   tscDestroyBlockArrayList(SArray* pDataBlockList);
114
void*   tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta);
H
Haojun Liao 已提交
115

H
hjxilinx 已提交
116
int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock);
H
Haojun Liao 已提交
117
int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap);
118
int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, SName* pName, STableMeta* pTableMeta,
H
Haojun Liao 已提交
119
                                STableDataBlocks** dataBlocks, SArray* pBlockList);
H
hjxilinx 已提交
120

H
hzcheng 已提交
121 122
/**
 * for the projection query on metric or point interpolation query on metric,
S
slguan 已提交
123
 * we iterate all the meters, instead of invoke query on all qualified meters simultaneously.
H
hzcheng 已提交
124 125 126 127
 *
 * @param pSql  sql object
 * @return
 */
128 129
bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo);
bool tscIsTWAQuery(SQueryInfo* pQueryInfo);
H
Haojun Liao 已提交
130
bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo);
131
bool tscGroupbyColumn(SQueryInfo* pQueryInfo);
H
Haojun Liao 已提交
132 133
bool tscIsTopBotQuery(SQueryInfo* pQueryInfo);
bool hasTagValOutput(SQueryInfo* pQueryInfo);
H
Haojun Liao 已提交
134
bool timeWindowInterpoRequired(SQueryInfo *pQueryInfo);
H
Haojun Liao 已提交
135
bool isStabledev(SQueryInfo* pQueryInfo);
H
Haojun Liao 已提交
136 137
bool isTsCompQuery(SQueryInfo* pQueryInfo);
bool isSimpleAggregate(SQueryInfo* pQueryInfo);
H
Haojun Liao 已提交
138
bool isBlockDistQuery(SQueryInfo* pQueryInfo);
139
int32_t tscGetTopbotQueryParam(SQueryInfo* pQueryInfo);
140

141
bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo *pQueryInfo, int32_t tableIndex);
142 143 144
bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex);
bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex);

H
Haojun Liao 已提交
145
bool tscIsProjectionQuery(SQueryInfo* pQueryInfo);
H
hjxilinx 已提交
146

H
hjxilinx 已提交
147
bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex);
H
hjxilinx 已提交
148
bool tscQueryTags(SQueryInfo* pQueryInfo);
149
bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t tableIndex);
H
Haojun Liao 已提交
150
bool tscQueryBlockInfo(SQueryInfo* pQueryInfo);
S
slguan 已提交
151

H
Haojun Liao 已提交
152
SExprInfo* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId,
H
Haojun Liao 已提交
153
                                       SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType);
H
hzcheng 已提交
154

H
Haojun Liao 已提交
155
int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql);
156
void    tscClearInterpInfo(SQueryInfo* pQueryInfo);
H
hzcheng 已提交
157

H
hjxilinx 已提交
158
bool tscIsInsertData(char* sqlstr);
H
hzcheng 已提交
159

S
slguan 已提交
160
int tscAllocPayload(SSqlCmd* pCmd, int size);
H
hzcheng 已提交
161

H
hjxilinx 已提交
162 163
TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes);

H
Haojun Liao 已提交
164 165
SInternalField* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField);
SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field);
H
hjxilinx 已提交
166

H
Haojun Liao 已提交
167
SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index);
H
hjxilinx 已提交
168
TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index);
H
hzcheng 已提交
169

H
Haojun Liao 已提交
170
void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo);
H
hzcheng 已提交
171

H
hjxilinx 已提交
172 173
int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index);
void    tscFieldInfoClear(SFieldInfo* pFieldInfo);
H
Haojun Liao 已提交
174 175 176

static FORCE_INLINE int32_t tscNumOfFields(SQueryInfo* pQueryInfo) { return pQueryInfo->fieldsInfo.numOfOutput; }

D
fix bug  
dapan1121 已提交
177 178
int32_t tscFieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2, int32_t *diffSize);
int32_t tscFieldInfoSetSize(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2);
H
hzcheng 已提交
179

H
Haojun Liao 已提交
180
void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes);
H
hzcheng 已提交
181

H
hjxilinx 已提交
182
int32_t   tscGetResRowLength(SArray* pExprList);
H
hjxilinx 已提交
183

H
Haojun Liao 已提交
184
SExprInfo* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
H
Haojun Liao 已提交
185
    int16_t size, int16_t resColId, int16_t interSize, bool isTagCol);
H
hjxilinx 已提交
186

H
Haojun Liao 已提交
187
SExprInfo* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
H
Haojun Liao 已提交
188
                           int16_t size, int16_t resColId, int16_t interSize, bool isTagCol);
H
hjxilinx 已提交
189

H
Haojun Liao 已提交
190
SExprInfo* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type,
H
hzcheng 已提交
191
                           int16_t size);
192
size_t   tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo);
H
Haojun Liao 已提交
193
void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, uint64_t uid);
H
hzcheng 已提交
194

H
Haojun Liao 已提交
195
SExprInfo* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index);
H
Haojun Liao 已提交
196
int32_t   tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy);
H
Haojun Liao 已提交
197
void      tscSqlExprAssign(SExprInfo* dst, const SExprInfo* src);
H
hjxilinx 已提交
198
void      tscSqlExprInfoDestroy(SArray* pExprInfo);
S
slguan 已提交
199

200
SColumn* tscColumnClone(const SColumn* src);
H
Haojun Liao 已提交
201 202
bool tscColumnExists(SArray* pColumnList, int32_t columnIndex, uint64_t uid);
SColumn* tscColumnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema);
203
void tscColumnListDestroy(SArray* pColList);
H
hzcheng 已提交
204

H
Haojun Liao 已提交
205 206
void convertQueryResult(SSqlRes* pRes, SQueryInfo* pQueryInfo);

B
Bomin Zhang 已提交
207
void tscDequoteAndTrimToken(SStrToken* pToken);
H
Haojun Liao 已提交
208
int32_t tscValidateName(SStrToken* pToken);
H
hzcheng 已提交
209 210 211

void tscIncStreamExecutionCount(void* pStream);

212
bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams);
H
hzcheng 已提交
213

S
slguan 已提交
214
// get starter position of metric query condition (query on tags) in SSqlCmd.payload
215
SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid);
216
void   tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw);
S
slguan 已提交
217

H
Haojun Liao 已提交
218
int32_t tscTagCondCopy(STagCond* dest, const STagCond* src);
H
hzcheng 已提交
219 220
void tscTagCondRelease(STagCond* pCond);

221
void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo);
H
hzcheng 已提交
222

H
hjxilinx 已提交
223
bool tscShouldBeFreed(SSqlObj* pSql);
S
slguan 已提交
224

225
STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex, int32_t tableIndex);
H
hjxilinx 已提交
226
STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex);
227

H
Haojun Liao 已提交
228
void        tscInitQueryInfo(SQueryInfo* pQueryInfo);
H
Haojun Liao 已提交
229 230
void        tscClearSubqueryInfo(SSqlCmd* pCmd);
int32_t     tscAddQueryInfo(SSqlCmd *pCmd);
H
Haojun Liao 已提交
231 232
SQueryInfo *tscGetQueryInfo(SSqlCmd* pCmd, int32_t subClauseIndex);
SQueryInfo *tscGetQueryInfoS(SSqlCmd *pCmd, int32_t subClauseIndex);
233

234
void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo);
S
slguan 已提交
235

236 237
STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableMeta* pTableMeta,
                                    SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables);
238

H
hjxilinx 已提交
239
STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo);
H
hjxilinx 已提交
240

241
void tscFreeVgroupTableInfo(SArray* pVgroupTables);
H
Haojun Liao 已提交
242
SArray* tscVgroupTableInfoDup(SArray* pVgroupTables);
H
Haojun Liao 已提交
243
void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index);
244
void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo);
S
slguan 已提交
245

H
hjxilinx 已提交
246
int  tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex);
H
hjxilinx 已提交
247
int  tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo);
248
int  tscGetTableMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists);
S
slguan 已提交
249 250

void tscResetForNextRetrieve(SSqlRes* pRes);
H
hzcheng 已提交
251
void tscDoQuery(SSqlObj* pSql);
H
Haojun Liao 已提交
252 253
void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo);
void doExecuteQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo);
H
hzcheng 已提交
254

H
Haojun Liao 已提交
255 256
SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *pInfo);
void* tscVgroupInfoClear(SVgroupsInfo *pInfo);
S
TD-1732  
Shengliang Guan 已提交
257
void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src);
S
slguan 已提交
258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
/**
 * The create object function must be successful expect for the out of memory issue.
 *
 * Therefore, the metermeta/metricmeta object is directly passed to the newly created subquery object from the
 * previous sql object, instead of retrieving the metermeta/metricmeta from cache.
 *
 * Because the metermeta/metricmeta may have been released by other threads, resulting in the retrieving failed as
 * well as the create function.
 *
 * @param pSql
 * @param vnodeIndex
 * @param tableIndex
 * @param fp
 * @param param
 * @param pPrevSql
 * @return
 */
H
Haojun Liao 已提交
275
SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, int32_t cmd);
H
Haojun Liao 已提交
276

277 278
void registerSqlObj(SSqlObj* pSql);

H
Haojun Liao 已提交
279
SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t fp, void* param, int32_t cmd, SSqlObj* pPrevSql);
280
void     addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex);
S
slguan 已提交
281

282
void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex);
S
slguan 已提交
283

H
Haojun Liao 已提交
284
int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid);
H
Haojun Liao 已提交
285
int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId);
S
slguan 已提交
286

H
Haojun Liao 已提交
287
void tscPrintSelNodeList(SSqlObj* pSql, int32_t subClauseIndex);
H
hjxilinx 已提交
288

289
bool hasMoreVnodesToTry(SSqlObj *pSql);
H
Haojun Liao 已提交
290 291
bool hasMoreClauseToTry(SSqlObj* pSql);

D
fix bug  
dapan1121 已提交
292
void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta);
H
Haojun Liao 已提交
293

294 295
void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp);
void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
H
Haojun Liao 已提交
296
void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp);
dengyihao's avatar
TD-2257  
dengyihao 已提交
297
int  tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corEpSet);
298

299 300
bool tscSetSqlOwner(SSqlObj* pSql);
void tscClearSqlOwner(SSqlObj* pSql);
H
Haojun Liao 已提交
301
int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize);
302

303 304 305
char*   serializeTagData(STagData* pTagData, char* pMsg);
int32_t copyTagData(STagData* dst, const STagData* src);

306 307 308 309
STableMeta* createSuperTableMeta(STableMetaMsg* pChild);
uint32_t tscGetTableMetaSize(STableMeta* pTableMeta);
CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta);
uint32_t tscGetTableMetaMaxSize();
310
int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf);
H
Haojun Liao 已提交
311
STableMeta* tscTableMetaDup(STableMeta* pTableMeta);
H
Haojun Liao 已提交
312
int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr);
313

H
Haojun Liao 已提交
314
void tsCreateSQLFunctionCtx(SQueryInfo* pQueryInfo, SQLFunctionCtx* pCtx, SSchema* pSchema);
H
Haojun Liao 已提交
315
void* createQueryInfoFromQueryNode(SQueryInfo* pQueryInfo, SExprInfo* pExprs, STableGroupInfo* pTableGroupInfo, SOperatorInfo* pOperator, char* sql, void* addr, int32_t stage);
316

weixin_48148422's avatar
weixin_48148422 已提交
317 318 319 320
void* malloc_throw(size_t size);
void* calloc_throw(size_t nmemb, size_t size);
char* strdup_throw(const char* str);

H
hzcheng 已提交
321 322 323 324 325
#ifdef __cplusplus
}
#endif

#endif  // TDENGINE_TSCUTIL_H