qExecutor.h 8.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
#ifndef TDENGINE_QUERYEXECUTOR_H
#define TDENGINE_QUERYEXECUTOR_H

#include "os.h"

#include "hash.h"
H
Haojun Liao 已提交
21
#include "qAggMain.h"
H
Haojun Liao 已提交
22 23
#include "qFill.h"
#include "qResultbuf.h"
24
#include "qSqlparser.h"
H
Haojun Liao 已提交
25
#include "qTsbuf.h"
26
#include "query.h"
27
#include "taosdef.h"
H
Haojun Liao 已提交
28
#include "tarray.h"
B
Bomin Zhang 已提交
29
#include "tlockfree.h"
H
Haojun Liao 已提交
30
#include "tsdb.h"
31 32

struct SColumnFilterElem;
33
typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, const char* val1, const char* val2, int16_t type);
H
hjxilinx 已提交
34
typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
35

H
Haojun Liao 已提交
36
typedef struct SResultRowPool {
37 38 39 40 41 42 43 44 45 46
  int32_t elemSize;
  int32_t blockSize;
  int32_t numOfElemPerBlock;

  struct {
    int32_t blockIndex;
    int32_t pos;
  } position;

  SArray* pData;    // SArray<void*>
H
Haojun Liao 已提交
47
} SResultRowPool;
48

49
typedef struct SSqlGroupbyExpr {
H
Haojun Liao 已提交
50 51 52 53 54
  int16_t tableIndex;
  SArray* columnInfo;  // SArray<SColIndex>, group by columns information
  int16_t numOfGroupCols;
  int16_t orderIndex;  // order by column index
  int16_t orderType;   // order by type: asc/desc
55 56
} SSqlGroupbyExpr;

H
Haojun Liao 已提交
57
typedef struct SResultRow {
58
  int32_t       pageId;      // pageId & rowId is the position of current result in disk-based output buffer
59 60 61 62 63
  int32_t       rowId:29;    // row index in buffer page
  bool          startInterp; // the time window start timestamp has done the interpolation already.
  bool          endInterp;   // the time window end timestamp has done the interpolation already.
  bool          closed;      // this result status: closed or opened
  uint32_t      numOfRows;   // number of rows of current time window
H
Haojun Liao 已提交
64
  SResultRowCellInfo*  pCellInfo;  // For each result column, there is a resultInfo
65
  union {STimeWindow win; char* key;};  // start key of current time window
H
Haojun Liao 已提交
66
} SResultRow;
67

68 69 70 71 72 73
typedef struct SGroupResInfo {
  int32_t rowId;
  int32_t index;
  SArray* pRows;      // SArray<SResultRow*>
} SGroupResInfo;

H
Haojun Liao 已提交
74 75 76 77
/**
 * If the number of generated results is greater than this value,
 * query query will be halt and return results to client immediate.
 */
78
typedef struct SResultRec {
H
Haojun Liao 已提交
79 80 81 82
  int64_t total;      // total generated result size in rows
  int64_t rows;       // current result set size in rows
  int64_t capacity;   // capacity of current result output buffer
  int32_t threshold;  // result size threshold in rows.
83 84
} SResultRec;

H
Haojun Liao 已提交
85
typedef struct SResultRowInfo {
H
Haojun Liao 已提交
86 87 88 89 90 91
  SResultRow** pResult;    // result list
  int16_t      type:8;     // data type for hash key
  int32_t      size:24;    // number of result set
  int32_t      capacity;   // max capacity
  int32_t      curIndex;   // current start active index
  int64_t      prevSKey;   // previous (not completed) sliding window start key
H
Haojun Liao 已提交
92
} SResultRowInfo;
93 94 95 96 97 98 99 100

typedef struct SColumnFilterElem {
  int16_t           bytes;  // column length
  __filter_func_t   fp;
  SColumnFilterInfo filterInfo;
} SColumnFilterElem;

typedef struct SSingleColumnFilterInfo {
H
Haojun Liao 已提交
101
  void*              pData;
102
  int32_t            numOfFilters;
H
Haojun Liao 已提交
103
  SColumnInfo        info;
104 105 106
  SColumnFilterElem* pFilters;
} SSingleColumnFilterInfo;

H
Haojun Liao 已提交
107
typedef struct STableQueryInfo {
H
hjxilinx 已提交
108
  TSKEY       lastKey;
H
Haojun Liao 已提交
109
  int32_t     groupIndex;     // group id in table list
110
  int16_t     queryRangeSet;  // denote if the query range is set, only available for interval query
111
  tVariant    tag;
H
hjxilinx 已提交
112
  STimeWindow win;
113
  STSCursor   cur;
H
Haojun Liao 已提交
114
  void*       pTable;         // for retrieve the page id list
H
Haojun Liao 已提交
115
  SResultRowInfo windowResInfo;
116 117
} STableQueryInfo;

H
Haojun Liao 已提交
118 119 120 121 122 123 124 125 126
typedef struct SQueryCostInfo {
  uint64_t loadStatisTime;
  uint64_t loadFileBlockTime;
  uint64_t loadDataInCacheTime;
  uint64_t loadStatisSize;
  uint64_t loadFileBlockSize;
  uint64_t loadDataInCacheSize;
  
  uint64_t loadDataTime;
127 128 129 130
  uint64_t totalRows;
  uint64_t totalCheckedRows;
  uint32_t totalBlocks;
  uint32_t loadBlocks;
H
Haojun Liao 已提交
131 132
  uint32_t loadBlockStatis;
  uint32_t discardBlocks;
133
  uint64_t elapsedTime;
H
Haojun Liao 已提交
134
  uint64_t firstStageMergeTime;
H
Haojun Liao 已提交
135 136
  uint64_t winInfoSize;
  uint64_t tableInfoSize;
H
Haojun Liao 已提交
137
  uint64_t hashSize;
138
  uint64_t numOfTimeWindows;
H
Haojun Liao 已提交
139
} SQueryCostInfo;
H
hjxilinx 已提交
140

H
Haojun Liao 已提交
141 142 143 144 145
typedef struct {
  int64_t vgroupLimit;
  int64_t ts;
} SOrderedPrjQueryInfo;

146
typedef struct SQuery {
H
Haojun Liao 已提交
147 148 149 150
  int16_t          numOfCols;
  int16_t          numOfTags;
  SOrderVal        order;
  STimeWindow      window;
151
  SInterval        interval;
152
  int16_t          precision;
H
Haojun Liao 已提交
153 154
  int16_t          numOfOutput;
  int16_t          fillType;
H
Haojun Liao 已提交
155
  int16_t          checkResultBuf;  // check if the buffer is full during scan each block
H
Haojun Liao 已提交
156 157 158
  SLimitVal        limit;
  int32_t          rowSize;
  SSqlGroupbyExpr* pGroupbyExpr;
H
Haojun Liao 已提交
159
  SExprInfo*       pExpr1;
H
Haojun Liao 已提交
160 161 162
  SExprInfo*       pExpr2;
  int32_t          numOfExpr2;

H
Haojun Liao 已提交
163 164 165 166
  SColumnInfo*     colList;
  SColumnInfo*     tagColList;
  int32_t          numOfFilterCols;
  int64_t*         fillVal;
167
  uint32_t         status;             // query status
H
Haojun Liao 已提交
168 169 170 171
  SResultRec       rec;
  int32_t          pos;
  tFilePage**      sdata;
  STableQueryInfo* current;
172
  int32_t          numOfCheckedBlocks; // number of check data blocks
H
Haojun Liao 已提交
173

174
  SOrderedPrjQueryInfo prjInfo;        // limit value for each vgroup, only available in global order projection query.
175 176 177 178
  SSingleColumnFilterInfo* pFilterInfo;
} SQuery;

typedef struct SQueryRuntimeEnv {
H
Haojun Liao 已提交
179
  jmp_buf              env;
H
Haojun Liao 已提交
180 181
  SQuery*              pQuery;
  SQLFunctionCtx*      pCtx;
182
  int32_t              numOfRowsPerPage;
H
Haojun Liao 已提交
183
  uint16_t*            offset;
H
Haojun Liao 已提交
184
  uint16_t             scanFlag;         // denotes reversed scan of data or not
H
Haojun Liao 已提交
185
  SFillInfo*           pFillInfo;
H
Haojun Liao 已提交
186
  SResultRowInfo       windowResInfo;
H
Haojun Liao 已提交
187
  STSBuf*              pTsBuf;
H
Haojun Liao 已提交
188
  STSCursor            cur;
189
  SQueryCostInfo       summary;
H
Haojun Liao 已提交
190 191
  void*                pQueryHandle;
  void*                pSecQueryHandle;  // another thread for
H
Haojun Liao 已提交
192
  bool                 stableQuery;      // super table query or not
193
  bool                 topBotQuery;      // TODO used bitwise flag
H
Haojun Liao 已提交
194
  bool                 groupbyColumn; // denote if this is a groupby normal column query
H
Haojun Liao 已提交
195
  bool                 hasTagResults;    // if there are tag values in final result or not
196
  bool                 timeWindowInterpo;// if the time window start/end required interpolation
197
  bool                 queryWindowIdentical; // all query time windows are identical for all tables in one group
Y
yihaoDeng 已提交
198
  bool                 queryBlockDist;    // if query data block distribution  
H
Haojun Liao 已提交
199
  int32_t              interBufSize;     // intermediate buffer sizse
H
Haojun Liao 已提交
200
  int32_t              prevGroupId;      // previous executed group id
H
Haojun Liao 已提交
201
  SDiskbasedResultBuf* pResultBuf;       // query result buffer based on blocked-wised disk file
H
Haojun Liao 已提交
202
  SHashObj*            pResultRowHashTable; // quick locate the window object for each result
H
Haojun Liao 已提交
203
  char*                keyBuf;           // window key buffer
204
  SResultRowPool*      pool;             // window result object pool
H
Haojun Liao 已提交
205 206

  int32_t*             rowCellInfoOffset;// offset value for each row result cell info
207 208
  char**               prevRow;
  char**               nextRow;
209 210

  SArithmeticSupport  *sasArray;
211 212
} SQueryRuntimeEnv;

213 214 215 216 217
enum {
  QUERY_RESULT_NOT_READY = 1,
  QUERY_RESULT_READY     = 2,
};

218
typedef struct SQInfo {
H
Haojun Liao 已提交
219
  void*            signature;
H
Haojun Liao 已提交
220
  int32_t          code;   // error code to returned to client
221
  int64_t          owner;  // if it is in execution
H
Haojun Liao 已提交
222
  void*            tsdb;
Y
TD-1733  
yihaoDeng 已提交
223
  SMemRef          memRef; 
H
Haojun Liao 已提交
224
  int32_t          vgId;
H
Haojun Liao 已提交
225
  STableGroupInfo  tableGroupInfo;       // table <tid, last_key> list  SArray<STableKeyInfo>
226
  STableGroupInfo  tableqinfoGroupInfo;  // this is a group array list, including SArray<STableQueryInfo*> structure
227
  SQueryRuntimeEnv runtimeEnv;
228
  SHashObj*        arrTableIdInfo;
H
Haojun Liao 已提交
229
  int32_t          groupIndex;
H
Haojun Liao 已提交
230

231 232 233 234
  /*
   * the query is executed position on which meter of the whole list.
   * when the index reaches the last one of the list, it means the query is completed.
   */
H
Haojun Liao 已提交
235
  int32_t          tableIndex;
H
Haojun Liao 已提交
236
  SGroupResInfo    groupResInfo;
237
  void*            pBuf;        // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables;
H
Haojun Liao 已提交
238

239
  pthread_mutex_t  lock;        // used to synchronize the rsp/query threads
H
Haojun Liao 已提交
240
  tsem_t           ready;
241 242
  int32_t          dataReady;   // denote if query result is ready or not
  void*            rspContext;  // response context
243
  int64_t          startExecTs; // start to exec timestamp
244
  char*            sql;         // query sql string
245 246 247
} SQInfo;

#endif  // TDENGINE_QUERYEXECUTOR_H