tsclient.h 14.0 KB
Newer Older
H
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef TDENGINE_TSCLIENT_H
#define TDENGINE_TSCLIENT_H

#ifdef __cplusplus
extern "C" {
#endif

S
slguan 已提交
23
#include "os.h"
24

25 26 27
#include "qsqlparser.h"
#include "qsqltype.h"
#include "qtsbuf.h"
H
hzcheng 已提交
28
#include "taos.h"
29
#include "taosdef.h"
H
hzcheng 已提交
30
#include "taosmsg.h"
H
hjxilinx 已提交
31
#include "tarray.h"
S
slguan 已提交
32
#include "tglobal.h"
33
#include "trpc.h"
H
hzcheng 已提交
34 35 36
#include "tsqlfunction.h"
#include "tutil.h"

H
hjxilinx 已提交
37 38
#define TSC_GET_RESPTR_BASE(res, _queryinfo, col) (res->data + ((_queryinfo)->fieldsInfo.pSqlExpr[col]->offset) * res->numOfRows)

H
hzcheng 已提交
39 40 41
// forward declaration
struct SSqlInfo;

42 43
typedef SCMSTableVgroupRspMsg SVgroupsInfo;

H
hzcheng 已提交
44
typedef struct SSqlGroupbyExpr {
H
hjxilinx 已提交
45
  int16_t     tableIndex;
S
slguan 已提交
46
  int16_t     numOfGroupCols;
47
  SColIndex   columnInfo[TSDB_MAX_TAGS];  // group by columns information
H
hjxilinx 已提交
48 49
  int16_t     orderIndex;                 // order by column index
  int16_t     orderType;                  // order by type: asc/desc
S
slguan 已提交
50
} SSqlGroupbyExpr;
H
hzcheng 已提交
51

H
hjxilinx 已提交
52
typedef struct STableComInfo {
H
hjxilinx 已提交
53 54 55 56
  uint8_t numOfTags;
  uint8_t precision;
  int16_t numOfColumns;
  int16_t rowSize;
H
hjxilinx 已提交
57
} STableComInfo;
H
hjxilinx 已提交
58 59

typedef struct STableMeta {
H
hjxilinx 已提交
60 61 62 63
  //super table if it is created according to super table, otherwise, tableInfo is used
  union { struct STableMeta* pSTable; STableComInfo tableInfo; };
  uint8_t    tableType;
  int16_t    sversion;
H
hjxilinx 已提交
64
  SCMVgroupInfo vgroupInfo;
H
hjxilinx 已提交
65 66 67
  int32_t    sid;      // the index of one table in a virtual node
  uint64_t   uid;      // unique id of a table
  SSchema    schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
H
hjxilinx 已提交
68 69 70
} STableMeta;

typedef struct STableMetaInfo {
H
hjxilinx 已提交
71
  STableMeta * pTableMeta;       // table meta, cached in client side and acquried by name
72
  SVgroupsInfo* vgroupList;
73
  
H
hjxilinx 已提交
74 75 76 77
  /*
   * 1. keep the vnode index during the multi-vnode super table projection query
   * 2. keep the vnode index for multi-vnode insertion
   */
H
hjxilinx 已提交
78
  int32_t vgroupIndex;
H
hjxilinx 已提交
79
  char    name[TSDB_TABLE_ID_LEN];        // (super) table name
80 81
  int16_t numOfTags;                      // total required tags in query, including groupby tags
  int16_t tagColumnIndex[TSDB_MAX_TAGS];  // clause + tag projection
H
hjxilinx 已提交
82
} STableMetaInfo;
H
hzcheng 已提交
83

S
slguan 已提交
84 85
/* the structure for sql function in select clause */
typedef struct SSqlExpr {
86
  char        aliasName[TSDB_COL_NAME_LEN];  // as aliasName
87
  SColIndex   colInfo;
S
slguan 已提交
88 89 90 91 92 93 94
  int64_t     uid;            // refactor use the pointer
  int16_t     functionId;     // function id in aAgg array
  int16_t     resType;        // return value type
  int16_t     resBytes;       // length of return value
  int16_t     interResBytes;  // inter result buffer size
  int16_t     numOfParams;    // argument value of each function
  tVariant    param[3];       // parameters are not more than 3
H
hjxilinx 已提交
95
  int32_t     offset;         // sub result column value of arithmetic expression.
H
hzcheng 已提交
96 97
} SSqlExpr;

H
hjxilinx 已提交
98 99 100 101 102
typedef struct SColumnIndex {
  int16_t tableIndex;
  int16_t columnIndex;
} SColumnIndex;

H
hzcheng 已提交
103 104 105 106
typedef struct SFieldInfo {
  int16_t     numOfOutputCols;  // number of column in result
  int16_t     numOfAlloc;       // allocated size
  TAOS_FIELD *pFields;
S
slguan 已提交
107 108 109 110 111 112 113 114

  /*
   * define if this column is belong to the queried result, it may be add by parser to faciliate
   * the query process
   *
   * NOTE: these hidden columns always locate at the end of the output columns
   */
  bool *  pVisibleCols;
H
hjxilinx 已提交
115 116 117
  int32_t numOfHiddenCols;   // the number of column not belongs to the queried result columns
  SSqlFunctionExpr** pExpr;  // used for aggregation arithmetic express,such as count(*)+count(*)
  SSqlExpr** pSqlExpr;
H
hzcheng 已提交
118 119 120
} SFieldInfo;

typedef struct SSqlExprInfo {
H
hjxilinx 已提交
121 122 123
  int16_t    numOfAlloc;
  int16_t    numOfExprs;
  SSqlExpr** pExprs;
H
hzcheng 已提交
124 125
} SSqlExprInfo;

S
slguan 已提交
126 127 128 129
typedef struct SColumnBase {
  SColumnIndex       colIndex;
  int32_t            numOfFilters;
  SColumnFilterInfo *filterInfo;
H
hzcheng 已提交
130 131
} SColumnBase;

S
slguan 已提交
132
typedef struct SColumnBaseInfo {
H
hzcheng 已提交
133 134 135
  int16_t      numOfAlloc;
  int16_t      numOfCols;
  SColumnBase *pColList;
S
slguan 已提交
136
} SColumnBaseInfo;
H
hzcheng 已提交
137 138 139

struct SLocalReducer;

S
slguan 已提交
140 141
typedef struct SCond {
  uint64_t uid;
142
  int32_t  len;   // length of tag query condition data
H
hjxilinx 已提交
143
  char *   cond;
S
slguan 已提交
144 145 146
} SCond;

typedef struct SJoinNode {
S
slguan 已提交
147
  char     tableId[TSDB_TABLE_ID_LEN];
S
slguan 已提交
148 149 150 151 152 153 154 155 156 157
  uint64_t uid;
  int16_t  tagCol;
} SJoinNode;

typedef struct SJoinInfo {
  bool      hasJoin;
  SJoinNode left;
  SJoinNode right;
} SJoinInfo;

H
hzcheng 已提交
158
typedef struct STagCond {
S
slguan 已提交
159 160 161 162 163 164 165 166 167 168
  // relation between tbname list and query condition, including : TK_AND or TK_OR
  int16_t relType;

  // tbname query condition, only support tbname query condition on one table
  SCond tbnameCond;

  // join condition, only support two tables join currently
  SJoinInfo joinInfo;

  // for different table, the query condition must be seperated
169
  SArray*  pCond;
H
hzcheng 已提交
170 171
} STagCond;

S
slguan 已提交
172
typedef struct SParamInfo {
H
hjxilinx 已提交
173 174 175 176
  int32_t  idx;
  char     type;
  uint8_t  timePrec;
  short    bytes;
S
slguan 已提交
177 178 179
  uint32_t offset;
} SParamInfo;

S
slguan 已提交
180
typedef struct STableDataBlocks {
S
slguan 已提交
181
  char    tableId[TSDB_TABLE_ID_LEN];
182 183
  int8_t  tsSource;     // where does the UNIX timestamp come from, server or client
  bool    ordered;      // if current rows are ordered or not
H
hjxilinx 已提交
184
  int64_t vgId;         // virtual group id
185
  int64_t prevTS;       // previous timestamp, recorded to decide if the records array is ts ascending
S
slguan 已提交
186
  int32_t numOfTables;  // number of tables in current submit block
187 188

  int32_t  rowSize;  // row size for current table
S
slguan 已提交
189
  uint32_t nAllocSize;
H
hjxilinx 已提交
190
  uint32_t headerSize;  // header for metadata (submit metadata)
S
slguan 已提交
191
  uint32_t size;
192

H
hjxilinx 已提交
193 194 195 196
  /*
   * the metermeta for current table, the metermeta will be used during submit stage, keep a ref
   * to avoid it to be removed from cache
   */
H
hjxilinx 已提交
197
  STableMeta *pTableMeta;
198

H
hzcheng 已提交
199 200 201 202
  union {
    char *filename;
    char *pData;
  };
S
slguan 已提交
203 204

  // for parameter ('?') binding
H
hjxilinx 已提交
205 206 207
  uint32_t    numOfAllocedParams;
  uint32_t    numOfParams;
  SParamInfo *params;
S
slguan 已提交
208
} STableDataBlocks;
H
hzcheng 已提交
209 210

typedef struct SDataBlockList {
H
hjxilinx 已提交
211 212
  uint32_t           nSize;
  uint32_t           nAlloc;
S
slguan 已提交
213
  STableDataBlocks **pData;
H
hzcheng 已提交
214 215
} SDataBlockList;

216
typedef struct SQueryInfo {
217 218
  int16_t  command;  // the command may be different for each subclause, so keep it seperately.
  uint16_t type;     // query/insert/import type
H
hjxilinx 已提交
219
  char     slidingTimeUnit;
220

221
  int64_t         etime, stime;
222
  int64_t         intervalTime;  // aggregation time interval
H
hjxilinx 已提交
223 224
  int64_t         slidingTime;   // sliding window in mseconds
  SSqlGroupbyExpr groupbyExpr;   // group by tags info
225 226 227 228 229 230 231 232 233 234

  SColumnBaseInfo  colList;
  SFieldInfo       fieldsInfo;
  SSqlExprInfo     exprsInfo;
  SLimitVal        limit;
  SLimitVal        slimit;
  STagCond         tagCond;
  SOrderVal        order;
  int16_t          interpoType;  // interpolate type
  int16_t          numOfTables;
H
hjxilinx 已提交
235
  STableMetaInfo **pTableMetaInfo;
236
  struct STSBuf *  tsBuf;
237 238
  int64_t *        defaultVal;   // default value for interpolation
  char *           msg;          // pointer to the pCmd->payload to keep error message temporarily
239
  int64_t          clauseLimit;  // limit for current sub clause
H
hjxilinx 已提交
240

241
  // offset value in the original sql expression, NOT sent to virtual node, only applied at client side
H
hjxilinx 已提交
242
  int64_t prjOffset;
243 244
} SQueryInfo;

245 246 247 248 249 250
// data source from sql string or from file
enum {
  DATA_FROM_SQL_STRING = 1,
  DATA_FROM_DATA_FILE = 2,
};

H
hzcheng 已提交
251
typedef struct {
252 253
  int     command;
  uint8_t msgType;
S
slguan 已提交
254 255

  union {
256
    bool   existsCheck;     // check if the table exists or not
H
hjxilinx 已提交
257
    bool   autoCreated;  // if the table is missing, on-the-fly create it. during getmeterMeta
258
    int8_t dataSourceType;  // load data from file or not
S
slguan 已提交
259
  };
H
hjxilinx 已提交
260

261 262 263 264
  union {
    int32_t count;
    int32_t numOfTablesInSubmit;
  };
H
hzcheng 已提交
265

266 267 268 269 270 271
  int32_t      clauseIndex;  // index of multiple subclause query
  int8_t       parseFinished;
  short        numOfCols;
  uint32_t     allocSize;
  char *       payload;
  int32_t      payloadLen;
272 273
  SQueryInfo **pQueryInfo;
  int32_t      numOfClause;
274 275 276 277 278
  
  SDataBlockList *pDataBlocks; // submit data blocks after parsing sql
  char *          curSql; // current sql, resume position of sql after parsing paused
  void *          pTableList;  // referred table involved in sql
  
S
slguan 已提交
279 280 281
  // for parameter ('?') binding and batch processing
  int32_t batchSize;
  int32_t numOfParams;
H
hzcheng 已提交
282 283 284 285 286 287 288
} SSqlCmd;

typedef struct SResRec {
  int numOfRows;
  int numOfTotal;
} SResRec;

S
slguan 已提交
289 290
struct STSBuf;

H
hzcheng 已提交
291
typedef struct {
H
hjxilinx 已提交
292 293 294 295
  int64_t       numOfRows;                  // num of results in current retrieved
  int64_t       numOfTotal;                 // num of total results
  int64_t       numOfTotalInCurrentClause;  // num of total result in current subclause
  char *        pRsp;
H
hjxilinx 已提交
296 297
  int32_t       rspType;
  int32_t       rspLen;
H
hjxilinx 已提交
298 299 300 301
  uint64_t      qhandle;
  int64_t       uid;
  int64_t       useconds;
  int64_t       offset;  // offset value from vnode during projection query of stable
H
hjxilinx 已提交
302
  int32_t       row;
H
hjxilinx 已提交
303
  int16_t       numOfCols;
H
hjxilinx 已提交
304
  int16_t       precision;
H
hjxilinx 已提交
305 306
  bool          completed;
  int32_t       code;
H
hjxilinx 已提交
307 308 309 310 311 312
  int32_t       numOfGroups;
  SResRec *     pGroupRec;
  char *        data;
  void **       tsrow;
  char **       buffer;  // Buffer used to put multibytes encoded using unicode (wchar_t)
  SColumnIndex *pColumnIndex;
H
hzcheng 已提交
313 314 315
  struct SLocalReducer *pLocalReducer;
} SSqlRes;

H
hjxilinx 已提交
316
typedef struct STscObj {
H
hzcheng 已提交
317 318 319
  void *           signature;
  void *           pTimer;
  char             mgmtIp[TSDB_USER_LEN];
L
lihui 已提交
320
  uint16_t         mgmtPort;
H
hzcheng 已提交
321 322 323
  char             user[TSDB_USER_LEN];
  char             pass[TSDB_KEY_LEN];
  char             acctId[TSDB_DB_NAME_LEN];
S
slguan 已提交
324
  char             db[TSDB_TABLE_ID_LEN];
H
hzcheng 已提交
325 326 327
  char             sversion[TSDB_VERSION_LEN];
  char             writeAuth : 1;
  char             superAuth : 1;
328 329 330
  struct SSqlObj *pSql;
  struct SSqlObj *pHb;
  struct SSqlObj *sqlList;
H
hjxilinx 已提交
331
  struct SSqlStream *streamList;
H
hzcheng 已提交
332 333 334
  pthread_mutex_t  mutex;
} STscObj;

335
typedef struct SSqlObj {
H
hzcheng 已提交
336 337 338 339
  void *   signature;
  STscObj *pTscObj;
  void (*fp)();
  void (*fetchFp)();
H
hjxilinx 已提交
340 341 342 343 344 345
  void *            param;
  uint32_t          ip;
  short             vnode;
  int64_t           stime;
  uint32_t          queryId;
  void *            pStream;
weixin_48148422's avatar
weixin_48148422 已提交
346
  void *            pSubscription;
H
hjxilinx 已提交
347 348 349
  char *            sqlstr;
  char              retry;
  char              maxRetry;
H
hjxilinx 已提交
350
  SRpcIpSet         ipList;
H
hjxilinx 已提交
351 352 353 354 355
  char              freed : 4;
  char              listed : 4;
  tsem_t            rspSem;
  SSqlCmd           cmd;
  SSqlRes           res;
H
hjxilinx 已提交
356
  uint8_t           numOfSubs;
357 358
  struct SSqlObj **pSubs;
  struct SSqlObj * prev, *next;
H
hzcheng 已提交
359 360
} SSqlObj;

H
hjxilinx 已提交
361
typedef struct SSqlStream {
H
hzcheng 已提交
362 363 364 365 366 367
  SSqlObj *pSql;
  uint32_t streamId;
  char     listed;
  int64_t  num;  // number of computing count

  /*
S
slguan 已提交
368
   * keep the number of current result in computing,
H
hzcheng 已提交
369 370 371 372 373 374 375
   * the value will be set to 0 before set timer for next computing
   */
  int64_t numOfRes;

  int64_t useconds;  // total  elapsed time
  int64_t ctime;     // stream created time
  int64_t stime;     // stream next executed time
S
slguan 已提交
376
  int64_t etime;     // stream end query time, when time is larger then etime, the stream will be closed
H
hzcheng 已提交
377 378
  int64_t interval;
  int64_t slidingTime;
379
  int16_t precision;
H
hzcheng 已提交
380 381 382 383 384
  void *  pTimer;

  void (*fp)();
  void *param;

S
slguan 已提交
385
  void (*callback)(void *);  // Callback function when stream is stopped from client level
H
hjxilinx 已提交
386
  struct SSqlStream *prev, *next;
H
hzcheng 已提交
387 388
} SSqlStream;

S
slguan 已提交
389 390
int32_t tscInitRpc(const char *user, const char *secret);

H
hzcheng 已提交
391
// tscSql API
392
int tsParseSql(SSqlObj *pSql, bool multiVnodeInsertion);
H
hzcheng 已提交
393

394
void tscInitMsgsFp();
395 396
extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);

397
void tscProcessMsgFromServer(SRpcMsg *rpcMsg);
S
slguan 已提交
398
int  tscProcessSql(SSqlObj *pSql);
H
hzcheng 已提交
399

S
slguan 已提交
400
int  tscRenewMeterMeta(SSqlObj *pSql, char *tableId);
H
hzcheng 已提交
401
void tscQueueAsyncRes(SSqlObj *pSql);
S
slguan 已提交
402

H
[td-99]  
hjxilinx 已提交
403
void tscQueueAsyncError(void(*fp), void *param, int32_t code);
H
hzcheng 已提交
404 405 406 407 408 409 410 411 412

int tscProcessLocalCmd(SSqlObj *pSql);
int tscCfgDynamicOptions(char *msg);
int taos_retrieve(TAOS_RES *res);

/*
 * transfer function for metric query in stream computing, the function need to be change
 * before send query message to vnode
 */
413 414
int32_t tscTansformSQLFunctionForSTableQuery(SQueryInfo *pQueryInfo);
void    tscRestoreSQLFunctionForMetricQuery(SQueryInfo *pQueryInfo);
H
hzcheng 已提交
415

416
int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo);
H
hjxilinx 已提交
417
void    tscDestroyResPointerInfo(SSqlRes *pRes);
H
hzcheng 已提交
418

419
void tscResetSqlCmdObj(SSqlCmd *pCmd);
H
hjxilinx 已提交
420
void tscFreeResData(SSqlObj *pSql);
H
hzcheng 已提交
421

weixin_48148422's avatar
weixin_48148422 已提交
422 423 424 425
/**
 * free query result of the sql object
 * @param pObj
 */
H
hjxilinx 已提交
426
void tscFreeSqlResult(SSqlObj *pSql);
weixin_48148422's avatar
weixin_48148422 已提交
427

H
hzcheng 已提交
428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443
/**
 * only free part of resources allocated during query.
 * Note: this function is multi-thread safe.
 * @param pObj
 */
void tscFreeSqlObjPartial(SSqlObj *pObj);

/**
 * free sql object, release allocated resource
 * @param pObj  Free metric/meta information, dynamically allocated payload, and
 * response buffer, object itself
 */
void tscFreeSqlObj(SSqlObj *pObj);

void tscCloseTscObj(STscObj *pObj);

H
[td-99]  
hjxilinx 已提交
444
void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen);
445

446 447 448 449
void tscProcessMultiVnodesInsertFromFile(SSqlObj *pSql);
void tscKillMetricQuery(SSqlObj *pSql);
void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen);
bool tscIsUpdateQuery(STscObj *pObj);
450
bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes);
H
hjxilinx 已提交
451

452
char *tscGetErrorMsgPayload(SSqlCmd *pCmd);
453

H
hjxilinx 已提交
454
int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql);
H
hzcheng 已提交
455 456

void tscQueueAsyncFreeResult(SSqlObj *pSql);
H
hjxilinx 已提交
457
int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo);
H
hzcheng 已提交
458

S
slguan 已提交
459 460 461 462 463 464 465 466 467
extern void *     pVnodeConn;
extern void *     pTscMgmtConn;
extern void *     tscCacheHandle;
extern int        slaveIndex;
extern void *     tscTmr;
extern void *     tscQhandle;
extern int        tscKeepConn[];
extern int        tsInsertHeadSize;
extern int        tscNumOfThreads;
S
slguan 已提交
468
extern SRpcIpSet  tscMgmtIpList;
H
hzcheng 已提交
469

470 471
typedef void (*__async_cb_func_t)(void *param, TAOS_RES *tres, int numOfRows);

H
hzcheng 已提交
472 473 474 475 476
#ifdef __cplusplus
}
#endif

#endif