tsclient.h 13.9 KB
Newer Older
H
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef TDENGINE_TSCLIENT_H
#define TDENGINE_TSCLIENT_H

#ifdef __cplusplus
extern "C" {
#endif

#include <errno.h>
#include <pthread.h>
#include <semaphore.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

S
slguan 已提交
31
#include "os.h"
H
hzcheng 已提交
32 33 34 35 36
#include "taos.h"
#include "taosmsg.h"
#include "tglobalcfg.h"
#include "tlog.h"
#include "tscCache.h"
H
hjxilinx 已提交
37
#include "tscSQLParser.h"
H
hjxilinx 已提交
38
#include "tsdb.h"
H
hzcheng 已提交
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
#include "tsqlfunction.h"
#include "tutil.h"

#define TSC_GET_RESPTR_BASE(res, cmd, col, ord)                     \
  ((res->data + tscFieldInfoGetOffset(cmd, col) * res->numOfRows) + \
   (1 - ord.order) * (res->numOfRows - 1) * tscFieldInfoGetField(cmd, col)->bytes)

enum _sql_cmd {
  TSDB_SQL_SELECT,
  TSDB_SQL_FETCH,
  TSDB_SQL_INSERT,

  TSDB_SQL_MGMT,  // the SQL below is for mgmt node
  TSDB_SQL_CREATE_DB,
  TSDB_SQL_CREATE_TABLE,
  TSDB_SQL_DROP_DB,
  TSDB_SQL_DROP_TABLE,
  TSDB_SQL_CREATE_ACCT,
  TSDB_SQL_CREATE_USER,
  TSDB_SQL_DROP_ACCT,  // 10
  TSDB_SQL_DROP_USER,
  TSDB_SQL_ALTER_USER,
  TSDB_SQL_ALTER_ACCT,
  TSDB_SQL_ALTER_TABLE,
  TSDB_SQL_ALTER_DB,
  TSDB_SQL_CREATE_MNODE,
  TSDB_SQL_DROP_MNODE,
S
slguan 已提交
66 67 68
  TSDB_SQL_CREATE_DNODE,
  TSDB_SQL_DROP_DNODE,
  TSDB_SQL_CFG_DNODE,  // 20
H
hzcheng 已提交
69 70 71 72 73 74 75 76 77 78 79 80
  TSDB_SQL_CFG_MNODE,
  TSDB_SQL_SHOW,
  TSDB_SQL_RETRIEVE,
  TSDB_SQL_KILL_QUERY,
  TSDB_SQL_KILL_STREAM,
  TSDB_SQL_KILL_CONNECTION,

  TSDB_SQL_READ,  // SQL below is for read operation
  TSDB_SQL_CONNECT,
  TSDB_SQL_USE_DB,
  TSDB_SQL_META,  // 30
  TSDB_SQL_METRIC,
S
slguan 已提交
81
  TSDB_SQL_MULTI_META,
H
hzcheng 已提交
82 83 84 85 86
  TSDB_SQL_HB,

  TSDB_SQL_LOCAL,  // SQL below for client local
  TSDB_SQL_DESCRIBE_TABLE,
  TSDB_SQL_RETRIEVE_METRIC,
S
slguan 已提交
87
  TSDB_SQL_METRIC_JOIN_RETRIEVE,
H
hzcheng 已提交
88
  TSDB_SQL_RETRIEVE_TAGS,
S
slguan 已提交
89 90 91 92 93 94
  /*
   * build empty result instead of accessing dnode to fetch result
   * reset the client cache
   */
  TSDB_SQL_RETRIEVE_EMPTY_RESULT,

H
hjxilinx 已提交
95 96 97 98 99 100
  TSDB_SQL_RESET_CACHE,  // 40
  TSDB_SQL_SERV_STATUS,
  TSDB_SQL_CURRENT_DB,
  TSDB_SQL_SERV_VERSION,
  TSDB_SQL_CLI_VERSION,
  TSDB_SQL_CURRENT_USER,
H
hzcheng 已提交
101 102 103 104 105 106 107 108 109
  TSDB_SQL_CFG_LOCAL,

  TSDB_SQL_MAX
};

// forward declaration
struct SSqlInfo;

typedef struct SSqlGroupbyExpr {
H
hjxilinx 已提交
110
  int16_t     tableIndex;
S
slguan 已提交
111 112
  int16_t     numOfGroupCols;
  SColIndexEx columnInfo[TSDB_MAX_TAGS];  // group by columns information
H
hjxilinx 已提交
113 114
  int16_t     orderIndex;                 // order by column index
  int16_t     orderType;                  // order by type: asc/desc
S
slguan 已提交
115
} SSqlGroupbyExpr;
H
hzcheng 已提交
116

S
slguan 已提交
117
typedef struct SMeterMetaInfo {
H
hjxilinx 已提交
118 119 120 121 122 123 124 125 126 127 128
  SMeterMeta * pMeterMeta;                     // metermeta
  SMetricMeta *pMetricMeta;                    // metricmeta
  
  /*
   * 1. keep the vnode index during the multi-vnode super table projection query
   * 2. keep the vnode index for multi-vnode insertion
   */
  int32_t      vnodeIndex;
  char         name[TSDB_METER_ID_LEN + 1];    // table(super table) name
  int16_t      numOfTags;                      // total required tags in query, including groupby tags
  int16_t      tagColumnIndex[TSDB_MAX_TAGS];  // clause + tag projection
S
slguan 已提交
129
} SMeterMetaInfo;
H
hzcheng 已提交
130

S
slguan 已提交
131 132 133 134 135 136 137 138 139 140 141
/* the structure for sql function in select clause */
typedef struct SSqlExpr {
  char        aliasName[TSDB_COL_NAME_LEN + 1];  // as aliasName
  SColIndexEx colInfo;
  int64_t     uid;            // refactor use the pointer
  int16_t     functionId;     // function id in aAgg array
  int16_t     resType;        // return value type
  int16_t     resBytes;       // length of return value
  int16_t     interResBytes;  // inter result buffer size
  int16_t     numOfParams;    // argument value of each function
  tVariant    param[3];       // parameters are not more than 3
H
hzcheng 已提交
142 143 144 145 146 147 148
} SSqlExpr;

typedef struct SFieldInfo {
  int16_t     numOfOutputCols;  // number of column in result
  int16_t     numOfAlloc;       // allocated size
  TAOS_FIELD *pFields;
  short *     pOffset;
S
slguan 已提交
149 150 151 152 153 154 155 156 157

  /*
   * define if this column is belong to the queried result, it may be add by parser to faciliate
   * the query process
   *
   * NOTE: these hidden columns always locate at the end of the output columns
   */
  bool *  pVisibleCols;
  int32_t numOfHiddenCols;  // the number of column not belongs to the queried result columns
H
hzcheng 已提交
158 159 160 161 162 163 164 165
} SFieldInfo;

typedef struct SSqlExprInfo {
  int16_t   numOfAlloc;
  int16_t   numOfExprs;
  SSqlExpr *pExprs;
} SSqlExprInfo;

S
slguan 已提交
166 167 168 169
typedef struct SColumnIndex {
  int16_t tableIndex;
  int16_t columnIndex;
} SColumnIndex;
H
hzcheng 已提交
170

S
slguan 已提交
171 172 173 174
typedef struct SColumnBase {
  SColumnIndex       colIndex;
  int32_t            numOfFilters;
  SColumnFilterInfo *filterInfo;
H
hzcheng 已提交
175 176
} SColumnBase;

S
slguan 已提交
177
typedef struct SColumnBaseInfo {
H
hzcheng 已提交
178 179 180
  int16_t      numOfAlloc;
  int16_t      numOfCols;
  SColumnBase *pColList;
S
slguan 已提交
181
} SColumnBaseInfo;
H
hzcheng 已提交
182 183 184

struct SLocalReducer;

S
slguan 已提交
185 186 187 188 189 190 191 192 193
// todo move to utility
typedef struct SString {
  int32_t alloc;
  int32_t n;
  char *  z;
} SString;

typedef struct SCond {
  uint64_t uid;
H
hjxilinx 已提交
194
  char *   cond;
S
slguan 已提交
195 196 197 198 199 200 201 202 203 204 205 206 207 208
} SCond;

typedef struct SJoinNode {
  char     meterId[TSDB_METER_ID_LEN];
  uint64_t uid;
  int16_t  tagCol;
} SJoinNode;

typedef struct SJoinInfo {
  bool      hasJoin;
  SJoinNode left;
  SJoinNode right;
} SJoinInfo;

H
hzcheng 已提交
209
typedef struct STagCond {
S
slguan 已提交
210 211 212 213 214 215 216 217 218 219 220 221
  // relation between tbname list and query condition, including : TK_AND or TK_OR
  int16_t relType;

  // tbname query condition, only support tbname query condition on one table
  SCond tbnameCond;

  // join condition, only support two tables join currently
  SJoinInfo joinInfo;

  // for different table, the query condition must be seperated
  SCond   cond[TSDB_MAX_JOIN_TABLE_NUM];
  int16_t numOfTagCond;
H
hzcheng 已提交
222 223
} STagCond;

S
slguan 已提交
224
typedef struct SParamInfo {
H
hjxilinx 已提交
225 226 227 228
  int32_t  idx;
  char     type;
  uint8_t  timePrec;
  short    bytes;
S
slguan 已提交
229 230 231
  uint32_t offset;
} SParamInfo;

S
slguan 已提交
232
typedef struct STableDataBlocks {
S
slguan 已提交
233 234
  char   meterId[TSDB_METER_ID_LEN];
  int8_t tsSource;
H
hjxilinx 已提交
235
  bool   ordered;
S
slguan 已提交
236

S
slguan 已提交
237 238
  int64_t vgid;
  int64_t prevTS;
S
slguan 已提交
239

H
hjxilinx 已提交
240
  int32_t numOfMeters;
S
slguan 已提交
241

S
slguan 已提交
242 243 244
  int32_t  rowSize;
  uint32_t nAllocSize;
  uint32_t size;
H
hzcheng 已提交
245 246 247 248
  union {
    char *filename;
    char *pData;
  };
S
slguan 已提交
249 250

  // for parameter ('?') binding
H
hjxilinx 已提交
251 252 253
  uint32_t    numOfAllocedParams;
  uint32_t    numOfParams;
  SParamInfo *params;
S
slguan 已提交
254
} STableDataBlocks;
H
hzcheng 已提交
255 256

typedef struct SDataBlockList {
S
slguan 已提交
257 258 259 260 261
  int32_t            idx;
  int32_t            nSize;
  int32_t            nAlloc;
  char *             userParam; /* user assigned parameters for async query */
  void *             udfp;      /* user defined function pointer, used in async model */
S
slguan 已提交
262
  STableDataBlocks **pData;
H
hzcheng 已提交
263 264 265
} SDataBlockList;

typedef struct {
S
slguan 已提交
266 267
  SOrderVal order;
  int       command;
H
hjxilinx 已提交
268
  int       count;  // TODO refactor
S
slguan 已提交
269 270

  union {
H
hjxilinx 已提交
271 272
    bool   existsCheck;  // check if the table exists
    int8_t showType;     // show command type
S
slguan 已提交
273
  };
H
hjxilinx 已提交
274

H
hjxilinx 已提交
275
  int8_t          isInsertFromFile;  // load data from file or not
H
hjxilinx 已提交
276
  bool            import;            // import/insert type
H
hzcheng 已提交
277
  char            msgType;
H
hjxilinx 已提交
278
  uint16_t        type;  // query type
H
hzcheng 已提交
279
  char            intervalTimeUnit;
S
slguan 已提交
280
  int64_t         etime, stime;
H
hzcheng 已提交
281 282 283 284 285 286 287 288 289 290
  int64_t         nAggTimeInterval;  // aggregation time interval
  int64_t         nSlidingTime;      // sliding window in mseconds
  SSqlGroupbyExpr groupbyExpr;       // group by tags info

  /*
   * use to keep short request msg and error msg, in such case, SSqlCmd->payload == SSqlCmd->ext;
   * create table/query/insert operations will exceed the TSDB_SQLCMD_SIZE.
   *
   * In such cases, allocate the memory dynamically, and need to free the memory
   */
H
hjxilinx 已提交
291 292 293 294
  uint32_t        allocSize;
  char *          payload;
  int             payloadLen;
  short           numOfCols;
S
slguan 已提交
295
  SColumnBaseInfo colList;
H
hjxilinx 已提交
296 297 298 299 300 301 302 303
  SFieldInfo      fieldsInfo;
  SSqlExprInfo    exprsInfo;
  SLimitVal       limit;
  SLimitVal       slimit;
  int64_t         globalLimit;
  STagCond        tagCond;
  int16_t         interpoType;  // interpolate type
  int16_t         numOfTables;
H
hzcheng 已提交
304

S
slguan 已提交
305 306 307 308
  // submit data blocks branched according to vnode
  SDataBlockList * pDataBlocks;
  SMeterMetaInfo **pMeterInfo;
  struct STSBuf *  tsBuf;
H
hzcheng 已提交
309 310
  // todo use dynamic allocated memory for defaultVal
  int64_t defaultVal[TSDB_MAX_COLUMNS];  // default value for interpolation
S
slguan 已提交
311 312 313 314

  // for parameter ('?') binding and batch processing
  int32_t batchSize;
  int32_t numOfParams;
H
hzcheng 已提交
315 316 317 318 319 320 321
} SSqlCmd;

typedef struct SResRec {
  int numOfRows;
  int numOfTotal;
} SResRec;

S
slguan 已提交
322 323
struct STSBuf;

H
hzcheng 已提交
324
typedef struct {
S
slguan 已提交
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
  uint8_t               code;
  int                   numOfRows;   // num of results in current retrieved
  int                   numOfTotal;  // num of total results
  char *                pRsp;
  int                   rspType;
  int                   rspLen;
  uint64_t              qhandle;
  int64_t               useconds;
  int64_t               offset;  // offset value from vnode during projection query of stable
  int                   row;
  int16_t               numOfnchar;
  int16_t               precision;
  int32_t               numOfGroups;
  SResRec *             pGroupRec;
  char *                data;
  short *               bytes;
  void **               tsrow;
  char **               buffer;  // Buffer used to put multibytes encoded using unicode (wchar_t)
H
hzcheng 已提交
343
  struct SLocalReducer *pLocalReducer;
S
slguan 已提交
344
  SColumnIndex *        pColumnIndex;
H
hzcheng 已提交
345 346 347 348 349 350
} SSqlRes;

typedef struct _tsc_obj {
  void *           signature;
  void *           pTimer;
  char             mgmtIp[TSDB_USER_LEN];
L
lihui 已提交
351
  uint16_t         mgmtPort;
H
hzcheng 已提交
352 353 354
  char             user[TSDB_USER_LEN];
  char             pass[TSDB_KEY_LEN];
  char             acctId[TSDB_DB_NAME_LEN];
L
lihui 已提交
355
  char             db[TSDB_METER_ID_LEN];
H
hzcheng 已提交
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
  char             sversion[TSDB_VERSION_LEN];
  char             writeAuth : 1;
  char             superAuth : 1;
  struct _sql_obj *pSql;
  struct _sql_obj *pHb;
  struct _sql_obj *sqlList;
  struct _sstream *streamList;
  pthread_mutex_t  mutex;
} STscObj;

typedef struct _sql_obj {
  void *   signature;
  STscObj *pTscObj;
  void (*fp)();
  void (*fetchFp)();
H
hjxilinx 已提交
371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387
  void *            param;
  uint32_t          ip;
  short             vnode;
  int64_t           stime;
  uint32_t          queryId;
  void *            thandle;
  void *            pStream;
  char *            sqlstr;
  char              retry;
  char              maxRetry;
  char              index;
  char              freed : 4;
  char              listed : 4;
  tsem_t            rspSem;
  tsem_t            emptyRspSem;
  SSqlCmd           cmd;
  SSqlRes           res;
H
hzcheng 已提交
388 389 390 391 392 393 394 395 396 397 398 399
  char              numOfSubs;
  struct _sql_obj **pSubs;
  struct _sql_obj * prev, *next;
} SSqlObj;

typedef struct _sstream {
  SSqlObj *pSql;
  uint32_t streamId;
  char     listed;
  int64_t  num;  // number of computing count

  /*
S
slguan 已提交
400
   * keep the number of current result in computing,
H
hzcheng 已提交
401 402 403 404 405 406 407
   * the value will be set to 0 before set timer for next computing
   */
  int64_t numOfRes;

  int64_t useconds;  // total  elapsed time
  int64_t ctime;     // stream created time
  int64_t stime;     // stream next executed time
S
slguan 已提交
408
  int64_t etime;     // stream end query time, when time is larger then etime, the stream will be closed
H
hzcheng 已提交
409 410
  int64_t interval;
  int64_t slidingTime;
411
  int16_t precision;
H
hzcheng 已提交
412 413 414 415 416
  void *  pTimer;

  void (*fp)();
  void *param;

S
slguan 已提交
417
  void (*callback)(void *);  // Callback function when stream is stopped from client level
H
hzcheng 已提交
418 419 420 421 422 423 424 425 426 427 428 429 430 431
  struct _sstream *prev, *next;
} SSqlStream;

typedef struct {
  char     numOfIps;
  uint32_t ip[TSDB_MAX_MGMT_IPS];
  char     ipstr[TSDB_MAX_MGMT_IPS][TSDB_IPv4ADDR_LEN];
} SIpStrList;

// tscSql API
int tsParseSql(SSqlObj *pSql, char *acct, char *db, bool multiVnodeInsertion);

void  tscInitMsgs();
void *tscProcessMsgFromServer(char *msg, void *ahandle, void *thandle);
H
hjxilinx 已提交
432
int   tscProcessSql(SSqlObj *pSql);
H
hzcheng 已提交
433 434 435

void tscAsyncInsertMultiVnodesProxy(void *param, TAOS_RES *tres, int numOfRows);

H
hjxilinx 已提交
436
int  tscRenewMeterMeta(SSqlObj *pSql, char *meterId);
H
hzcheng 已提交
437
void tscQueueAsyncRes(SSqlObj *pSql);
S
slguan 已提交
438

H
hzcheng 已提交
439 440 441 442 443 444 445 446 447 448
void tscQueueAsyncError(void(*fp), void *param);

int tscProcessLocalCmd(SSqlObj *pSql);
int tscCfgDynamicOptions(char *msg);
int taos_retrieve(TAOS_RES *res);

/*
 * transfer function for metric query in stream computing, the function need to be change
 * before send query message to vnode
 */
S
slguan 已提交
449
int32_t tscTansformSQLFunctionForMetricQuery(SSqlCmd *pCmd);
H
hjxilinx 已提交
450
void    tscRestoreSQLFunctionForMetricQuery(SSqlCmd *pCmd);
H
hzcheng 已提交
451 452 453 454

void tscClearSqlMetaInfoForce(SSqlCmd *pCmd);

int32_t tscCreateResPointerInfo(SSqlCmd *pCmd, SSqlRes *pRes);
H
hjxilinx 已提交
455
void    tscDestroyResPointerInfo(SSqlRes *pRes);
H
hzcheng 已提交
456

S
slguan 已提交
457
void tscFreeSqlCmdData(SSqlCmd *pCmd);
H
hzcheng 已提交
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474

/**
 * only free part of resources allocated during query.
 * Note: this function is multi-thread safe.
 * @param pObj
 */
void tscFreeSqlObjPartial(SSqlObj *pObj);

/**
 * free sql object, release allocated resource
 * @param pObj  Free metric/meta information, dynamically allocated payload, and
 * response buffer, object itself
 */
void tscFreeSqlObj(SSqlObj *pObj);

void tscCloseTscObj(STscObj *pObj);

H
hjxilinx 已提交
475 476 477 478 479
void    tscProcessMultiVnodesInsert(SSqlObj *pSql);
void    tscProcessMultiVnodesInsertForFile(SSqlObj *pSql);
void    tscKillMetricQuery(SSqlObj *pSql);
void    tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen);
bool    tscIsUpdateQuery(STscObj *pObj);
H
hjxilinx 已提交
480 481
bool    tscHasReachLimitation(SSqlObj* pSql);

H
hjxilinx 已提交
482
int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql);
H
hzcheng 已提交
483 484 485 486 487 488

// transfer SSqlInfo to SqlCmd struct
int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo);

void tscQueueAsyncFreeResult(SSqlObj *pSql);

S
slguan 已提交
489 490 491 492 493 494 495 496 497 498 499 500
extern void *     pVnodeConn;
extern void *     pTscMgmtConn;
extern void *     tscCacheHandle;
extern uint8_t    globalCode;
extern int        slaveIndex;
extern void *     tscTmr;
extern void *     tscConnCache;
extern void *     tscQhandle;
extern int        tscKeepConn[];
extern int        tsInsertHeadSize;
extern int        tscNumOfThreads;
extern SIpStrList tscMgmtIpList;
H
hzcheng 已提交
501 502 503 504 505 506

#ifdef __cplusplus
}
#endif

#endif