tsdb.h 9.9 KB
Newer Older
H
more  
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
hzcheng 已提交
15
#ifndef _TD_TSDB_H_
H
Hongze Cheng 已提交
16 17 18 19 20 21
#define _TD_TSDB_H_

#include <pthread.h>
#include <stdbool.h>
#include <stdint.h>

S
slguan 已提交
22 23
#include "tdataformat.h"
#include "tname.h"
H
hzcheng 已提交
24
#include "taosdef.h"
H
hzcheng 已提交
25
#include "taosmsg.h"
H
hjxilinx 已提交
26
#include "tarray.h"
H
Hongze Cheng 已提交
27

H
more  
hzcheng 已提交
28 29 30 31
#ifdef __cplusplus
extern "C" {
#endif

H
Hongze Cheng 已提交
32 33 34
#define TSDB_VERSION_MAJOR 1
#define TSDB_VERSION_MINOR 0

H
hzcheng 已提交
35 36
#define TSDB_INVALID_SUPER_TABLE_ID -1

37 38 39
#define TSDB_STATUS_COMMIT_START 1
#define TSDB_STATUS_COMMIT_OVER  2

H
hzcheng 已提交
40 41 42
// --------- TSDB APPLICATION HANDLE DEFINITION
typedef struct {
  void *appH;
J
jtao1735 已提交
43
  void *cqH;
44
  int (*notifyStatus)(void *, int status);
H
hzcheng 已提交
45 46 47
  int (*eventCallBack)(void *);
} STsdbAppH;

H
hzcheng 已提交
48 49 50
// --------- TSDB REPOSITORY CONFIGURATION DEFINITION
typedef struct {
  int32_t tsdbId;
S
slguan 已提交
51 52
  int32_t cacheBlockSize;
  int32_t totalBlocks;
H
hzcheng 已提交
53 54
  int32_t maxTables;            // maximum number of tables this repository can have
  int32_t daysPerFile;          // day per file sharding policy
S
slguan 已提交
55 56 57
  int32_t keep;                 // day of data to keep
  int32_t keep1;
  int32_t keep2;
H
hzcheng 已提交
58 59
  int32_t minRowsPerFileBlock;  // minimum rows per file block
  int32_t maxRowsPerFileBlock;  // maximum rows per file block
S
slguan 已提交
60 61 62
  int32_t commitTime;
  int8_t  precision;
  int8_t  compression;
H
hzcheng 已提交
63 64 65 66 67 68 69
} STsdbCfg;

void      tsdbSetDefaultCfg(STsdbCfg *pCfg);
STsdbCfg *tsdbCreateDefaultCfg();
void      tsdbFreeCfg(STsdbCfg *pCfg);

// --------- TSDB REPOSITORY DEFINITION
H
hzcheng 已提交
70
typedef void TsdbRepoT;  // use void to hide implementation details from outside
H
hzcheng 已提交
71

H
hzcheng 已提交
72 73 74
int        tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter);
int32_t    tsdbDropRepo(TsdbRepoT *repo);
TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH);
75
int32_t    tsdbCloseRepo(TsdbRepoT *repo, int toCommit);
H
hzcheng 已提交
76
int32_t    tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg);
H
hzcheng 已提交
77 78

// --------- TSDB TABLE DEFINITION
H
more  
hzcheng 已提交
79
typedef struct {
80 81
  uint64_t uid;  // the unique table ID
  int32_t  tid;  // the table ID in the repository.
H
more  
hzcheng 已提交
82 83
} STableId;

H
hzcheng 已提交
84 85
// --------- TSDB TABLE configuration
typedef struct {
H
hzcheng 已提交
86 87 88 89 90
  ETableType type;
  char *     name;
  STableId   tableId;
  int32_t    sversion;
  char *     sname;  // super table name
91
  uint64_t   superUid;
H
hzcheng 已提交
92 93 94
  STSchema * schema;
  STSchema * tagSchema;
  SDataRow   tagValues;
H
TD-354  
Hongze Cheng 已提交
95
  char *     sql;
H
hzcheng 已提交
96 97
} STableCfg;

98 99
int  tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid);
int  tsdbTableSetSuperUid(STableCfg *config, uint64_t uid);
H
hzcheng 已提交
100 101 102
int  tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup);
int  tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup);
int  tsdbTableSetTagValue(STableCfg *config, SDataRow row, bool dup);
H
hzcheng 已提交
103 104
int  tsdbTableSetName(STableCfg *config, char *name, bool dup);
int  tsdbTableSetSName(STableCfg *config, char *sname, bool dup);
H
TD-354  
Hongze Cheng 已提交
105
int  tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup);
H
hzcheng 已提交
106 107
void tsdbClearTableCfg(STableCfg *config);

H
hjxilinx 已提交
108
int32_t tsdbGetTableTagVal(TsdbRepoT *repo, STableId* id, int32_t colId, int16_t *type, int16_t *bytes, char **val);
H
hjxilinx 已提交
109
char* tsdbGetTableName(TsdbRepoT *repo, const STableId* id, int16_t* bytes);
110

H
hzcheng 已提交
111 112 113
int   tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg);
int   tsdbDropTable(TsdbRepoT *pRepo, STableId tableId);
int   tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg);
114
TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, uint64_t uid);
H
hzcheng 已提交
115

116 117
uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, int32_t *size);

H
Hongze Cheng 已提交
118
// the TSDB repository info
H
more  
hzcheng 已提交
119
typedef struct STsdbRepoInfo {
H
more  
hzcheng 已提交
120
  STsdbCfg tsdbCfg;
H
Hongze Cheng 已提交
121 122 123 124
  int64_t  version;            // version of the repository
  int64_t  tsdbTotalDataSize;  // the original inserted data size
  int64_t  tsdbTotalDiskSize;  // the total disk size taken by this TSDB repository
  // TODO: Other informations to add
H
more  
hzcheng 已提交
125
} STsdbRepoInfo;
H
hzcheng 已提交
126
STsdbRepoInfo *tsdbGetStatus(TsdbRepoT *pRepo);
H
hzcheng 已提交
127

H
Hongze Cheng 已提交
128 129 130 131
// the meter information report structure
typedef struct {
  STableCfg tableCfg;
  int64_t   version;
H
more  
hzcheng 已提交
132 133
  int64_t   tableTotalDataSize;  // In bytes
  int64_t   tableTotalDiskSize;  // In bytes
H
Hongze Cheng 已提交
134
} STableInfo;
H
hzcheng 已提交
135
STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tid);
H
Hongze Cheng 已提交
136 137 138 139 140 141 142 143 144

// -- FOR INSERT DATA
/**
 * Insert data to a table in a repository
 * @param pRepo the TSDB repository handle
 * @param pData the data to insert (will give a more specific description)
 *
 * @return the number of points inserted, -1 for failure and the error number is set
 */
145
int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) ;
H
Hongze Cheng 已提交
146 147 148

// -- FOR QUERY TIME SERIES DATA

H
hzcheng 已提交
149
typedef void *TsdbQueryHandleT;  // Use void to hide implementation details
H
Hongze Cheng 已提交
150 151

// query condition to build vnode iterator
H
hjxilinx 已提交
152
typedef struct STsdbQueryCond {
H
hzcheng 已提交
153
  STimeWindow      twindow;
H
hjxilinx 已提交
154
  int32_t          order;  // desc|asc order to iterate the data block
155
  int32_t          numOfCols;
156
  SColumnInfo     *colList;
H
hjxilinx 已提交
157
} STsdbQueryCond;
H
Hongze Cheng 已提交
158

H
hjxilinx 已提交
159 160
typedef struct SDataBlockInfo {
  STimeWindow window;
161
  int32_t     rows;
H
hjxilinx 已提交
162 163
  int32_t     numOfCols;
  int64_t     uid;
H
hjxilinx 已提交
164
  int32_t     tid;
H
hjxilinx 已提交
165
} SDataBlockInfo;
H
Hongze Cheng 已提交
166

167 168
typedef struct {
  size_t  numOfTables;
H
hzcheng 已提交
169
  SArray *pGroupList;
170 171
} STableGroupInfo;

H
hjxilinx 已提交
172 173 174 175 176
typedef struct SQueryRowCond {
  int32_t rel;
  TSKEY   ts;
} SQueryRowCond;

H
hzcheng 已提交
177
typedef void *TsdbPosT;
H
hjxilinx 已提交
178

H
Hongze Cheng 已提交
179 180
/**
 * Get the data block iterator, starting from position according to the query condition
H
hjxilinx 已提交
181 182 183 184
 *
 * @param tsdb       tsdb handle
 * @param pCond      query condition, including time window, result set order, and basic required columns for each block
 * @param groupInfo  tableId list in the form of set, seperated into different groups according to group by condition
H
Hongze Cheng 已提交
185 186
 * @return
 */
H
hjxilinx 已提交
187
TsdbQueryHandleT *tsdbQueryTables(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo);
H
Hongze Cheng 已提交
188 189

/**
H
hjxilinx 已提交
190 191 192 193 194 195 196 197 198 199 200
 * Get the last row of the given query time window for all the tables in STableGroupInfo object.
 * Note that only one data block with only row will be returned while invoking retrieve data block function for
 * all tables in this group.
 *
 * @param tsdb        tsdb handle
 * @param pCond       query condition, including time window, result set order, and basic required columns for each block
 * @param groupInfo   tableId list.
 * @return
 */
TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo);

201 202
TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TsdbRepoT *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList);

H
hjxilinx 已提交
203 204 205
/**
 * move to next block if exists
 *
H
hjxilinx 已提交
206
 * @param pQueryHandle
H
Hongze Cheng 已提交
207 208
 * @return
 */
H
hzcheng 已提交
209
bool tsdbNextDataBlock(TsdbQueryHandleT *pQueryHandle);
H
Hongze Cheng 已提交
210 211

/**
H
hjxilinx 已提交
212
 * Get current data block information
H
Hongze Cheng 已提交
213 214 215 216
 *
 * @param pQueryHandle
 * @return
 */
H
hzcheng 已提交
217
SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle);
H
Hongze Cheng 已提交
218 219

/**
H
hjxilinx 已提交
220 221 222 223 224 225 226
 *
 * Get the pre-calculated information w.r.t. current data block.
 *
 * In case of data block in cache, the pBlockStatis will always be NULL.
 * If a block is not completed loaded from disk, the pBlockStatis will be NULL.

 * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0
H
Hongze Cheng 已提交
227 228
 * @return
 */
H
hzcheng 已提交
229
int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataStatis **pBlockStatis);
H
Hongze Cheng 已提交
230 231

/**
H
hjxilinx 已提交
232
 *
H
hjxilinx 已提交
233 234 235
 * The query condition with primary timestamp is passed to iterator during its constructor function,
 * the returned data block must be satisfied with the time window condition in any cases,
 * which means the SData data block is not actually the completed disk data blocks.
H
Hongze Cheng 已提交
236
 *
H
hjxilinx 已提交
237 238
 * @param pQueryHandle      query handle
 * @param pColumnIdList     required data columns id list
H
Hongze Cheng 已提交
239 240
 * @return
 */
H
hjxilinx 已提交
241
SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdList);
H
Hongze Cheng 已提交
242 243

/**
H
hjxilinx 已提交
244 245 246 247 248
 * todo remove this function later
 * @param pQueryHandle
 * @param pIdList
 * @return
 */
H
hzcheng 已提交
249
SArray *tsdbRetrieveDataRow(TsdbQueryHandleT *pQueryHandle, SArray *pIdList, SQueryRowCond *pCond);
H
hjxilinx 已提交
250 251 252

/**
 *  Get iterator for super tables, of which tags values satisfy the tag filter info
H
Hongze Cheng 已提交
253
 *
H
hjxilinx 已提交
254 255 256
 *  NOTE: the tagFilterStr is an bin-expression for tag filter, such as ((tag_col = 5) and (tag_col2 > 7))
 *  The filter string is sent from client directly.
 *  The build of the tags filter expression from string is done in the iterator generating function.
H
Hongze Cheng 已提交
257
 *
H
hjxilinx 已提交
258 259
 * @param pCond         query condition
 * @param pTagFilterStr tag filter info
H
Hongze Cheng 已提交
260 261
 * @return
 */
H
hzcheng 已提交
262
TsdbQueryHandleT *tsdbQueryFromTagConds(STsdbQueryCond *pCond, int16_t stableId, const char *pTagFilterStr);
H
Hongze Cheng 已提交
263 264 265 266 267 268 269 270 271

/**
 * Get the qualified tables for (super) table query.
 * Used to handle the super table projection queries, the last_row query, the group by on normal columns query,
 * the interpolation query, and timestamp-comp query for join processing.
 *
 * @param pQueryHandle
 * @return table sid list. the invoker is responsible for the release of this the sid list.
 */
H
hzcheng 已提交
272
SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle);
H
Hongze Cheng 已提交
273 274

/**
275
 * Get the qualified table id for a super table according to the tag query expression.
H
Hongze Cheng 已提交
276 277 278
 * @param stableid. super table sid
 * @param pTagCond. tag query condition
 */
H
Hongze Cheng 已提交
279 280 281
int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, uint64_t uid, const char *pTagCond, size_t len,
                                 int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList,
                                 SColIndex *pColIndex, int32_t numOfCols);
H
hjxilinx 已提交
282 283 284 285 286 287 288 289 290

/**
 * create the table group result including only one table, used to handle the normal table query
 *
 * @param tsdb        tsdbHandle
 * @param uid         table uid
 * @param pGroupInfo  the generated result
 * @return
 */
H
Hongze Cheng 已提交
291
int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, uint64_t uid, STableGroupInfo *pGroupInfo);
H
Hongze Cheng 已提交
292

293 294 295 296
/**
 * clean up the query handle
 * @param queryHandle
 */
H
hzcheng 已提交
297
void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
298

H
more  
hzcheng 已提交
299 300 301 302
#ifdef __cplusplus
}
#endif

陶建辉(Jeff)'s avatar
陶建辉(Jeff) 已提交
303
#endif  // _TD_TSDB_H_