tsdb.h 9.7 KB
Newer Older
H
more  
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
hzcheng 已提交
15
#ifndef _TD_TSDB_H_
H
Hongze Cheng 已提交
16 17 18 19 20 21
#define _TD_TSDB_H_

#include <pthread.h>
#include <stdbool.h>
#include <stdint.h>

H
hzcheng 已提交
22
#include "taosdef.h"
H
hzcheng 已提交
23
#include "taosmsg.h"
H
hjxilinx 已提交
24
#include "tarray.h"
H
Hongze Cheng 已提交
25 26
#include "tdataformat.h"
#include "tname.h"
H
Haojun Liao 已提交
27
#include "hash.h"
H
Hongze Cheng 已提交
28

H
more  
hzcheng 已提交
29 30 31 32
#ifdef __cplusplus
extern "C" {
#endif

H
Hongze Cheng 已提交
33 34 35
#define TSDB_VERSION_MAJOR 1
#define TSDB_VERSION_MINOR 0

H
hzcheng 已提交
36 37
#define TSDB_INVALID_SUPER_TABLE_ID -1

38
#define TSDB_STATUS_COMMIT_START 1
H
Hongze Cheng 已提交
39
#define TSDB_STATUS_COMMIT_OVER 2
40

H
hzcheng 已提交
41 42 43
// --------- TSDB APPLICATION HANDLE DEFINITION
typedef struct {
  void *appH;
J
jtao1735 已提交
44
  void *cqH;
45
  int (*notifyStatus)(void *, int status);
H
hzcheng 已提交
46
  int (*eventCallBack)(void *);
B
Bomin Zhang 已提交
47
  void *(*cqCreateFunc)(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema);
H
TD-354  
Hongze Cheng 已提交
48
  void (*cqDropFunc)(void *handle);
H
hzcheng 已提交
49 50
} STsdbAppH;

H
hzcheng 已提交
51 52 53
// --------- TSDB REPOSITORY CONFIGURATION DEFINITION
typedef struct {
  int32_t tsdbId;
S
slguan 已提交
54 55
  int32_t cacheBlockSize;
  int32_t totalBlocks;
H
Hongze Cheng 已提交
56 57
  int32_t daysPerFile;  // day per file sharding policy
  int32_t keep;         // day of data to keep
S
slguan 已提交
58 59
  int32_t keep1;
  int32_t keep2;
H
hzcheng 已提交
60 61
  int32_t minRowsPerFileBlock;  // minimum rows per file block
  int32_t maxRowsPerFileBlock;  // maximum rows per file block
S
slguan 已提交
62 63
  int8_t  precision;
  int8_t  compression;
H
hzcheng 已提交
64 65
} STsdbCfg;

T
Tao Liu 已提交
66 67
// --------- TSDB REPOSITORY USAGE STATISTICS
typedef struct {
68 69 70 71
  int64_t totalStorage;  // total bytes occupie
  int64_t compStorage;
  int64_t pointsWritten;  // total data points written
} STsdbStat;
T
Tao Liu 已提交
72

H
TD-353  
Hongze Cheng 已提交
73
typedef void TSDB_REPO_T;  // use void to hide implementation details from outside
74

H
TD-353  
Hongze Cheng 已提交
75
STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo);
H
hzcheng 已提交
76 77

// --------- TSDB REPOSITORY DEFINITION
H
TD-353  
Hongze Cheng 已提交
78
int          tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg);
H
TD-353  
Hongze Cheng 已提交
79
int32_t      tsdbDropRepo(char *rootDir);
H
TD-353  
Hongze Cheng 已提交
80
TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH);
H
TD-353  
Hongze Cheng 已提交
81
void         tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit);
H
Hongze Cheng 已提交
82
int32_t      tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg);
H
hzcheng 已提交
83 84

// --------- TSDB TABLE DEFINITION
H
more  
hzcheng 已提交
85
typedef struct {
86 87
  uint64_t uid;  // the unique table ID
  int32_t  tid;  // the table ID in the repository.
H
more  
hzcheng 已提交
88 89
} STableId;

H
hzcheng 已提交
90 91
// --------- TSDB TABLE configuration
typedef struct {
H
hzcheng 已提交
92 93 94 95 96
  ETableType type;
  char *     name;
  STableId   tableId;
  int32_t    sversion;
  char *     sname;  // super table name
97
  uint64_t   superUid;
H
hzcheng 已提交
98 99 100
  STSchema * schema;
  STSchema * tagSchema;
  SDataRow   tagValues;
H
TD-354  
Hongze Cheng 已提交
101
  char *     sql;
H
hzcheng 已提交
102 103 104 105
} STableCfg;

void tsdbClearTableCfg(STableCfg *config);

106 107
void* tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_t bytes);
char* tsdbGetTableName(void *pTable);
H
Haojun Liao 已提交
108 109 110

#define TSDB_TABLEID(_table) ((STableId*) (_table))

H
Hongze Cheng 已提交
111
STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg);
112

H
TD-353  
Hongze Cheng 已提交
113 114
int   tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg);
int   tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId);
H
Hongze Cheng 已提交
115
int   tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg);
H
TD-353  
Hongze Cheng 已提交
116
TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid);
H
hzcheng 已提交
117

118
uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size);
119

H
Hongze Cheng 已提交
120
// the TSDB repository info
H
more  
hzcheng 已提交
121
typedef struct STsdbRepoInfo {
H
more  
hzcheng 已提交
122
  STsdbCfg tsdbCfg;
H
Hongze Cheng 已提交
123 124 125 126
  int64_t  version;            // version of the repository
  int64_t  tsdbTotalDataSize;  // the original inserted data size
  int64_t  tsdbTotalDiskSize;  // the total disk size taken by this TSDB repository
  // TODO: Other informations to add
H
more  
hzcheng 已提交
127
} STsdbRepoInfo;
H
TD-353  
Hongze Cheng 已提交
128
STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo);
H
hzcheng 已提交
129

H
Hongze Cheng 已提交
130 131 132 133
// the meter information report structure
typedef struct {
  STableCfg tableCfg;
  int64_t   version;
H
more  
hzcheng 已提交
134 135
  int64_t   tableTotalDataSize;  // In bytes
  int64_t   tableTotalDiskSize;  // In bytes
H
Hongze Cheng 已提交
136
} STableInfo;
H
TD-353  
Hongze Cheng 已提交
137
STableInfo *tsdbGetTableInfo(TSDB_REPO_T *pRepo, STableId tid);
H
Hongze Cheng 已提交
138 139 140 141 142 143 144 145 146

// -- FOR INSERT DATA
/**
 * Insert data to a table in a repository
 * @param pRepo the TSDB repository handle
 * @param pData the data to insert (will give a more specific description)
 *
 * @return the number of points inserted, -1 for failure and the error number is set
 */
H
Hongze Cheng 已提交
147
int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp);
H
Hongze Cheng 已提交
148 149 150

// -- FOR QUERY TIME SERIES DATA

H
hzcheng 已提交
151
typedef void *TsdbQueryHandleT;  // Use void to hide implementation details
H
Hongze Cheng 已提交
152 153

// query condition to build vnode iterator
H
hjxilinx 已提交
154
typedef struct STsdbQueryCond {
H
Hongze Cheng 已提交
155 156 157 158
  STimeWindow  twindow;
  int32_t      order;  // desc|asc order to iterate the data block
  int32_t      numOfCols;
  SColumnInfo *colList;
H
hjxilinx 已提交
159
} STsdbQueryCond;
H
Hongze Cheng 已提交
160

H
hjxilinx 已提交
161 162
typedef struct SDataBlockInfo {
  STimeWindow window;
163
  int32_t     rows;
H
hjxilinx 已提交
164 165
  int32_t     numOfCols;
  int64_t     uid;
H
hjxilinx 已提交
166
  int32_t     tid;
H
hjxilinx 已提交
167
} SDataBlockInfo;
H
Hongze Cheng 已提交
168

169
typedef struct {
H
Haojun Liao 已提交
170 171 172 173 174 175
  void  *pTable;
  TSKEY  lastKey;
} STableKeyInfo;

typedef struct {
  size_t    numOfTables;
H
Haojun Liao 已提交
176
  SArray   *pGroupList;
H
Haojun Liao 已提交
177
  SHashObj *map;         // speedup acquire the tableQueryInfo by table uid
178 179
} STableGroupInfo;

H
Hongze Cheng 已提交
180 181
/**
 * Get the data block iterator, starting from position according to the query condition
H
hjxilinx 已提交
182 183 184
 *
 * @param tsdb       tsdb handle
 * @param pCond      query condition, including time window, result set order, and basic required columns for each block
H
Haojun Liao 已提交
185 186
 * @param tableInfoGroup  table object list in the form of set, grouped into different sets according to the
 *                        group by condition
H
Haojun Liao 已提交
187
 * @param qinfo      query info handle from query processor
H
Hongze Cheng 已提交
188 189
 * @return
 */
H
Haojun Liao 已提交
190
TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo);
H
Hongze Cheng 已提交
191 192

/**
H
hjxilinx 已提交
193 194 195 196
 * Get the last row of the given query time window for all the tables in STableGroupInfo object.
 * Note that only one data block with only row will be returned while invoking retrieve data block function for
 * all tables in this group.
 *
H
Haojun Liao 已提交
197 198 199
 * @param tsdb   tsdb handle
 * @param pCond  query condition, including time window, result set order, and basic required columns for each block
 * @param tableInfo  table list.
H
hjxilinx 已提交
200 201
 * @return
 */
H
Haojun Liao 已提交
202
TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo);
H
hjxilinx 已提交
203

204 205 206 207 208 209
/**
 * get the queried table object list
 * @param pHandle
 * @return
 */
SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
210

211 212 213 214 215 216 217 218
/**
 * get the group list according to table id from client
 * @param tsdb
 * @param pCond
 * @param groupList
 * @param qinfo
 * @return
 */
H
Hongze Cheng 已提交
219 220
TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
                                               void *qinfo);
221

H
hjxilinx 已提交
222 223 224
/**
 * move to next block if exists
 *
H
hjxilinx 已提交
225
 * @param pQueryHandle
H
Hongze Cheng 已提交
226 227
 * @return
 */
H
hzcheng 已提交
228
bool tsdbNextDataBlock(TsdbQueryHandleT *pQueryHandle);
H
Hongze Cheng 已提交
229 230

/**
H
hjxilinx 已提交
231
 * Get current data block information
H
Hongze Cheng 已提交
232 233
 *
 * @param pQueryHandle
H
Haojun Liao 已提交
234
 * @param pBlockInfo
H
Hongze Cheng 已提交
235 236
 * @return
 */
H
Haojun Liao 已提交
237
void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo* pBlockInfo);
H
Hongze Cheng 已提交
238 239

/**
H
hjxilinx 已提交
240 241 242 243 244 245 246
 *
 * Get the pre-calculated information w.r.t. current data block.
 *
 * In case of data block in cache, the pBlockStatis will always be NULL.
 * If a block is not completed loaded from disk, the pBlockStatis will be NULL.

 * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0
H
Hongze Cheng 已提交
247 248
 * @return
 */
H
hzcheng 已提交
249
int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataStatis **pBlockStatis);
H
Hongze Cheng 已提交
250 251

/**
H
hjxilinx 已提交
252
 *
H
hjxilinx 已提交
253 254 255
 * The query condition with primary timestamp is passed to iterator during its constructor function,
 * the returned data block must be satisfied with the time window condition in any cases,
 * which means the SData data block is not actually the completed disk data blocks.
H
Hongze Cheng 已提交
256
 *
H
hjxilinx 已提交
257 258
 * @param pQueryHandle      query handle
 * @param pColumnIdList     required data columns id list
H
Hongze Cheng 已提交
259 260
 * @return
 */
H
hjxilinx 已提交
261
SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdList);
H
Hongze Cheng 已提交
262 263

/**
264
 * Get the qualified table id for a super table according to the tag query expression.
H
Hongze Cheng 已提交
265 266 267
 * @param stableid. super table sid
 * @param pTagCond. tag query condition
 */
H
Haojun Liao 已提交
268
int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len,
H
Hongze Cheng 已提交
269 270
                                 int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList,
                                 SColIndex *pColIndex, int32_t numOfCols);
H
hjxilinx 已提交
271

272 273 274 275
/**
 * destory the created table group list, which is generated by tag query
 * @param pGroupList
 */
H
Haojun Liao 已提交
276
void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
277

H
hjxilinx 已提交
278 279 280 281 282 283 284 285
/**
 * create the table group result including only one table, used to handle the normal table query
 *
 * @param tsdb        tsdbHandle
 * @param uid         table uid
 * @param pGroupInfo  the generated result
 * @return
 */
H
Haojun Liao 已提交
286
int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
H
Hongze Cheng 已提交
287

288 289 290 291 292 293 294 295 296
/**
 *
 * @param tsdb
 * @param pTableIdList
 * @param pGroupInfo
 * @return
 */
int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo);

297 298 299 300
/**
 * clean up the query handle
 * @param queryHandle
 */
H
hzcheng 已提交
301
void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle);
302

T
Tao Liu 已提交
303 304 305 306 307 308 309 310 311
/**
 * get the statistics of repo usage
 * @param repo. point to the tsdbrepo
 * @param totalPoints. total data point written
 * @param totalStorage. total bytes took by the tsdb
 * @param compStorage. total bytes took by the tsdb after compressed
 */
void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage);

H
more  
hzcheng 已提交
312 313 314 315
#ifdef __cplusplus
}
#endif

陶建辉(Jeff)'s avatar
陶建辉(Jeff) 已提交
316
#endif  // _TD_TSDB_H_