tsdb.h 11.5 KB
Newer Older
H
more  
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
Hongze Cheng 已提交
15 16 17 18 19 20 21
#if !defined(_TD_TSDB_H_)
#define _TD_TSDB_H_

#include <pthread.h>
#include <stdbool.h>
#include <stdint.h>

H
hzcheng 已提交
22
#include "taosdef.h"
H
hzcheng 已提交
23
#include "taosmsg.h"
H
more  
hzcheng 已提交
24
#include "dataformat.h"
H
Hongze Cheng 已提交
25

H
more  
hzcheng 已提交
26 27 28 29
#ifdef __cplusplus
extern "C" {
#endif

H
Hongze Cheng 已提交
30 31 32
#define TSDB_VERSION_MAJOR 1
#define TSDB_VERSION_MINOR 0

H
more  
hzcheng 已提交
33
typedef void tsdb_repo_t;  // use void to hide implementation details from outside
H
Hongze Cheng 已提交
34

H
hzcheng 已提交
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
// --------- TSDB REPOSITORY CONFIGURATION DEFINITION
typedef struct {
  int8_t  precision;
  int32_t tsdbId;
  int32_t maxTables;            // maximum number of tables this repository can have
  int32_t daysPerFile;          // day per file sharding policy
  int32_t minRowsPerFileBlock;  // minimum rows per file block
  int32_t maxRowsPerFileBlock;  // maximum rows per file block
  int32_t keep;                 // day of data to keep
  int64_t maxCacheSize;         // maximum cache size this TSDB can use
} STsdbCfg;

void      tsdbSetDefaultCfg(STsdbCfg *pCfg);
STsdbCfg *tsdbCreateDefaultCfg();
void      tsdbFreeCfg(STsdbCfg *pCfg);

// --------- TSDB REPOSITORY DEFINITION

H
more  
hzcheng 已提交
53
typedef struct {
H
more  
hzcheng 已提交
54 55
  int64_t uid;  // the unique table ID
  int32_t tid;  // the table ID in the repository.
H
more  
hzcheng 已提交
56 57 58
} STableId;

// Submit message for this TSDB
H
more  
hzcheng 已提交
59 60
typedef struct {
  int32_t numOfTables;
H
more  
hzcheng 已提交
61
  int32_t compressed;
H
more  
hzcheng 已提交
62 63 64 65 66
  char    data[];
} SSubmitMsg;

// Submit message for one table
typedef struct {
H
more  
hzcheng 已提交
67
  STableId tableId;
H
hzcheng 已提交
68
  int32_t  padding;    // TODO just for padding here
H
more  
hzcheng 已提交
69
  int32_t  sversion;   // data schema version
H
hzcheng 已提交
70
  int32_t  len;        // message length
H
more  
hzcheng 已提交
71
  char     data[];
H
more  
hzcheng 已提交
72 73
} SSubmitBlock;

H
more  
hzcheng 已提交
74 75
enum { TSDB_PRECISION_MILLI, TSDB_PRECISION_MICRO, TSDB_PRECISION_NANO };

H
Hongze Cheng 已提交
76
// the TSDB repository info
H
more  
hzcheng 已提交
77
typedef struct STsdbRepoInfo {
H
more  
hzcheng 已提交
78
  STsdbCfg tsdbCfg;
H
Hongze Cheng 已提交
79 80 81 82
  int64_t  version;            // version of the repository
  int64_t  tsdbTotalDataSize;  // the original inserted data size
  int64_t  tsdbTotalDiskSize;  // the total disk size taken by this TSDB repository
  // TODO: Other informations to add
H
more  
hzcheng 已提交
83
} STsdbRepoInfo;
H
Hongze Cheng 已提交
84 85 86

// the meter configuration
typedef struct {
H
more  
hzcheng 已提交
87
  STableId tableId;
H
Hongze Cheng 已提交
88 89

  int64_t stableUid;
H
more  
hzcheng 已提交
90
  int64_t createdTime;
H
Hongze Cheng 已提交
91 92

  int32_t  numOfCols;  // number of columns. For table form super table, not includes the tag schema
H
hzcheng 已提交
93
  STSchema *schema;     // If numOfCols == schema_->numOfCols, it is a normal table, stableName = NULL
H
Hongze Cheng 已提交
94 95 96
                       // If numOfCols < schema->numOfCols, it is a table created from super table
                       // assert(numOfCols <= schema->numOfCols);

H
more  
hzcheng 已提交
97 98
  SDataRow tagValues;  // NULL if it is normal table
                       // otherwise, it contains the tag values.
H
Hongze Cheng 已提交
99 100 101 102 103 104
} STableCfg;

// the meter information report structure
typedef struct {
  STableCfg tableCfg;
  int64_t   version;
H
more  
hzcheng 已提交
105 106
  int64_t   tableTotalDataSize;  // In bytes
  int64_t   tableTotalDiskSize;  // In bytes
H
Hongze Cheng 已提交
107 108
} STableInfo;

H
hzcheng 已提交
109

H
Hongze Cheng 已提交
110 111
/**
 * Create a new TSDB repository
H
more  
hzcheng 已提交
112
 * @param rootDir the TSDB repository root directory
H
Hongze Cheng 已提交
113 114 115 116
 * @param pCfg the TSDB repository configuration, upper layer to free the pointer
 *
 * @return a TSDB repository handle on success, NULL for failure and the error number is set
 */
H
hzcheng 已提交
117
tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter);
H
Hongze Cheng 已提交
118 119 120

/**
 * Close and free all resources taken by the repository
H
more  
hzcheng 已提交
121
 * @param repo the TSDB repository handle. The interface will free the handle too, so upper
H
Hongze Cheng 已提交
122 123 124 125
 *              layer do NOT need to free the repo handle again.
 *
 * @return 0 for success, -1 for failure and the error number is set
 */
H
more  
hzcheng 已提交
126
int32_t tsdbDropRepo(tsdb_repo_t *repo);
H
Hongze Cheng 已提交
127 128 129 130 131 132 133

/**
 * Open an existing TSDB storage repository
 * @param tsdbDir the existing TSDB root directory
 *
 * @return a TSDB repository handle on success, NULL for failure and the error number is set
 */
H
more  
hzcheng 已提交
134
tsdb_repo_t *tsdbOpenRepo(char *tsdbDir);
H
Hongze Cheng 已提交
135 136 137

/**
 * Close a TSDB repository. Only free memory resources, and keep the files.
H
more  
hzcheng 已提交
138
 * @param repo the opened TSDB repository handle. The interface will free the handle too, so upper
H
Hongze Cheng 已提交
139 140 141 142
 *              layer do NOT need to free the repo handle again.
 *
 * @return 0 for success, -1 for failure and the error number is set
 */
H
more  
hzcheng 已提交
143
int32_t tsdbCloseRepo(tsdb_repo_t *repo);
H
Hongze Cheng 已提交
144 145 146 147 148 149 150

/**
 * Change the configuration of a repository
 * @param pCfg the repository configuration, the upper layer should free the pointer
 *
 * @return 0 for success, -1 for failure and the error number is set
 */
H
more  
hzcheng 已提交
151
int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg);
H
Hongze Cheng 已提交
152 153 154 155 156 157 158 159 160

/**
 * Get the TSDB repository information, including some statistics
 * @param pRepo the TSDB repository handle
 * @param error the error number to set when failure occurs
 *
 * @return a info struct handle on success, NULL for failure and the error number is set. The upper
 *         layers should free the info handle themselves or memory leak will occur
 */
H
more  
hzcheng 已提交
161
STsdbRepoInfo *tsdbGetStatus(tsdb_repo_t *pRepo);
H
Hongze Cheng 已提交
162 163 164 165 166

// -- For table manipulation

/**
 * Create/Alter a table in a TSDB repository handle
H
more  
hzcheng 已提交
167
 * @param repo the TSDB repository handle
H
Hongze Cheng 已提交
168 169 170 171
 * @param pCfg the table configurations, the upper layer should free the pointer
 *
 * @return 0 for success, -1 for failure and the error number is set
 */
H
more  
hzcheng 已提交
172 173
int32_t tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg);
int32_t tsdbAlterTable(tsdb_repo_t *repo, STableCfg *pCfg);
H
Hongze Cheng 已提交
174 175 176 177 178 179 180 181 182

/**
 * Drop a table in a repository and free all the resources it takes
 * @param pRepo the TSDB repository handle
 * @param tid the ID of the table to drop
 * @param error the error number to set when failure occurs
 *
 * @return 0 for success, -1 for failure and the error number is set
 */
H
hzcheng 已提交
183
int32_t tsdbDropTable(tsdb_repo_t *pRepo, STableId tableId);
H
Hongze Cheng 已提交
184 185 186 187 188 189 190 191 192

/**
 * Get the information of a table in the repository
 * @param pRepo the TSDB repository handle
 * @param tid the ID of the table to drop
 * @param error the error number to set when failure occurs
 *
 * @return a table information handle for success, NULL for failure and the error number is set
 */
H
more  
hzcheng 已提交
193
STableInfo *tsdbGetTableInfo(tsdb_repo_t *pRepo, STableId tid);
H
Hongze Cheng 已提交
194 195 196 197 198 199 200 201 202

// -- FOR INSERT DATA
/**
 * Insert data to a table in a repository
 * @param pRepo the TSDB repository handle
 * @param pData the data to insert (will give a more specific description)
 *
 * @return the number of points inserted, -1 for failure and the error number is set
 */
H
hzcheng 已提交
203
int32_t tsdbInsertData(tsdb_repo_t *pRepo, SSubmitMsg *pMsg);
H
Hongze Cheng 已提交
204 205 206

// -- FOR QUERY TIME SERIES DATA

H
more  
hzcheng 已提交
207
typedef void tsdb_query_handle_t;  // Use void to hide implementation details
H
Hongze Cheng 已提交
208 209 210 211 212 213 214

// time window
typedef struct STimeWindow {
  int64_t skey;
  int64_t ekey;
} STimeWindow;

H
hzcheng 已提交
215 216
// typedef struct {
// } SColumnFilterInfo;
H
more  
hzcheng 已提交
217

H
Hongze Cheng 已提交
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
// query condition to build vnode iterator
typedef struct STSDBQueryCond {
  STimeWindow       twindow;
  int32_t           order;  // desc/asc order to iterate the data block
  SColumnFilterInfo colFilterInfo;
} STSDBQueryCond;

typedef struct SBlockInfo {
  STimeWindow window;

  int32_t numOfRows;
  int32_t numOfCols;

  STableId tableId;
} SBlockInfo;

//  TODO: move this data struct out of the module
typedef struct SData {
  int32_t num;
  char *  data;
} SData;

typedef struct SDataBlock {
  int32_t numOfCols;
  SData **pData;
} SDataBlock;

typedef struct STableIDList {
  STableId *tableIds;
  int32_t   num;
} STableIDList;

H
more  
hzcheng 已提交
250 251 252
typedef struct {
} SFields;

H
Hongze Cheng 已提交
253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
/**
 * Get the data block iterator, starting from position according to the query condition
 * @param pRepo  the TSDB repository to query on
 * @param pCond  query condition, only includes the filter on primary time stamp
 * @param pTableList    table sid list
 * @return
 */
tsdb_query_handle_t *tsdbQueryFromTableID(tsdb_repo_t *pRepo, STSDBQueryCond *pCond, const STableIDList *pTableList);

/**
 *  Get iterator for super tables, of which tags values satisfy the tag filter info
 *
 *  NOTE: the tagFilterStr is an bin-expression for tag filter, such as ((tag_col = 5) and (tag_col2 > 7))
 *  The filter string is sent from client directly.
 *  The build of the tags filter expression from string is done in the iterator generating function.
 *
 * @param pRepo         the repository to query on
 * @param pCond         query condition
 * @param pTagFilterStr tag filter info
 * @return
 */
H
more  
hzcheng 已提交
274 275
tsdb_query_handle_t *tsdbQueryFromTagConds(tsdb_repo_t *pRepo, STSDBQueryCond *pCond, int16_t stableId,
                                           const char *pTagFilterStr);
H
Hongze Cheng 已提交
276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348

/**
 *  Reset to the start(end) position of current query, from which the iterator starts.
 *
 * @param pQueryHandle
 * @param position  set the iterator traverses position. (TSDB_POS_START|TSDB_POS_END)
 * @return
 */
int32_t tsdbResetQuery(tsdb_query_handle_t *pQueryHandle, int16_t position);

/**
 *  move to next block
 * @param pQueryHandle
 * @param pCond
 * @return
 */
bool tsdbIterNext(tsdb_query_handle_t *pQueryHandle);

/**
 * 当前数据块的信息,调用next函数后,只会获得block的信息,包括:行数、列数、skey/ekey信息。注意该信息并不是现在的SCompBlockInfo信息。
 * 因为SCompBlockInfo是完整的数据块信息,但是迭代器返回并不是。
 * 查询处理引擎会自己决定需要blockInfo, 还是预计算数据,亦或是完整的数据。
 * Get current data block information
 *
 * @param pQueryHandle
 * @return
 */
SBlockInfo tsdbRetrieveDataBlockInfo(tsdb_query_handle_t *pQueryHandle);

/**
 * 获取当前数据块的预计算信息,如果块不完整,无预计算信息,如果是cache块,无预计算信息。
 *
 * Get the pre-calculated information w.r.t. current data block.
 *
 * In case of data block in cache, the pBlockStatis will always be NULL.
 * If a block is not completed loaded from disk, the pBlockStatis will be NULL.

 * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0
 * @return
 */
int32_t tsdbRetrieveDataBlockStatisInfo(tsdb_query_handle_t *pQueryHandle, SFields *pBlockStatis);

/**
 * 返回加载到缓存中的数据,可能是磁盘数据也可能是内存数据,对客户透明。即使是磁盘数据,返回的结果也是磁盘块中,满足查询时间范围要求的数据行,并不是一个完整的磁盘数
 * 据块。
 *
 * The query condition with primary timestamp is passed to iterator during its constructor function,
 * the returned data block must be satisfied with the time window condition in any cases,
 * which means the SData data block is not actually the completed disk data blocks.
 *
 * @param pQueryHandle
 * @return
 */
SDataBlock *tsdbRetrieveDataBlock(tsdb_query_handle_t *pQueryHandle);

/**
 * Get the qualified tables for (super) table query.
 * Used to handle the super table projection queries, the last_row query, the group by on normal columns query,
 * the interpolation query, and timestamp-comp query for join processing.
 *
 * @param pQueryHandle
 * @return table sid list. the invoker is responsible for the release of this the sid list.
 */
STableIDList *tsdbGetTableList(tsdb_query_handle_t *pQueryHandle);

/**
 * Get the qualified table sid for a super table according to the tag query expression.
 * @param stableid. super table sid
 * @param pTagCond. tag query condition
 *
 */
STableIDList *tsdbQueryTableList(int16_t stableId, const char *pTagCond);

H
more  
hzcheng 已提交
349 350 351 352
#ifdef __cplusplus
}
#endif

H
Hongze Cheng 已提交
353
#endif  // _TD_TSDB_H_