tsdbMain.h 12.1 KB
Newer Older
H
more  
hzcheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
H
TD-34  
hzcheng 已提交
15 16
#ifndef _TD_TSDB_MAIN_H_
#define _TD_TSDB_MAIN_H_
H
more  
Hongze Cheng 已提交
17

H
TD-353  
Hongze Cheng 已提交
18 19
#include "hash.h"
#include "tcoding.h"
S
slguan 已提交
20
#include "tglobal.h"
H
TD-353  
Hongze Cheng 已提交
21
#include "tkvstore.h"
H
hzcheng 已提交
22
#include "tlist.h"
H
TD-353  
Hongze Cheng 已提交
23 24
#include "tlog.h"
#include "tref.h"
H
hzcheng 已提交
25
#include "tsdb.h"
H
TD-34  
hzcheng 已提交
26 27
#include "tskiplist.h"
#include "tutil.h"
H
more  
Hongze Cheng 已提交
28

H
more  
hzcheng 已提交
29 30 31 32
#ifdef __cplusplus
extern "C" {
#endif

H
hzcheng 已提交
33 34
extern int tsdbDebugFlag;

S
Shengliang Guan 已提交
35 36 37 38
#define tsdbError(...) { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("ERROR TDB ", tsdbDebugFlag, __VA_ARGS__); }}
#define tsdbWarn(...)  { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("WARN TDB ", tsdbDebugFlag, __VA_ARGS__); }}
#define tsdbTrace(...) { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }}
#define tsdbPrint(...) { taosPrintLog("TDB ", 255, __VA_ARGS__); }
H
hzcheng 已提交
39

H
TD-90  
Hongze Cheng 已提交
40
#define TSDB_MAX_TABLE_SCHEMAS 16
H
TD-353  
Hongze Cheng 已提交
41 42 43 44 45
#define TSDB_FILE_HEAD_SIZE 512
#define TSDB_FILE_DELIMITER 0xF00AFA0F

// Definitions
// ------------------ tsdbMeta.c
H
TD-34  
hzcheng 已提交
46
typedef struct STable {
H
TD-353  
Hongze Cheng 已提交
47 48 49 50 51 52
  ETableType     type;
  tstr*          name;  // NOTE: there a flexible string here
  STableId       tableId;
  uint64_t       suid;
  struct STable* pSuper;  // super table pointer
  uint8_t        numOfSchemas;
H
TD-353  
Hongze Cheng 已提交
53
  STSchema*      schema[TSDB_MAX_TABLE_SCHEMAS];
H
TD-353  
Hongze Cheng 已提交
54 55
  STSchema*      tagSchema;
  SKVRow         tagVal;
H
TD-353  
Hongze Cheng 已提交
56
  SSkipList*     pIndex;         // For TSDB_SUPER_TABLE, it is the skiplist index
H
TD-353  
Hongze Cheng 已提交
57 58 59 60 61
  void*          eventHandler;   // TODO
  void*          streamHandler;  // TODO
  TSKEY          lastKey;        // lastkey inserted in this table, initialized as 0, TODO: make a structure
  char*          sql;
  void*          cqhandle;
H
TD-353  
Hongze Cheng 已提交
62
  T_REF_DECLARE();
H
TD-34  
hzcheng 已提交
63 64 65
} STable;

typedef struct {
H
TD-353  
Hongze Cheng 已提交
66 67 68 69 70 71 72
  pthread_rwlock_t rwLock;

  int32_t   nTables;
  STable**  tables;
  SList*    superList;
  SHashObj* uidMap;
  SKVStore* pStore;
H
TD-353  
Hongze Cheng 已提交
73 74
  int       maxRowBytes;
  int       maxCols;
H
TD-34  
hzcheng 已提交
75 76
} STsdbMeta;

H
TD-353  
Hongze Cheng 已提交
77
// ------------------ tsdbBuffer.c
H
TD-34  
hzcheng 已提交
78
typedef struct {
H
TD-353  
Hongze Cheng 已提交
79 80 81 82 83
  int64_t blockId;
  int     offset;
  int     remain;
  char    data[];
} STsdbBufBlock;
H
TD-34  
hzcheng 已提交
84 85

typedef struct {
H
TD-353  
Hongze Cheng 已提交
86 87 88 89 90 91 92 93 94
  pthread_cond_t poolNotEmpty;
  int            bufBlockSize;
  int            tBufBlocks;
  int            nBufBlocks;
  int64_t        index;
  SList*         bufBlockList;
} STsdbBufPool;

// ------------------ tsdbMemTable.c
H
TD-34  
hzcheng 已提交
95
typedef struct {
H
TD-353  
Hongze Cheng 已提交
96 97 98 99 100 101
  uint64_t   uid;
  TSKEY      keyFirst;
  TSKEY      keyLast;
  int64_t    numOfRows;
  SSkipList* pData;
} STableData;
H
TD-34  
hzcheng 已提交
102 103

typedef struct {
H
TD-353  
Hongze Cheng 已提交
104 105 106 107 108 109 110 111
  T_REF_DECLARE();
  TSKEY        keyFirst;
  TSKEY        keyLast;
  int64_t      numOfRows;
  STableData** tData;
  SList*       actList;
  SList*       bufBlockList;
} SMemTable;
H
TD-34  
hzcheng 已提交
112

H
TD-353  
Hongze Cheng 已提交
113 114 115 116 117 118 119 120 121 122 123
enum { TSDB_UPDATE_META, TSDB_DROP_META };
typedef struct __attribute__((packed)){
  char     act;
  uint64_t uid;
} SActObj;

typedef struct {
  int  len;
  char cont[];
} SActCont;

H
TD-353  
Hongze Cheng 已提交
124
// ------------------ tsdbFile.c
H
TD-353  
Hongze Cheng 已提交
125
extern const char* tsdbFileSuffix[];
H
TD-353  
Hongze Cheng 已提交
126 127 128 129
typedef enum {
  TSDB_FILE_TYPE_HEAD = 0,
  TSDB_FILE_TYPE_DATA,
  TSDB_FILE_TYPE_LAST,
H
TD-353  
Hongze Cheng 已提交
130
  TSDB_FILE_TYPE_MAX,
H
TD-353  
Hongze Cheng 已提交
131 132
  TSDB_FILE_TYPE_NHEAD,
  TSDB_FILE_TYPE_NLAST
H
TD-353  
Hongze Cheng 已提交
133
} TSDB_FILE_TYPE;
H
Hongze Cheng 已提交
134

H
more  
Hongze Cheng 已提交
135
typedef struct {
H
TD-353  
Hongze Cheng 已提交
136 137 138 139 140 141
  uint32_t offset;
  uint32_t len;
  uint64_t size;      // total size of the file
  uint64_t tombSize;  // unused file size
  uint32_t totalBlocks;
  uint32_t totalSubBlocks;
142
} STsdbFileInfo;
H
TD-34  
hzcheng 已提交
143 144

typedef struct {
H
TD-353  
Hongze Cheng 已提交
145
  char  fname[TSDB_FILENAME_LEN];
H
TD-353  
Hongze Cheng 已提交
146 147
  int   fd;

H
TD-353  
Hongze Cheng 已提交
148
  STsdbFileInfo info;
H
hzcheng 已提交
149
} SFile;
H
more  
Hongze Cheng 已提交
150

H
hzcheng 已提交
151
typedef struct {
H
TD-353  
Hongze Cheng 已提交
152 153
  int   fileId;
  SFile files[TSDB_FILE_TYPE_MAX];
H
hzcheng 已提交
154 155 156
} SFileGroup;

typedef struct {
H
TD-353  
Hongze Cheng 已提交
157 158
  pthread_rwlock_t fhlock;

H
TD-353  
Hongze Cheng 已提交
159 160 161
  int         maxFGroups;
  int         nFGroups;
  SFileGroup* pFGroup;
H
hzcheng 已提交
162
} STsdbFileH;
H
more  
Hongze Cheng 已提交
163

H
TD-34  
hzcheng 已提交
164 165 166 167 168 169 170
typedef struct {
  int         numOfFGroups;
  SFileGroup *base;
  SFileGroup *pFileGroup;
  int         direction;
} SFileGroupIter;

H
TD-353  
Hongze Cheng 已提交
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
// ------------------ tsdbMain.c
typedef struct {
  int8_t state;

  char*           rootDir;
  STsdbCfg        config;
  STsdbAppH       appH;
  STsdbStat       stat;
  STsdbMeta*      tsdbMeta;
  STsdbBufPool*   pPool;
  SMemTable*      mem;
  SMemTable*      imem;
  STsdbFileH*     tsdbFileH;
  int             commit;
  pthread_t       commitThread;
  pthread_mutex_t mutex;
  bool            repoLocked;
} STsdbRepo;

H
TD-353  
Hongze Cheng 已提交
190
// ------------------ tsdbRWHelper.c
H
TD-34  
hzcheng 已提交
191
typedef struct {
H
TD-185  
Hongze Cheng 已提交
192 193
  uint32_t len;
  uint32_t offset;
H
TD-353  
Hongze Cheng 已提交
194
  // uint32_t padding;
H
TD-185  
Hongze Cheng 已提交
195 196
  uint32_t hasLast : 2;
  uint32_t numOfBlocks : 30;
197 198
  uint64_t uid;
  TSKEY    maxKey;
H
TD-353  
Hongze Cheng 已提交
199 200
} SCompIdx;

201
typedef struct {
H
TD-353  
Hongze Cheng 已提交
202 203 204 205 206 207 208
  int64_t last : 1;
  int64_t offset : 63;
  int32_t algorithm : 8;
  int32_t numOfRows : 24;
  int32_t sversion;
  int32_t len;
  int16_t numOfSubBlocks;
209 210 211 212 213
  int16_t numOfCols;
  TSKEY   keyFirst;
  TSKEY   keyLast;
} SCompBlock;

H
TD-34  
hzcheng 已提交
214 215 216
typedef struct {
  int32_t    delimiter;  // For recovery usage
  int32_t    checksum;   // TODO: decide if checksum logic in this file or make it one API
217
  uint64_t   uid;
H
TD-34  
hzcheng 已提交
218 219
  SCompBlock blocks[];
} SCompInfo;
H
hzcheng 已提交
220

H
TD-34  
hzcheng 已提交
221
typedef struct {
H
TD-353  
Hongze Cheng 已提交
222 223
  int16_t colId;
  int16_t len;
H
TD-34  
hzcheng 已提交
224 225
  int32_t type : 8;
  int32_t offset : 24;
H
TD-321  
Hongze Cheng 已提交
226 227 228 229 230 231 232
  int64_t sum;
  int64_t max;
  int64_t min;
  int16_t maxIndex;
  int16_t minIndex;
  int16_t numOfNull;
  char    padding[2];
H
TD-34  
hzcheng 已提交
233 234 235 236 237
} SCompCol;

typedef struct {
  int32_t  delimiter;  // For recovery usage
  int32_t  numOfCols;  // For recovery usage
238
  uint64_t uid;        // For recovery usage
H
TD-34  
hzcheng 已提交
239 240
  SCompCol cols[];
} SCompData;
H
TD-34  
hzcheng 已提交
241

H
hzcheng 已提交
242 243 244
typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t;

typedef struct {
H
TD-353  
Hongze Cheng 已提交
245
  int   fid;
H
hzcheng 已提交
246 247 248 249 250 251 252 253 254 255 256 257
  TSKEY minKey;
  TSKEY maxKey;
  // For read/write purpose
  SFile headF;
  SFile dataF;
  SFile lastF;
  // For write purpose only
  SFile nHeadF;
  SFile nLastF;
} SHelperFile;

typedef struct {
258 259 260
  uint64_t uid;
  int32_t  tid;
  int32_t  sversion;
H
hzcheng 已提交
261 262 263
} SHelperTable;

typedef struct {
H
TD-353  
Hongze Cheng 已提交
264
  tsdb_rw_helper_t type;
H
TD-100  
hzcheng 已提交
265

H
TD-353  
Hongze Cheng 已提交
266 267
  STsdbRepo* pRepo;
  int8_t     state;
H
TD-100  
hzcheng 已提交
268
  // For file set usage
H
hzcheng 已提交
269
  SHelperFile files;
H
TD-353  
Hongze Cheng 已提交
270
  SCompIdx*   pCompIdx;
H
TD-100  
hzcheng 已提交
271
  // For table set usage
H
hzcheng 已提交
272
  SHelperTable tableInfo;
H
TD-353  
Hongze Cheng 已提交
273
  SCompInfo*   pCompInfo;
H
TD-100  
hzcheng 已提交
274 275
  bool         hasOldLastBlock;
  // For block set usage
H
TD-353  
Hongze Cheng 已提交
276
  SCompData* pCompData;
H
TD-353  
Hongze Cheng 已提交
277 278 279
  SDataCols* pDataCols[2];
  void*      pBuffer;     // Buffer to hold the whole data block
  void*      compBuffer;  // Buffer for temperary compress/decompress purpose
H
hzcheng 已提交
280 281
} SRWHelper;

H
TD-353  
Hongze Cheng 已提交
282 283 284 285 286 287

// Operations
// ------------------ tsdbMeta.c
#define TABLE_TYPE(t) (t)->type
#define TABLE_NAME(t) (t)->name
#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data
H
TD-353  
Hongze Cheng 已提交
288
#define TABLE_UID(t) (t)->tableId.uid
H
TD-353  
Hongze Cheng 已提交
289
#define TABLE_TID(t) (t)->tableId.tid
H
TD-353  
Hongze Cheng 已提交
290
#define TABLE_SUID(t) (t)->suid
H
TD-353  
Hongze Cheng 已提交
291 292 293 294
#define TABLE_LASTKEY(t) (t)->lastKey

STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg);
void       tsdbFreeMeta(STsdbMeta* pMeta);
H
TD-353  
Hongze Cheng 已提交
295 296 297 298 299 300
int        tsdbOpenMeta(STsdbRepo* pRepo);
int        tsdbCloseMeta(STsdbRepo* pRepo);
STSchema*  tsdbGetTableSchema(STable* pTable);
STable*    tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid);
STSchema*  tsdbGetTableSchemaByVersion(STable* pTable, int16_t version);
STSchema*  tsdbGetTableTagSchema(STable* pTable);
H
TD-353  
Hongze Cheng 已提交
301
int        tsdbUpdateTable(STsdbRepo* pRepo, STable* pTable, STableCfg* pCfg);
H
TD-353  
Hongze Cheng 已提交
302 303 304
int        tsdbWLockRepoMeta(STsdbRepo* pRepo);
int        tsdbRLockRepoMeta(STsdbRepo* pRepo);
int        tsdbUnlockRepoMeta(STsdbRepo* pRepo);
H
TD-353  
Hongze Cheng 已提交
305 306
void       tsdbRefTable(STable* pTable);
void       tsdbUnRefTable(STable* pTable);
H
TD-353  
Hongze Cheng 已提交
307 308 309 310 311

// ------------------ tsdbBuffer.c
STsdbBufPool* tsdbNewBufPool();
void          tsdbFreeBufPool(STsdbBufPool* pBufPool);
int           tsdbOpenBufPool(STsdbRepo* pRepo);
H
TD-353  
Hongze Cheng 已提交
312
void          tsdbCloseBufPool(STsdbRepo* pRepo);
H
TD-353  
Hongze Cheng 已提交
313 314 315
SListNode*    tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);

// ------------------ tsdbMemTable.c
H
TD-353  
Hongze Cheng 已提交
316 317 318 319 320
int   tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
int   tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int   tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
int   tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
H
TD-353  
Hongze Cheng 已提交
321
int   tsdbAsyncCommit(STsdbRepo* pRepo);
H
TD-353  
Hongze Cheng 已提交
322 323 324 325 326 327

// ------------------ tsdbFile.c
#define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile))
#define TSDB_MAX_FILE(keep, daysPerFile) ((keep) / (daysPerFile) + 3)
#define TSDB_MIN_FILE_ID(fh) (fh)->pFGroup[0].fileId
#define TSDB_MAX_FILE_ID(fh) (fh)->pFGroup[(fh)->nFGroups - 1].fileId
H
TD-353  
Hongze Cheng 已提交
328
#define TSDB_IS_FILE_OPENED(f) ((f)->fd > 0)
H
TD-353  
Hongze Cheng 已提交
329 330 331
#define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC
#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC

H
TD-353  
Hongze Cheng 已提交
332 333
STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg);
void        tsdbFreeFileH(STsdbFileH* pFileH);
H
TD-353  
Hongze Cheng 已提交
334
int         tsdbOpenFileH(STsdbRepo* pRepo);
H
TD-353  
Hongze Cheng 已提交
335
void        tsdbCloseFileH(STsdbRepo* pRepo);
H
TD-353  
Hongze Cheng 已提交
336
SFileGroup* tsdbCreateFGroupIfNeed(STsdbRepo* pRepo, char* dataDir, int fid, int maxTables);
H
TD-353  
Hongze Cheng 已提交
337 338 339 340 341 342 343 344
void        tsdbInitFileGroupIter(STsdbFileH* pFileH, SFileGroupIter* pIter, int direction);
void        tsdbSeekFileGroupIter(SFileGroupIter* pIter, int fid);
SFileGroup* tsdbGetFileGroupNext(SFileGroupIter* pIter);
int         tsdbOpenFile(SFile* pFile, int oflag);
void        tsdbCloseFile(SFile* pFile);
int         tsdbCreateFile(SFile* pFile, STsdbRepo* pRepo, int fid, int type);
SFileGroup* tsdbSearchFGroup(STsdbFileH* pFileH, int fid, int flags);
void        tsdbFitRetention(STsdbRepo* pRepo);
H
TD-353  
Hongze Cheng 已提交
345
int         tsdbUpdateFileHeader(SFile* pFile, uint32_t version);
H
TD-353  
Hongze Cheng 已提交
346
int         tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo);
H
TD-353  
Hongze Cheng 已提交
347
void*       tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo);
H
TD-353  
Hongze Cheng 已提交
348
void        tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup);
H
TD-353  
Hongze Cheng 已提交
349 350

// ------------------ tsdbRWHelper.c
H
TD-353  
Hongze Cheng 已提交
351 352 353 354 355 356
#define TSDB_HELPER_CLEAR_STATE 0x0        // Clear state
#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1  // File is set
#define TSDB_HELPER_IDX_LOAD 0x2           // SCompIdx part is loaded
#define TSDB_HELPER_TABLE_SET 0x4          // Table is set
#define TSDB_HELPER_INFO_LOAD 0x8          // SCompInfo part is loaded
#define TSDB_HELPER_FILE_DATA_LOAD 0x10    // SCompData part is loaded
H
TD-353  
Hongze Cheng 已提交
357 358 359 360
#define helperSetState(h, s) (((h)->state) |= (s))
#define helperClearState(h, s) ((h)->state &= (~(s)))
#define helperHasState(h, s) ((((h)->state) & (s)) == (s))
#define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx)
H
TD-353  
Hongze Cheng 已提交
361 362
#define TSDB_MAX_SUBBLOCKS 8
#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0)
H
TD-353  
Hongze Cheng 已提交
363 364 365 366
#define helperType(h) (h)->type
#define helperRepo(h) (h)->pRepo
#define helperState(h) (h)->state

H
TD-353  
Hongze Cheng 已提交
367 368 369 370 371 372 373 374 375 376 377 378 379
int   tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
int   tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
void  tsdbDestroyHelper(SRWHelper* pHelper);
void  tsdbResetHelper(SRWHelper* pHelper);
int   tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup);
int   tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError);
void  tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo);
int   tsdbWriteDataBlock(SRWHelper* pHelper, SDataCols* pDataCols);
int   tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper);
int   tsdbWriteCompInfo(SRWHelper* pHelper);
int   tsdbWriteCompIdx(SRWHelper* pHelper);
int   tsdbLoadCompIdx(SRWHelper* pHelper, void* target);
int   tsdbLoadCompInfo(SRWHelper* pHelper, void* target);
H
TD-353  
Hongze Cheng 已提交
380
int   tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target);
H
TD-353  
Hongze Cheng 已提交
381 382 383 384
void  tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols);
int   tsdbLoadBlockDataCols(SRWHelper* pHelper, SDataCols* pDataCols, int blkIdx, int16_t* colIds, int numOfColIds);
int   tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SDataCols* target);

H
TD-353  
Hongze Cheng 已提交
385 386
// ------------------ tsdbMain.c
#define REPO_ID(r) (r)->config.tsdbId
H
TD-353  
Hongze Cheng 已提交
387
#define IS_REPO_LOCKED(r) (r)->repoLocked
H
TD-353  
Hongze Cheng 已提交
388
#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
H
TD-353  
Hongze Cheng 已提交
389

H
TD-353  
Hongze Cheng 已提交
390
char*       tsdbGetMetaFileName(char* rootDir);
H
TD-353  
Hongze Cheng 已提交
391
void        tsdbGetDataFileName(STsdbRepo* pRepo, int fid, int type, char* fname);
H
TD-353  
Hongze Cheng 已提交
392 393 394 395 396
int         tsdbLockRepo(STsdbRepo* pRepo);
int         tsdbUnlockRepo(STsdbRepo* pRepo);
char*       tsdbGetDataDirName(char* rootDir);
STsdbMeta*  tsdbGetMeta(TSDB_REPO_T* pRepo);
STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo);
H
TD-353  
Hongze Cheng 已提交
397

H
more  
hzcheng 已提交
398 399 400 401
#ifdef __cplusplus
}
#endif

H
TD-34  
hzcheng 已提交
402
#endif