wal.h 6.7 KB
Newer Older
H
refact  
Hongze Cheng 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
#ifndef _TD_WAL_H_
#define _TD_WAL_H_

L
Liu Jicong 已提交
18
#include "os.h"
L
Liu Jicong 已提交
19
#include "tarray.h"
L
Liu Jicong 已提交
20 21
#include "tdef.h"
#include "tlog.h"
L
Liu Jicong 已提交
22
#include "tmsg.h"
H
refact  
Hongze Cheng 已提交
23 24 25 26
#ifdef __cplusplus
extern "C" {
#endif

27 28 29 30 31
#define wFatal(...)                                              \
  {                                                              \
    if (wDebugFlag & DEBUG_FATAL) {                              \
      taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
    }                                                            \
L
Liu Jicong 已提交
32
  }
33 34 35 36 37
#define wError(...)                                              \
  {                                                              \
    if (wDebugFlag & DEBUG_ERROR) {                              \
      taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
    }                                                            \
L
Liu Jicong 已提交
38
  }
39 40 41 42 43
#define wWarn(...)                                             \
  {                                                            \
    if (wDebugFlag & DEBUG_WARN) {                             \
      taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
    }                                                          \
L
Liu Jicong 已提交
44
  }
45 46 47 48 49
#define wInfo(...)                                        \
  {                                                       \
    if (wDebugFlag & DEBUG_INFO) {                        \
      taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); \
    }                                                     \
L
Liu Jicong 已提交
50
  }
51 52 53 54 55
#define wDebug(...)                                               \
  {                                                               \
    if (wDebugFlag & DEBUG_DEBUG) {                               \
      taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); \
    }                                                             \
L
Liu Jicong 已提交
56
  }
57 58 59 60 61
#define wTrace(...)                                               \
  {                                                               \
    if (wDebugFlag & DEBUG_TRACE) {                               \
      taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); \
    }                                                             \
L
Liu Jicong 已提交
62 63
  }

L
Liu Jicong 已提交
64
#define WAL_PROTO_VER    0
L
Liu Jicong 已提交
65
#define WAL_NOSUFFIX_LEN 20
H
Hongze Cheng 已提交
66 67
#define WAL_SUFFIX_AT    (WAL_NOSUFFIX_LEN + 1)
#define WAL_LOG_SUFFIX   "log"
L
Liu Jicong 已提交
68
#define WAL_INDEX_SUFFIX "idx"
H
Hongze Cheng 已提交
69
#define WAL_REFRESH_MS   1000
L
Liu Jicong 已提交
70
#define WAL_MAX_SIZE     (TSDB_MAX_WAL_SIZE + sizeof(SWalCkHead))
H
Hongze Cheng 已提交
71 72 73
#define WAL_PATH_LEN     (TSDB_FILENAME_LEN + 12)
#define WAL_FILE_LEN     (WAL_PATH_LEN + 32)
#define WAL_MAGIC        0xFAFBFCFDULL
L
Liu Jicong 已提交
74

L
Liu Jicong 已提交
75 76 77 78 79
typedef enum {
  TAOS_WAL_NOLOG = 0,
  TAOS_WAL_WRITE = 1,
  TAOS_WAL_FSYNC = 2,
} EWalType;
80

L
Liu Jicong 已提交
81 82 83 84 85
typedef struct {
  int32_t  vgId;
  int32_t  fsyncPeriod;      // millisecond
  int32_t  retentionPeriod;  // secs
  int32_t  rollPeriod;       // secs
L
Liu Jicong 已提交
86
  int64_t  retentionSize;
L
Liu Jicong 已提交
87
  int64_t  segSize;
L
Liu Jicong 已提交
88
  EWalType level;  // wal level
L
Liu Jicong 已提交
89 90
} SWalCfg;

L
Liu Jicong 已提交
91
typedef struct {
L
Liu Jicong 已提交
92 93 94 95 96 97 98
  int64_t firstVer;
  int64_t verInSnapshotting;
  int64_t snapshotVer;
  int64_t commitVer;
  int64_t lastVer;
} SWalVer;

L
Liu Jicong 已提交
99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
#pragma pack(push, 1)
// used by sync module
typedef struct {
  int8_t   isWeek;
  uint64_t seqNum;
  uint64_t term;
} SSyncLogMeta;

typedef struct {
  int8_t  protoVer;
  int64_t version;
  int16_t msgType;
  int32_t bodyLen;
  int64_t ingestTs;  // not implemented

  // sync meta
  SSyncLogMeta syncMeta;

  char body[];
} SWalCont;

typedef struct {
  uint64_t magic;
  uint32_t cksumHead;
  uint32_t cksumBody;
  SWalCont head;
} SWalCkHead;
#pragma pack(pop)

L
Liu Jicong 已提交
128
typedef struct SWal {
L
Liu Jicong 已提交
129
  // cfg
L
Liu Jicong 已提交
130
  SWalCfg cfg;
L
Liu Jicong 已提交
131
  int32_t fsyncSeq;
L
Liu Jicong 已提交
132
  // meta
L
Liu Jicong 已提交
133
  SWalVer   vers;
134 135
  TdFilePtr pWriteLogTFile;
  TdFilePtr pWriteIdxTFile;
L
Liu Jicong 已提交
136
  int32_t   writeCur;
L
Liu Jicong 已提交
137
  SArray   *fileInfoSet;  // SArray<SWalFileInfo>
L
Liu Jicong 已提交
138
  // status
L
Liu Jicong 已提交
139 140
  int64_t totSize;
  int64_t lastRollSeq;
L
Liu Jicong 已提交
141
  // ctl
L
Liu Jicong 已提交
142
  int64_t       refId;
wafwerar's avatar
wafwerar 已提交
143
  TdThreadMutex mutex;
144 145
  // ref
  SHashObj *pRefHash;  // ref -> SWalRef
L
Liu Jicong 已提交
146
  // path
L
Liu Jicong 已提交
147
  char path[WAL_PATH_LEN];
L
Liu Jicong 已提交
148
  // reusable write head
L
Liu Jicong 已提交
149
  SWalCkHead writeHead;
L
Liu Jicong 已提交
150 151
} SWal;  // WAL HANDLE

L
Liu Jicong 已提交
152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
typedef struct {
  int8_t scanUncommited;
  int8_t scanMeta;
} SWalFilterCond;

typedef struct {
  SWal          *pWal;
  TdFilePtr      pLogFile;
  TdFilePtr      pIdxFile;
  int64_t        curFileFirstVer;
  int64_t        curVersion;
  int64_t        capacity;
  TdThreadMutex  mutex;
  SWalFilterCond cond;
  SWalCkHead    *pHead;
} SWalReader;
168

S
Shengliang Guan 已提交
169 170 171
// module initialization
int32_t walInit();
void    walCleanUp();
172

S
Shengliang Guan 已提交
173
// handle open and ctl
L
Liu Jicong 已提交
174
SWal   *walOpen(const char *path, SWalCfg *pCfg);
S
Shengliang Guan 已提交
175 176
int32_t walAlter(SWal *, SWalCfg *pCfg);
void    walClose(SWal *);
177

S
Shengliang Guan 已提交
178
// write
L
Liu Jicong 已提交
179
int32_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SSyncLogMeta syncMeta, const void *body,
L
Liu Jicong 已提交
180
                             int32_t bodyLen);
L
Liu Jicong 已提交
181
int32_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen);
L
Liu Jicong 已提交
182
void    walFsync(SWal *, bool force);
L
Liu Jicong 已提交
183

S
Shengliang Guan 已提交
184 185 186
// apis for lifecycle management
int32_t walCommit(SWal *, int64_t ver);
int32_t walRollback(SWal *, int64_t ver);
L
Liu Jicong 已提交
187
// notify that previous logs can be pruned safely
L
Liu Jicong 已提交
188 189
int32_t walBeginSnapshot(SWal *, int64_t ver);
int32_t walEndSnapshot(SWal *);
190
int32_t walRestoreFromSnapshot(SWal *, int64_t ver);
L
Liu Jicong 已提交
191
// int32_t  walDataCorrupted(SWal*);
192

S
Shengliang Guan 已提交
193
// read
L
Liu Jicong 已提交
194 195 196
SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond);
void        walCloseReader(SWalReader *pRead);
int32_t     walReadVer(SWalReader *pRead, int64_t ver);
L
Liu Jicong 已提交
197
int32_t     walReadSeekVer(SWalReader *pRead, int64_t ver);
L
Liu Jicong 已提交
198
int32_t     walNextValidMsg(SWalReader *pRead);
199 200

// only for tq usage
L
Liu Jicong 已提交
201 202 203 204
void    walSetReaderCapacity(SWalReader *pRead, int32_t capacity);
int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead);
int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead);
int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead);
L
Liu Jicong 已提交
205

206 207 208 209 210 211 212 213 214 215
typedef struct {
  int64_t refId;
  int64_t ver;
} SWalRef;

SWalRef *walOpenRef(SWal *);
void     walCloseRef(SWalRef *);
int32_t  walRefVer(SWalRef *, int64_t ver);
int32_t  walUnrefVer(SWal *);

216
// help function for raft
217
bool walLogExist(SWal *, int64_t ver);
218
bool walIsEmpty(SWal *);
219

S
Shengliang Guan 已提交
220
// lifecycle check
L
Liu Jicong 已提交
221 222 223
int64_t walGetFirstVer(SWal *);
int64_t walGetSnapshotVer(SWal *);
int64_t walGetLastVer(SWal *);
L
Liu Jicong 已提交
224
int64_t walGetCommittedVer(SWal *);
225

H
refact  
Hongze Cheng 已提交
226 227 228 229
#ifdef __cplusplus
}
#endif

230
#endif  // _TD_WAL_H_