Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
T
TDengine
提交 e25637fe 编写于 作者: S Shengliang Guan
浏览文件
操作

Merge remote-tracking branch 'origin/3.0' into feature/config

上级 6bf4eb6f c9c75bdf
展开全部 显示空白变更内容
内联 并排
Showing with 2585 addition and 1342 deletion
contrib/CMakeLists.txt
浏览文件 @ e25637fe
......@@ -193,6 +193,7 @@ endif(${BUILD_WITH_TRAFT})
# LIBUV
if(${BUILD_WITH_UV})
add_compile_options(-Wno-sign-compare)
add_subdirectory(libuv)
endif(${BUILD_WITH_UV})
......
include/common/tep.h
浏览文件 @ e25637fe
......@@ -68,8 +68,8 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
}
#define colDataGet(p1_, r_) \
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? (p1_)->pData + (p1_)->varmeta.offset[(r_)] \
: (p1_)->pData + ((r_) * (p1_)->info.bytes));
((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \
: ((p1_)->pData + ((r_) * (p1_)->info.bytes)))
int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2);
......
include/common/ttszip.h
浏览文件 @ e25637fe
......@@ -73,7 +73,7 @@ typedef struct STSGroupBlockInfoEx {
} STSGroupBlockInfoEx;
typedef struct STSBuf {
FILE* f;
TdFilePtr pFile;
char path[PATH_MAX];
uint32_t fileSize;
......
include/libs/nodes/nodes.h
浏览文件 @ e25637fe
......@@ -62,7 +62,6 @@ typedef enum ENodeType {
QUERY_NODE_NODE_LIST,
QUERY_NODE_FILL,
QUERY_NODE_RAW_EXPR, // Only be used in parser module.
QUERY_NODE_COLUMN_REF,
QUERY_NODE_TARGET,
QUERY_NODE_TUPLE_DESC,
QUERY_NODE_SLOT_DESC,
......@@ -81,7 +80,9 @@ typedef enum ENodeType {
// physical plan node
QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN,
QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
QUERY_NODE_PHYSICAL_PLAN_PROJECT
QUERY_NODE_PHYSICAL_PLAN_PROJECT,
QUERY_NODE_PHYSICAL_PLAN_JOIN,
QUERY_NODE_PHYSICAL_PLAN_AGG
} ENodeType;
/**
......
include/libs/nodes/plannodes.h
浏览文件 @ e25637fe
......@@ -69,8 +69,6 @@ typedef struct SSlotDescNode {
ENodeType type;
int16_t slotId;
SDataType dataType;
int16_t srcTupleId;
int16_t srcSlotId;
bool reserve;
bool output;
} SSlotDescNode;
......@@ -115,6 +113,20 @@ typedef struct SProjectPhysiNode {
SNodeList* pProjections;
} SProjectPhysiNode;
typedef struct SJoinPhysiNode {
SPhysiNode node;
EJoinType joinType;
SNode* pOnConditions; // in or out tuple ?
SNodeList* pTargets;
} SJoinPhysiNode;
typedef struct SAggPhysiNode {
SPhysiNode node;
SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function
SNodeList* pGroupKeys; // SColumnRefNode list
SNodeList* pAggFuncs;
} SAggPhysiNode;
#ifdef __cplusplus
}
#endif
......
include/libs/nodes/querynodes.h
浏览文件 @ e25637fe
......@@ -58,15 +58,17 @@ typedef struct SColumnNode {
char tableAlias[TSDB_TABLE_NAME_LEN];
char colName[TSDB_COL_NAME_LEN];
SNode* pProjectRef;
} SColumnNode;
typedef struct SColumnRefNode {
ENodeType type;
SDataType dataType;
int16_t tupleId;
int16_t slotId;
int16_t columnId;
} SColumnRefNode;
} SColumnNode;
// typedef struct SColumnRefNode {
// ENodeType type;
// SDataType dataType;
// int16_t tupleId;
// int16_t slotId;
// int16_t columnId;
// } SColumnRefNode;
typedef struct STargetNode {
ENodeType type;
......
include/libs/scalar/filter.h
浏览文件 @ e25637fe
......@@ -37,8 +37,8 @@ typedef struct SFilterColumnParam{
extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options);
extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param, filer_get_col_from_id fp);
extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param, filer_get_col_from_id fp);
extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param);
extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param);
extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
......
include/libs/wal/wal.h
浏览文件 @ e25637fe
......@@ -120,8 +120,8 @@ typedef struct SWal {
int32_t fsyncSeq;
// meta
SWalVer vers;
int64_t writeLogTfd;
int64_t writeIdxTfd;
TdFilePtr pWriteLogTFile;
TdFilePtr pWriteIdxTFile;
int32_t writeCur;
SArray *fileInfoSet;
// status
......@@ -138,8 +138,8 @@ typedef struct SWal {
typedef struct SWalReadHandle {
SWal *pWal;
int64_t readLogTfd;
int64_t readIdxTfd;
TdFilePtr pReadLogTFile;
TdFilePtr pReadIdxTFile;
int64_t curFileFirstVer;
int64_t curVersion;
int64_t capacity;
......
include/os/osFile.h
浏览文件 @ e25637fe
......@@ -22,49 +22,63 @@ extern "C" {
#include "osSocket.h"
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
typedef int32_t FileFd;
#else
typedef int32_t FileFd;
#ifndef ALLOW_FORBID_FUNC
#define open OPEN_FUNC_TAOS_FORBID
#define fopen FOPEN_FUNC_TAOS_FORBID
// #define close CLOSE_FUNC_TAOS_FORBID
// #define fclose FCLOSE_FUNC_TAOS_FORBID
#endif
#define FD_INITIALIZER ((int32_t)-1)
#ifndef PATH_MAX
#define PATH_MAX 256
#endif
int32_t taosLockFile(FileFd fd);
int32_t taosUnLockFile(FileFd fd);
typedef struct TdFile *TdFilePtr;
#define TD_FILE_CTEATE 0x0001
#define TD_FILE_WRITE 0x0002
#define TD_FILE_READ 0x0004
#define TD_FILE_TRUNC 0x0008
#define TD_FILE_APPEND 0x0010
#define TD_FILE_TEXT 0x0020
#define TD_FILE_AUTO_DEL 0x0040
#define TD_FILE_EXCL 0x0080
int32_t taosLockFile(TdFilePtr pFile);
int32_t taosUnLockFile(TdFilePtr pFile);
int32_t taosUmaskFile(FileFd fd);
int32_t taosUmaskFile(int32_t maskVal);
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
int32_t taosFStatFile(FileFd fd, int64_t *size, int32_t *mtime);
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
FileFd taosOpenFileWrite(const char *path);
FileFd taosOpenFileCreateWrite(const char *path);
FileFd taosOpenFileCreateWriteTrunc(const char *path);
FileFd taosOpenFileCreateWriteAppend(const char *path);
FileFd taosOpenFileRead(const char *path);
FileFd taosOpenFileReadWrite(const char *path);
TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions);
int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence);
int32_t taosFtruncateFile(FileFd fd, int64_t length);
int32_t taosFsyncFile(FileFd fd);
int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence);
int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length);
int32_t taosFsyncFile(TdFilePtr pFile);
int64_t taosReadFile(FileFd fd, void *buf, int64_t count);
int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count);
int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count);
int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset);
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
void taosFprintfFile(TdFilePtr pFile, const char *format, ...);
size_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf);
int32_t taosEOFFile(TdFilePtr pFile);
void taosCloseFile(FileFd fd);
int64_t taosCloseFile(TdFilePtr *ppFile);
int32_t taosRenameFile(const char *oldName, const char *newName);
int64_t taosCopyFile(const char *from, const char *to);
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size);
int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size);
int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size);
int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
bool taosValidFile(TdFilePtr pFile);
int taosGetErrorFile(TdFilePtr pFile);
#ifdef __cplusplus
}
......
include/util/tfile.h
浏览文件 @ e25637fe
......@@ -30,20 +30,20 @@ void tfCleanup();
// the same syntax as UNIX standard open/close/read/write
// but FD is int64_t and will never be reused
int64_t tfOpenRead(const char *pathname);
int64_t tfOpenReadWrite(const char *pathname);
int64_t tfOpenCreateWrite(const char *pathname);
int64_t tfOpenCreateWriteAppend(const char *pathname);
int64_t tfClose(int64_t tfd);
int64_t tfWrite(int64_t tfd, void *buf, int64_t count);
int64_t tfRead(int64_t tfd, void *buf, int64_t count);
int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset);
int32_t tfFsync(int64_t tfd);
bool tfValid(int64_t tfd);
int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence);
int32_t tfFtruncate(int64_t tfd, int64_t length);
void * tfMmapReadOnly(int64_t tfd, int64_t length);
// int64_t tfOpenRead(const char *pathname);
// int64_t tfOpenReadWrite(const char *pathname);
// int64_t tfOpenCreateWrite(const char *pathname);
// int64_t tfOpenCreateWriteAppend(const char *pathname);
// int64_t tfClose(int64_t tfd);
// int64_t tfWrite(int64_t tfd, void *buf, int64_t count);
// int64_t tfRead(int64_t tfd, void *buf, int64_t count);
// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset);
// int32_t tfFsync(int64_t tfd);
// bool tfValid(int64_t tfd);
// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence);
// int32_t tfFtruncate(int64_t tfd, int64_t length);
// void * tfMmapReadOnly(int64_t tfd, int64_t length);
#ifdef __cplusplus
}
#endif
......
source/common/src/ttszip.c
浏览文件 @ e25637fe
......@@ -24,8 +24,9 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
pTSBuf->autoDelete = autoDelete;
taosGetTmpfilePath(tsTempDir, "join", pTSBuf->path);
pTSBuf->f = fopen(pTSBuf->path, "wb+");
if (pTSBuf->f == NULL) {
// pTSBuf->pFile = fopen(pTSBuf->path, "wb+");
pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (pTSBuf->pFile == NULL) {
free(pTSBuf);
return NULL;
}
......@@ -60,8 +61,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path));
pTSBuf->f = fopen(pTSBuf->path, "rb+");
if (pTSBuf->f == NULL) {
// pTSBuf->pFile = fopen(pTSBuf->path, "rb+");
pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ);
if (pTSBuf->pFile == NULL) {
free(pTSBuf);
return NULL;
}
......@@ -72,9 +74,9 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
// validate the file magic number
STSBufFileHeader header = {0};
int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET);
int32_t ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET);
UNUSED(ret);
size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f);
size_t sz = taosReadFile(pTSBuf->pFile, &header, sizeof(STSBufFileHeader));
UNUSED(sz);
// invalid file
......@@ -112,8 +114,8 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
return NULL;
}
//int64_t pos = ftell(pTSBuf->f); //pos not used
sz = fread(buf, infoSize, 1, pTSBuf->f);
//int64_t pos = ftell(pTSBuf->pFile); //pos not used
sz = taosReadFile(pTSBuf->pFile, buf, infoSize);
UNUSED(sz);
// the length value for each vnode is not kept in file, so does not set the length value
......@@ -123,22 +125,22 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
}
free(buf);
ret = fseek(pTSBuf->f, 0, SEEK_END);
ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END);
UNUSED(ret);
struct stat fileStat;
if (fstat(fileno(pTSBuf->f), &fileStat) != 0) {
int64_t file_size;
if (taosFStatFile(pTSBuf->pFile, &file_size, NULL) != 0) {
tsBufDestroy(pTSBuf);
return NULL;
}
pTSBuf->fileSize = (uint32_t)fileStat.st_size;
pTSBuf->fileSize = (uint32_t)file_size;
tsBufResetPos(pTSBuf);
// ascending by default
pTSBuf->cur.order = TSDB_ORDER_ASC;
// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f),
// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->pFile),
// pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete);
return pTSBuf;
......@@ -156,7 +158,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) {
tfree(pTSBuf->block.payload);
if (!pTSBuf->remainOpen) {
fclose(pTSBuf->f);
taosCloseFile(&pTSBuf->pFile);
}
if (pTSBuf->autoDelete) {
......@@ -253,7 +255,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize,
TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize);
int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET);
int64_t r = taosLSeekFile(pTSBuf->pFile, pTSBuf->fileSize, SEEK_SET);
assert(r == 0);
/*
......@@ -264,30 +266,30 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
* both side has the compressed length is used to support load data forwards/backwords.
*/
int32_t metaLen = 0;
metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType));
int32_t trueLen = pBlock->tag.nLen;
if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen);
} else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
float tfloat = (float)pBlock->tag.d;
metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen);
} else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)fwrite(&pBlock->tag.i, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen);
} else {
trueLen = 0;
metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen));
}
fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
taosWriteFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem));
taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
taosWriteFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen);
taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t) taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen));
assert(metaLen == getTagAreaLength(&pBlock->tag));
int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen;
......@@ -332,20 +334,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
* the end of each comp data block
*/
int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen));
int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR);
size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f);
sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
int32_t ret = taosLSeekFile(pTSBuf->pFile, prev, SEEK_CUR);
size_t sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding));
sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
UNUSED(sz);
pBlock->compLen = pBlock->padding;
offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
ret = fseek(pTSBuf->f, -offset, SEEK_CUR);
ret = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR);
UNUSED(ret);
}
int32_t ret = fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f);
ret = fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
int32_t ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType));
ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen));
// NOTE: mix types tags are not supported
size_t sz = 0;
......@@ -356,23 +358,23 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
memset(tp, 0, pBlock->tag.nLen + 1);
pBlock->tag.pz = tp;
sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen);
UNUSED(sz);
} else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
float tfloat = 0;
sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen);
pBlock->tag.d = (double)tfloat;
UNUSED(sz);
} else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value
sz = fread(&pBlock->tag.i, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen);
UNUSED(sz);
}
sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem));
UNUSED(sz);
sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen));
UNUSED(sz);
sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen);
if (decomp) {
pTSBuf->tsData.len =
......@@ -381,11 +383,11 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
}
// read the comp length at the length of comp block
sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding));
assert(pBlock->padding == pBlock->compLen);
int32_t n = 0;
sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
sz = taosReadFile(pTSBuf->pFile, &n, sizeof(pBlock->tag.nLen));
if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) {
assert(n == 0);
} else {
......@@ -396,7 +398,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
// for backwards traverse, set the start position at the end of previous block
if (order == TSDB_ORDER_DESC) {
int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR);
int32_t r = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR);
UNUSED(r);
}
......@@ -512,7 +514,7 @@ static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t num
// todo opt performance by cache blocks info
static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) {
if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) {
return -1;
}
......@@ -531,7 +533,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int
STSBlock* pBlock = &pTSBuf->block;
int32_t compBlockSize =
pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR);
int32_t ret = taosLSeekFile(pTSBuf->pFile, -compBlockSize, SEEK_CUR);
UNUSED(ret);
}
......@@ -548,7 +550,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo
offset = pBlockInfo->offset + pBlockInfo->compLen;
}
if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) {
if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) {
return -1;
}
......@@ -618,11 +620,11 @@ static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockIn
return -1;
}
if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) {
if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) {
return -1;
}
fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f);
taosWriteFile(pTSBuf->pFile, pVInfo, sizeof(STSGroupBlockInfo));
return 0;
}
......@@ -636,19 +638,19 @@ STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) {
}
int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) {
if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
if ((pTSBuf->pFile == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
return -1;
}
assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC);
int32_t r = fseek(pTSBuf->f, 0, SEEK_SET);
int32_t r = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET);
if (r != 0) {
// qError("fseek failed, errno:%d", errno);
return -1;
}
size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f);
size_t ws = taosWriteFile(pTSBuf->pFile, pHeader, sizeof(STSBufFileHeader));
if (ws != 1) {
// qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader));
return -1;
......@@ -823,12 +825,12 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
pBlockInfoEx->info.id = id;
}
int32_t r = fseek(pDestBuf->f, 0, SEEK_END);
int32_t r = taosLSeekFile(pDestBuf->pFile, 0, SEEK_END);
assert(r == 0);
int64_t offset = getDataStartOffset();
int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset;
int64_t written = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size);
int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size);
if (written == -1 || written != size) {
return -1;
......@@ -839,17 +841,18 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
int32_t oldSize = pDestBuf->fileSize;
// file meta data may be cached, close and reopen the file for accurate file size.
fclose(pDestBuf->f);
pDestBuf->f = fopen(pDestBuf->path, "rb+");
if (pDestBuf->f == NULL) {
taosCloseFile(&pDestBuf->pFile);
// pDestBuf->pFile = fopen(pDestBuf->path, "rb+");
pDestBuf->pFile = taosOpenFile(pDestBuf->path, TD_FILE_WRITE | TD_FILE_READ);
if (pDestBuf->pFile == NULL) {
return -1;
}
struct stat fileStat;
if (fstat(fileno(pDestBuf->f), &fileStat) != 0) {
int64_t file_size;
if (taosFStatFile(pDestBuf->pFile, &file_size, NULL) != 0) {
return -1;
}
pDestBuf->fileSize = (uint32_t)fileStat.st_size;
pDestBuf->fileSize = (uint32_t)file_size;
assert(pDestBuf->fileSize == oldSize + size);
......@@ -868,13 +871,13 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
// update prev vnode length info in file
TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo);
int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET);
int32_t ret = taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET);
if (ret == -1) {
// qError("fseek failed, errno:%d", errno);
tsBufDestroy(pTSBuf);
return NULL;
}
size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f);
size_t sz = taosWriteFile(pTSBuf->pFile, (void*)pData, len);
if (sz != len) {
// qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len);
tsBufDestroy(pTSBuf);
......@@ -893,7 +896,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_
}
// TODO taosFsync??
// if (taosFsync(fileno(pTSBuf->f)) == -1) {
// if (taosFsync(fileno(pTSBuf->pFile)) == -1) {
//// qError("fsync failed, errno:%d", errno);
// tsBufDestroy(pTSBuf);
// return NULL;
......@@ -1071,15 +1074,15 @@ int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, in
*len = 0;
*numOfBlocks = 0;
if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) {
int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile));
// qError("%p: fseek failed: %s", pSql, tstrerror(code));
return code;
}
size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f);
size_t s = taosReadFile(pTSBuf->pFile, buf, pBlockInfo->compLen);
if (s != pBlockInfo->compLen) {
int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile));
// tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
return code;
}
......
source/dnode/mgmt/impl/inc/dndEnv.h
浏览文件 @ e25637fe
......@@ -125,7 +125,7 @@ typedef struct SDnode {
EStat stat;
SDnodeObjCfg cfg;
SDnodeDir dir;
FileFd lockFd;
TdFilePtr pLockFile;
SDnodeMgmt dmgmt;
SMnodeMgmt mmgmt;
SQnodeMgmt qmgmt;
......
source/dnode/mgmt/impl/src/dndBnode.c
浏览文件 @ e25637fe
......@@ -62,14 +62,15 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
// FILE *fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_BNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_BNODE_OVER;
......@@ -102,7 +103,7 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) {
PRASE_BNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
terrno = code;
return code;
......@@ -114,8 +115,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
......@@ -130,9 +132,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
char realfile[PATH_MAX + 20];
......
source/dnode/mgmt/impl/src/dndEnv.c
浏览文件 @ e25637fe
......@@ -59,31 +59,31 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING);
}
static FileFd dndCheckRunning(char *dataDir) {
static TdFilePtr dndCheckRunning(char *dataDir) {
char filepath[PATH_MAX] = {0};
snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);
FileFd fd = taosOpenFileCreateWriteTrunc(filepath);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to lock file:%s since %s, quit", filepath, terrstr());
return -1;
return NULL;
}
int32_t ret = taosLockFile(fd);
int32_t ret = taosLockFile(pFile);
if (ret != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to lock file:%s since %s, quit", filepath, terrstr());
taosCloseFile(fd);
return -1;
taosCloseFile(&pFile);
return NULL;
}
return fd;
return pFile;
}
static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) {
pDnode->lockFd = dndCheckRunning(pCfg->dataDir);
if (pDnode->lockFd < 0) {
pDnode->pLockFile = dndCheckRunning(pCfg->dataDir);
if (pDnode->pLockFile == NULL) {
return -1;
}
......@@ -147,10 +147,10 @@ static void dndCloseImp(SDnode *pDnode) {
tfree(pDnode->dir.snode);
tfree(pDnode->dir.bnode);
if (pDnode->lockFd >= 0) {
taosUnLockFile(pDnode->lockFd);
taosCloseFile(pDnode->lockFd);
pDnode->lockFd = 0;
if (pDnode->pLockFile != NULL) {
taosUnLockFile(pDnode->pLockFile);
taosCloseFile(&pDnode->pLockFile);
pDnode->pLockFile = NULL;
}
}
......
source/dnode/mgmt/impl/src/dndMgmt.c
浏览文件 @ e25637fe
......@@ -185,16 +185,16 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
int32_t maxLen = 256 * 1024;
char *content = calloc(1, maxLen + 1);
cJSON *root = NULL;
FILE *fp = NULL;
fp = fopen(pMgmt->file, "r");
if (fp == NULL) {
// fp = fopen(pMgmt->file, "r");
TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", pMgmt->file);
code = 0;
goto PRASE_DNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", pMgmt->file);
goto PRASE_DNODE_OVER;
......@@ -286,7 +286,7 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
PRASE_DNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) {
dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file);
......@@ -309,8 +309,9 @@ PRASE_DNODE_OVER:
static int32_t dndWriteDnodes(SDnode *pDnode) {
SDnodeMgmt *pMgmt = &pDnode->dmgmt;
FILE *fp = fopen(pMgmt->file, "w");
if (fp == NULL) {
// FILE *fp = fopen(pMgmt->file, "w");
TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
dError("failed to write %s since %s", pMgmt->file, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
......@@ -341,9 +342,9 @@ static int32_t dndWriteDnodes(SDnode *pDnode) {
}
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
terrno = 0;
......
source/dnode/mgmt/impl/src/dndMnode.c
浏览文件 @ e25637fe
......@@ -63,14 +63,15 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
// FILE *fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_MNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_MNODE_OVER;
......@@ -143,7 +144,7 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) {
PRASE_MNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
terrno = code;
return code;
......@@ -155,8 +156,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
......@@ -184,9 +186,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) {
}
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
char realfile[PATH_MAX + 20];
......
source/dnode/mgmt/impl/src/dndQnode.c
浏览文件 @ e25637fe
......@@ -62,14 +62,15 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
// FILE *fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_QNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_QNODE_OVER;
......@@ -102,7 +103,7 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) {
PRASE_QNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
terrno = code;
return code;
......@@ -114,8 +115,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
......@@ -130,9 +132,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
char realfile[PATH_MAX + 20];
......
source/dnode/mgmt/impl/src/dndSnode.c
浏览文件 @ e25637fe
......@@ -62,14 +62,15 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
// FILE *fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_SNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_SNODE_OVER;
......@@ -102,7 +103,7 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) {
PRASE_SNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
terrno = code;
return code;
......@@ -114,8 +115,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) {
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
......@@ -130,9 +132,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
char realfile[PATH_MAX + 20];
......
source/dnode/mgmt/impl/src/dndVnodes.c
浏览文件 @ e25637fe
......@@ -219,14 +219,15 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_
snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
fp = fopen(file, "r");
if (fp == NULL) {
// fp = fopen(file, "r");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_VNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
len = (int32_t)taosReadFile(pFile, content, maxLen);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_VNODE_OVER;
......@@ -304,7 +305,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_
PRASE_VNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
if (pFile != NULL) taosCloseFile(&pFile);
return code;
}
......@@ -315,8 +316,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes);
snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
// FILE *fp = fopen(file, "w");
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to write %s since %s", file, terrstr());
return -1;
......@@ -347,9 +349,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, " ]\n");
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
free(content);
terrno = 0;
......
source/dnode/mnode/impl/src/mndTelem.c
浏览文件 @ e25637fe
......@@ -62,12 +62,13 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) {
size_t size = 0;
int32_t done = 0;
FILE* fp = fopen("/proc/cpuinfo", "r");
if (fp == NULL) {
// FILE* fp = fopen("/proc/cpuinfo", "r");
TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ);
if (pFile == NULL) {
return;
}
while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) {
while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) {
const char* v = strchr(line, ':') + 2;
......@@ -83,20 +84,21 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) {
}
}
free(line);
fclose(fp);
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) {
char* line = NULL;
size_t size = 0;
FILE* fp = fopen("/etc/os-release", "r");
if (fp == NULL) {
// FILE* fp = fopen("/etc/os-release", "r");
TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ);
if (pFile == NULL) {
return;
}
while ((size = tgetline(&line, &size, fp)) != -1) {
while ((size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (strncmp(line, "PRETTY_NAME", 11) == 0) {
const char* p = strchr(line, '=') + 1;
......@@ -109,20 +111,21 @@ static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) {
}
}
free(line);
fclose(fp);
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) {
char* line = NULL;
size_t size = 0;
FILE* fp = fopen("/proc/meminfo", "r");
if (fp == NULL) {
// FILE* fp = fopen("/proc/meminfo", "r");
TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ);
if (pFile == NULL) {
return;
}
while ((size = tgetline(&line, &size, fp)) != -1) {
while ((size = taosGetLineFile(pFile, &line)) != -1) {
line[size - 1] = '\0';
if (strncmp(line, "MemTotal", 8) == 0) {
const char* p = strchr(line, ':') + 1;
......@@ -132,8 +135,8 @@ static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) {
}
}
free(line);
fclose(fp);
if(line != NULL) free(line);
taosCloseFile(&pFile);
}
static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) {
......@@ -252,16 +255,16 @@ static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) {
static void mndGetEmail(SMnode* pMnode, char* filepath) {
STelemMgmt* pMgmt = &pMnode->telemMgmt;
int32_t fd = taosOpenFileRead(filepath);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ);
if (pFile == NULL) {
return;
}
if (taosReadFile(fd, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) {
if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) {
mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
}
taosCloseFile(fd);
taosCloseFile(&pFile);
}
int32_t mndInitTelem(SMnode* pMnode) {
......
source/dnode/mnode/impl/test/trans/trans.cpp
浏览文件 @ e25637fe
......@@ -27,25 +27,25 @@ class MndTestTrans : public ::testing::Test {
static void KillThenRestartServer() {
char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data";
FileFd fd = taosOpenFileRead(file);
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
int32_t size = 3 * 1024 * 1024;
void* buffer = malloc(size);
int32_t readLen = taosReadFile(fd, buffer, size);
int32_t readLen = taosReadFile(pFile, buffer, size);
if (readLen < 0 || readLen == size) {
ASSERT(1);
}
taosCloseFile(fd);
taosCloseFile(&pFile);
test.ServerStop();
fd = taosOpenFileCreateWriteTrunc(file);
int32_t writeLen = taosWriteFile(fd, buffer, readLen);
pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
int32_t writeLen = taosWriteFile(pFile, buffer, readLen);
if (writeLen < 0 || writeLen == readLen) {
ASSERT(1);
}
free(buffer);
taosFsyncFile(fd);
taosCloseFile(fd);
taosFsyncFile(pFile);
taosCloseFile(&pFile);
taosMsleep(1000);
test.ServerStart();
......
source/dnode/mnode/sdb/src/sdbFile.c
浏览文件 @ e25637fe
......@@ -37,8 +37,8 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) {
return 0;
}
static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
int32_t ret = taosReadFile(fd, &pSdb->curVer, sizeof(int64_t));
static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
int32_t ret = taosReadFile(pFile, &pSdb->curVer, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
......@@ -50,7 +50,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
int64_t maxId = -1;
ret = taosReadFile(fd, &maxId, sizeof(int64_t));
ret = taosReadFile(pFile, &maxId, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
......@@ -66,7 +66,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
int64_t ver = -1;
ret = taosReadFile(fd, &ver, sizeof(int64_t));
ret = taosReadFile(pFile, &ver, sizeof(int64_t));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
......@@ -81,7 +81,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
}
char reserve[SDB_RESERVE_SIZE] = {0};
ret = taosReadFile(fd, reserve, sizeof(reserve));
ret = taosReadFile(pFile, reserve, sizeof(reserve));
if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
......@@ -94,8 +94,8 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) {
return 0;
}
static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) {
if (taosWriteFile(fd, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) {
static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
if (taosWriteFile(pFile, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
......@@ -105,7 +105,7 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) {
if (i < SDB_MAX) {
maxId = pSdb->maxId[i];
}
if (taosWriteFile(fd, &maxId, sizeof(int64_t)) != sizeof(int64_t)) {
if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
......@@ -116,14 +116,14 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) {
if (i < SDB_MAX) {
ver = pSdb->tableVer[i];
}
if (taosWriteFile(fd, &ver, sizeof(int64_t)) != sizeof(int64_t)) {
if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
char reserve[SDB_RESERVE_SIZE] = {0};
if (taosWriteFile(fd, reserve, sizeof(reserve)) != sizeof(reserve)) {
if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
......@@ -148,25 +148,25 @@ int32_t sdbReadFile(SSdb *pSdb) {
snprintf(file, sizeof(file), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP);
mDebug("start to read file:%s", file);
FileFd fd = taosOpenFileRead(file);
if (fd <= 0) {
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
free(pRaw);
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to read file:%s since %s", file, terrstr());
return 0;
}
if (sdbReadFileHead(pSdb, fd) != 0) {
if (sdbReadFileHead(pSdb, pFile) != 0) {
mError("failed to read file:%s head since %s", file, terrstr());
pSdb->curVer = -1;
free(pRaw);
taosCloseFile(fd);
taosCloseFile(&pFile);
return -1;
}
while (1) {
readLen = sizeof(SSdbRaw);
ret = taosReadFile(fd, pRaw, readLen);
ret = taosReadFile(pFile, pRaw, readLen);
if (ret == 0) break;
if (ret < 0) {
......@@ -182,7 +182,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
}
readLen = pRaw->dataLen + sizeof(int32_t);
ret = taosReadFile(fd, pRaw->pData, readLen);
ret = taosReadFile(pFile, pRaw->pData, readLen);
if (ret < 0) {
code = TAOS_SYSTEM_ERROR(errno);
mError("failed to read file:%s since %s", file, tstrerror(code));
......@@ -214,7 +214,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
mDebug("read file:%s successfully, ver:%" PRId64, file, pSdb->lastCommitVer);
PARSE_SDB_DATA_ERROR:
taosCloseFile(fd);
taosCloseFile(&pFile);
sdbFreeRaw(pRaw);
terrno = code;
......@@ -232,16 +232,16 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer,
pSdb->lastCommitVer);
FileFd fd = taosOpenFileCreateWriteTrunc(tmpfile);
if (fd <= 0) {
TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to open file:%s for write since %s", tmpfile, terrstr());
return -1;
}
if (sdbWriteFileHead(pSdb, fd) != 0) {
if (sdbWriteFileHead(pSdb, pFile) != 0) {
mError("failed to write file:%s head since %s", tmpfile, terrstr());
taosCloseFile(fd);
taosCloseFile(&pFile);
return -1;
}
......@@ -269,7 +269,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
if (pRaw != NULL) {
pRaw->status = pRow->status;
int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen;
if (taosWriteFile(fd, pRaw, writeLen) != writeLen) {
if (taosWriteFile(pFile, pRaw, writeLen) != writeLen) {
code = TAOS_SYSTEM_ERROR(errno);
taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw);
......@@ -277,7 +277,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
}
int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen);
if (taosWriteFile(fd, &cksum, sizeof(int32_t)) != sizeof(int32_t)) {
if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) {
code = TAOS_SYSTEM_ERROR(errno);
taosHashCancelIterate(hash, ppRow);
sdbFreeRaw(pRaw);
......@@ -296,14 +296,14 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
}
if (code == 0) {
code = taosFsyncFile(fd);
code = taosFsyncFile(pFile);
if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
mError("failed to write file:%s since %s", tmpfile, tstrerror(code));
}
}
taosCloseFile(fd);
taosCloseFile(&pFile);
if (code == 0) {
code = taosRenameFile(tmpfile, curfile);
......
source/dnode/vnode/src/inc/tqInt.h
浏览文件 @ e25637fe
......@@ -141,9 +141,9 @@ typedef struct {
STqMetaList* unconnectTopic;
// TODO:temporaral use, to be replaced by unified tfile
int fileFd;
TdFilePtr pFile;
// TODO:temporaral use, to be replaced by unified tfile
int idxFd;
TdFilePtr pIdxFile;
char* dirPath;
int32_t tqConfigFlag;
......
source/dnode/vnode/src/inc/tsdbFile.h
浏览文件 @ e25637fe
......@@ -28,17 +28,17 @@
#define TSDB_FILE_INFO(tf) (&((tf)->info))
#define TSDB_FILE_F(tf) (&((tf)->f))
#define TSDB_FILE_FD(tf) ((tf)->fd)
#define TSDB_FILE_PFILE(tf) ((tf)->pFile)
#define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname)
#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0)
#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL)
#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1)
#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL)
#define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level)
#define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id)
#define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did)
#define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname)
#define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname)
#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf))
#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf))
#define TSDB_FILE_STATE(tf) ((tf)->state)
#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s))
#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK)
......@@ -180,7 +180,7 @@ typedef struct {
typedef struct {
SDFInfo info;
STfsFile f;
int fd;
TdFilePtr pFile;
uint8_t state;
} SDFile;
......@@ -198,8 +198,8 @@ static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFi
static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
ASSERT(!TSDB_FILE_OPENED(pDFile));
pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags);
if (pDFile->fd < 0) {
pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags);
if (pDFile->pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
......@@ -209,7 +209,7 @@ static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
if (TSDB_FILE_OPENED(pDFile)) {
close(pDFile->fd);
taosCloseFile(&pDFile->pFile);
TSDB_FILE_SET_CLOSED(pDFile);
}
}
......@@ -217,7 +217,7 @@ static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
ASSERT(TSDB_FILE_OPENED(pDFile));
int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pDFile), offset, whence);
int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence);
if (loffset < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
......@@ -229,7 +229,7 @@ static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int wh
static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
ASSERT(TSDB_FILE_OPENED(pDFile));
int64_t nwrite = taosWriteFile(pDFile->fd, buf, nbyte);
int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte);
if (nwrite < nbyte) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
......@@ -271,7 +271,7 @@ static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(T
static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
ASSERT(TSDB_FILE_OPENED(pDFile));
int64_t nread = taosReadFile(pDFile->fd, buf, nbyte);
int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte);
if (nread < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
......
source/dnode/vnode/src/meta/metaBDBImpl.c
浏览文件 @ e25637fe
......@@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define ALLOW_FORBID_FUNC
#include "db.h"
#include "metaDef.h"
......
source/dnode/vnode/src/tq/tqMetaStore.c
浏览文件 @ e25637fe
......@@ -34,11 +34,11 @@ static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) {
}
}
static inline int tqSeekLastPage(int fd) {
int offset = lseek(fd, 0, SEEK_END);
static inline int64_t tqSeekLastPage(TdFilePtr pFile) {
int offset = taosLSeekFile(pFile, 0, SEEK_END);
int pageNo = offset / TQ_PAGE_SIZE;
int curPageOffset = pageNo * TQ_PAGE_SIZE;
return lseek(fd, curPageOffset, SEEK_SET);
return taosLSeekFile(pFile, curPageOffset, SEEK_SET);
}
// TODO: the struct is tightly coupled with index entry
......@@ -52,10 +52,10 @@ typedef struct STqIdxPageBuf {
char buffer[TQ_IDX_PAGE_BODY_SIZE];
} STqIdxPageBuf;
static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
int offset = tqSeekLastPage(fd);
static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) {
int offset = tqSeekLastPage(pFile);
int nBytes;
if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) {
if ((nBytes = taosReadFile(pFile, pBuf, TQ_PAGE_SIZE)) == -1) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
......@@ -65,7 +65,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
}
ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset);
return lseek(fd, offset, SEEK_SET);
return taosLSeekFile(pFile, offset, SEEK_SET);
}
STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer,
......@@ -95,15 +95,15 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
tqError("failed to create dir:%s since %s ", name, terrstr());
}
strcat(name, "/" TQ_IDX_NAME);
int idxFd = open(name, O_RDWR | O_CREAT, 0755);
if (idxFd < 0) {
TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pIdxFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s ", name, terrstr());
// free memory
return NULL;
}
pMeta->idxFd = idxFd;
pMeta->pIdxFile = pIdxFile;
pMeta->unpersistHead = calloc(1, sizeof(STqMetaList));
if (pMeta->unpersistHead == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
......@@ -113,14 +113,14 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
strcpy(name, path);
strcat(name, "/" TQ_META_NAME);
int fileFd = open(name, O_RDWR | O_CREAT, 0755);
if (fileFd < 0) {
TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s", name, terrstr());
return NULL;
}
pMeta->fileFd = fileFd;
pMeta->pFile = pFile;
pMeta->pSerializer = serializer;
pMeta->pDeserializer = deserializer;
......@@ -136,7 +136,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
int idxRead;
int allocated = TQ_PAGE_SIZE;
bool readEnd = false;
while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) {
while ((idxRead = taosReadFile(pIdxFile, &idxBuf, TQ_PAGE_SIZE))) {
if (idxRead == -1) {
// TODO: handle error
terrno = TAOS_SYSTEM_ERROR(errno);
......@@ -152,7 +152,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
}
memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE);
lseek(fileFd, pNode->handle.offset, SEEK_SET);
taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET);
if (allocated < pNode->handle.serializedSize) {
void* ptr = realloc(serializedObj, pNode->handle.serializedSize);
if (ptr == NULL) {
......@@ -163,7 +163,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
allocated = pNode->handle.serializedSize;
}
serializedObj->ssize = pNode->handle.serializedSize;
if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
if (taosReadFile(pFile, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
// TODO: read error
}
if (serializedObj->action == TQ_ACTION_INUSE) {
......@@ -237,8 +237,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) {
// commit data and idx
tqStorePersist(pMeta);
ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL);
close(pMeta->fileFd);
close(pMeta->idxFd);
taosCloseFile(&pMeta->pFile);
taosCloseFile(&pMeta->pIdxFile);
// free memory
for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
STqMetaList* pNode = pMeta->bucket[i];
......@@ -263,8 +263,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) {
}
int32_t tqStoreDelete(STqMetaStore* pMeta) {
close(pMeta->fileFd);
close(pMeta->idxFd);
taosCloseFile(&pMeta->pFile);
taosCloseFile(&pMeta->pIdxFile);
// free memory
for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
STqMetaList* pNode = pMeta->bucket[i];
......@@ -302,12 +302,12 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
pSHead->checksum = 0;
pSHead->ssize = sizeof(STqSerializedHead);
/*int allocatedSize = sizeof(STqSerializedHead);*/
int offset = lseek(pMeta->fileFd, 0, SEEK_CUR);
int offset = taosLSeekFile(pMeta->pFile, 0, SEEK_CUR);
tqReadLastPage(pMeta->idxFd, &idxBuf);
tqReadLastPage(pMeta->pIdxFile, &idxBuf);
if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) {
lseek(pMeta->idxFd, 0, SEEK_END);
taosLSeekFile(pMeta->pIdxFile, 0, SEEK_END);
memset(&idxBuf, 0, TQ_PAGE_SIZE);
idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
} else {
......@@ -329,7 +329,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
} else {
pMeta->pSerializer(pNode->handle.valueInUse, &pSHead);
}
nBytes = write(pMeta->fileFd, pSHead, pSHead->ssize);
nBytes = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize);
ASSERT(nBytes == pSHead->ssize);
}
......@@ -340,7 +340,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
} else {
pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead);
}
int nBytesTxn = write(pMeta->fileFd, pSHead, pSHead->ssize);
int nBytesTxn = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize);
ASSERT(nBytesTxn == pSHead->ssize);
nBytes += nBytesTxn;
}
......@@ -355,7 +355,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
idxBuf.head.writeOffset += TQ_IDX_SIZE;
if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) {
nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE);
nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, TQ_PAGE_SIZE);
// TODO: handle error with tfile
ASSERT(nBytes == TQ_PAGE_SIZE);
memset(&idxBuf, 0, TQ_PAGE_SIZE);
......@@ -391,13 +391,13 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
free(pSHead);
// TODO: write new version in tfile
if ((char*)bufPtr != idxBuf.buffer) {
int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset);
int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset);
// TODO: handle error in tfile
ASSERT(nBytes == idxBuf.head.writeOffset);
}
// TODO: using fsync in tfile
fsync(pMeta->idxFd);
fsync(pMeta->fileFd);
taosFsyncFile(pMeta->pIdxFile);
taosFsyncFile(pMeta->pFile);
return 0;
}
......
source/dnode/vnode/src/tsdb/tsdbFS.c
浏览文件 @ e25637fe
......@@ -416,8 +416,8 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname);
tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname);
int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(tfname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
......@@ -436,9 +436,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE);
if (taosWriteFile(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
terrno = TAOS_SYSTEM_ERROR(errno);
close(fd);
taosCloseFile(&pFile);
remove(tfname);
return -1;
}
......@@ -446,7 +446,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
// Encode file status and write to file
if (fsheader.len > 0) {
if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) {
close(fd);
taosCloseFile(&pFile);
remove(tfname);
return -1;
}
......@@ -455,9 +455,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
tsdbEncodeFSStatus(&ptr, pStatus);
taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len);
if (taosWriteFile(fd, pBuf, fsheader.len) < fsheader.len) {
if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) {
terrno = TAOS_SYSTEM_ERROR(errno);
close(fd);
taosCloseFile(&pFile);
(void)remove(tfname);
taosTZfree(pBuf);
return -1;
......@@ -465,15 +465,15 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
}
// fsync, close and rename
if (taosFsyncFile(fd) < 0) {
if (taosFsyncFile(pFile) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
close(fd);
taosCloseFile(&pFile);
remove(tfname);
taosTZfree(pBuf);
return -1;
}
(void)close(fd);
(void)taosCloseFile(&pFile);
(void)taosRenameFile(tfname, cfname);
taosTZfree(pBuf);
......@@ -652,7 +652,7 @@ static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) {
static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
STsdbFS * pfs = REPO_FS(pRepo);
int fd = -1;
TdFilePtr pFile = NULL;
void * buffer = NULL;
SFSHeader fsheader;
char current[TSDB_FILENAME_LEN] = "\0";
......@@ -661,8 +661,8 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current);
// current file exists, try to recover
fd = open(current, O_RDONLY | O_BINARY);
if (fd < 0) {
pFile = taosOpenFile(current, TD_FILE_READ);
if (pFile == NULL) {
tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
......@@ -672,7 +672,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
goto _err;
}
int nread = (int)taosReadFile(fd, buffer, TSDB_FILE_HEAD_SIZE);
int nread = (int)taosReadFile(pFile, buffer, TSDB_FILE_HEAD_SIZE);
if (nread < 0) {
tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current,
strerror(errno));
......@@ -706,7 +706,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
goto _err;
}
nread = (int)taosReadFile(fd, buffer, fsheader.len);
nread = (int)taosReadFile(pFile, buffer, fsheader.len);
if (nread < 0) {
tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
......@@ -732,13 +732,13 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
}
taosTZfree(buffer);
close(fd);
taosCloseFile(&pFile);
return 0;
_err:
if (fd >= 0) {
close(fd);
if (pFile != NULL) {
taosCloseFile(&pFile);
}
taosTZfree(buffer);
return -1;
......@@ -1244,18 +1244,17 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
}
if (tsdbForceKeepFile) {
struct stat tfstat;
int64_t file_size;
// Get real file size
if (fstat(pDFile->fd, &tfstat) < 0) {
if (taosFStatFile(pDFile->pFile, &file_size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosArrayDestroy(fArray);
return -1;
}
if (pDFile->info.size != tfstat.st_size) {
if (pDFile->info.size != file_size) {
int64_t tfsize = pDFile->info.size;
pDFile->info.size = tfstat.st_size;
pDFile->info.size = file_size;
tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo),
TSDB_FILE_FULL_NAME(pDFile), tfsize, pDFile->info.size);
}
......
source/dnode/vnode/src/tsdb/tsdbFile.c
浏览文件 @ e25637fe
......@@ -355,8 +355,8 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) {
int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) {
ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC);
pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
if (pDFile->fd < 0) {
pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pDFile->pFile < 0) {
if (errno == ENOENT) {
// Try to create directory recursively
char *s = strdup(TSDB_FILE_REL_NAME(pDFile));
......@@ -366,8 +366,8 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) {
}
tfree(s);
pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
if (pDFile->fd < 0) {
pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pDFile->pFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
......@@ -460,7 +460,7 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) {
return -1;
}
if (taosFtruncateFile(df.fd, df.info.size) < 0) {
if (taosFtruncateFile(df.pFile, df.info.size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbCloseDFile(&df);
return -1;
......@@ -541,7 +541,7 @@ static int tsdbRollBackDFile(SDFile *pDFile) {
return -1;
}
if (taosFtruncateFile(TSDB_FILE_FD(&df), pDFile->info.size) < 0) {
if (taosFtruncateFile(TSDB_FILE_PFILE(&df), pDFile->info.size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
tsdbCloseDFile(&df);
return -1;
......
source/libs/CMakeLists.txt
浏览文件 @ e25637fe
add_definitions("-D ALLOW_FORBID_FUNC")
add_subdirectory(transport)
add_subdirectory(sync)
add_subdirectory(tdb)
......
source/libs/function/src/taggfunction.c
浏览文件 @ e25637fe
......@@ -3960,14 +3960,14 @@ static void ts_comp_finalize(SqlFunctionCtx *pCtx) {
// qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal);
// TODO refactor transfer ownership of current file
*(FILE **)pCtx->pOutput = pTSbuf->f;
*(TdFilePtr *)pCtx->pOutput = pTSbuf->pFile;
pResInfo->complete = true;
// get the file size
struct stat fStat;
if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) {
pResInfo->numOfRes = fStat.st_size;
int64_t file_size;
if (taosFStatFile(pTSbuf->pFile, &file_size, NULL) == 0) {
pResInfo->numOfRes = (uint32_t )file_size;
}
pTSbuf->remainOpen = true;
......
source/libs/index/inc/index_fst_counting_writer.h
浏览文件 @ e25637fe
......@@ -38,7 +38,7 @@ typedef struct WriterCtx {
WriterType type;
union {
struct {
int fd;
TdFilePtr pFile;
bool readOnly;
char buf[256];
int size;
......
source/libs/index/inc/index_util.h
浏览文件 @ e25637fe
......@@ -15,19 +15,21 @@
#ifndef __INDEX_UTIL_H__
#define __INDEX_UTIL_H__
#include "tarray.h"
#ifdef __cplusplus
extern "C" {
#endif
#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \
do { \
memcpy((void*)buf, (void*)(&key->mem), sizeof(key->mem)); \
memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \
buf += sizeof(key->mem); \
} while (0)
#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \
do { \
memcpy((void*)buf, (void*)key->mem, len); \
memcpy((void *)buf, (void *)key->mem, len); \
buf += len; \
} while (0)
......@@ -35,16 +37,24 @@ extern "C" {
do { \
type c = var; \
assert(sizeof(type) == sizeof(c)); \
memcpy((void*)buf, (void*)&c, sizeof(c)); \
memcpy((void *)buf, (void *)&c, sizeof(c)); \
buf += sizeof(c); \
} while (0)
#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \
do { \
memcpy((void*)buf, (void*)var, len); \
memcpy((void *)buf, (void *)var, len); \
buf += len; \
} while (0)
/* multi sorted result intersection
* input: [1, 2, 4, 5]
* [2, 3, 4, 5]
* [1, 4, 5]
* output:[4, 5]
*/
void iIntersection(SArray *interResults, SArray *finalResult);
void iUnion(SArray *interResults, SArray *finalResult);
#ifdef __cplusplus
}
#endif
......
source/libs/index/src/index.c
浏览文件 @ e25637fe
......@@ -370,22 +370,23 @@ static void indexInterResultsDestroy(SArray* results) {
}
taosArrayDestroy(results);
}
static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) {
// refactor, merge interResults into fResults by oType
SArray* first = taosArrayGetP(interResults, 0);
taosArraySort(first, uidCompare);
taosArrayRemoveDuplicate(first, uidCompare, NULL);
for (int i = 0; i < taosArrayGetSize(interResults); i--) {
SArray* t = taosArrayGetP(interResults, i);
taosArraySort(t, uidCompare);
taosArrayRemoveDuplicate(t, uidCompare, NULL);
}
if (oType == MUST) {
// just one column index, enhance later
taosArrayAddAll(fResults, first);
iIntersection(interResults, fResults);
} else if (oType == SHOULD) {
// just one column index, enhance later
taosArrayAddAll(fResults, first);
// tag1 condistion || tag2 condition
iUnion(interResults, fResults);
} else if (oType == NOT) {
// just one column index, enhance later
taosArrayAddAll(fResults, first);
taosArrayAddAll(fResults, interResults);
// not use currently
}
return 0;
......
source/libs/index/src/index_fst_counting_writer.c
浏览文件 @ e25637fe
......@@ -19,7 +19,7 @@
static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) {
if (ctx->type == TFile) {
assert(len == tfWrite(ctx->file.fd, buf, len));
assert(len == taosWriteFile(ctx->file.pFile, buf, len));
} else {
memcpy(ctx->mem.buf + ctx->offset, buf, len);
}
......@@ -33,7 +33,7 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
nRead = len < ctx->file.size ? len : ctx->file.size;
memcpy(buf, ctx->file.ptr, nRead);
#else
nRead = tfRead(ctx->file.fd, buf, len);
nRead = taosReadFile(ctx->file.pFile, buf, len);
#endif
} else {
memcpy(buf, ctx->mem.buf + ctx->offset, len);
......@@ -45,13 +45,13 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) {
int nRead = 0;
if (ctx->type == TFile) {
// tfLseek(ctx->file.fd, offset, 0);
// tfLseek(ctx->file.pFile, offset, 0);
#ifdef USE_MMAP
int32_t last = ctx->file.size - offset;
nRead = last >= len ? len : last;
memcpy(buf, ctx->file.ptr + offset, nRead);
#else
nRead = tfPread(ctx->file.fd, buf, len, offset);
nRead = taosPReadFile(ctx->file.pFile, buf, len, offset);
#endif
} else {
// refactor later
......@@ -69,9 +69,9 @@ static int writeCtxGetSize(WriterCtx* ctx) {
}
static int writeCtxDoFlush(WriterCtx* ctx) {
if (ctx->type == TFile) {
// taosFsyncFile(ctx->file.fd);
tfFsync(ctx->file.fd);
// tfFlush(ctx->file.fd);
// taosFsyncFile(ctx->file.pFile);
taosFsyncFile(ctx->file.pFile);
// tfFlush(ctx->file.pFile);
} else {
// do nothing
}
......@@ -87,25 +87,25 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
// ugly code, refactor later
ctx->file.readOnly = readOnly;
if (readOnly == false) {
// ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.fd = tfOpenCreateWriteAppend(path);
tfFtruncate(ctx->file.fd, 0);
struct stat fstat;
stat(path, &fstat);
ctx->file.size = fstat.st_size;
// ctx->file.pFile = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.pFile = taosOpenFile(path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
taosFtruncateFile(ctx->file.pFile, 0);
int64_t file_size;
taosStatFile(path, &file_size, NULL);
ctx->file.size = (int)file_size;
} else {
// ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.fd = tfOpenRead(path);
// ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
ctx->file.pFile = taosOpenFile(path, TD_FILE_READ);
struct stat fstat;
stat(path, &fstat);
ctx->file.size = fstat.st_size;
#ifdef USE_MMAP
ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.fd, ctx->file.size);
ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size);
#endif
}
memcpy(ctx->file.buf, path, strlen(path));
if (ctx->file.fd < 0) {
if (ctx->file.pFile < 0) {
indexError("failed to open file, error %d", errno);
goto END;
}
......@@ -133,7 +133,7 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) {
free(ctx->mem.buf);
} else {
ctx->flush(ctx);
tfClose(ctx->file.fd);
taosCloseFile(&ctx->file.pFile);
if (ctx->file.readOnly) {
#ifdef USE_MMAP
munmap(ctx->file.ptr, ctx->file.size);
......
source/libs/index/src/index_tfile.c
浏览文件 @ e25637fe
......@@ -371,7 +371,7 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
return ret;
}
IndexTFile* pTfile = (IndexTFile*)tfile;
IndexTFile* pTfile = tfile;
SIndexTerm* term = query->term;
ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName};
......@@ -586,11 +586,11 @@ static int tfileReaderLoadHeader(TFileReader* reader) {
int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0);
if (nread == -1) {
indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf),
errno, reader->ctx->file.fd, reader->ctx->file.buf);
indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf),
errno, reader->ctx->file.buf);
} else {
indexInfo("actual Read: %d, to read: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf),
reader->ctx->file.fd, reader->ctx->file.buf);
indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf),
reader->ctx->file.buf);
}
// assert(nread == sizeof(buf));
memcpy(&reader->header, buf, sizeof(buf));
......
source/libs/index/src/index_util.c 0 → 100644
浏览文件 @ e25637fe
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "index_util.h"
#include "index.h"
typedef struct MergeIndex {
int idx;
int len;
} MergeIndex;
static int iBinarySearch(SArray *arr, int s, int e, uint64_t k) {
uint64_t v;
int32_t m;
while (s <= e) {
m = s + (e - s) / 2;
v = *(uint64_t *)taosArrayGet(arr, m);
if (v >= k) {
e = m - 1;
} else {
s = m + 1;
}
}
return s;
}
void iIntersection(SArray *inters, SArray *final) {
int32_t sz = taosArrayGetSize(inters);
if (sz <= 0) {
return;
}
MergeIndex *mi = calloc(sz, sizeof(MergeIndex));
for (int i = 0; i < sz; i++) {
SArray *t = taosArrayGetP(inters, i);
mi[i].len = taosArrayGetSize(t);
mi[i].idx = 0;
}
SArray *base = taosArrayGetP(inters, 0);
for (int i = 0; i < taosArrayGetSize(base); i++) {
uint64_t tgt = *(uint64_t *)taosArrayGet(base, i);
bool has = true;
for (int j = 1; j < taosArrayGetSize(inters); j++) {
SArray *oth = taosArrayGetP(inters, j);
int mid = iBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt);
if (mid >= 0 && mid < mi[j].len) {
uint64_t val = *(uint64_t *)taosArrayGet(oth, mid);
has = (val == tgt ? true : false);
mi[j].idx = mid;
} else {
has = false;
}
}
if (has == true) {
taosArrayPush(final, &tgt);
}
}
tfree(mi);
}
void iUnion(SArray *inters, SArray *final) {
int32_t sz = taosArrayGetSize(inters);
if (sz <= 0) {
return;
}
MergeIndex *mi = calloc(sz, sizeof(MergeIndex));
for (int i = 0; i < sz; i++) {
SArray *t = taosArrayGetP(inters, i);
mi[i].len = taosArrayGetSize(t);
mi[i].idx = 0;
}
while (1) {
uint64_t mVal = UINT_MAX;
int mIdx = -1;
for (int j = 0; j < sz; j++) {
SArray *t = taosArrayGetP(inters, j);
if (mi[j].idx >= mi[j].len) {
continue;
}
uint64_t cVal = *(uint64_t *)taosArrayGet(t, mi[j].idx);
if (cVal < mVal) {
mVal = cVal;
mIdx = j;
}
}
if (mIdx != -1) {
mi[mIdx].idx++;
if (taosArrayGetSize(final) > 0) {
uint64_t lVal = *(uint64_t *)taosArrayGetLast(final);
if (lVal == mVal) {
continue;
}
}
taosArrayPush(final, &mVal);
} else {
break;
}
}
tfree(mi);
}
source/libs/index/test/CMakeLists.txt
浏览文件 @ e25637fe
add_executable(indexTest "")
add_executable(fstTest "")
add_executable(fstUT "")
add_executable(UtilUT "")
target_sources(indexTest
PRIVATE
......@@ -15,6 +16,11 @@ target_sources(fstUT
PRIVATE
"fstUT.cc"
)
target_sources(UtilUT
PRIVATE
"utilUT.cc"
)
target_include_directories ( indexTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/index"
......@@ -31,6 +37,12 @@ target_include_directories ( fstUT
"${CMAKE_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories ( UtilUT
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries (indexTest
os
util
......@@ -53,6 +65,14 @@ target_link_libraries (fstUT
index
)
target_link_libraries (UtilUT
os
util
common
gtest_main
index
)
#add_test(
# NAME index_test
......
source/libs/index/test/utilUT.cc 0 → 100644
浏览文件 @ e25637fe
#include <gtest/gtest.h>
#include <algorithm>
#include <iostream>
#include <string>
#include <thread>
#include <vector>
#include "index.h"
#include "indexInt.h"
#include "index_cache.h"
#include "index_fst.h"
#include "index_fst_counting_writer.h"
#include "index_fst_util.h"
#include "index_tfile.h"
#include "index_util.h"
#include "tglobal.h"
#include "tskiplist.h"
#include "tutil.h"
class UtilEnv : public ::testing::Test {
protected:
virtual void SetUp() {
src = (SArray *)taosArrayInit(2, sizeof(void *));
for (int i = 0; i < 3; i++) {
SArray *m = taosArrayInit(10, sizeof(uint64_t));
taosArrayPush(src, &m);
}
rslt = (SArray *)taosArrayInit(10, sizeof(uint64_t));
}
virtual void TearDown() {
for (int i = 0; i < taosArrayGetSize(src); i++) {
SArray *m = (SArray *)taosArrayGetP(src, i);
taosArrayDestroy(m);
}
taosArrayDestroy(src);
}
SArray *src;
SArray *rslt;
};
static void clearSourceArray(SArray *p) {
for (int i = 0; i < taosArrayGetSize(p); i++) {
SArray *m = (SArray *)taosArrayGetP(p, i);
taosArrayClear(m);
}
}
static void clearFinalArray(SArray *p) { taosArrayClear(p); }
TEST_F(UtilEnv, intersectionSimpleResult) {
SArray *f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 10);
clearSourceArray(src);
clearFinalArray(rslt);
}
TEST_F(UtilEnv, intersectMultiEmptyResult) {
SArray *f = (SArray *)taosArrayGetP(src, 0);
for (int i = 10; i < 20; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < 10; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 20; i < 30; i++) {
uint64_t val = i;
taosArrayPush(f, &val);
}
// empty source
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 0);
clearSourceArray(src);
clearFinalArray(rslt);
}
TEST_F(UtilEnv, intersectSimpleEmpty) {
clearSourceArray(src);
clearFinalArray(rslt);
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 0);
}
TEST_F(UtilEnv, intersect01) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {2, 3, 4, 5};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
uint64_t arr2[] = {1, 2, 3, 5};
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
taosArrayPush(f, &arr2[i]);
}
uint64_t arr3[] = {3, 5, 10, 11};
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
taosArrayPush(f, &arr3[i]);
}
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 2);
}
TEST_F(UtilEnv, intersect02) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {13, 14, 15};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
uint64_t arr2[] = {8, 10, 12, 13};
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
taosArrayPush(f, &arr2[i]);
}
uint64_t arr3[] = {9, 10, 11};
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
taosArrayPush(f, &arr3[i]);
}
iIntersection(src, rslt);
assert(taosArrayGetSize(rslt) == 0);
}
TEST_F(UtilEnv, 01union) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {13, 14, 15};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
iUnion(src, rslt);
assert(taosArrayGetSize(rslt) == 3);
}
TEST_F(UtilEnv, 02union) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {13, 14, 15};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
uint64_t arr2[] = {13, 14, 15};
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
taosArrayPush(f, &arr2[i]);
}
iUnion(src, rslt);
assert(taosArrayGetSize(rslt) == 3);
}
TEST_F(UtilEnv, 03union) {
clearSourceArray(src);
clearFinalArray(rslt);
uint64_t arr1[] = {13, 16, 18, 20};
SArray * f = (SArray *)taosArrayGetP(src, 0);
for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) {
taosArrayPush(f, &arr1[i]);
}
uint64_t arr2[] = {0, 12, 13, 20, 23};
f = (SArray *)taosArrayGetP(src, 1);
for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) {
taosArrayPush(f, &arr2[i]);
}
uint64_t arr3[] = {1, 12, 13, 16, 17};
f = (SArray *)taosArrayGetP(src, 2);
for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) {
taosArrayPush(f, &arr3[i]);
}
iUnion(src, rslt);
assert(taosArrayGetSize(rslt) == 9);
}
source/libs/nodes/src/nodesCloneFuncs.c
浏览文件 @ e25637fe
......@@ -142,14 +142,6 @@ static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) {
return (SNode*)pDst;
}
static SNode* columnRefNodeCopy(const SColumnRefNode* pSrc, SColumnRefNode* pDst) {
dataTypeCopy(&pSrc->dataType, &pDst->dataType);
COPY_SCALAR_FIELD(tupleId);
COPY_SCALAR_FIELD(slotId);
COPY_SCALAR_FIELD(columnId);
return (SNode*)pDst;
}
static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) {
COPY_SCALAR_FIELD(tupleId);
COPY_SCALAR_FIELD(slotId);
......@@ -183,8 +175,6 @@ SNode* nodesCloneNode(const SNode* pNode) {
return logicConditionNodeCopy((const SLogicConditionNode*)pNode, (SLogicConditionNode*)pDst);
case QUERY_NODE_FUNCTION:
return functionNodeCopy((const SFunctionNode*)pNode, (SFunctionNode*)pDst);
case QUERY_NODE_COLUMN_REF:
return columnRefNodeCopy((const SColumnRefNode*)pNode, (SColumnRefNode*)pDst);
case QUERY_NODE_TARGET:
return targetNodeCopy((const STargetNode*)pNode, (STargetNode*)pDst);
case QUERY_NODE_REAL_TABLE:
......
source/libs/nodes/src/nodesCodeFuncs.c
浏览文件 @ e25637fe
......@@ -55,8 +55,6 @@ static char* nodeName(ENodeType type) {
return "NodeList";
case QUERY_NODE_FILL:
return "Fill";
case QUERY_NODE_COLUMN_REF:
return "ColumnRef";
case QUERY_NODE_TARGET:
return "Target";
case QUERY_NODE_RAW_EXPR:
......@@ -503,28 +501,6 @@ static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) {
return code;
}
static const char* jkColumnRefDataType = "DataType";
static const char* jkColumnRefTupleId = "TupleId";
static const char* jkColumnRefSlotId = "SlotId";
static const char* jkColumnRefColumnId = "ColumnId";
static int32_t columnRefNodeToJson(const void* pObj, SJson* pJson) {
const SColumnRefNode* pNode = (const SColumnRefNode*)pObj;
int32_t code = tjsonAddObject(pJson, jkColumnRefDataType, dataTypeToJson, &pNode->dataType);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnRefTupleId, pNode->tupleId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnRefSlotId, pNode->slotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnRefColumnId, pNode->columnId);
}
return code;
}
static const char* jkTargetTupleId = "TupleId";
static const char* jkTargetSlotId = "SlotId";
static const char* jkTargetExpr = "Expr";
......@@ -646,8 +622,6 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_INTERVAL_WINDOW:
case QUERY_NODE_NODE_LIST:
case QUERY_NODE_FILL:
case QUERY_NODE_COLUMN_REF:
return columnRefNodeToJson(pObj, pJson);
case QUERY_NODE_TARGET:
return targetNodeToJson(pObj, pJson);
case QUERY_NODE_RAW_EXPR:
......
source/libs/nodes/src/nodesUtilFuncs.c
浏览文件 @ e25637fe
......@@ -63,8 +63,6 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SNodeListNode));
case QUERY_NODE_FILL:
return makeNode(type, sizeof(SFillNode));
case QUERY_NODE_COLUMN_REF:
return makeNode(type, sizeof(SColumnRefNode));
case QUERY_NODE_RAW_EXPR:
return makeNode(type, sizeof(SRawExprNode));
case QUERY_NODE_SET_OPERATOR:
......@@ -93,6 +91,10 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(STableScanPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_PROJECT:
return makeNode(type, sizeof(SProjectPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_JOIN:
return makeNode(type, sizeof(SJoinPhysiNode));
case QUERY_NODE_PHYSICAL_PLAN_AGG:
return makeNode(type, sizeof(SAggPhysiNode));
default:
break;
}
......
source/libs/planner/test/newPlannerTest.cpp
浏览文件 @ e25637fe
......@@ -123,8 +123,8 @@ TEST_F(NewPlannerTest, simple) {
TEST_F(NewPlannerTest, groupBy) {
setDatabase("root", "test");
bind("SELECT count(*) FROM t1");
ASSERT_TRUE(run());
// bind("SELECT count(*) FROM t1");
// ASSERT_TRUE(run());
bind("SELECT c1, count(*) FROM t1 GROUP BY c1");
ASSERT_TRUE(run());
......
source/libs/scalar/inc/filterInt.h
浏览文件 @ e25637fe
......@@ -26,6 +26,7 @@ extern "C" {
#include "scalar.h"
#include "querynodes.h"
#include "query.h"
#include "tep.h"
#define FILTER_DEFAULT_GROUP_SIZE 4
#define FILTER_DEFAULT_UNIT_SIZE 4
......@@ -199,7 +200,7 @@ typedef struct SFilterUnit {
} SFilterUnit;
typedef struct SFilterComUnit {
void *colData;
void *colData; // pointer to SColumnInfoData
void *valData;
void *valData2;
uint16_t colId;
......@@ -282,7 +283,7 @@ typedef struct SFilterInfo {
#define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0)
#define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0)
#define FLT_IS_COMPARISON_OPERATOR(_op) ((_op) >= OP_TYPE_GREATER_THAN && (_op) < OP_TYPE_IS_NOT_NULL)
#define FLT_IS_COMPARISON_OPERATOR(_op) ((_op) >= OP_TYPE_GREATER_THAN && (_op) < OP_TYPE_IS_NOT_UNKNOWN)
#define fltFatal(...) qFatal(__VA_ARGS__)
#define fltError(...) qError(__VA_ARGS__)
......@@ -300,12 +301,12 @@ typedef struct SFilterInfo {
#define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
#define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.type)
#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.bytes)
#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnRefNode *)((fi)->desc))->columnId)
#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnRefNode *)((fi)->desc))->slotId)
#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnRefNode *)((fi)->desc))
#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SColumnRefNode *)((fi)->desc))->dataType.bytes * (ri))
#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnNode *)((fi)->desc))->node.resType.type)
#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnNode *)((fi)->desc))->node.resType.bytes)
#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnNode *)((fi)->desc))->colId)
#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnNode *)((fi)->desc))->slotId)
#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnNode *)((fi)->desc))
#define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGet(((SColumnInfoData *)(fi)->data), (ri)))
#define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type)
#define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data)
#define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc)
......
source/libs/scalar/src/filter.c
浏览文件 @ e25637fe
......@@ -914,14 +914,14 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
if (nodeType(node) != QUERY_NODE_COLUMN_REF && nodeType(node) != QUERY_NODE_VALUE) {
if (nodeType(node) != QUERY_NODE_COLUMN && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) {
FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
int32_t type;
void *v;
if (nodeType(node) == QUERY_NODE_COLUMN_REF) {
if (nodeType(node) == QUERY_NODE_COLUMN) {
type = FLD_TYPE_COLUMN;
v = node;
} else {
......@@ -968,8 +968,10 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi
SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u);
assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE));
} else {
if(optr != OP_TYPE_IS_NULL && optr != OP_TYPE_IS_NOT_NULL && optr != FILTER_DUMMY_EMPTY_OPTR){
return -1;
int32_t paramNum = scalarGetOperatorParamNum(optr);
if (1 != paramNum) {
fltError("invalid right field in unit, operator:%s, rightType:%d", gOptrStr[optr].str, u->right.type);
return TSDB_CODE_QRY_APP_ERROR;
}
}
......@@ -1016,7 +1018,6 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) {
if (node->opType == OP_TYPE_IN && (!IS_VAR_DATA_TYPE(type))) {
SNodeListNode *listNode = (SNodeListNode *)node->pRight;
void *fdata = NULL;
SListCell *cell = listNode->pNodeList->pHead;
SScalarParam in = {.num = 1}, out = {.num = 1, .type = type};
......@@ -1036,7 +1037,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) {
len = tDataTypes[type].bytes;
filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true);
filterAddField(info, NULL, &out.data, FLD_TYPE_VALUE, &right, len, true);
filterAddUnit(info, OP_TYPE_EQUAL, &left, &right, &uidx);
......@@ -1337,6 +1338,8 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) {
return DEAL_RES_IGNORE_CHILD;
}
ctx->code = TSDB_CODE_QRY_APP_ERROR;
fltError("invalid condition type, type:%d", node->condType);
return DEAL_RES_ERROR;
......@@ -1451,7 +1454,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num);
for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) {
SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i];
SColumnRefNode *refNode = (SColumnRefNode *)field->desc;
SColumnNode *refNode = (SColumnNode *)field->desc;
qDebug("COL%d => [%d][%d]", i, refNode->tupleId, refNode->slotId);
}
......@@ -1480,7 +1483,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options)
char str[512] = {0};
SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit);
SColumnRefNode *refNode = (SColumnRefNode *)left->desc;
SColumnNode *refNode = (SColumnNode *)left->desc;
if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS){
len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str);
}
......@@ -1736,7 +1739,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
for (uint32_t i = 0; i < info->unitNum; ++i) {
SFilterUnit* unit = &info->units[i];
if (unit->right.type != FLD_TYPE_VALUE) {
assert(unit->compare.optr == OP_TYPE_IS_NULL || unit->compare.optr == OP_TYPE_IS_NOT_NULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR);
assert(unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR || scalarGetOperatorParamNum(unit->compare.optr) == 1);
continue;
}
......@@ -1792,8 +1795,9 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
}
if(type != TSDB_DATA_TYPE_JSON){
bool converted = false;
char extInfo = 0;
if (dType->type == type) {
assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type);
} else {
SScalarParam in = {.data = nodesGetValueFromNode(var), .num = 1, .type = dType->type, .bytes = dType->bytes};
SScalarParam out = {.data = fi->data, .num = 1, .type = type};
if (vectorConvertImpl(&in, &out)) {
......@@ -1801,6 +1805,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
}
}
// match/nmatch for nchar type need convert from ucs4 to mbs
if(type == TSDB_DATA_TYPE_NCHAR &&
......@@ -1809,7 +1814,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) {
int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData));
if (len < 0){
qError("filterInitValFieldData taosUcs4ToMbs error 1");
return TSDB_CODE_FAILED;
return TSDB_CODE_QRY_APP_ERROR;
}
varDataSetLen(newValData, len);
varDataCopy(fi->data, newValData);
......@@ -2565,7 +2570,8 @@ int32_t filterUpdateComUnits(SFilterInfo *info) {
for (uint32_t i = 0; i < info->unitNum; ++i) {
SFilterUnit *unit = &info->units[i];
info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0);
SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, unit);
info->cunits[i].colData = col->data;
}
return TSDB_CODE_SUCCESS;
......@@ -2770,7 +2776,7 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p,
uint32_t unitNum = *(unitIdx++);
for (uint32_t u = 0; u < unitNum; ++u) {
SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)];
void *colData = (char *)cunit->colData + cunit->dataSize * i;
void *colData = colDataGet((SColumnInfoData *)cunit->colData, i);
//if (FILTER_UNIT_GET_F(info, uidx)) {
// p[i] = FILTER_UNIT_GET_R(info, uidx);
......@@ -2868,7 +2874,7 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows,
for (int32_t i = 0; i < numOfRows; ++i) {
uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
void *colData = colDataGet((SColumnInfoData *)info->cunits[uidx].colData, i);
if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
if (!colData){ // for json->'key' is null
(*p)[i] = 1;
......@@ -2902,7 +2908,7 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows
for (int32_t i = 0; i < numOfRows; ++i) {
uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
void *colData = colDataGet((SColumnInfoData *)info->cunits[uidx].colData, i);
if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){
if (!colData) { // for json->'key' is not null
......@@ -2929,7 +2935,6 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD
SFilterInfo *info = (SFilterInfo *)pinfo;
bool all = true;
uint16_t dataSize = info->cunits[0].dataSize;
char *colData = (char *)info->cunits[0].colData;
rangeCompFunc rfunc = gRangeCompare[info->cunits[0].rfunc];
void *valData = info->cunits[0].valData;
void *valData2 = info->cunits[0].valData2;
......@@ -2944,9 +2949,10 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD
}
for (int32_t i = 0; i < numOfRows; ++i) {
void *colData = colDataGet((SColumnInfoData *)info->cunits[0].colData, i);
if (colData == NULL || isNull(colData, info->cunits[0].dataType)) {
all = false;
colData += dataSize;
continue;
}
......@@ -2955,8 +2961,6 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD
if ((*p)[i] == 0) {
all = false;
}
colData += dataSize;
}
return all;
......@@ -2976,7 +2980,7 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa
for (int32_t i = 0; i < numOfRows; ++i) {
uint32_t uidx = info->groups[0].unitIdxs[0];
void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i;
void *colData = colDataGet((SColumnInfoData *)info->cunits[uidx].colData, i);
if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) {
(*p)[i] = 0;
all = false;
......@@ -3027,7 +3031,7 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg
for (uint32_t u = 0; u < group->unitNum; ++u) {
uint32_t uidx = group->unitIdxs[u];
SFilterComUnit *cunit = &info->cunits[uidx];
void *colData = (char *)cunit->colData + cunit->dataSize * i;
void *colData = colDataGet((SColumnInfoData *)(cunit->colData), i);
//if (FILTER_UNIT_GET_F(info, uidx)) {
// p[i] = FILTER_UNIT_GET_R(info, uidx);
......@@ -3483,11 +3487,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
if (stat->scalarMode) {
return DEAL_RES_CONTINUE;
}
if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN_REF == nodeType(*pNode)) {
if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
return DEAL_RES_CONTINUE;
}
......@@ -3505,22 +3505,28 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
if (NULL == node->pRight) {
if (scalarGetOperatorParamNum(node->opType) > 1) {
fltError("invalid operator, pRight:%p, type:%d", node->pRight, nodeType(node));
fltError("invalid operator, pRight:%p, nodeType:%d, opType:%d", node->pRight, nodeType(node), node->opType);
stat->code = TSDB_CODE_QRY_APP_ERROR;
return DEAL_RES_ERROR;
}
if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) {
if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
if (OP_TYPE_IS_TRUE == node->opType || OP_TYPE_IS_FALSE == node->opType || OP_TYPE_IS_UNKNOWN == node->opType
|| OP_TYPE_IS_NOT_TRUE == node->opType || OP_TYPE_IS_NOT_FALSE == node->opType || OP_TYPE_IS_NOT_UNKNOWN == node->opType) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
} else {
if ((QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) {
if ((QUERY_NODE_COLUMN != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
if ((QUERY_NODE_COLUMN_REF != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) {
if ((QUERY_NODE_COLUMN != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
......@@ -3530,7 +3536,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) {
if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) {
SNode *t = node->pLeft;
node->pLeft = node->pRight;
node->pRight = t;
......@@ -3543,10 +3549,10 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
}
if (OP_TYPE_IN != node->opType) {
SColumnRefNode *refNode = (SColumnRefNode *)node->pLeft;
SColumnNode *refNode = (SColumnNode *)node->pLeft;
SValueNode *valueNode = (SValueNode *)node->pRight;
int32_t type = vectorGetConvertType(refNode->dataType.type, valueNode->node.resType.type);
if (0 != type && type != refNode->dataType.type) {
int32_t type = vectorGetConvertType(refNode->node.resType.type, valueNode->node.resType.type);
if (0 != type && type != refNode->node.resType.type) {
stat->scalarMode = true;
return DEAL_RES_CONTINUE;
}
......@@ -3570,18 +3576,19 @@ int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {
}
int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) {
//TODO
return TSDB_CODE_SUCCESS;
}
int32_t filterGetDataFromColId(void *param, int32_t id, void **data) {
int32_t fltGetDataFromColId(void *param, int32_t id, void **data) {
int32_t numOfCols = ((SFilterColumnParam *)param)->numOfCols;
SArray* pDataBlock = ((SFilterColumnParam *)param)->pDataBlock;
for (int32_t j = 0; j < numOfCols; ++j) {
SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j);
if (id == pColInfo->info.colId) {
*data = pColInfo->pData;
*data = pColInfo;
break;
}
}
......@@ -3589,13 +3596,28 @@ int32_t filterGetDataFromColId(void *param, int32_t id, void **data) {
return TSDB_CODE_SUCCESS;
}
int32_t fltGetDataFromSlotId(void *param, int32_t id, void **data) {
int32_t numOfCols = ((SFilterColumnParam *)param)->numOfCols;
SArray* pDataBlock = ((SFilterColumnParam *)param)->pDataBlock;
if (id < 0 || id >= numOfCols || id >= taosArrayGetSize(pDataBlock)) {
fltError("invalid slot id, id:%d, numOfCols:%d, arraySize:%d", id, numOfCols, (int32_t)taosArrayGetSize(pDataBlock));
return TSDB_CODE_QRY_APP_ERROR;
}
SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, id);
*data = pColInfo;
return TSDB_CODE_SUCCESS;
}
int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param, filer_get_col_from_id fp) {
return fltSetColFieldDataImpl(info, param, fp, false);
int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param) {
return fltSetColFieldDataImpl(info, param, fltGetDataFromSlotId, false);
}
int32_t filterSetDataFromColId(SFilterInfo *info, void *param, filer_get_col_from_id fp) {
return fltSetColFieldDataImpl(info, param, fp, true);
int32_t filterSetDataFromColId(SFilterInfo *info, void *param) {
return fltSetColFieldDataImpl(info, param, fltGetDataFromColId, true);
}
......@@ -3623,6 +3645,8 @@ int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options)
SFltTreeStat stat = {0};
FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat));
info->scalarMode = stat.scalarMode;
if (!info->scalarMode) {
FLT_ERR_JRET(fltInitFromNode(pNode, info, options));
} else {
......@@ -3647,7 +3671,15 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData
FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pSrc, &output));
*p = output.data;
return TSDB_CODE_SUCCESS;
int8_t *r = output.data;
for (int32_t i = 0; i < output.num; ++i) {
if (0 == *(r+i)) {
return false;
}
}
return true;
}
return (*info->func)(info, pSrc->info.rows, p, statis, numOfCols);
......
source/libs/scalar/src/scalar.c
浏览文件 @ e25637fe
......@@ -134,13 +134,13 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
break;
}
case QUERY_NODE_COLUMN_REF: {
case QUERY_NODE_COLUMN: {
if (NULL == ctx) {
sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx);
SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
SColumnRefNode *ref = (SColumnRefNode *)node;
SColumnNode *ref = (SColumnNode *)node;
if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) {
sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock));
SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
......@@ -281,7 +281,8 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu
SScalarFuncExecFuncs ffpSet = {0};
int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet);
if (code) {
sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
sclError(
"fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
SCL_ERR_RET(code);
}
......@@ -299,7 +300,8 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu
for (int32_t i = 0; i < rowNum; ++i) {
code = (*ffpSet.process)(params, node->pParameterList->length, output);
if (code) {
sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
sclError(
"scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
SCL_ERR_JRET(code);
}
......@@ -599,7 +601,7 @@ EDealRes sclWalkOperator(SNode* pNode, void* pContext) {
EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN_REF == nodeType(pNode)) {
if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) {
return DEAL_RES_CONTINUE;
}
......
source/libs/scalar/test/scalar/scalarTests.cpp
浏览文件 @ e25637fe
......@@ -75,10 +75,10 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) {
}
void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) {
SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF);
SColumnRefNode *rnode = (SColumnRefNode *)node;
rnode->dataType.type = dataType;
rnode->dataType.bytes = dataBytes;
SNode *node = nodesMakeNode(QUERY_NODE_COLUMN);
SColumnNode *rnode = (SColumnNode *)node;
rnode->node.resType.type = dataType;
rnode->node.resType.bytes = dataBytes;
rnode->tupleId = 0;
if (NULL == *block) {
......@@ -111,7 +111,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
taosArrayPush(res->pDataBlock, &idata);
rnode->slotId = 2;
rnode->columnId = 55;
rnode->colId = 55;
*block = res;
} else {
......@@ -126,7 +126,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
taosArrayPush(res->pDataBlock, &idata);
rnode->slotId = idx;
rnode->columnId = 55 + idx;
rnode->colId = 55 + idx;
}
*pNode = (SNode *)rnode;
......
source/libs/tdb/src/db/tdbEnv.c
浏览文件 @ e25637fe
......@@ -18,7 +18,7 @@
struct STDbEnv {
char * rootDir; // root directory of the environment
char * jname; // journal file name
int jfd; // journal file fd
TdFilePtr jpFile; // journal file fd
pgsz_t pgSize; // page size
cachesz_t cacheSize; // total cache size
STDbList dbList; // TDB List
......@@ -55,7 +55,7 @@ int tdbEnvCreate(TENV **ppEnv, const char *rootDir) {
pEnv->rootDir = (char *)(&pEnv[1]);
pEnv->jname = pEnv->rootDir + slen + 1;
pEnv->jfd = -1;
pEnv->jpFile = NULL;
pEnv->pgSize = TDB_DEFAULT_PGSIZE;
pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE;
......@@ -139,8 +139,8 @@ static int tdbEnvDestroy(TENV *pEnv) {
}
int tdbEnvBeginTxn(TENV *pEnv) {
pEnv->jfd = open(pEnv->jname, O_CREAT | O_RDWR, 0755);
if (pEnv->jfd < 0) {
pEnv->jpFile = taosOpenFile(pEnv->jname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pEnv->jpFile < 0) {
return -1;
}
......@@ -149,8 +149,8 @@ int tdbEnvBeginTxn(TENV *pEnv) {
int tdbEnvCommit(TENV *pEnv) {
/* TODO */
close(pEnv->jfd);
pEnv->jfd = -1;
taosCloseFile(&pEnv->jpFile);
pEnv->jpFile = NULL;
return 0;
}
......
source/libs/tdb/src/db/tdbPgFile.c
浏览文件 @ e25637fe
......@@ -51,10 +51,10 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) {
pPgFile->fname = (char *)(&(pPgFile[1]));
memcpy(pPgFile->fname, fname, fnameLen);
pPgFile->fname[fnameLen] = '\0';
pPgFile->fd = -1;
pPgFile->pFile = NULL;
pPgFile->fd = open(fname, O_CREAT | O_RDWR, 0755);
if (pPgFile->fd < 0) {
pPgFile->pFile = taosOpenFile(fname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
if (pPgFile->pFile == NULL) {
// TODO: handle error
return -1;
}
......@@ -95,8 +95,8 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) {
int pgFileClose(SPgFile *pPgFile) {
if (pPgFile) {
if (pPgFile->fd >= 0) {
close(pPgFile->fd);
if (pPgFile->pFile >= 0) {
taosCloseFile(&pPgFile->pFile);
}
tfree(pPgFile->fname);
......@@ -201,7 +201,7 @@ static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) {
pTData = pData;
szToRead = pgSize;
for (; szToRead > 0;) {
rsize = pread(pPgFile->fd, pTData, szToRead, pgno * pgSize);
rsize = pread(pPgFile->pFile, pTData, szToRead, pgno * pgSize);
if (rsize < 0) {
if (errno == EINTR) {
continue;
......
source/libs/tdb/src/inc/tdbPgFile.h
浏览文件 @ e25637fe
......@@ -38,7 +38,7 @@ struct SPgFile {
uint8_t fileid[TDB_FILE_ID_LEN]; // file id
pgno_t lsize; // page file logical size (for count)
pgno_t fsize; // real file size on disk (for rollback)
int fd;
TdFilePtr pFile;
SPgFileListNode envHash;
SPgFileListNode envPgfList;
};
......
source/libs/tfs/test/tfsTest.cpp
浏览文件 @ e25637fe
......@@ -230,10 +230,11 @@ TEST_F(TfsTest, 04_File) {
EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0);
FILE *fp = fopen(f1.aname, "w");
ASSERT_NE(fp, nullptr);
fwrite("12345678", 1, 5, fp);
fclose(fp);
// FILE *fp = fopen(f1.aname, "w");
TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
ASSERT_NE(pFile, nullptr);
taosWriteFile(pFile, "12345678", 5);
taosCloseFile(&pFile);
char base[128] = {0};
tfsBasename(&f1, base);
......@@ -638,10 +639,11 @@ TEST_F(TfsTest, 05_MultiDisk) {
EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0);
FILE *fp = fopen(f1.aname, "w");
ASSERT_NE(fp, nullptr);
fwrite("12345678", 1, 5, fp);
fclose(fp);
// FILE *fp = fopen(f1.aname, "w");
TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
ASSERT_NE(pFile, nullptr);
taosWriteFile(pFile, "12345678", 5);
taosCloseFile(&pFile);
char base[128] = {0};
tfsBasename(&f1, base);
......
source/libs/transport/inc/rpcUdp.h
浏览文件 @ e25637fe
......@@ -30,7 +30,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t
void taosFreeMsgHdr(void *hdr);
int taosMsgHdrSize(void *hdr);
void taosSendMsgHdr(void *hdr, int fd);
void taosSendMsgHdr(void *hdr, TdFilePtr pFile);
void taosInitMsgHdr(void **hdr, void *dest, int maxPkts);
void taosSetMsgHdrData(void *hdr, char *data, int dataLen);
......
source/libs/transport/test/pushServer.c
浏览文件 @ e25637fe
......@@ -22,7 +22,7 @@
int msgSize = 128;
int commit = 0;
int dataFd = -1;
TdFilePtr pDataFile = NULL;
STaosQueue *qhandle = NULL;
STaosQset * qset = NULL;
......@@ -43,8 +43,8 @@ void processShellMsg() {
for (int i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pRpcMsg);
if (dataFd >= 0) {
if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
if (pDataFile != NULL) {
if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
tInfo("failed to write data file, reason:%s", strerror(errno));
}
}
......@@ -52,7 +52,7 @@ void processShellMsg() {
if (commit >= 2) {
num += numOfMsgs;
// if (taosFsync(dataFd) < 0) {
// if (taosFsync(pDataFile) < 0) {
// tInfo("failed to flush data to file, reason:%s", strerror(errno));
//}
......@@ -181,8 +181,8 @@ int main(int argc, char *argv[]) {
tInfo("RPC server is running, ctrl-c to exit");
if (commit) {
dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno));
pDataFile = taosOpenFile(dataName, TD_FILE_APPEND | TD_FILE_CTEATE | TD_FILE_WRITE);
if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno));
}
qhandle = taosOpenQueue();
qset = taosOpenQset();
......@@ -190,8 +190,8 @@ int main(int argc, char *argv[]) {
processShellMsg();
if (dataFd >= 0) {
close(dataFd);
if (pDataFile != NULL) {
taosCloseFile(&pDataFile);
remove(dataName);
}
......
source/libs/transport/test/rserver.c
浏览文件 @ e25637fe
......@@ -22,7 +22,7 @@
int msgSize = 128;
int commit = 0;
int dataFd = -1;
TdFilePtr pDataFile = NULL;
STaosQueue *qhandle = NULL;
STaosQset * qset = NULL;
......@@ -43,8 +43,8 @@ void processShellMsg() {
for (int i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pRpcMsg);
if (dataFd >= 0) {
if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
if (pDataFile >= 0) {
if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) {
tInfo("failed to write data file, reason:%s", strerror(errno));
}
}
......@@ -52,7 +52,7 @@ void processShellMsg() {
if (commit >= 2) {
num += numOfMsgs;
// if (taosFsync(dataFd) < 0) {
// if (taosFsync(pDataFile) < 0) {
// tInfo("failed to flush data to file, reason:%s", strerror(errno));
//}
......@@ -170,8 +170,8 @@ int main(int argc, char *argv[]) {
tInfo("RPC server is running, ctrl-c to exit");
if (commit) {
dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno));
pDataFile = taosOpenFile(dataName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno));
}
qhandle = taosOpenQueue();
qset = taosOpenQset();
......@@ -179,8 +179,8 @@ int main(int argc, char *argv[]) {
processShellMsg();
if (dataFd >= 0) {
close(dataFd);
if (pDataFile != NULL) {
taosCloseFile(&pDataFile);
remove(dataName);
}
......
source/libs/wal/src/walMeta.c
浏览文件 @ e25637fe
......@@ -69,8 +69,8 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size);
pLastFileInfo->fileSize = statbuf.st_size;
FileFd fd = taosOpenFileRead(fnameStr);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
......@@ -79,15 +79,15 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
char* buf = malloc(readSize + 5);
if (buf == NULL) {
taosCloseFile(fd);
taosCloseFile(&pFile);
terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
return -1;
}
taosLSeekFile(fd, -readSize, SEEK_END);
if (readSize != taosReadFile(fd, buf, readSize)) {
taosLSeekFile(pFile, -readSize, SEEK_END);
if (readSize != taosReadFile(pFile, buf, readSize)) {
free(buf);
taosCloseFile(fd);
taosCloseFile(&pFile);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
......@@ -108,12 +108,12 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) {
// file has to be deleted
free(buf);
taosCloseFile(fd);
taosCloseFile(&pFile);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
}
taosCloseFile(fd);
taosCloseFile(&pFile);
SWalHead *lastEntry = (SWalHead*)found;
return lastEntry->head.version;
......@@ -364,18 +364,18 @@ int walSaveMeta(SWal* pWal) {
int metaVer = walFindCurMetaVer(pWal);
char fnameStr[WAL_FILE_LEN];
walBuildMetaName(pWal, metaVer + 1, fnameStr);
FileFd metaFd = taosOpenFileCreateWrite(fnameStr);
if (metaFd < 0) {
TdFilePtr pMataFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE);
if (pMataFile == NULL) {
return -1;
}
char* serialized = walMetaSerialize(pWal);
int len = strlen(serialized);
if (len != taosWriteFile(metaFd, serialized, len)) {
if (len != taosWriteFile(pMataFile, serialized, len)) {
// TODO:clean file
return -1;
}
taosCloseFile(metaFd);
taosCloseFile(&pMataFile);
// delete old file
if (metaVer > -1) {
walBuildMetaName(pWal, metaVer, fnameStr);
......@@ -404,20 +404,20 @@ int walLoadMeta(SWal* pWal) {
return -1;
}
memset(buf, 0, size + 5);
FileFd fd = taosOpenFileRead(fnameStr);
if (fd < 0) {
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pFile == NULL) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
if (taosReadFile(fd, buf, size) != size) {
if (taosReadFile(pFile, buf, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(fd);
taosCloseFile(&pFile);
free(buf);
return -1;
}
// load into fileInfoSet
int code = walMetaDeserialize(pWal, buf);
taosCloseFile(fd);
taosCloseFile(&pFile);
free(buf);
return code;
}
source/libs/wal/src/walMgmt.c
浏览文件 @ e25637fe
......@@ -89,8 +89,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
// open meta
walResetVer(&pWal->vers);
pWal->writeLogTfd = -1;
pWal->writeIdxTfd = -1;
pWal->pWriteLogTFile = NULL;
pWal->pWriteIdxTFile = NULL;
pWal->writeCur = -1;
pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo));
if (pWal->fileInfoSet == NULL) {
......@@ -164,10 +164,10 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) {
void walClose(SWal *pWal) {
pthread_mutex_lock(&pWal->mutex);
tfClose(pWal->writeLogTfd);
pWal->writeLogTfd = -1;
tfClose(pWal->writeIdxTfd);
pWal->writeIdxTfd = -1;
taosCloseFile(&pWal->pWriteLogTFile);
pWal->pWriteLogTFile = NULL;
taosCloseFile(&pWal->pWriteIdxTFile);
pWal->pWriteIdxTFile = NULL;
walSaveMeta(pWal);
taosArrayDestroy(pWal->fileInfoSet);
pWal->fileInfoSet = NULL;
......@@ -207,7 +207,7 @@ static void walFsyncAll() {
if (walNeedFsync(pWal)) {
wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq,
atomic_load_32(&tsWal.seq));
int32_t code = tfFsync(pWal->writeLogTfd);
int32_t code = taosFsyncFile(pWal->pWriteLogTFile);
if (code != 0) {
wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(code));
......
source/libs/wal/src/walRead.c
浏览文件 @ e25637fe
......@@ -25,8 +25,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) {
}
pRead->pWal = pWal;
pRead->readIdxTfd = -1;
pRead->readLogTfd = -1;
pRead->pReadIdxTFile = NULL;
pRead->pReadLogTFile = NULL;
pRead->curVersion = -1;
pRead->curFileFirstVer = -1;
pRead->capacity = 0;
......@@ -41,8 +41,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) {
}
void walCloseReadHandle(SWalReadHandle *pRead) {
tfClose(pRead->readIdxTfd);
tfClose(pRead->readLogTfd);
taosCloseFile(&pRead->pReadIdxTFile);
taosCloseFile(&pRead->pReadLogTFile);
tfree(pRead->pHead);
free(pRead);
}
......@@ -52,24 +52,24 @@ int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; }
static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) {
int code = 0;
int64_t idxTfd = pRead->readIdxTfd;
int64_t logTfd = pRead->readLogTfd;
TdFilePtr pIdxTFile = pRead->pReadIdxTFile;
TdFilePtr pLogTFile = pRead->pReadLogTFile;
// seek position
int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry);
code = tfLseek(idxTfd, offset, SEEK_SET);
code = taosLSeekFile(pIdxTFile, offset, SEEK_SET);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
SWalIdxEntry entry;
if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
// TODO:deserialize
ASSERT(entry.ver == ver);
code = tfLseek(logTfd, entry.offset, SEEK_SET);
code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
......@@ -80,24 +80,24 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i
static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
char fnameStr[WAL_FILE_LEN];
tfClose(pRead->readIdxTfd);
tfClose(pRead->readLogTfd);
taosCloseFile(&pRead->pReadIdxTFile);
taosCloseFile(&pRead->pReadLogTFile);
walBuildLogName(pRead->pWal, fileFirstVer, fnameStr);
int64_t logTfd = tfOpenRead(fnameStr);
if (logTfd < 0) {
TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pLogTFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr);
int64_t idxTfd = tfOpenRead(fnameStr);
if (idxTfd < 0) {
TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ);
if (pIdxTFile == NULL) {
return -1;
}
pRead->readLogTfd = logTfd;
pRead->readIdxTfd = idxTfd;
pRead->pReadLogTFile = pLogTFile;
pRead->pReadIdxTFile = pIdxTFile;
return 0;
}
......@@ -145,9 +145,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
}
}
if (!tfValid(pRead->readLogTfd)) return -1;
if (!taosValidFile(pRead->pReadLogTFile)) return -1;
code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead));
code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead));
if (code != sizeof(SWalHead)) {
return -1;
}
......@@ -165,7 +165,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
pRead->pHead = ptr;
pRead->capacity = pRead->pHead->head.len;
}
if (pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) {
if (pRead->pHead->head.len != taosReadFile(pRead->pReadLogTFile, pRead->pHead->head.body, pRead->pHead->head.len)) {
return -1;
}
......@@ -202,7 +202,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
}
*ppHead = ptr;
}
if (tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
if (tfRead(pWal->pWriteLogTFile, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
return -1;
}
// TODO: endian compatibility processing after read
......@@ -215,7 +215,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
*ppHead = NULL;
return -1;
}
if (tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
if (tfRead(pWal->pWriteLogTFile, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
return -1;
}
// TODO: endian compatibility processing after read
......
source/libs/wal/src/walSeek.c
浏览文件 @ e25637fe
......@@ -23,25 +23,25 @@
static int walSeekWritePos(SWal* pWal, int64_t ver) {
int code = 0;
int64_t idxTfd = pWal->writeIdxTfd;
int64_t logTfd = pWal->writeLogTfd;
TdFilePtr pIdxTFile = pWal->pWriteIdxTFile;
TdFilePtr pLogTFile = pWal->pWriteLogTFile;
// seek position
int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = tfLseek(idxTfd, idxOff, SEEK_SET);
code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
SWalIdxEntry entry;
// TODO:deserialize
code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry));
code = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry));
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
ASSERT(entry.ver == ver);
code = tfLseek(logTfd, entry.offset, SEEK_SET);
code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
......@@ -50,43 +50,43 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) {
}
int walSetWrite(SWal* pWal) {
int64_t idxTfd, logTfd;
TdFilePtr pIdxTFile, pLogTFile;
SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
ASSERT(pRet != NULL);
int64_t fileFirstVer = pRet->firstVer;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, fileFirstVer, fnameStr);
idxTfd = tfOpenCreateWriteAppend(fnameStr);
if (idxTfd < 0) {
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
logTfd = tfOpenCreateWriteAppend(fnameStr);
if (logTfd < 0) {
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
// switch file
pWal->writeIdxTfd = idxTfd;
pWal->writeLogTfd = logTfd;
pWal->pWriteIdxTFile = pIdxTFile;
pWal->pWriteLogTFile = pLogTFile;
return 0;
}
int walChangeWrite(SWal* pWal, int64_t ver) {
int code = 0;
int64_t idxTfd, logTfd;
TdFilePtr pIdxTFile, pLogTFile;
char fnameStr[WAL_FILE_LEN];
if (pWal->writeLogTfd != -1) {
code = tfClose(pWal->writeLogTfd);
if (pWal->pWriteLogTFile != NULL) {
code = taosCloseFile(&pWal->pWriteLogTFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
}
if (pWal->writeIdxTfd != -1) {
code = tfClose(pWal->writeIdxTfd);
if (pWal->pWriteIdxTFile != NULL) {
code = taosCloseFile(&pWal->pWriteIdxTFile);
if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
......@@ -103,23 +103,23 @@ int walChangeWrite(SWal* pWal, int64_t ver) {
int64_t fileFirstVer = pFileInfo->firstVer;
walBuildIdxName(pWal, fileFirstVer, fnameStr);
idxTfd = tfOpenCreateWriteAppend(fnameStr);
if (idxTfd < 0) {
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
pWal->writeIdxTfd = -1;
pWal->pWriteIdxTFile = NULL;
return -1;
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
logTfd = tfOpenCreateWriteAppend(fnameStr);
if (logTfd < 0) {
tfClose(idxTfd);
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile < 0) {
taosCloseFile(&pIdxTFile);
terrno = TAOS_SYSTEM_ERROR(errno);
pWal->writeLogTfd = -1;
pWal->pWriteLogTFile = NULL;
return -1;
}
pWal->writeLogTfd = logTfd;
pWal->writeIdxTfd = idxTfd;
pWal->pWriteLogTFile = pLogTFile;
pWal->pWriteIdxTFile = pIdxTFile;
pWal->writeCur = idx;
return fileFirstVer;
}
......
source/libs/wal/src/walWrite.c
浏览文件 @ e25637fe
......@@ -65,15 +65,15 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
}
walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
int64_t idxTfd = tfOpenReadWrite(fnameStr);
TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ);
// TODO:change to deserialize function
if (idxTfd < 0) {
if (pIdxTFile == NULL) {
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = tfLseek(idxTfd, idxOff, SEEK_SET);
code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET);
if (code < 0) {
pthread_mutex_unlock(&pWal->mutex);
return -1;
......@@ -81,20 +81,20 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
// read idx file and get log file pos
// TODO:change to deserialize function
SWalIdxEntry entry;
if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
ASSERT(entry.ver == ver);
walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
int64_t logTfd = tfOpenReadWrite(fnameStr);
if (logTfd < 0) {
TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ);
if (pLogTFile < 0) {
// TODO
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
code = tfLseek(logTfd, entry.offset, SEEK_SET);
code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (code < 0) {
// TODO
pthread_mutex_unlock(&pWal->mutex);
......@@ -102,8 +102,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
}
// validate offset
SWalHead head;
ASSERT(tfValid(logTfd));
int size = tfRead(logTfd, &head, sizeof(SWalHead));
ASSERT(taosValidFile(pLogTFile));
int size = taosReadFile(pLogTFile, &head, sizeof(SWalHead));
if (size != sizeof(SWalHead)) {
return -1;
}
......@@ -118,11 +118,11 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
return -1;
}
// truncate old files
code = tfFtruncate(logTfd, entry.offset);
code = taosFtruncateFile(pLogTFile, entry.offset);
if (code < 0) {
return -1;
}
code = tfFtruncate(idxTfd, idxOff);
code = taosFtruncateFile(pIdxTFile, idxOff);
if (code < 0) {
return -1;
}
......@@ -203,31 +203,31 @@ int32_t walEndSnapshot(SWal *pWal) {
int walRoll(SWal *pWal) {
int code = 0;
if (pWal->writeIdxTfd != -1) {
code = tfClose(pWal->writeIdxTfd);
if (pWal->pWriteIdxTFile != NULL) {
code = taosCloseFile(&pWal->pWriteIdxTFile);
if (code != 0) {
return -1;
}
}
if (pWal->writeLogTfd != -1) {
code = tfClose(pWal->writeLogTfd);
if (pWal->pWriteLogTFile != NULL) {
code = taosCloseFile(&pWal->pWriteLogTFile);
if (code != 0) {
return -1;
}
}
int64_t idxTfd, logTfd;
TdFilePtr pIdxTFile, pLogTFile;
// create new file
int64_t newFileFirstVersion = pWal->vers.lastVer + 1;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, newFileFirstVersion, fnameStr);
idxTfd = tfOpenCreateWriteAppend(fnameStr);
if (idxTfd < 0) {
pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pIdxTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
walBuildLogName(pWal, newFileFirstVersion, fnameStr);
logTfd = tfOpenCreateWriteAppend(fnameStr);
if (logTfd < 0) {
pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND);
if (pLogTFile < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
......@@ -237,8 +237,8 @@ int walRoll(SWal *pWal) {
}
// switch file
pWal->writeIdxTfd = idxTfd;
pWal->writeLogTfd = logTfd;
pWal->pWriteIdxTFile = pIdxTFile;
pWal->pWriteLogTFile = pLogTFile;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
ASSERT(pWal->writeCur >= 0);
......@@ -248,7 +248,7 @@ int walRoll(SWal *pWal) {
static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
SWalIdxEntry entry = {.ver = ver, .offset = offset};
int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(SWalIdxEntry));
int size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry));
if (size != sizeof(SWalIdxEntry)) {
terrno = TAOS_SYSTEM_ERROR(errno);
// TODO truncate
......@@ -282,16 +282,16 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
// must truncate explicitly first
return -1;
}
/*if (!tfValid(pWal->writeLogTfd)) return -1;*/
/*if (!tfValid(pWal->pWriteLogTFile)) return -1;*/
ASSERT(pWal->writeCur >= 0);
pthread_mutex_lock(&pWal->mutex);
if (pWal->writeIdxTfd == -1 || pWal->writeLogTfd == -1) {
if (pWal->pWriteIdxTFile == NULL || pWal->pWriteLogTFile == NULL) {
walSetWrite(pWal);
tfLseek(pWal->writeLogTfd, 0, SEEK_END);
tfLseek(pWal->writeIdxTfd, 0, SEEK_END);
taosLSeekFile(pWal->pWriteLogTFile, 0, SEEK_END);
taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END);
}
pWal->writeHead.head.version = index;
......@@ -302,14 +302,14 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead);
pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen);
if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
if (taosWriteFile(pWal->pWriteLogTFile, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
// ftruncate
code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
}
if (tfWrite(pWal->writeLogTfd, (char *)body, bodyLen) != bodyLen) {
if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) {
// ftruncate
code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
......@@ -336,7 +336,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in
void walFsync(SWal *pWal, bool forceFsync) {
if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) {
wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
if (tfFsync(pWal->writeLogTfd) < 0) {
if (taosFsyncFile(pWal->pWriteLogTFile) < 0) {
wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),
strerror(errno));
}
......
source/os/src/osFile.c
浏览文件 @ e25637fe
此差异已折叠。
source/os/src/osRand.c
浏览文件 @ e25637fe
......@@ -24,18 +24,18 @@
uint32_t taosRand(void) { return rand(); }
uint32_t taosSafeRand(void) {
int fd;
TdFilePtr pFile;
int seed;
fd = open("/dev/urandom", 0);
if (fd < 0) {
pFile = taosOpenFile("/dev/urandom", TD_FILE_READ);
if (pFile == NULL) {
seed = (int)time(0);
} else {
int len = read(fd, &seed, sizeof(seed));
int len = taosReadFile(pFile, &seed, sizeof(seed));
if (len < 0) {
seed = (int)time(0);
}
close(fd);
taosCloseFile(&pFile);
}
return (uint32_t)seed;
......
source/os/src/osSocket.c
浏览文件 @ e25637fe
......@@ -14,6 +14,7 @@
*/
#define _DEFAULT_SOURCE
#define ALLOW_FORBID_FUNC
#include "os.h"
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
......
source/os/src/osSysinfo.c
浏览文件 @ e25637fe
此差异已折叠。
source/os/src/osTimezone.c
浏览文件 @ e25637fe
此差异已折叠。
source/util/src/tconfig.c
浏览文件 @ e25637fe
此差异已折叠。
source/util/src/tfile.c
浏览文件 @ e25637fe
此差异已折叠。
source/util/src/tlog.c
浏览文件 @ e25637fe
此差异已折叠。
source/util/src/tpagedbuf.c
浏览文件 @ e25637fe
此差异已折叠。
tests @ 12233db3
比较 498e17e2...12233db3
Subproject commit 498e17e225c454f2b45ff16eaa6d3eb595b107cf
Subproject commit 12233db374f1fe97b327e89a3442c631578ad38d
tools/shell/src/backup/shellCheck.c
浏览文件 @ e25637fe
此差异已折叠。
tools/shell/src/shellEngine.c
浏览文件 @ e25637fe
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或