Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Tonvia
TDengine
提交
253c7e64
T
TDengine
项目概览
Tonvia
/
TDengine
与 Fork 源项目一致
Fork自
taosdata / TDengine
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
253c7e64
编写于
4月 07, 2020
作者:
H
hzcheng
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
first version of committing data
上级
ecd1e289
变更
8
展开全部
隐藏空白更改
内联
并排
Showing
8 changed file
with
1489 addition
and
429 deletion
+1489
-429
src/common/inc/dataformat.h
src/common/inc/dataformat.h
+1
-0
src/common/src/dataformat.c
src/common/src/dataformat.c
+2
-1
src/vnode/tsdb/inc/tsdbMain.h
src/vnode/tsdb/inc/tsdbMain.h
+132
-28
src/vnode/tsdb/src/tsdbFile.c
src/vnode/tsdb/src/tsdbFile.c
+69
-57
src/vnode/tsdb/src/tsdbMain.c
src/vnode/tsdb/src/tsdbMain.c
+193
-309
src/vnode/tsdb/src/tsdbRWHelper.c
src/vnode/tsdb/src/tsdbRWHelper.c
+1057
-0
src/vnode/tsdb/src/tsdbRead.c
src/vnode/tsdb/src/tsdbRead.c
+11
-11
src/vnode/tsdb/tests/tsdbTests.cpp
src/vnode/tsdb/tests/tsdbTests.cpp
+24
-23
未找到文件。
src/common/inc/dataformat.h
浏览文件 @
253c7e64
...
...
@@ -136,6 +136,7 @@ void tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
void
tdFreeDataCols
(
SDataCols
*
pCols
);
void
tdAppendDataRowToDataCol
(
SDataRow
row
,
SDataCols
*
pCols
);
void
tdPopDataColsPoints
(
SDataCols
*
pCols
,
int
pointsToPop
);
int
tdMergeDataCols
(
SDataCols
*
target
,
SDataCols
*
src
,
int
rowsToMerge
);
#ifdef __cplusplus
}
...
...
src/common/src/dataformat.c
浏览文件 @
253c7e64
...
...
@@ -382,6 +382,7 @@ static int tdFLenFromSchema(STSchema *pSchema) {
return
ret
;
}
int
tdMergeDataCols
(
SDataCols
*
target
,
SDataCols
*
source
)
{
int
tdMergeDataCols
(
SDataCols
*
target
,
SDataCols
*
source
,
int
rowsToMerge
)
{
// TODO
return
0
;
}
\ No newline at end of file
src/vnode/tsdb/inc/tsdbMain.h
浏览文件 @
253c7e64
...
...
@@ -90,9 +90,9 @@ typedef struct {
STable
*
superList
;
// super table list TODO: change it to list container
void
*
map
;
// table map of (uid ===> table)
void
*
map
;
// table map of (uid ===> table)
SMetaFile
*
mfh
;
// meta file handle
SMetaFile
*
mfh
;
// meta file handle
int
maxRowBytes
;
int
maxCols
;
}
STsdbMeta
;
...
...
@@ -118,14 +118,14 @@ STSchema * tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable);
#define TSDB_TABLE_OF_ID(pHandle, id) ((pHandle)->pTables)[id]
#define TSDB_GET_TABLE_OF_NAME(pHandle, name)
/* TODO */
STsdbMeta
*
tsdbGetMeta
(
tsdb_repo_t
*
pRepo
);
STsdbMeta
*
tsdbGetMeta
(
tsdb_repo_t
*
pRepo
);
int32_t
tsdbCreateTableImpl
(
STsdbMeta
*
pMeta
,
STableCfg
*
pCfg
);
int32_t
tsdbDropTableImpl
(
STsdbMeta
*
pMeta
,
STableId
tableId
);
STable
*
tsdbIsValidTableToInsert
(
STsdbMeta
*
pMeta
,
STableId
tableId
);
// int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable);
STable
*
tsdbGetTableByUid
(
STsdbMeta
*
pMeta
,
int64_t
uid
);
char
*
getTupleKey
(
const
void
*
data
);
char
*
getTupleKey
(
const
void
*
data
);
// ------------------------------ TSDB CACHE INTERFACES ------------------------------
#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 * 1024 * 1024
/* 16M */
...
...
@@ -191,8 +191,8 @@ typedef struct {
}
SFileInfo
;
typedef
struct
{
int
fd
;
char
fname
[
128
];
int
fd
;
char
fname
[
128
];
SFileInfo
info
;
}
SFile
;
...
...
@@ -216,11 +216,14 @@ typedef struct {
STsdbFileH
*
tsdbInitFileH
(
char
*
dataDir
,
int
maxFiles
);
void
tsdbCloseFileH
(
STsdbFileH
*
pFileH
);
int
tsdbCreateFile
(
char
*
dataDir
,
int
fileId
,
const
char
*
suffix
,
int
maxTables
,
SFile
*
pFile
,
int
writeHeader
,
int
toClose
);
int
tsdbCreateFGroup
(
STsdbFileH
*
pFileH
,
char
*
dataDir
,
int
fid
,
int
maxTables
);
int
tsdbCreateFile
(
char
*
dataDir
,
int
fileId
,
const
char
*
suffix
,
int
maxTables
,
SFile
*
pFile
,
int
writeHeader
,
int
toClose
);
SFileGroup
*
tsdbCreateFGroup
(
STsdbFileH
*
pFileH
,
char
*
dataDir
,
int
fid
,
int
maxTables
);
int
tsdbOpenFile
(
SFile
*
pFile
,
int
oflag
);
int
tsdbCloseFile
(
SFile
*
pFile
);
SFileGroup
*
tsdbOpenFilesForCommit
(
STsdbFileH
*
pFileH
,
int
fid
);
int
tsdbCloseFile
(
SFile
*
pFile
);
SFileGroup
*
tsdbOpenFilesForCommit
(
STsdbFileH
*
pFileH
,
int
fid
);
int
tsdbRemoveFileGroup
(
STsdbFileH
*
pFile
,
int
fid
);
int
tsdbGetFileName
(
char
*
dataDir
,
int
fileId
,
const
char
*
suffix
,
char
*
fname
);
#define TSDB_FGROUP_ITER_FORWARD 0
#define TSDB_FGROUP_ITER_BACKWARD 1
...
...
@@ -265,6 +268,8 @@ typedef struct {
TSKEY
keyLast
;
}
SCompBlock
;
// Maximum number of sub-blocks a super-block can have
#define TSDB_MAX_SUBBLOCKS 8
#define IS_SUPER_BLOCK(pBlock) ((pBlock)->numOfSubBlocks >= 1)
#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0)
...
...
@@ -276,15 +281,15 @@ typedef struct {
}
SCompInfo
;
#define TSDB_COMPBLOCK_AT(pCompInfo, idx) ((pCompInfo)->blocks + (idx))
#define TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pCompBlock, size)\
do {
\
if (pCompBlock->numOfSubBlocks > 1) {
\
pCompBlock = pCompInfo->blocks + pCompBlock->offset;
\
size = pCompBlock->numOfSubBlocks;
\
} else {
\
size = 1;
\
}
\
} while (0)
#define TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pCompBlock, size)
\
do {
\
if (pCompBlock->numOfSubBlocks > 1) {
\
pCompBlock = pCompInfo->blocks + pCompBlock->offset;
\
size = pCompBlock->numOfSubBlocks;
\
} else {
\
size = 1;
\
}
\
} while (0)
// TODO: take pre-calculation into account
typedef
struct
{
...
...
@@ -302,18 +307,11 @@ typedef struct {
SCompCol
cols
[];
}
SCompData
;
STsdbFileH
*
tsdbGetFile
(
tsdb_repo_t
*
pRepo
);
int
tsdbCopyBlockDataInFile
(
SFile
*
pOutFile
,
SFile
*
pInFile
,
SCompInfo
*
pCompInfo
,
int
idx
,
int
isLast
,
SDataCols
*
pCols
);
int
tsdbLoadCompIdx
(
SFileGroup
*
pGroup
,
void
*
buf
,
int
maxTables
);
int
tsdbLoadCompBlocks
(
SFileGroup
*
pGroup
,
SCompIdx
*
pIdx
,
void
*
buf
);
int
tsdbLoadCompCols
(
SFile
*
pFile
,
SCompBlock
*
pBlock
,
void
*
buf
);
int
tsdbLoadColData
(
SFile
*
pFile
,
SCompCol
*
pCol
,
int64_t
blockBaseOffset
,
void
*
buf
);
int
tsdbLoadDataBlock
(
SFile
*
pFile
,
SCompBlock
*
pStartBlock
,
int
numOfBlocks
,
SDataCols
*
pCols
,
SCompData
*
pCompData
);
STsdbFileH
*
tsdbGetFile
(
tsdb_repo_t
*
pRepo
);
int
tsdbCopyBlockDataInFile
(
SFile
*
pOutFile
,
SFile
*
pInFile
,
SCompInfo
*
pCompInfo
,
int
idx
,
int
isLast
,
SDataCols
*
pCols
);
SFileGroup
*
tsdbSearchFGroup
(
STsdbFileH
*
pFileH
,
int
fid
);
void
tsdbGetKeyRangeOfFileId
(
int32_t
daysPerFile
,
int8_t
precision
,
int32_t
fileId
,
TSKEY
*
minKey
,
TSKEY
*
maxKey
);
// TSDB repository definition
...
...
@@ -348,6 +346,112 @@ typedef struct _tsdb_repo {
}
STsdbRepo
;
typedef
enum
{
TSDB_WRITE_HELPER
,
TSDB_READ_HELPER
}
tsdb_rw_helper_t
;
typedef
struct
{
tsdb_rw_helper_t
type
;
// helper type
int
maxTables
;
int
maxRowSize
;
int
maxRows
;
int
maxCols
;
int
minRowsPerFileBlock
;
int
maxRowsPerFileBlock
;
int8_t
compress
;
}
SHelperCfg
;
typedef
struct
{
int
fid
;
TSKEY
minKey
;
TSKEY
maxKey
;
// For read/write purpose
SFile
headF
;
SFile
dataF
;
SFile
lastF
;
// For write purpose only
SFile
nHeadF
;
SFile
nLastF
;
}
SHelperFile
;
typedef
struct
{
int64_t
uid
;
int32_t
tid
;
int32_t
sversion
;
}
SHelperTable
;
typedef
struct
{
// Global configuration
SHelperCfg
config
;
SHelperFile
files
;
SHelperTable
tableInfo
;
// ---------- For read purpose
int8_t
state
;
// current loading state
SCompIdx
*
pCompIdx
;
size_t
compIdxSize
;
SCompInfo
*
pCompInfo
;
size_t
compInfoSize
;
int
blockIter
;
// For write purpose
SCompData
*
pCompData
;
size_t
compDataSize
;
SDataCols
*
pDataCols
[
2
];
// ---------- For read purpose
bool
hasLast
;
int
newBlocks
;
SCompIdx
*
pWCompIdx
;
size_t
wCompIdxSize
;
SCompInfo
*
pWCompInfo
;
size_t
wCompInfoSize
;
SCompData
*
pWCompData
;
size_t
wCompDataSize
;
}
SRWHelper
;
// --------- Helper state
#define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state
#define TSDB_HELPER_FILE_SET 0x1 // File is set
#define TSDB_HELPER_FILE_OPEN 0x2 // File is opened
#define TSDB_HELPER_IDX_LOAD 0x4 // SCompIdx part is loaded
#define TSDB_HELPER_INFO_LOAD 0x8 // SCompInfo part is loaded
#define TSDB_HELPER_FILE_DATA_LOAD 0x10 // SCompData part is loaded
#define TSDB_HELPER_TYPE(h) ((h)->config.type)
#define helperSetState(h, s) (((h)->state) |= (s))
#define helperClearState(h, s) ((h)->state &= (~(s)))
#define helperHasState(h, s) ((((h)->state) & (s)) == (s))
int
tsdbInitHelper
(
SRWHelper
*
pHelper
,
SHelperCfg
*
pCfg
);
void
tsdbDestroyHelper
(
SRWHelper
*
pHelper
);
void
tsdbClearHelper
(
SRWHelper
*
pHelper
);
// --------- For set operations
int
tsdbSetHelperFile
(
SRWHelper
*
pHelper
,
SFileGroup
*
pGroup
);
int
tsdbOpenHelperFile
(
SRWHelper
*
pHelper
);
void
tsdbSetHelperTable
(
SRWHelper
*
pHelper
,
SHelperTable
*
pHelperTable
,
STSchema
*
pSchema
);
int
tsdbCloseHelperFile
(
SRWHelper
*
pHelper
,
bool
hasError
);
// --------- For read operations
int
tsdbLoadCompIdx
(
SRWHelper
*
pHelper
,
void
*
target
);
int
tsdbLoadCompInfo
(
SRWHelper
*
pHelper
,
void
*
target
);
int
tsdbLoadCompData
(
SRWHelper
*
pHelper
,
int
blkIdx
,
void
*
target
);
int
tsdbLoadBlockDataCols
(
SRWHelper
*
pHelper
,
SDataCols
*
pDataCols
,
int32_t
*
colIds
,
int
numOfColIds
);
int
tsdbLoadBlockData
(
SRWHelper
*
pHelper
,
int
blkIdx
,
SDataCols
*
pDataCols
);
// --------- For write operations
int
tsdbWriteDataBlock
(
SRWHelper
*
pHelper
,
SDataCols
*
pDataCols
);
int
tsdbMoveLastBlockIfNeccessary
(
SRWHelper
*
pHelper
);
int
tsdbWriteCompInfo
(
SRWHelper
*
pHelper
);
int
tsdbWriteCompIdx
(
SRWHelper
*
pHelper
);
#ifdef __cplusplus
}
...
...
src/vnode/tsdb/src/tsdbFile.c
浏览文件 @
253c7e64
...
...
@@ -21,6 +21,7 @@
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <libgen.h>
#include "tutil.h"
#include "tsdbMain.h"
...
...
@@ -33,7 +34,6 @@ const char *tsdbFileSuffix[] = {
static
int
compFGroupKey
(
const
void
*
key
,
const
void
*
fgroup
);
static
int
compFGroup
(
const
void
*
arg1
,
const
void
*
arg2
);
static
int
tsdbGetFileName
(
char
*
dataDir
,
int
fileId
,
const
char
*
suffix
,
char
*
fname
);
static
int
tsdbWriteFileHead
(
SFile
*
pFile
);
static
int
tsdbWriteHeadFileIdx
(
SFile
*
pFile
,
int
maxTables
);
static
int
tsdbOpenFGroup
(
STsdbFileH
*
pFileH
,
char
*
dataDir
,
int
fid
);
...
...
@@ -91,24 +91,36 @@ static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid) {
return
0
;
}
int
tsdbCreateFGroup
(
STsdbFileH
*
pFileH
,
char
*
dataDir
,
int
fid
,
int
maxTables
)
{
if
(
pFileH
->
numOfFGroups
>=
pFileH
->
maxFGroups
)
return
-
1
;
/**
* Create the file group if the file group not exists.
*
* @return A pointer to
*/
SFileGroup
*
tsdbCreateFGroup
(
STsdbFileH
*
pFileH
,
char
*
dataDir
,
int
fid
,
int
maxTables
)
{
if
(
pFileH
->
numOfFGroups
>=
pFileH
->
maxFGroups
)
return
NULL
;
SFileGroup
fGroup
;
SFileGroup
*
pFGroup
=
&
fGroup
;
if
(
tsdbSearchFGroup
(
pFileH
,
fid
)
==
NULL
)
{
// if not exists, create one
SFileGroup
*
pGroup
=
tsdbSearchFGroup
(
pFileH
,
fid
);
if
(
pGroup
==
NULL
)
{
// if not exists, create one
pFGroup
->
fileId
=
fid
;
for
(
int
type
=
TSDB_FILE_TYPE_HEAD
;
type
<
TSDB_FILE_TYPE_MAX
;
type
++
)
{
if
(
tsdbCreateFile
(
dataDir
,
fid
,
tsdbFileSuffix
[
type
],
maxTables
,
&
(
pFGroup
->
files
[
type
]),
type
==
TSDB_FILE_TYPE_HEAD
?
1
:
0
,
1
)
<
0
)
{
// TODO: deal with the ERROR here, remove those creaed file
return
-
1
;
}
if
(
tsdbCreateFile
(
dataDir
,
fid
,
tsdbFileSuffix
[
type
],
maxTables
,
&
(
pFGroup
->
files
[
type
]),
type
==
TSDB_FILE_TYPE_HEAD
?
1
:
0
,
1
)
<
0
)
goto
_err
;
}
pFileH
->
fGroup
[
pFileH
->
numOfFGroups
++
]
=
fGroup
;
qsort
((
void
*
)(
pFileH
->
fGroup
),
pFileH
->
numOfFGroups
,
sizeof
(
SFileGroup
),
compFGroup
);
return
tsdbSearchFGroup
(
pFileH
,
fid
);
}
return
0
;
return
pGroup
;
_err:
// TODO: deal with the err here
return
NULL
;
}
int
tsdbRemoveFileGroup
(
STsdbFileH
*
pFileH
,
int
fid
)
{
...
...
@@ -181,27 +193,27 @@ SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) {
return
ret
;
}
int
tsdbLoadDataBlock
(
SFile
*
pFile
,
SCompBlock
*
pStartBlock
,
int
numOfBlocks
,
SDataCols
*
pCols
,
SCompData
*
pCompData
)
{
SCompBlock
*
pBlock
=
pStartBlock
;
for
(
int
i
=
0
;
i
<
numOfBlocks
;
i
++
)
{
if
(
tsdbLoadCompCols
(
pFile
,
pBlock
,
(
void
*
)
pCompData
)
<
0
)
return
-
1
;
pCols
->
numOfPoints
+=
(
pCompData
->
cols
[
0
].
len
/
8
);
for
(
int
iCol
=
0
;
iCol
<
pBlock
->
numOfCols
;
iCol
++
)
{
SCompCol
*
pCompCol
=
&
(
pCompData
->
cols
[
iCol
]);
// pCols->numOfPoints += pBlock->numOfPoints;
int
k
=
0
;
for
(;
k
<
pCols
->
numOfCols
;
k
++
)
{
if
(
pCompCol
->
colId
==
pCols
->
cols
[
k
].
colId
)
break
;
}
if
(
tsdbLoadColData
(
pFile
,
pCompCol
,
pBlock
->
offset
,
(
void
*
)((
char
*
)(
pCols
->
cols
[
k
].
pData
)
+
pCols
->
cols
[
k
].
len
))
<
0
)
return
-
1
;
}
pStartBlock
++
;
}
return
0
;
}
//
int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData) {
//
SCompBlock *pBlock = pStartBlock;
//
for (int i = 0; i < numOfBlocks; i++) {
//
if (tsdbLoadCompCols(pFile, pBlock, (void *)pCompData) < 0) return -1;
//
pCols->numOfPoints += (pCompData->cols[0].len / 8);
//
for (int iCol = 0; iCol < pBlock->numOfCols; iCol++) {
//
SCompCol *pCompCol = &(pCompData->cols[iCol]);
//
// pCols->numOfPoints += pBlock->numOfPoints;
//
int k = 0;
//
for (; k < pCols->numOfCols; k++) {
//
if (pCompCol->colId == pCols->cols[k].colId) break;
//
}
//
if (tsdbLoadColData(pFile, pCompCol, pBlock->offset,
//
(void *)((char *)(pCols->cols[k].pData) + pCols->cols[k].len)) < 0)
//
return -1;
//
}
//
pStartBlock++;
//
}
//
return 0;
//
}
int
tsdbCopyBlockDataInFile
(
SFile
*
pOutFile
,
SFile
*
pInFile
,
SCompInfo
*
pCompInfo
,
int
idx
,
int
isLast
,
SDataCols
*
pCols
)
{
SCompBlock
*
pSuperBlock
=
TSDB_COMPBLOCK_AT
(
pCompInfo
,
idx
);
...
...
@@ -237,42 +249,42 @@ int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInf
return
0
;
}
int
tsdbLoadCompIdx
(
SFileGroup
*
pGroup
,
void
*
buf
,
int
maxTables
)
{
SFile
*
pFile
=
&
(
pGroup
->
files
[
TSDB_FILE_TYPE_HEAD
]);
if
(
lseek
(
pFile
->
fd
,
TSDB_FILE_HEAD_SIZE
,
SEEK_SET
)
<
0
)
return
-
1
;
//
int tsdbLoadCompIdx(SFileGroup *pGroup, void *buf, int maxTables) {
//
SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]);
//
if (lseek(pFile->fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1;
if
(
read
(
pFile
->
fd
,
buf
,
sizeof
(
SCompIdx
)
*
maxTables
)
<
0
)
return
-
1
;
// TODO: need to check the correctness
return
0
;
}
//
if (read(pFile->fd, buf, sizeof(SCompIdx) * maxTables) < 0) return -1;
//
// TODO: need to check the correctness
//
return 0;
//
}
int
tsdbLoadCompBlocks
(
SFileGroup
*
pGroup
,
SCompIdx
*
pIdx
,
void
*
buf
)
{
SFile
*
pFile
=
&
(
pGroup
->
files
[
TSDB_FILE_TYPE_HEAD
]);
//
int tsdbLoadCompBlocks(SFileGroup *pGroup, SCompIdx *pIdx, void *buf) {
//
SFile *pFile = &(pGroup->files[TSDB_FILE_TYPE_HEAD]);
if
(
lseek
(
pFile
->
fd
,
pIdx
->
offset
,
SEEK_SET
)
<
0
)
return
-
1
;
//
if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) return -1;
if
(
read
(
pFile
->
fd
,
buf
,
pIdx
->
len
)
<
0
)
return
-
1
;
//
if (read(pFile->fd, buf, pIdx->len) < 0) return -1;
// TODO: need to check the correctness
//
// TODO: need to check the correctness
return
0
;
}
//
return 0;
//
}
int
tsdbLoadCompCols
(
SFile
*
pFile
,
SCompBlock
*
pBlock
,
void
*
buf
)
{
// assert(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
//
int tsdbLoadCompCols(SFile *pFile, SCompBlock *pBlock, void *buf) {
//
// assert(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
if
(
lseek
(
pFile
->
fd
,
pBlock
->
offset
,
SEEK_SET
)
<
0
)
return
-
1
;
size_t
size
=
sizeof
(
SCompData
)
+
sizeof
(
SCompCol
)
*
pBlock
->
numOfCols
;
if
(
read
(
pFile
->
fd
,
buf
,
size
)
<
0
)
return
-
1
;
//
if (lseek(pFile->fd, pBlock->offset, SEEK_SET) < 0) return -1;
//
size_t size = sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols;
//
if (read(pFile->fd, buf, size) < 0) return -1;
return
0
;
}
//
return 0;
//
}
int
tsdbLoadColData
(
SFile
*
pFile
,
SCompCol
*
pCol
,
int64_t
blockBaseOffset
,
void
*
buf
)
{
if
(
lseek
(
pFile
->
fd
,
blockBaseOffset
+
pCol
->
offset
,
SEEK_SET
)
<
0
)
return
-
1
;
if
(
read
(
pFile
->
fd
,
buf
,
pCol
->
len
)
<
0
)
return
-
1
;
return
0
;
}
//
int tsdbLoadColData(SFile *pFile, SCompCol *pCol, int64_t blockBaseOffset, void *buf) {
//
if (lseek(pFile->fd, blockBaseOffset + pCol->offset, SEEK_SET) < 0) return -1;
//
if (read(pFile->fd, buf, pCol->len) < 0) return -1;
//
return 0;
//
}
static
int
compFGroupKey
(
const
void
*
key
,
const
void
*
fgroup
)
{
int
fid
=
*
(
int
*
)
key
;
...
...
@@ -317,7 +329,7 @@ static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) {
return
0
;
}
static
int
tsdbGetFileName
(
char
*
dataDir
,
int
fileId
,
const
char
*
suffix
,
char
*
fname
)
{
int
tsdbGetFileName
(
char
*
dataDir
,
int
fileId
,
const
char
*
suffix
,
char
*
fname
)
{
if
(
dataDir
==
NULL
||
fname
==
NULL
)
return
-
1
;
sprintf
(
fname
,
"%s/f%d%s"
,
dataDir
,
fileId
,
suffix
);
...
...
src/vnode/tsdb/src/tsdbMain.c
浏览文件 @
253c7e64
此差异已折叠。
点击以展开。
src/vnode/tsdb/src/tsdbRWHelper.c
0 → 100644
浏览文件 @
253c7e64
此差异已折叠。
点击以展开。
src/vnode/tsdb/src/tsdbRead.c
浏览文件 @
253c7e64
...
...
@@ -369,14 +369,14 @@ static int32_t getFileCompInfo(STableCheckInfo* pCheckInfo, SFileGroup* fileGrou
fileGroup
->
files
[
TSDB_FILE_TYPE_HEAD
].
fd
=
open
(
fileGroup
->
files
[
TSDB_FILE_TYPE_HEAD
].
fname
,
O_RDONLY
);
}
tsdbLoadCompIdx
(
fileGroup
,
pCheckInfo
->
compIndex
,
10000
);
// todo set dynamic max tables
SCompIdx
*
compIndex
=
&
pCheckInfo
->
compIndex
[
pCheckInfo
->
tableId
.
tid
];
//
tsdbLoadCompIdx(fileGroup, pCheckInfo->compIndex, 10000); // todo set dynamic max tables
//
SCompIdx* compIndex = &pCheckInfo->compIndex[pCheckInfo->tableId.tid];
if
(
compIndex
->
len
==
0
||
compIndex
->
numOfSuperBlocks
==
0
)
{
// no data block in this file, try next file
//
if (compIndex->len == 0 || compIndex->numOfSuperBlocks == 0) { // no data block in this file, try next file
}
else
{
tsdbLoadCompBlocks
(
fileGroup
,
compIndex
,
pCheckInfo
->
pCompInfo
);
}
//
} else {
//
tsdbLoadCompBlocks(fileGroup, compIndex, pCheckInfo->pCompInfo);
//
}
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -444,7 +444,7 @@ static bool doLoadDataFromFileBlock(STsdbQueryHandle *pQueryHandle) {
pFile
->
fd
=
open
(
pFile
->
fname
,
O_RDONLY
);
}
tsdbLoadDataBlock
(
pFile
,
pBlock
,
1
,
pCheckInfo
->
pDataCols
,
data
);
//
tsdbLoadDataBlock(pFile, pBlock, 1, pCheckInfo->pDataCols, data);
return
true
;
}
...
...
@@ -810,10 +810,10 @@ static bool getQualifiedDataBlock(STsdbQueryHandle *pQueryHandle, STableCheckInf
pFile
->
fd
=
open
(
pFile
->
fname
,
O_RDONLY
);
}
if
(
tsdbLoadDataBlock
(
pFile
,
&
pCheckInfo
->
pCompInfo
->
blocks
[
cur
->
slot
],
1
,
pCheckInfo
->
pDataCols
,
data
)
==
0
)
{
blockLoaded
=
true
;
}
//
if (tsdbLoadDataBlock(pFile, &pCheckInfo->pCompInfo->blocks[cur->slot], 1,
//
pCheckInfo->pDataCols, data) == 0) {
//
blockLoaded = true;
//
}
// dError("QInfo:%p fileId:%d total numOfBlks:%d blockId:%d load into memory failed due to error in disk files",
// GET_QINFO_ADDR(pQuery), pQuery->fileId, pQuery->numOfBlocks, blkIdx);
...
...
src/vnode/tsdb/tests/tsdbTests.cpp
浏览文件 @
253c7e64
...
...
@@ -54,7 +54,8 @@ TEST(TsdbTest, createRepo) {
// 1. Create a tsdb repository
tsdbSetDefaultCfg
(
&
config
);
tsdb_repo_t
*
pRepo
=
tsdbCreateRepo
(
"/home/ubuntu/work/ttest/vnode0"
,
&
config
,
NULL
);
tsdbCreateRepo
(
"/home/ubuntu/work/ttest/vnode0"
,
&
config
,
NULL
);
tsdb_repo_t
*
pRepo
=
tsdbOpenRepo
(
"/home/ubuntu/work/ttest/vnode0"
,
NULL
);
ASSERT_NE
(
pRepo
,
nullptr
);
// 2. Create a normal table
...
...
@@ -139,42 +140,42 @@ TEST(TsdbTest, createRepo) {
}
// TEST(TsdbTest, DISABLED_openRepo) {
TEST
(
TsdbTest
,
openRepo
)
{
tsdb_repo_t
*
repo
=
tsdbOpenRepo
(
"/home/ubuntu/work/ttest/vnode0"
,
NULL
);
ASSERT_NE
(
repo
,
nullptr
);
//
TEST(TsdbTest, openRepo) {
//
tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL);
//
ASSERT_NE(repo, nullptr);
STsdbRepo
*
pRepo
=
(
STsdbRepo
*
)
repo
;
//
STsdbRepo *pRepo = (STsdbRepo *)repo;
SFileGroup
*
pGroup
=
tsdbSearchFGroup
(
pRepo
->
tsdbFileH
,
1833
);
//
SFileGroup *pGroup = tsdbSearchFGroup(pRepo->tsdbFileH, 1833);
for
(
int
type
=
TSDB_FILE_TYPE_HEAD
;
type
<
TSDB_FILE_TYPE_MAX
;
type
++
)
{
tsdbOpenFile
(
&
pGroup
->
files
[
type
],
O_RDONLY
);
}
//
for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) {
//
tsdbOpenFile(&pGroup->files[type], O_RDONLY);
//
}
SCompIdx
*
pIdx
=
(
SCompIdx
*
)
calloc
(
pRepo
->
config
.
maxTables
,
sizeof
(
SCompIdx
));
tsdbLoadCompIdx
(
pGroup
,
(
void
*
)
pIdx
,
pRepo
->
config
.
maxTables
);
//
SCompIdx *pIdx = (SCompIdx *)calloc(pRepo->config.maxTables, sizeof(SCompIdx));
//
tsdbLoadCompIdx(pGroup, (void *)pIdx, pRepo->config.maxTables);
SCompInfo
*
pCompInfo
=
(
SCompInfo
*
)
malloc
(
sizeof
(
SCompInfo
)
+
pIdx
[
1
].
len
);
//
SCompInfo *pCompInfo = (SCompInfo *)malloc(sizeof(SCompInfo) + pIdx[1].len);
tsdbLoadCompBlocks
(
pGroup
,
&
pIdx
[
0
],
(
void
*
)
pCompInfo
);
//
tsdbLoadCompBlocks(pGroup, &pIdx[0], (void *)pCompInfo);
int
blockIdx
=
0
;
SCompBlock
*
pBlock
=
&
(
pCompInfo
->
blocks
[
blockIdx
]);
//
int blockIdx = 0;
//
SCompBlock *pBlock = &(pCompInfo->blocks[blockIdx]);
SCompData
*
pCompData
=
(
SCompData
*
)
malloc
(
sizeof
(
SCompData
)
+
sizeof
(
SCompCol
)
*
pBlock
->
numOfCols
);
//
SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pBlock->numOfCols);
tsdbLoadCompCols
(
&
pGroup
->
files
[
TSDB_FILE_TYPE_DATA
],
pBlock
,
(
void
*
)
pCompData
);
//
tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData);
STable
*
pTable
=
tsdbGetTableByUid
(
pRepo
->
tsdbMeta
,
pCompData
->
uid
);
SDataCols
*
pDataCols
=
tdNewDataCols
(
tdMaxRowBytesFromSchema
(
pTable
->
schema
),
5
,
10
);
tdInitDataCols
(
pDataCols
,
pTable
->
schema
);
//
STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid);
//
SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(pTable->schema), 5, 10);
//
tdInitDataCols(pDataCols, pTable->schema);
tsdbLoadDataBlock
(
&
pGroup
->
files
[
TSDB_FILE_TYPE_DATA
],
pBlock
,
1
,
pDataCols
,
pCompData
);
//
tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData);
int
k
=
0
;
//
int k = 0;
}
//
}
TEST
(
TsdbTest
,
DISABLED_createFileGroup
)
{
SFileGroup
fGroup
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录