Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
codyshao
TDengine
提交
d5059564
T
TDengine
项目概览
codyshao
/
TDengine
与 Fork 源项目一致
Fork自
taosdata / TDengine
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d5059564
编写于
4月 16, 2020
作者:
H
hzcheng
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
TD-100
上级
53f1d0a2
变更
5
展开全部
隐藏空白更改
内联
并排
Showing
5 changed file
with
213 addition
and
173 deletion
+213
-173
src/tsdb/inc/tsdbMain.h
src/tsdb/inc/tsdbMain.h
+8
-10
src/tsdb/src/tsdbFile.c
src/tsdb/src/tsdbFile.c
+4
-1
src/tsdb/src/tsdbMain.c
src/tsdb/src/tsdbMain.c
+6
-1
src/tsdb/src/tsdbRWHelper.c
src/tsdb/src/tsdbRWHelper.c
+105
-72
src/tsdb/tests/tsdbTests.cpp
src/tsdb/tests/tsdbTests.cpp
+90
-89
未找到文件。
src/tsdb/inc/tsdbMain.h
浏览文件 @
d5059564
...
...
@@ -351,13 +351,14 @@ typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t;
typedef
struct
{
tsdb_rw_helper_t
type
;
// helper type
int
maxTables
;
int
maxRowSize
;
int
maxRows
;
int
maxCols
;
int
minRowsPerFileBlock
;
int
maxRowsPerFileBlock
;
int8_t
compress
;
int
maxTables
;
int
maxRowSize
;
int
maxRows
;
int
maxCols
;
int
minRowsPerFileBlock
;
int
maxRowsPerFileBlock
;
int8_t
compress
;
}
SHelperCfg
;
typedef
struct
{
...
...
@@ -388,17 +389,14 @@ typedef struct {
// For file set usage
SHelperFile
files
;
SCompIdx
*
pCompIdx
;
// size_t compIdxSize;
// For table set usage
SHelperTable
tableInfo
;
SCompInfo
*
pCompInfo
;
// size_t compInfoSize;
bool
hasOldLastBlock
;
// For block set usage
SCompData
*
pCompData
;
// size_t compDataSize;
SDataCols
*
pDataCols
[
2
];
}
SRWHelper
;
...
...
src/tsdb/src/tsdbFile.c
浏览文件 @
d5059564
...
...
@@ -25,6 +25,7 @@
#include "tutil.h"
#include "tsdbMain.h"
#include "tchecksum.h"
const
char
*
tsdbFileSuffix
[]
=
{
".head"
,
// TSDB_FILE_TYPE_HEAD
...
...
@@ -310,7 +311,7 @@ static int tsdbWriteFileHead(SFile *pFile) {
}
static
int
tsdbWriteHeadFileIdx
(
SFile
*
pFile
,
int
maxTables
)
{
int
size
=
sizeof
(
SCompIdx
)
*
maxTables
;
int
size
=
sizeof
(
SCompIdx
)
*
maxTables
+
sizeof
(
TSCKSUM
)
;
void
*
buf
=
calloc
(
1
,
size
);
if
(
buf
==
NULL
)
return
-
1
;
...
...
@@ -319,6 +320,8 @@ static int tsdbWriteHeadFileIdx(SFile *pFile, int maxTables) {
return
-
1
;
}
taosCalcChecksumAppend
(
0
,
(
uint8_t
*
)
buf
,
size
);
if
(
write
(
pFile
->
fd
,
buf
,
size
)
<
0
)
{
free
(
buf
);
return
-
1
;
...
...
src/tsdb/src/tsdbMain.c
浏览文件 @
d5059564
...
...
@@ -56,7 +56,8 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock);
static
int32_t
tsdbRestoreCfg
(
STsdbRepo
*
pRepo
,
STsdbCfg
*
pCfg
);
static
int32_t
tsdbGetDataDirName
(
STsdbRepo
*
pRepo
,
char
*
fname
);
static
void
*
tsdbCommitData
(
void
*
arg
);
static
int
tsdbCommitToFile
(
STsdbRepo
*
pRepo
,
int
fid
,
SSkipListIterator
**
iters
,
SRWHelper
*
pHelper
,
SDataCols
*
pDataCols
);
static
int
tsdbCommitToFile
(
STsdbRepo
*
pRepo
,
int
fid
,
SSkipListIterator
**
iters
,
SRWHelper
*
pHelper
,
SDataCols
*
pDataCols
);
static
TSKEY
tsdbNextIterKey
(
SSkipListIterator
*
pIter
);
static
int
tsdbHasDataToCommit
(
SSkipListIterator
**
iters
,
int
nIters
,
TSKEY
minKey
,
TSKEY
maxKey
);
// static int tsdbWriteBlockToFileImpl(SFile *pFile, SDataCols *pCols, int pointsToWrite, int64_t *offset, int32_t *len,
...
...
@@ -847,6 +848,7 @@ static void *tsdbCommitData(void *arg) {
.
maxRows
=
pCfg
->
maxRowsPerFileBlock
,
.
maxCols
=
pMeta
->
maxCols
,
.
minRowsPerFileBlock
=
pCfg
->
minRowsPerFileBlock
,
.
maxRowsPerFileBlock
=
pCfg
->
maxRowsPerFileBlock
,
.
compress
=
2
// TODO make it a configuration
};
if
(
tsdbInitHelper
(
&
whelper
,
&
hcfg
)
<
0
)
goto
_exit
;
...
...
@@ -911,6 +913,8 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
// Loop to commit data in each table
for
(
int
tid
=
0
;
tid
<
pCfg
->
maxTables
;
tid
++
)
{
STable
*
pTable
=
pMeta
->
tables
[
tid
];
if
(
pTable
==
NULL
)
continue
;
SSkipListIterator
*
pIter
=
iters
[
tid
];
// Set the helper and the buffer dataCols object to help to write this table
...
...
@@ -929,6 +933,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
ASSERT
(
dataColsKeyLast
(
pDataCols
)
>=
minKey
&&
dataColsKeyLast
(
pDataCols
)
<=
maxKey
);
int
rowsWritten
=
tsdbWriteDataBlock
(
pHelper
,
pDataCols
);
ASSERT
(
rowsWritten
!=
0
);
if
(
rowsWritten
<
0
)
goto
_err
;
ASSERT
(
rowsWritten
<=
pDataCols
->
numOfPoints
);
...
...
src/tsdb/src/tsdbRWHelper.c
浏览文件 @
d5059564
此差异已折叠。
点击以展开。
src/tsdb/tests/tsdbTests.cpp
浏览文件 @
d5059564
...
...
@@ -48,98 +48,99 @@ TEST(TsdbTest, DISABLED_tableEncodeDecode) {
ASSERT_EQ
(
memcmp
(
pTable
->
schema
,
tTable
->
schema
,
sizeof
(
STSchema
)
+
sizeof
(
STColumn
)
*
nCols
),
0
);
}
TEST
(
TsdbTest
,
DISABLED_createRepo
)
{
// TEST(TsdbTest, createRepo) {
// STsdbCfg config;
// // 1. Create a tsdb repository
// tsdbSetDefaultCfg(&config);
// tsdb_repo_t *pRepo = tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL);
// ASSERT_NE(pRepo, nullptr);
// // 2. Create a normal table
// STableCfg tCfg;
// ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_SUPER_TABLE, 987607499877672L, 0), -1);
// ASSERT_EQ(tsdbInitTableCfg(&tCfg, TSDB_NORMAL_TABLE, 987607499877672L, 0), 0);
// int nCols = 5;
// STSchema *schema = tdNewSchema(nCols);
// for (int i = 0; i < nCols; i++) {
// if (i == 0) {
// tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
// } else {
// tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
// }
// }
// tsdbTableSetSchema(&tCfg, schema, true);
// tsdbCreateTable(pRepo, &tCfg);
// // // 3. Loop to write some simple data
// int nRows = 1;
// int rowsPerSubmit = 1;
// int64_t start_time = 1584081000000;
// SSubmitMsg *pMsg = (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(schema) * rowsPerSubmit);
// double stime = getCurTime();
// for (int k = 0; k < nRows/rowsPerSubmit; k++) {
// memset((void *)pMsg, 0, sizeof(SSubmitMsg));
// SSubmitBlk *pBlock = pMsg->blocks;
// pBlock->uid = 987607499877672L;
// pBlock->tid = 0;
// pBlock->sversion = 0;
// pBlock->len = 0;
// for (int i = 0; i < rowsPerSubmit; i++) {
// // start_time += 1000;
// start_time += 1000;
// SDataRow row = (SDataRow)(pBlock->data + pBlock->len);
// tdInitDataRow(row, schema);
// for (int j = 0; j < schemaNCols(schema); j++) {
// if (j == 0) { // Just for timestamp
// tdAppendColVal(row, (void *)(&start_time), schemaColAt(schema, j));
// } else { // For int
// int val = 10;
// tdAppendColVal(row, (void *)(&val), schemaColAt(schema, j));
// }
// }
// pBlock->len += dataRowLen(row);
// }
// pMsg->length = pMsg->length + sizeof(SSubmitBlk) + pBlock->len;
// pMsg->numOfBlocks = 1;
// pBlock->len = htonl(pBlock->len);
// pBlock->numOfRows = htonl(pBlock->numOfRows);
// pBlock->uid = htobe64(pBlock->uid);
// pBlock->tid = htonl(pBlock->tid);
// pBlock->sversion = htonl(pBlock->sversion);
// pBlock->padding = htonl(pBlock->padding);
// pMsg->length = htonl(pMsg->length);
// pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
// pMsg->compressed = htonl(pMsg->numOfBlocks);
// tsdbInsertData(pRepo, pMsg);
// }
// double etime = getCurTime();
// void *ptr = malloc(150000);
// free(ptr);
// printf("Spent %f seconds to write %d records\n", etime - stime, nRows);
// tsdbCloseRepo(pRepo);
// TEST(TsdbTest, DISABLED_createRepo) {
TEST
(
TsdbTest
,
createRepo
)
{
STsdbCfg
config
;
// 1. Create a tsdb repository
tsdbSetDefaultCfg
(
&
config
);
ASSERT_EQ
(
tsdbCreateRepo
(
"/home/ubuntu/work/ttest/vnode0"
,
&
config
,
NULL
),
0
);
tsdb_repo_t
*
pRepo
=
tsdbOpenRepo
(
"/home/ubuntu/work/ttest/vnode0"
,
NULL
);
ASSERT_NE
(
pRepo
,
nullptr
);
// 2. Create a normal table
STableCfg
tCfg
;
ASSERT_EQ
(
tsdbInitTableCfg
(
&
tCfg
,
TSDB_SUPER_TABLE
,
987607499877672L
,
0
),
-
1
);
ASSERT_EQ
(
tsdbInitTableCfg
(
&
tCfg
,
TSDB_NORMAL_TABLE
,
987607499877672L
,
0
),
0
);
int
nCols
=
5
;
STSchema
*
schema
=
tdNewSchema
(
nCols
);
for
(
int
i
=
0
;
i
<
nCols
;
i
++
)
{
if
(
i
==
0
)
{
tdSchemaAppendCol
(
schema
,
TSDB_DATA_TYPE_TIMESTAMP
,
i
,
-
1
);
}
else
{
tdSchemaAppendCol
(
schema
,
TSDB_DATA_TYPE_INT
,
i
,
-
1
);
}
}
tsdbTableSetSchema
(
&
tCfg
,
schema
,
true
);
tsdbCreateTable
(
pRepo
,
&
tCfg
);
// // 3. Loop to write some simple data
int
nRows
=
10000000
;
int
rowsPerSubmit
=
10
;
int64_t
start_time
=
1584081000000
;
SSubmitMsg
*
pMsg
=
(
SSubmitMsg
*
)
malloc
(
sizeof
(
SSubmitMsg
)
+
sizeof
(
SSubmitBlk
)
+
tdMaxRowBytesFromSchema
(
schema
)
*
rowsPerSubmit
);
double
stime
=
getCurTime
();
for
(
int
k
=
0
;
k
<
nRows
/
rowsPerSubmit
;
k
++
)
{
memset
((
void
*
)
pMsg
,
0
,
sizeof
(
SSubmitMsg
));
SSubmitBlk
*
pBlock
=
pMsg
->
blocks
;
pBlock
->
uid
=
987607499877672L
;
pBlock
->
tid
=
0
;
pBlock
->
sversion
=
0
;
pBlock
->
len
=
0
;
for
(
int
i
=
0
;
i
<
rowsPerSubmit
;
i
++
)
{
// start_time += 1000;
start_time
+=
1000
;
SDataRow
row
=
(
SDataRow
)(
pBlock
->
data
+
pBlock
->
len
);
tdInitDataRow
(
row
,
schema
);
for
(
int
j
=
0
;
j
<
schemaNCols
(
schema
);
j
++
)
{
if
(
j
==
0
)
{
// Just for timestamp
tdAppendColVal
(
row
,
(
void
*
)(
&
start_time
),
schemaColAt
(
schema
,
j
));
}
else
{
// For int
int
val
=
10
;
tdAppendColVal
(
row
,
(
void
*
)(
&
val
),
schemaColAt
(
schema
,
j
));
}
}
pBlock
->
len
+=
dataRowLen
(
row
);
}
pMsg
->
length
=
pMsg
->
length
+
sizeof
(
SSubmitBlk
)
+
pBlock
->
len
;
pMsg
->
numOfBlocks
=
1
;
pBlock
->
len
=
htonl
(
pBlock
->
len
);
pBlock
->
numOfRows
=
htonl
(
pBlock
->
numOfRows
);
pBlock
->
uid
=
htobe64
(
pBlock
->
uid
);
pBlock
->
tid
=
htonl
(
pBlock
->
tid
);
pBlock
->
sversion
=
htonl
(
pBlock
->
sversion
);
pBlock
->
padding
=
htonl
(
pBlock
->
padding
);
pMsg
->
length
=
htonl
(
pMsg
->
length
);
pMsg
->
numOfBlocks
=
htonl
(
pMsg
->
numOfBlocks
);
pMsg
->
compressed
=
htonl
(
pMsg
->
numOfBlocks
);
tsdbInsertData
(
pRepo
,
pMsg
);
}
double
etime
=
getCurTime
();
void
*
ptr
=
malloc
(
150000
);
free
(
ptr
);
printf
(
"Spent %f seconds to write %d records
\n
"
,
etime
-
stime
,
nRows
);
tsdbCloseRepo
(
pRepo
);
}
//
TEST(TsdbTest, DISABLED_openRepo) {
TEST
(
TsdbTest
,
openRepo
)
{
TEST
(
TsdbTest
,
DISABLED_openRepo
)
{
//
TEST(TsdbTest, openRepo) {
// tsdb_repo_t *repo = tsdbOpenRepo("/home/ubuntu/work/build/test/data/vnode/vnode2/tsdb", NULL);
// ASSERT_NE(repo, nullptr);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录