Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
王长生-AnH
TDengine
提交
ddd05d9c
T
TDengine
项目概览
王长生-AnH
/
TDengine
与 Fork 源项目一致
Fork自
taosdata / TDengine
通知
5
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
ddd05d9c
编写于
3月 23, 2020
作者:
S
slguan
浏览文件
操作
浏览文件
下载
差异文件
Merge branch '2.0' into refact/slguan
上级
2e3e7f32
88f44b1a
变更
38
展开全部
隐藏空白更改
内联
并排
Showing
38 changed file
with
3597 addition
and
1565 deletion
+3597
-1565
src/client/inc/tscUtil.h
src/client/inc/tscUtil.h
+1
-1
src/client/inc/tsclient.h
src/client/inc/tsclient.h
+2
-2
src/client/src/tscLocal.c
src/client/src/tscLocal.c
+4
-3
src/client/src/tscParseInsert.c
src/client/src/tscParseInsert.c
+29
-32
src/client/src/tscPrepare.c
src/client/src/tscPrepare.c
+8
-8
src/client/src/tscSchemaUtil.c
src/client/src/tscSchemaUtil.c
+2
-2
src/client/src/tscServer.c
src/client/src/tscServer.c
+61
-58
src/client/src/tscSql.c
src/client/src/tscSql.c
+3
-1
src/client/src/tscSub.c
src/client/src/tscSub.c
+1
-1
src/client/src/tscSystem.c
src/client/src/tscSystem.c
+0
-3
src/client/src/tscUtil.c
src/client/src/tscUtil.c
+64
-16
src/common/inc/dataformat.h
src/common/inc/dataformat.h
+9
-17
src/common/src/dataformat.c
src/common/src/dataformat.c
+11
-1
src/dnode/src/dnodeRead.c
src/dnode/src/dnodeRead.c
+68
-25
src/dnode/src/dnodeWrite.c
src/dnode/src/dnodeWrite.c
+7
-1
src/inc/taosmsg.h
src/inc/taosmsg.h
+56
-37
src/mnode/src/mgmtChildTable.c
src/mnode/src/mgmtChildTable.c
+1
-1
src/mnode/src/mgmtDnode.c
src/mnode/src/mgmtDnode.c
+4
-1
src/mnode/src/mgmtNormalTable.c
src/mnode/src/mgmtNormalTable.c
+1
-1
src/mnode/src/mgmtSuperTable.c
src/mnode/src/mgmtSuperTable.c
+1
-1
src/query/CMakeLists.txt
src/query/CMakeLists.txt
+1
-1
src/query/inc/qextbuffer.h
src/query/inc/qextbuffer.h
+1
-2
src/query/inc/queryExecutor.h
src/query/inc/queryExecutor.h
+22
-11
src/query/src/queryExecutor.c
src/query/src/queryExecutor.c
+1963
-1091
src/rpc/src/rpcMain.c
src/rpc/src/rpcMain.c
+25
-25
src/util/inc/tlist.h
src/util/inc/tlist.h
+69
-0
src/util/src/tlist.c
src/util/src/tlist.c
+169
-0
src/util/src/tskiplist.c
src/util/src/tskiplist.c
+12
-12
src/vnode/tsdb/inc/tsdb.h
src/vnode/tsdb/inc/tsdb.h
+6
-17
src/vnode/tsdb/inc/tsdbCache.h
src/vnode/tsdb/inc/tsdbCache.h
+33
-28
src/vnode/tsdb/inc/tsdbFile.h
src/vnode/tsdb/inc/tsdbFile.h
+39
-15
src/vnode/tsdb/inc/tsdbMeta.h
src/vnode/tsdb/inc/tsdbMeta.h
+24
-13
src/vnode/tsdb/src/tsdbCache.c
src/vnode/tsdb/src/tsdbCache.c
+113
-11
src/vnode/tsdb/src/tsdbFile.c
src/vnode/tsdb/src/tsdbFile.c
+131
-69
src/vnode/tsdb/src/tsdbMain.c
src/vnode/tsdb/src/tsdbMain.c
+212
-13
src/vnode/tsdb/src/tsdbMeta.c
src/vnode/tsdb/src/tsdbMeta.c
+18
-19
src/vnode/tsdb/src/tsdbRead.c
src/vnode/tsdb/src/tsdbRead.c
+373
-0
src/vnode/tsdb/tests/tsdbTests.cpp
src/vnode/tsdb/tests/tsdbTests.cpp
+53
-26
未找到文件。
src/client/inc/tscUtil.h
浏览文件 @
ddd05d9c
...
...
@@ -85,7 +85,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList,
STableDataBlocks
**
dataBlocks
);
SVnodeSidList
*
tscGetVnodeSidList
(
SSuperTableMeta
*
pMetricmeta
,
int32_t
vnodeIdx
);
STable
SidExt
Info
*
tscGetMeterSidInfo
(
SVnodeSidList
*
pSidList
,
int32_t
idx
);
STable
Id
Info
*
tscGetMeterSidInfo
(
SVnodeSidList
*
pSidList
,
int32_t
idx
);
/**
*
...
...
src/client/inc/tsclient.h
浏览文件 @
ddd05d9c
...
...
@@ -62,7 +62,7 @@ typedef struct STableMeta {
int8_t
numOfVpeers
;
int16_t
sversion
;
SVnodeDesc
vpeerDesc
[
TSDB_VNODES_SUPPORT
];
int32_t
vg
i
d
;
// virtual group id, which current table belongs to
int32_t
vg
I
d
;
// virtual group id, which current table belongs to
int32_t
sid
;
// the index of one table in a virtual node
uint64_t
uid
;
// unique id of a table
SSchema
schema
[];
// if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info
...
...
@@ -182,7 +182,7 @@ typedef struct STableDataBlocks {
char
tableId
[
TSDB_TABLE_ID_LEN
];
int8_t
tsSource
;
// where does the UNIX timestamp come from, server or client
bool
ordered
;
// if current rows are ordered or not
int64_t
vg
i
d
;
// virtual group id
int64_t
vg
I
d
;
// virtual group id
int64_t
prevTS
;
// previous timestamp, recorded to decide if the records array is ts ascending
int32_t
numOfTables
;
// number of tables in current submit block
...
...
src/client/src/tscLocal.c
浏览文件 @
ddd05d9c
...
...
@@ -318,15 +318,16 @@ static int tscBuildMetricTagProjectionResult(SSqlObj *pSql) {
SVnodeSidList
*
pSidList
=
(
SVnodeSidList
*
)((
char
*
)
pMetricMeta
+
pMetricMeta
->
list
[
i
]);
for
(
int32_t
j
=
0
;
j
<
pSidList
->
numOfSids
;
++
j
)
{
STable
SidExt
Info
*
pSidExt
=
tscGetMeterSidInfo
(
pSidList
,
j
);
STable
Id
Info
*
pSidExt
=
tscGetMeterSidInfo
(
pSidList
,
j
);
for
(
int32_t
k
=
0
;
k
<
pQueryInfo
->
fieldsInfo
.
numOfOutputCols
;
++
k
)
{
SColIndexEx
*
pColIndex
=
&
tscSqlExprGet
(
pQueryInfo
,
k
)
->
colInfo
;
int16_t
offsetId
=
pColIndex
->
colIdx
;
assert
((
pColIndex
->
flag
&
TSDB_COL_TAG
)
!=
0
);
char
*
val
=
pSidExt
->
tags
+
vOffset
[
offsetId
];
assert
(
0
);
char
*
val
=
NULL
;
//pSidExt->tags + vOffset[offsetId];
TAOS_FIELD
*
pField
=
tscFieldInfoGetField
(
pQueryInfo
,
k
);
memcpy
(
pRes
->
data
+
tscFieldInfoGetOffset
(
pQueryInfo
,
k
)
*
totalNumOfResults
+
pField
->
bytes
*
rowIdx
,
val
,
...
...
src/client/src/tscParseInsert.c
浏览文件 @
ddd05d9c
...
...
@@ -18,9 +18,10 @@
#define _XOPEN_SOURCE
#include "hash.h"
#include "os.h"
#include "tscSecondaryMerge.h"
#include "hash.h"
//#include "tscSecondaryMerge.h"
#include "tscUtil.h"
#include "tschemautil.h"
#include "tsclient.h"
...
...
@@ -32,6 +33,8 @@
#include "tstoken.h"
#include "ttime.h"
#include "dataformat.h"
enum
{
TSDB_USE_SERVER_TS
=
0
,
TSDB_USE_CLI_TS
=
1
,
...
...
@@ -393,7 +396,6 @@ static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start
int
tsParseOneRowData
(
char
**
str
,
STableDataBlocks
*
pDataBlocks
,
SSchema
schema
[],
SParsedDataColInfo
*
spd
,
char
*
error
,
int16_t
timePrec
,
int32_t
*
code
,
char
*
tmpTokenBuf
)
{
int32_t
index
=
0
;
// bool isPrevOptr; //fang, never used
SSQLToken
sToken
=
{
0
};
char
*
payload
=
pDataBlocks
->
pData
+
pDataBlocks
->
size
;
...
...
@@ -604,8 +606,8 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3
return
TSDB_CODE_SUCCESS
;
}
static
void
tsSetBlockInfo
(
SS
hellSubmitBloc
k
*
pBlocks
,
const
STableMeta
*
pTableMeta
,
int32_t
numOfRows
)
{
pBlocks
->
s
id
=
pTableMeta
->
sid
;
static
void
tsSetBlockInfo
(
SS
ubmitBl
k
*
pBlocks
,
const
STableMeta
*
pTableMeta
,
int32_t
numOfRows
)
{
pBlocks
->
t
id
=
pTableMeta
->
sid
;
pBlocks
->
uid
=
pTableMeta
->
uid
;
pBlocks
->
sversion
=
pTableMeta
->
sversion
;
pBlocks
->
numOfRows
+=
numOfRows
;
...
...
@@ -613,10 +615,10 @@ static void tsSetBlockInfo(SShellSubmitBlock *pBlocks, const STableMeta *pTableM
// data block is disordered, sort it in ascending order
void
sortRemoveDuplicates
(
STableDataBlocks
*
dataBuf
)
{
SS
hellSubmitBlock
*
pBlocks
=
(
SShellSubmitBloc
k
*
)
dataBuf
->
pData
;
SS
ubmitBlk
*
pBlocks
=
(
SSubmitBl
k
*
)
dataBuf
->
pData
;
// size is less than the total size, since duplicated rows may be removed yet.
assert
(
pBlocks
->
numOfRows
*
dataBuf
->
rowSize
+
sizeof
(
SS
hellSubmitBloc
k
)
==
dataBuf
->
size
);
assert
(
pBlocks
->
numOfRows
*
dataBuf
->
rowSize
+
sizeof
(
SS
ubmitBl
k
)
==
dataBuf
->
size
);
// if use server time, this block must be ordered
if
(
dataBuf
->
tsSource
==
TSDB_USE_SERVER_TS
)
{
...
...
@@ -624,7 +626,7 @@ void sortRemoveDuplicates(STableDataBlocks *dataBuf) {
}
if
(
!
dataBuf
->
ordered
)
{
char
*
pBlockData
=
pBlocks
->
payLoad
;
char
*
pBlockData
=
pBlocks
->
data
;
qsort
(
pBlockData
,
pBlocks
->
numOfRows
,
dataBuf
->
rowSize
,
rowDataCompar
);
int32_t
i
=
0
;
...
...
@@ -650,7 +652,7 @@ void sortRemoveDuplicates(STableDataBlocks *dataBuf) {
dataBuf
->
ordered
=
true
;
pBlocks
->
numOfRows
=
i
+
1
;
dataBuf
->
size
=
sizeof
(
SS
hellSubmitBloc
k
)
+
dataBuf
->
rowSize
*
pBlocks
->
numOfRows
;
dataBuf
->
size
=
sizeof
(
SS
ubmitBl
k
)
+
dataBuf
->
rowSize
*
pBlocks
->
numOfRows
;
}
}
...
...
@@ -663,7 +665,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char
STableDataBlocks
*
dataBuf
=
NULL
;
int32_t
ret
=
tscGetDataBlockFromList
(
pTableHashList
,
pCmd
->
pDataBlocks
,
pTableMeta
->
uid
,
TSDB_DEFAULT_PAYLOAD_SIZE
,
sizeof
(
SS
hellSubmitBloc
k
),
tinfo
.
rowSize
,
pTableMetaInfo
->
name
,
sizeof
(
SS
ubmitBl
k
),
tinfo
.
rowSize
,
pTableMetaInfo
->
name
,
pTableMeta
,
&
dataBuf
);
if
(
ret
!=
TSDB_CODE_SUCCESS
)
{
return
ret
;
...
...
@@ -691,14 +693,14 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char
SParamInfo
*
param
=
dataBuf
->
params
+
i
;
if
(
param
->
idx
==
-
1
)
{
param
->
idx
=
pCmd
->
numOfParams
++
;
param
->
offset
-=
sizeof
(
SS
hellSubmitBloc
k
);
param
->
offset
-=
sizeof
(
SS
ubmitBl
k
);
}
}
SS
hellSubmitBlock
*
pBlocks
=
(
SShellSubmitBloc
k
*
)(
dataBuf
->
pData
);
SS
ubmitBlk
*
pBlocks
=
(
SSubmitBl
k
*
)(
dataBuf
->
pData
);
tsSetBlockInfo
(
pBlocks
,
pTableMeta
,
numOfRows
);
dataBuf
->
vg
id
=
pTableMeta
->
vgi
d
;
dataBuf
->
vg
Id
=
pTableMeta
->
vgI
d
;
dataBuf
->
numOfTables
=
1
;
/*
...
...
@@ -1058,7 +1060,6 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
goto
_error_clean
;
}
void
*
fp
=
pSql
->
fp
;
ptrdiff_t
pos
=
pSql
->
asyncTblPos
-
pSql
->
sqlstr
;
if
((
code
=
tscCheckIfCreateTable
(
&
str
,
pSql
))
!=
TSDB_CODE_SUCCESS
)
{
...
...
@@ -1068,17 +1069,15 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
* And during the getMeterMetaCallback function, the sql string will be parsed from the
* interrupted position.
*/
if
(
fp
!=
NULL
)
{
if
(
TSDB_CODE_ACTION_IN_PROGRESS
==
code
)
{
tscTrace
(
"async insert and waiting to get meter meta, then continue parse sql from offset: %"
PRId64
,
pos
);
return
code
;
}
// todo add to return
tscError
(
"async insert parse error, code:%d, %s"
,
code
,
tstrerror
(
code
));
pSql
->
asyncTblPos
=
NULL
;
if
(
TSDB_CODE_ACTION_IN_PROGRESS
==
code
)
{
tscTrace
(
"async insert and waiting to get meter meta, then continue parse sql from offset: %"
PRId64
,
pos
);
return
code
;
}
// todo add to return
tscError
(
"async insert parse error, code:%d, %s"
,
code
,
tstrerror
(
code
));
pSql
->
asyncTblPos
=
NULL
;
goto
_error_clean
;
// TODO: should _clean or _error_clean to async flow ????
}
...
...
@@ -1096,15 +1095,13 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
goto
_error_clean
;
}
int32_t
numOfCols
=
tscGetNumOfTags
(
pTableMetaInfo
->
pTableMeta
);
STableComInfo
tinfo
=
tscGetTableInfo
(
pTableMetaInfo
->
pTableMeta
);
if
(
sToken
.
type
==
TK_VALUES
)
{
SParsedDataColInfo
spd
=
{.
numOfCols
=
numOfCol
s
};
SParsedDataColInfo
spd
=
{.
numOfCols
=
tinfo
.
numOfColumn
s
};
SSchema
*
pSchema
=
tscGetTableSchema
(
pTableMetaInfo
->
pTableMeta
);
tscSetAssignedColumnInfo
(
&
spd
,
pSchema
,
numOfCols
);
tscSetAssignedColumnInfo
(
&
spd
,
pSchema
,
tinfo
.
numOfColumns
);
if
(
validateDataSource
(
pCmd
,
DATA_FROM_SQL_STRING
,
sToken
.
z
)
!=
TSDB_CODE_SUCCESS
)
{
goto
_error_clean
;
...
...
@@ -1146,7 +1143,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
STableDataBlocks
*
pDataBlock
=
NULL
;
STableMeta
*
pTableMeta
=
pTableMetaInfo
->
pTableMeta
;
int32_t
ret
=
tscCreateDataBlock
(
PATH_MAX
,
tinfo
.
rowSize
,
sizeof
(
SS
hellSubmitBloc
k
),
pTableMetaInfo
->
name
,
int32_t
ret
=
tscCreateDataBlock
(
PATH_MAX
,
tinfo
.
rowSize
,
sizeof
(
SS
ubmitBl
k
),
pTableMetaInfo
->
name
,
pTableMeta
,
&
pDataBlock
);
if
(
ret
!=
TSDB_CODE_SUCCESS
)
{
goto
_error_clean
;
...
...
@@ -1243,7 +1240,7 @@ int doParseInsertSql(SSqlObj *pSql, char *str) {
// submit to more than one vnode
if
(
pCmd
->
pDataBlocks
->
nSize
>
0
)
{
// merge according to vg
i
d
// merge according to vg
I
d
if
((
code
=
tscMergeTableDataBlocks
(
pSql
,
pCmd
->
pDataBlocks
))
!=
TSDB_CODE_SUCCESS
)
{
goto
_error_clean
;
}
...
...
@@ -1358,7 +1355,7 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock
assert
(
pCmd
->
numOfClause
==
1
);
STableMeta
*
pTableMeta
=
tscGetTableMetaInfoFromCmd
(
pCmd
,
pCmd
->
clauseIndex
,
0
)
->
pTableMeta
;
SS
hellSubmitBlock
*
pBlocks
=
(
SShellSubmitBloc
k
*
)(
pTableDataBlocks
->
pData
);
SS
ubmitBlk
*
pBlocks
=
(
SSubmitBl
k
*
)(
pTableDataBlocks
->
pData
);
tsSetBlockInfo
(
pBlocks
,
pTableMeta
,
numOfRows
);
if
((
code
=
tscMergeTableDataBlocks
(
pSql
,
pCmd
->
pDataBlocks
))
!=
TSDB_CODE_SUCCESS
)
{
...
...
@@ -1399,7 +1396,7 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) {
pCmd
->
pDataBlocks
=
tscCreateBlockArrayList
();
STableDataBlocks
*
pTableDataBlock
=
NULL
;
int32_t
ret
=
tscCreateDataBlock
(
TSDB_PAYLOAD_SIZE
,
rowSize
,
sizeof
(
SS
hellSubmitBloc
k
),
int32_t
ret
=
tscCreateDataBlock
(
TSDB_PAYLOAD_SIZE
,
rowSize
,
sizeof
(
SS
ubmitBl
k
),
pTableMetaInfo
->
name
,
pTableMeta
,
&
pTableDataBlock
);
if
(
ret
!=
TSDB_CODE_SUCCESS
)
{
return
-
1
;
...
...
@@ -1440,7 +1437,7 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) {
}
pTableDataBlock
=
pCmd
->
pDataBlocks
->
pData
[
0
];
pTableDataBlock
->
size
=
sizeof
(
SS
hellSubmitBloc
k
);
pTableDataBlock
->
size
=
sizeof
(
SS
ubmitBl
k
);
pTableDataBlock
->
rowSize
=
tinfo
.
rowSize
;
numOfRows
+=
pSql
->
res
.
numOfRows
;
...
...
src/client/src/tscPrepare.c
浏览文件 @
ddd05d9c
...
...
@@ -325,12 +325,12 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) {
for
(
int32_t
i
=
0
;
i
<
pCmd
->
pDataBlocks
->
nSize
;
++
i
)
{
STableDataBlocks
*
pBlock
=
pCmd
->
pDataBlocks
->
pData
[
i
];
uint32_t
totalDataSize
=
pBlock
->
size
-
sizeof
(
SS
hellSubmitBloc
k
);
uint32_t
totalDataSize
=
pBlock
->
size
-
sizeof
(
SS
ubmitBl
k
);
uint32_t
dataSize
=
totalDataSize
/
alloced
;
assert
(
dataSize
*
alloced
==
totalDataSize
);
if
(
alloced
==
binded
)
{
totalDataSize
+=
dataSize
+
sizeof
(
SS
hellSubmitBloc
k
);
totalDataSize
+=
dataSize
+
sizeof
(
SS
ubmitBl
k
);
if
(
totalDataSize
>
pBlock
->
nAllocSize
)
{
const
double
factor
=
1
.
5
;
void
*
tmp
=
realloc
(
pBlock
->
pData
,
(
uint32_t
)(
totalDataSize
*
factor
));
...
...
@@ -342,7 +342,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) {
}
}
char
*
data
=
pBlock
->
pData
+
sizeof
(
SS
hellSubmitBloc
k
)
+
dataSize
*
binded
;
char
*
data
=
pBlock
->
pData
+
sizeof
(
SS
ubmitBl
k
)
+
dataSize
*
binded
;
for
(
uint32_t
j
=
0
;
j
<
pBlock
->
numOfParams
;
++
j
)
{
SParamInfo
*
param
=
pBlock
->
params
+
j
;
int
code
=
doBindParam
(
data
,
param
,
bind
+
param
->
idx
);
...
...
@@ -365,10 +365,10 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) {
for
(
int32_t
i
=
0
;
i
<
pCmd
->
pDataBlocks
->
nSize
;
++
i
)
{
STableDataBlocks
*
pBlock
=
pCmd
->
pDataBlocks
->
pData
[
i
];
uint32_t
totalDataSize
=
pBlock
->
size
-
sizeof
(
SS
hellSubmitBloc
k
);
uint32_t
totalDataSize
=
pBlock
->
size
-
sizeof
(
SS
ubmitBl
k
);
pBlock
->
size
+=
totalDataSize
/
alloced
;
SS
hellSubmitBlock
*
pSubmit
=
(
SShellSubmitBloc
k
*
)
pBlock
->
pData
;
SS
ubmitBlk
*
pSubmit
=
(
SSubmitBl
k
*
)
pBlock
->
pData
;
pSubmit
->
numOfRows
+=
pSubmit
->
numOfRows
/
alloced
;
}
...
...
@@ -398,10 +398,10 @@ static int insertStmtReset(STscStmt* pStmt) {
for
(
int32_t
i
=
0
;
i
<
pCmd
->
pDataBlocks
->
nSize
;
++
i
)
{
STableDataBlocks
*
pBlock
=
pCmd
->
pDataBlocks
->
pData
[
i
];
uint32_t
totalDataSize
=
pBlock
->
size
-
sizeof
(
SS
hellSubmitBloc
k
);
pBlock
->
size
=
sizeof
(
SS
hellSubmitBloc
k
)
+
totalDataSize
/
alloced
;
uint32_t
totalDataSize
=
pBlock
->
size
-
sizeof
(
SS
ubmitBl
k
);
pBlock
->
size
=
sizeof
(
SS
ubmitBl
k
)
+
totalDataSize
/
alloced
;
SS
hellSubmitBlock
*
pSubmit
=
(
SShellSubmitBloc
k
*
)
pBlock
->
pData
;
SS
ubmitBlk
*
pSubmit
=
(
SSubmitBl
k
*
)
pBlock
->
pData
;
pSubmit
->
numOfRows
=
pSubmit
->
numOfRows
/
alloced
;
}
}
...
...
src/client/src/tscSchemaUtil.c
浏览文件 @
ddd05d9c
...
...
@@ -58,7 +58,7 @@ SSchema *tscGetTableSchema(const STableMeta *pTableMeta) {
return
pSTableMeta
->
schema
;
}
return
pTableMeta
->
schema
;
return
(
SSchema
*
)
pTableMeta
->
schema
;
}
SSchema
*
tscGetTableTagSchema
(
const
STableMeta
*
pTableMeta
)
{
...
...
@@ -165,7 +165,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
pTableMeta
->
sid
=
pTableMetaMsg
->
sid
;
pTableMeta
->
uid
=
pTableMetaMsg
->
uid
;
pTableMeta
->
vg
id
=
pTableMetaMsg
->
vgi
d
;
pTableMeta
->
vg
Id
=
pTableMetaMsg
->
vgI
d
;
pTableMeta
->
numOfVpeers
=
pTableMetaMsg
->
numOfVpeers
;
memcpy
(
pTableMeta
->
vpeerDesc
,
pTableMetaMsg
->
vpeerDesc
,
sizeof
(
SVnodeDesc
)
*
pTableMeta
->
numOfVpeers
);
...
...
src/client/src/tscServer.c
浏览文件 @
ddd05d9c
...
...
@@ -341,11 +341,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg) {
* the tscShouldFreeAsyncSqlObj will success and tscFreeSqlObj free it immediately.
*/
bool
shouldFree
=
tscShouldFreeAsyncSqlObj
(
pSql
);
if
(
command
==
TSDB_SQL_INSERT
)
{
// handle multi-vnode insertion situation
(
*
pSql
->
fp
)(
pSql
,
taosres
,
rpcMsg
->
code
);
}
else
{
(
*
pSql
->
fp
)(
pSql
->
param
,
taosres
,
rpcMsg
->
code
);
}
(
*
pSql
->
fp
)(
pSql
->
param
,
taosres
,
rpcMsg
->
code
);
if
(
shouldFree
)
{
// If it is failed, all objects allocated during execution taos_connect_a should be released
...
...
@@ -512,14 +508,17 @@ int tscBuildRetrieveMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pRetrieveMsg
->
free
=
htons
(
pQueryInfo
->
type
);
pMsg
+=
sizeof
(
pQueryInfo
->
type
);
pSql
->
cmd
.
payloadLen
=
pMsg
-
pStart
;
pRetrieveMsg
->
header
.
vgId
=
htonl
(
1
);
pMsg
+=
sizeof
(
SRetrieveTableMsg
);
pRetrieveMsg
->
header
.
contLen
=
htonl
(
pSql
->
cmd
.
payloadLen
);
pSql
->
cmd
.
msgType
=
TSDB_MSG_TYPE_RETRIEVE
;
return
TSDB_CODE_SUCCESS
;
}
void
tscUpdateVnodeInSubmitMsg
(
SSqlObj
*
pSql
,
char
*
buf
)
{
//SS
hellS
ubmitMsg *pShellMsg;
//SSubmitMsg *pShellMsg;
//char * pMsg;
//STableMetaInfo * pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0);
...
...
@@ -528,34 +527,38 @@ void tscUpdateVnodeInSubmitMsg(SSqlObj *pSql, char *buf) {
//pMsg = buf + tsRpcHeadSize;
//TODO set iplist
//pShellMsg = (SS
hellS
ubmitMsg *)pMsg;
//pShellMsg = (SSubmitMsg *)pMsg;
//pShellMsg->vnode = htons(pTableMeta->vpeerDesc[pSql->index].vnode);
//tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pSql->index].ip),
// htons(pShellMsg->vnode));
}
int
tscBuildSubmitMsg
(
SSqlObj
*
pSql
,
SSqlInfo
*
pInfo
)
{
SS
hellS
ubmitMsg
*
pShellMsg
;
SSubmitMsg
*
pShellMsg
;
char
*
pMsg
,
*
pStart
;
SQueryInfo
*
pQueryInfo
=
tscGetQueryInfoDetail
(
&
pSql
->
cmd
,
0
);
STableMeta
*
pTableMeta
=
tscGetMetaInfo
(
pQueryInfo
,
0
)
->
pTableMeta
;
pStart
=
pSql
->
cmd
.
payload
+
tsRpcHeadSize
;
pMsg
=
pStart
;
pShellMsg
=
(
SShellSubmitMsg
*
)
pMsg
;
pShellMsg
->
import
=
htons
(
TSDB_QUERY_HAS_TYPE
(
pQueryInfo
->
type
,
TSDB_QUERY_TYPE_INSERT
)
?
0
:
1
);
pShellMsg
->
vnode
=
0
;
//htons(pTableMeta->vpeerDesc[pTableMeta->index].vnode);
pShellMsg
->
numOfSid
=
htonl
(
pSql
->
cmd
.
numOfTablesInSubmit
);
// number of meters to be inserted
SMsgDesc
*
pMsgDesc
=
(
SMsgDesc
*
)
pMsg
;
pMsgDesc
->
numOfVnodes
=
htonl
(
1
);
//set the number of vnodes
pMsg
+=
sizeof
(
SMsgDesc
);
pShellMsg
=
(
SSubmitMsg
*
)
pMsg
;
pShellMsg
->
header
.
vgId
=
htonl
(
pTableMeta
->
vgId
);
pShellMsg
->
header
.
contLen
=
htonl
(
pSql
->
cmd
.
payloadLen
);
pShellMsg
->
length
=
pShellMsg
->
header
.
contLen
;
pShellMsg
->
numOfBlocks
=
htonl
(
pSql
->
cmd
.
numOfTablesInSubmit
);
// number of meters to be inserted
// pSql->cmd.payloadLen is set during parse sql routine, so we do not use it here
pSql
->
cmd
.
msgType
=
TSDB_MSG_TYPE_SUBMIT
;
// tscTrace("%p update submit msg vnode:%s:%d", pSql, taosIpStr(pTableMeta->vpeerDesc[pTableMeta->index].ip),
// htons(pShellMsg->vnode));
pSql
->
cmd
.
payloadLen
=
sizeof
(
SShellSubmitMsg
);
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -598,7 +601,7 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
SSuperTableMeta
*
pMetricMeta
=
pTableMetaInfo
->
pMetricMeta
;
SVnodeSidList
*
pVnodeSidList
=
tscGetVnodeSidList
(
pMetricMeta
,
pTableMetaInfo
->
vnodeIndex
);
int32_t
meterInfoSize
=
(
pMetricMeta
->
tagLen
+
sizeof
(
STable
SidExt
Info
))
*
pVnodeSidList
->
numOfSids
;
int32_t
meterInfoSize
=
(
pMetricMeta
->
tagLen
+
sizeof
(
STable
Id
Info
))
*
pVnodeSidList
->
numOfSids
;
int32_t
outputColumnSize
=
pQueryInfo
->
exprsInfo
.
numOfExprs
*
sizeof
(
SSqlFuncExprMsg
);
int32_t
size
=
meterInfoSize
+
outputColumnSize
+
srcColListSize
+
exprSize
+
MIN_QUERY_MSG_PKT_SIZE
;
...
...
@@ -609,37 +612,34 @@ static int32_t tscEstimateQueryMsgSize(SSqlCmd *pCmd, int32_t clauseIndex) {
return
size
;
}
static
char
*
doSerializeTableInfo
(
SSqlObj
*
pSql
,
int32_t
numOfTables
,
int32_t
v
node
Id
,
char
*
pMsg
)
{
static
char
*
doSerializeTableInfo
(
SSqlObj
*
pSql
,
int32_t
numOfTables
,
int32_t
v
g
Id
,
char
*
pMsg
)
{
STableMetaInfo
*
pTableMetaInfo
=
tscGetTableMetaInfoFromCmd
(
&
pSql
->
cmd
,
pSql
->
cmd
.
clauseIndex
,
0
);
STableMeta
*
pTableMeta
=
pTableMetaInfo
->
pTableMeta
;
SSuperTableMeta
*
pMetricMeta
=
pTableMetaInfo
->
pMetricMeta
;
tscTrace
(
"%p v
id:%d, query on %d meters"
,
pSql
,
vnode
Id
,
numOfTables
);
tscTrace
(
"%p v
gId:%d, query on %d tables"
,
pSql
,
vg
Id
,
numOfTables
);
if
(
UTIL_TABLE_IS_NOMRAL_TABLE
(
pTableMetaInfo
))
{
#ifdef _DEBUG_VIEW
tscTrace
(
"%p sid:%d, uid:%"
PRIu64
,
pSql
,
pTableMetaInfo
->
pTableMeta
->
sid
,
pTableMetaInfo
->
pTableMeta
->
uid
);
#endif
STable
SidExtInfo
*
pTableMetaInfo
=
(
STableSidExt
Info
*
)
pMsg
;
pTable
Meta
Info
->
sid
=
htonl
(
pTableMeta
->
sid
);
pTable
Meta
Info
->
uid
=
htobe64
(
pTableMeta
->
uid
);
pTable
Meta
Info
->
key
=
htobe64
(
tscGetSubscriptionProgress
(
pSql
->
pSubscription
,
pTableMeta
->
uid
));
pMsg
+=
sizeof
(
STable
SidExt
Info
);
STable
IdInfo
*
pTableIdInfo
=
(
STableId
Info
*
)
pMsg
;
pTable
Id
Info
->
sid
=
htonl
(
pTableMeta
->
sid
);
pTable
Id
Info
->
uid
=
htobe64
(
pTableMeta
->
uid
);
pTable
Id
Info
->
key
=
htobe64
(
tscGetSubscriptionProgress
(
pSql
->
pSubscription
,
pTableMeta
->
uid
));
pMsg
+=
sizeof
(
STable
Id
Info
);
}
else
{
SVnodeSidList
*
pVnodeSidList
=
tscGetVnodeSidList
(
pMetricMeta
,
pTableMetaInfo
->
vnodeIndex
);
for
(
int32_t
i
=
0
;
i
<
numOfTables
;
++
i
)
{
STable
SidExtInfo
*
pTableMetaInfo
=
(
STableSidExt
Info
*
)
pMsg
;
STable
SidExt
Info
*
pQueryMeterInfo
=
tscGetMeterSidInfo
(
pVnodeSidList
,
i
);
STable
IdInfo
*
pTableIdInfo
=
(
STableId
Info
*
)
pMsg
;
STable
Id
Info
*
pQueryMeterInfo
=
tscGetMeterSidInfo
(
pVnodeSidList
,
i
);
pTable
Meta
Info
->
sid
=
htonl
(
pQueryMeterInfo
->
sid
);
pTable
Meta
Info
->
uid
=
htobe64
(
pQueryMeterInfo
->
uid
);
pTable
Meta
Info
->
key
=
htobe64
(
tscGetSubscriptionProgress
(
pSql
->
pSubscription
,
pQueryMeterInfo
->
uid
));
pTable
Id
Info
->
sid
=
htonl
(
pQueryMeterInfo
->
sid
);
pTable
Id
Info
->
uid
=
htobe64
(
pQueryMeterInfo
->
uid
);
pTable
Id
Info
->
key
=
htobe64
(
tscGetSubscriptionProgress
(
pSql
->
pSubscription
,
pQueryMeterInfo
->
uid
));
pMsg
+=
sizeof
(
STableSidExtInfo
);
memcpy
(
pMsg
,
pQueryMeterInfo
->
tags
,
pMetricMeta
->
tagLen
);
pMsg
+=
pMetricMeta
->
tagLen
;
pMsg
+=
sizeof
(
STableIdInfo
);
#ifdef _DEBUG_VIEW
tscTrace
(
"%p sid:%d, uid:%"
PRId64
,
pSql
,
pQueryMeterInfo
->
sid
,
pQueryMeterInfo
->
uid
);
...
...
@@ -679,6 +679,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg
->
uid
=
pTableMeta
->
uid
;
pQueryMsg
->
numOfTagsCols
=
0
;
pQueryMsg
->
vgId
=
htonl
(
pTableMeta
->
vgId
);
tscTrace
(
"%p queried tables:%d, table id: %s"
,
pSql
,
1
,
pTableMetaInfo
->
name
);
}
else
{
// query on super table
if
(
pTableMetaInfo
->
vnodeIndex
<
0
)
{
...
...
@@ -696,7 +697,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
}
tscTrace
(
"%p query on vid:%d, number of tables:%d"
,
pSql
,
vnodeId
,
numOfTables
);
pQueryMsg
->
v
node
=
htons
(
vnodeId
);
pQueryMsg
->
v
gId
=
htons
(
vnodeId
);
}
pQueryMsg
->
numOfTables
=
htonl
(
numOfTables
);
...
...
@@ -764,14 +765,14 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SColumnBase
*
pCol
=
tscColumnBaseInfoGet
(
&
pQueryInfo
->
colList
,
i
);
SSchema
*
pColSchema
=
&
pSchema
[
pCol
->
colIndex
.
columnIndex
];
if
(
pCol
->
colIndex
.
columnIndex
>=
tscGetNumOfColumns
(
pTableMeta
)
||
pColSchema
->
type
<
TSDB_DATA_TYPE_BOOL
||
pColSchema
->
type
>
TSDB_DATA_TYPE_NCHAR
)
{
tscError
(
"%p vid:%d sid:%d id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s"
,
pSql
,
htons
(
pQueryMsg
->
vnode
),
pTableMeta
->
sid
,
pTableMetaInfo
->
name
,
tscGetNumOfColumns
(
pTableMeta
),
pCol
->
colIndex
,
pColSchema
->
name
);
return
-
1
;
// 0 means build msg failed
}
//
if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL ||
//
pColSchema->type > TSDB_DATA_TYPE_NCHAR) {
//
tscError("%p vid:%d sid:%d id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", pSql,
//
htons(pQueryMsg->vnode), pTableMeta->sid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex,
//
pColSchema->name);
//
//
return -1; // 0 means build msg failed
//
}
pQueryMsg
->
colList
[
i
].
colId
=
htons
(
pColSchema
->
colId
);
pQueryMsg
->
colList
[
i
].
bytes
=
htons
(
pColSchema
->
bytes
);
...
...
@@ -865,7 +866,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg
->
colNameLen
=
htonl
(
len
);
// serialize the table info (sid, uid, tags)
pMsg
=
doSerializeTableInfo
(
pSql
,
numOfTables
,
htons
(
pQueryMsg
->
v
node
),
pMsg
);
pMsg
=
doSerializeTableInfo
(
pSql
,
numOfTables
,
htons
(
pQueryMsg
->
v
gId
),
pMsg
);
// only include the required tag column schema. If a tag is not required, it won't be sent to vnode
if
(
pTableMetaInfo
->
numOfTags
>
0
)
{
...
...
@@ -946,7 +947,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
tscTrace
(
"%p msg built success,len:%d bytes"
,
pSql
,
msgLen
);
pCmd
->
payloadLen
=
msgLen
;
pSql
->
cmd
.
msgType
=
TSDB_MSG_TYPE_QUERY
;
pQueryMsg
->
contLen
=
htonl
(
msgLen
);
assert
(
msgLen
+
minMsgSize
()
<=
size
);
return
TSDB_CODE_SUCCESS
;
...
...
@@ -1849,12 +1852,12 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
pMetaMsg
->
sid
=
htonl
(
pMetaMsg
->
sid
);
pMetaMsg
->
sversion
=
htons
(
pMetaMsg
->
sversion
);
pMetaMsg
->
vg
id
=
htonl
(
pMetaMsg
->
vgi
d
);
pMetaMsg
->
vg
Id
=
htonl
(
pMetaMsg
->
vgI
d
);
pMetaMsg
->
uid
=
htobe64
(
pMetaMsg
->
uid
);
pMetaMsg
->
contLen
=
htons
(
pMetaMsg
->
contLen
);
if
(
pMetaMsg
->
sid
<
0
||
pMetaMsg
->
vg
i
d
<
0
)
{
tscError
(
"invalid meter vg
id:%d, sid%d"
,
pMetaMsg
->
vgi
d
,
pMetaMsg
->
sid
);
if
(
pMetaMsg
->
sid
<
0
||
pMetaMsg
->
vg
I
d
<
0
)
{
tscError
(
"invalid meter vg
Id:%d, sid%d"
,
pMetaMsg
->
vgI
d
,
pMetaMsg
->
sid
);
return
TSDB_CODE_INVALID_VALUE
;
}
...
...
@@ -1948,11 +1951,11 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
pMeta
->
sid
=
htonl
(
pMeta
->
sid
);
pMeta
->
sversion
=
htons
(
pMeta
->
sversion
);
pMeta
->
vg
id
=
htonl
(
pMeta
->
vgi
d
);
pMeta
->
vg
Id
=
htonl
(
pMeta
->
vgI
d
);
pMeta
->
uid
=
htobe64
(
pMeta
->
uid
);
if
(
pMeta
->
sid
<=
0
||
pMeta
->
vg
i
d
<
0
)
{
tscError
(
"invalid meter vg
id:%d, sid%d"
,
pMeta
->
vgi
d
,
pMeta
->
sid
);
if
(
pMeta
->
sid
<=
0
||
pMeta
->
vg
I
d
<
0
)
{
tscError
(
"invalid meter vg
Id:%d, sid%d"
,
pMeta
->
vgI
d
,
pMeta
->
sid
);
pSql
->
res
.
code
=
TSDB_CODE_INVALID_VALUE
;
pSql
->
res
.
numOfTotal
=
i
;
return
TSDB_CODE_OTHERS
;
...
...
@@ -2067,7 +2070,7 @@ int tscProcessMetricMetaRsp(SSqlObj *pSql) {
pMeta
->
numOfVnodes
=
htonl
(
pMeta
->
numOfVnodes
);
pMeta
->
tagLen
=
htons
(
pMeta
->
tagLen
);
size
+=
pMeta
->
numOfVnodes
*
sizeof
(
SVnodeSidList
*
)
+
pMeta
->
numOfTables
*
sizeof
(
STable
SidExt
Info
*
);
size
+=
pMeta
->
numOfVnodes
*
sizeof
(
SVnodeSidList
*
)
+
pMeta
->
numOfTables
*
sizeof
(
STable
Id
Info
*
);
char
*
pBuf
=
calloc
(
1
,
size
);
if
(
pBuf
==
NULL
)
{
...
...
@@ -2093,16 +2096,16 @@ int tscProcessMetricMetaRsp(SSqlObj *pSql) {
tscTrace
(
"%p metricmeta:vid:%d,numOfTables:%d"
,
pSql
,
i
,
pLists
->
numOfSids
);
pBuf
+=
sizeof
(
SVnodeSidList
)
+
sizeof
(
STable
SidExt
Info
*
)
*
pSidLists
->
numOfSids
;
pBuf
+=
sizeof
(
SVnodeSidList
)
+
sizeof
(
STable
Id
Info
*
)
*
pSidLists
->
numOfSids
;
rsp
+=
sizeof
(
SVnodeSidList
);
size_t
elemSize
=
sizeof
(
STable
SidExt
Info
)
+
pNewMetricMeta
->
tagLen
;
size_t
elemSize
=
sizeof
(
STable
Id
Info
)
+
pNewMetricMeta
->
tagLen
;
for
(
int32_t
j
=
0
;
j
<
pSidLists
->
numOfSids
;
++
j
)
{
pLists
->
pSidExtInfoList
[
j
]
=
pBuf
-
(
char
*
)
pLists
;
memcpy
(
pBuf
,
rsp
,
elemSize
);
((
STable
SidExtInfo
*
)
pBuf
)
->
uid
=
htobe64
(((
STableSidExt
Info
*
)
pBuf
)
->
uid
);
((
STable
SidExtInfo
*
)
pBuf
)
->
sid
=
htonl
(((
STableSidExt
Info
*
)
pBuf
)
->
sid
);
((
STable
IdInfo
*
)
pBuf
)
->
uid
=
htobe64
(((
STableId
Info
*
)
pBuf
)
->
uid
);
((
STable
IdInfo
*
)
pBuf
)
->
sid
=
htonl
(((
STableId
Info
*
)
pBuf
)
->
sid
);
rsp
+=
elemSize
;
pBuf
+=
elemSize
;
...
...
src/client/src/tscSql.c
浏览文件 @
ddd05d9c
...
...
@@ -130,7 +130,7 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
pSql
->
pTscObj
=
pObj
;
pSql
->
signature
=
pSql
;
tsem_init
(
&
pSql
->
rspSem
,
0
,
0
);
// tsem_init(&pSql->emptyRspSem, 0, 1);
pObj
->
pSql
=
pSql
;
pSql
->
fp
=
fp
;
pSql
->
param
=
param
;
...
...
@@ -146,6 +146,8 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
return
NULL
;
}
// tsRpcHeaderSize will be updated during RPC initialization, so only after it initialization, this value is valid
tsInsertHeadSize
=
tsRpcHeadSize
+
sizeof
(
SMsgDesc
)
+
sizeof
(
SSubmitMsg
);
return
pObj
;
}
...
...
src/client/src/tscSub.c
浏览文件 @
ddd05d9c
...
...
@@ -202,7 +202,7 @@ int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
for
(
int32_t
i
=
0
;
i
<
pMetricMeta
->
numOfVnodes
;
i
++
)
{
SVnodeSidList
*
pVnodeSidList
=
tscGetVnodeSidList
(
pMetricMeta
,
i
);
for
(
int32_t
j
=
0
;
j
<
pVnodeSidList
->
numOfSids
;
j
++
)
{
STable
SidExt
Info
*
pTableMetaInfo
=
tscGetMeterSidInfo
(
pVnodeSidList
,
j
);
STable
Id
Info
*
pTableMetaInfo
=
tscGetMeterSidInfo
(
pVnodeSidList
,
j
);
int64_t
uid
=
pTableMetaInfo
->
uid
;
progress
[
numOfTables
].
uid
=
uid
;
progress
[
numOfTables
++
].
key
=
tscGetSubscriptionProgress
(
pSub
,
uid
);
...
...
src/client/src/tscSystem.c
浏览文件 @
ddd05d9c
...
...
@@ -34,7 +34,6 @@ void * pTscMgmtConn;
void
*
pSlaveConn
;
void
*
tscCacheHandle
;
int32_t
globalCode
=
0
;
int
initialized
=
0
;
int
slaveIndex
;
void
*
tscTmr
;
void
*
tscQhandle
;
...
...
@@ -187,9 +186,7 @@ void taos_init_imp() {
if
(
tscCacheHandle
==
NULL
)
tscCacheHandle
=
taosCacheInit
(
tscTmr
,
refreshTime
);
initialized
=
1
;
tscTrace
(
"client is initialized successfully"
);
tsInsertHeadSize
=
tsRpcHeadSize
+
sizeof
(
SShellSubmitMsg
);
}
void
taos_init
()
{
pthread_once
(
&
tscinit
,
taos_init_imp
);
}
...
...
src/client/src/tscUtil.c
浏览文件 @
ddd05d9c
...
...
@@ -191,7 +191,7 @@ SVnodeSidList* tscGetVnodeSidList(SSuperTableMeta* pMetricmeta, int32_t vnodeIdx
return
(
SVnodeSidList
*
)(
pMetricmeta
->
list
[
vnodeIdx
]
+
(
char
*
)
pMetricmeta
);
}
STable
SidExt
Info
*
tscGetMeterSidInfo
(
SVnodeSidList
*
pSidList
,
int32_t
idx
)
{
STable
Id
Info
*
tscGetMeterSidInfo
(
SVnodeSidList
*
pSidList
,
int32_t
idx
)
{
if
(
pSidList
==
NULL
)
{
tscError
(
"illegal sidlist"
);
return
0
;
...
...
@@ -206,7 +206,7 @@ STableSidExtInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx) {
assert
(
pSidList
->
pSidExtInfoList
[
idx
]
>=
0
);
return
(
STable
SidExt
Info
*
)(
pSidList
->
pSidExtInfoList
[
idx
]
+
(
char
*
)
pSidList
);
return
(
STable
Id
Info
*
)(
pSidList
->
pSidExtInfoList
[
idx
]
+
(
char
*
)
pSidList
);
}
bool
tscIsTwoStageSTableQuery
(
SQueryInfo
*
pQueryInfo
,
int32_t
tableIndex
)
{
...
...
@@ -614,7 +614,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
*/
pCmd
->
payloadLen
=
pDataBlock
->
nAllocSize
-
tsRpcHeadSize
;
assert
(
pCmd
->
allocSize
>=
pCmd
->
payloadLen
+
tsRpcHeadSize
+
100
);
assert
(
pCmd
->
allocSize
>=
pCmd
->
payloadLen
+
tsRpcHeadSize
+
100
&&
pCmd
->
payloadLen
>
0
);
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -695,6 +695,49 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList,
return
TSDB_CODE_SUCCESS
;
}
static
void
trimDataBlock
(
void
*
pDataBlock
,
STableDataBlocks
*
pTableDataBlock
)
{
int32_t
firstPartLen
=
0
;
STableMeta
*
pTableMeta
=
pTableDataBlock
->
pTableMeta
;
STableComInfo
tinfo
=
tscGetTableInfo
(
pTableMeta
);
SSchema
*
pSchema
=
tscGetTableSchema
(
pTableMeta
);
memcpy
(
pDataBlock
,
pTableDataBlock
->
pData
,
sizeof
(
SSubmitBlk
));
pDataBlock
+=
sizeof
(
SSubmitBlk
);
int32_t
total
=
sizeof
(
int32_t
)
*
2
;
for
(
int32_t
i
=
0
;
i
<
tinfo
.
numOfColumns
;
++
i
)
{
switch
(
pSchema
[
i
].
type
)
{
case
TSDB_DATA_TYPE_NCHAR
:
case
TSDB_DATA_TYPE_BINARY
:
{
assert
(
0
);
// not support binary yet
firstPartLen
+=
sizeof
(
int32_t
);
break
;
}
default:
firstPartLen
+=
tDataTypeDesc
[
pSchema
[
i
].
type
].
nSize
;
total
+=
tDataTypeDesc
[
pSchema
[
i
].
type
].
nSize
;
}
}
char
*
p
=
pTableDataBlock
->
pData
+
sizeof
(
SSubmitBlk
);
SSubmitBlk
*
pBlock
=
pTableDataBlock
->
pData
;
int32_t
rows
=
htons
(
pBlock
->
numOfRows
);
for
(
int32_t
i
=
0
;
i
<
rows
;
++
i
)
{
*
(
int32_t
*
)
pDataBlock
=
total
;
pDataBlock
+=
sizeof
(
int32_t
);
*
(
int32_t
*
)
pDataBlock
=
firstPartLen
;
pDataBlock
+=
sizeof
(
int32_t
);
memcpy
(
pDataBlock
,
p
,
pTableDataBlock
->
rowSize
);
p
+=
pTableDataBlock
->
rowSize
;
pDataBlock
+=
pTableDataBlock
->
rowSize
;
}
}
int32_t
tscMergeTableDataBlocks
(
SSqlObj
*
pSql
,
SDataBlockList
*
pTableDataBlockList
)
{
SSqlCmd
*
pCmd
=
&
pSql
->
cmd
;
...
...
@@ -705,8 +748,9 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
STableDataBlocks
*
pOneTableBlock
=
pTableDataBlockList
->
pData
[
i
];
STableDataBlocks
*
dataBuf
=
NULL
;
int32_t
ret
=
tscGetDataBlockFromList
(
pVnodeDataBlockHashList
,
pVnodeDataBlockList
,
pOneTableBlock
->
vgid
,
TSDB_PAYLOAD_SIZE
,
int32_t
ret
=
tscGetDataBlockFromList
(
pVnodeDataBlockHashList
,
pVnodeDataBlockList
,
pOneTableBlock
->
vgId
,
TSDB_PAYLOAD_SIZE
,
tsInsertHeadSize
,
0
,
pOneTableBlock
->
tableId
,
pOneTableBlock
->
pTableMeta
,
&
dataBuf
);
if
(
ret
!=
TSDB_CODE_SUCCESS
)
{
tscError
(
"%p failed to prepare the data block buffer for merging table data, code:%d"
,
pSql
,
ret
);
...
...
@@ -715,7 +759,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
return
ret
;
}
int64_t
destSize
=
dataBuf
->
size
+
pOneTableBlock
->
size
;
int64_t
destSize
=
dataBuf
->
size
+
pOneTableBlock
->
size
+
pOneTableBlock
->
size
*
sizeof
(
int32_t
)
*
2
;
if
(
dataBuf
->
nAllocSize
<
destSize
)
{
while
(
dataBuf
->
nAllocSize
<
destSize
)
{
dataBuf
->
nAllocSize
=
dataBuf
->
nAllocSize
*
1
.
5
;
...
...
@@ -729,29 +773,33 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
tscError
(
"%p failed to allocate memory for merging submit block, size:%d"
,
pSql
,
dataBuf
->
nAllocSize
);
taosHashCleanup
(
pVnodeDataBlockHashList
);
tfree
(
dataBuf
->
pData
);
tscDestroyBlockArrayList
(
pVnodeDataBlockList
);
tfree
(
dataBuf
->
pData
);
return
TSDB_CODE_CLI_OUT_OF_MEMORY
;
}
}
SS
hellSubmitBlock
*
pBlocks
=
(
SShellSubmitBlock
*
)
pOneTableBlock
->
pData
;
SS
ubmitBlk
*
pBlocks
=
(
SSubmitBlk
*
)
pOneTableBlock
->
pData
;
sortRemoveDuplicates
(
pOneTableBlock
);
char
*
e
=
(
char
*
)
pBlocks
->
payLoad
+
pOneTableBlock
->
rowSize
*
(
pBlocks
->
numOfRows
-
1
);
char
*
e
=
(
char
*
)
pBlocks
->
data
+
pOneTableBlock
->
rowSize
*
(
pBlocks
->
numOfRows
-
1
);
tscTrace
(
"%p tableId:%s, sid:%d rows:%d sversion:%d skey:%"
PRId64
", ekey:%"
PRId64
,
pSql
,
pOneTableBlock
->
tableId
,
pBlocks
->
sid
,
pBlocks
->
numOfRows
,
pBlocks
->
sversion
,
GET_INT64_VAL
(
pBlocks
->
payLoad
),
GET_INT64_VAL
(
e
));
tscTrace
(
"%p tableId:%s, sid:%d rows:%d sversion:%d skey:%"
PRId64
", ekey:%"
PRId64
,
pSql
,
pOneTableBlock
->
tableId
,
pBlocks
->
tid
,
pBlocks
->
numOfRows
,
pBlocks
->
sversion
,
GET_INT64_VAL
(
pBlocks
->
data
),
GET_INT64_VAL
(
e
));
pBlocks
->
sid
=
htonl
(
pBlocks
->
sid
);
int32_t
len
=
pBlocks
->
numOfRows
*
(
pOneTableBlock
->
rowSize
+
sizeof
(
int32_t
)
*
2
);
pBlocks
->
tid
=
htonl
(
pBlocks
->
tid
);
pBlocks
->
uid
=
htobe64
(
pBlocks
->
uid
);
pBlocks
->
sversion
=
htonl
(
pBlocks
->
sversion
);
pBlocks
->
numOfRows
=
htons
(
pBlocks
->
numOfRows
);
memcpy
(
dataBuf
->
pData
+
dataBuf
->
size
,
pOneTableBlock
->
pData
,
pOneTableBlock
->
size
);
dataBuf
->
size
+=
pOneTableBlock
->
size
;
pBlocks
->
len
=
htonl
(
len
);
// erase the empty space reserved for binary data
trimDataBlock
(
dataBuf
->
pData
+
dataBuf
->
size
,
pOneTableBlock
);
dataBuf
->
size
+=
(
len
+
sizeof
(
SSubmitBlk
));
dataBuf
->
numOfTables
+=
1
;
}
...
...
src/common/inc/dataformat.h
浏览文件 @
ddd05d9c
...
...
@@ -81,11 +81,13 @@ STSchema *tdDecodeSchema(void **psrc);
*/
typedef
void
*
SDataRow
;
#define TD_DATA_ROW_HEAD_SIZE (2 * sizeof(int32_t))
#define dataRowLen(r) (*(int32_t *)(r))
#define dataRowFLen(r) (*(int32_t *)((char *)(r) + sizeof(int32_t)))
#define dataRowTuple(r) ((char *)(r) + TD_DATA_ROW_HEAD_SIZE)
#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
#define dataRowSetLen(r, l) (dataRowLen(r) = (l))
#define dataRowSetFLen(r, l) (dataRowFLen(r) = (l))
#define dataRowIdx(r, i) ((char *)(r) + i)
...
...
@@ -101,23 +103,13 @@ int tdAppendColVal(SDataRow row, void *value, STColumn *pCol);
void
tdDataRowReset
(
SDataRow
row
,
STSchema
*
pSchema
);
SDataRow
tdDataRowDup
(
SDataRow
row
);
/* Data column definition
* +---------+---------+-----------------------+
* | int32_t | int32_t | |
* +---------+---------+-----------------------+
* | len | npoints | data |
* +---------+---------+-----------------------+
*/
typedef
char
*
SDataCol
;
/* Data columns definition
* +---------+---------+-----------------------+--------+-----------------------+
* | int32_t | int32_t | | | |
* +---------+---------+-----------------------+--------+-----------------------+
* | len | npoints | SDataCol | .... | SDataCol |
* +---------+---------+-----------------------+--------+-----------------------+
*/
typedef
char
*
SDataCols
;
// ----------------- Data column structure
typedef
struct
SDataCol
{
int64_t
len
;
char
data
[];
}
SDataCol
;
void
tdConvertDataRowToCol
(
SDataCol
*
cols
,
STSchema
*
pSchema
,
int
*
iter
);
#ifdef __cplusplus
}
...
...
src/common/src/dataformat.c
浏览文件 @
ddd05d9c
...
...
@@ -150,7 +150,7 @@ void tdFreeSchema(STSchema *pSchema) {
*/
void
tdUpdateSchema
(
STSchema
*
pSchema
)
{
STColumn
*
pCol
=
NULL
;
int32_t
offset
=
0
;
int32_t
offset
=
TD_DATA_ROW_HEAD_SIZE
;
for
(
int
i
=
0
;
i
<
schemaNCols
(
pSchema
);
i
++
)
{
pCol
=
schemaColAt
(
pSchema
,
i
);
colSetOffset
(
pCol
,
offset
);
...
...
@@ -294,6 +294,16 @@ SDataRow tdDataRowDup(SDataRow row) {
return
trow
;
}
void
tdConvertDataRowToCol
(
SDataCol
*
cols
,
STSchema
*
pSchema
,
int
*
iter
)
{
int
row
=
*
iter
;
for
(
int
i
=
0
;
i
<
schemaNCols
(
pSchema
);
i
++
)
{
// TODO
}
*
iter
=
row
+
1
;
}
/**
* Return the first part length of a data row for a schema
*/
...
...
src/dnode/src/dnodeRead.c
浏览文件 @
ddd05d9c
...
...
@@ -15,13 +15,16 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tlog.h"
#include "tqueue.h"
#include "trpc.h"
#include "dnodeRead.h"
#include "dnodeMgmt.h"
#include "dnodeRead.h"
#include "queryExecutor.h"
typedef
struct
{
int32_t
code
;
...
...
@@ -74,7 +77,9 @@ void dnodeRead(SRpcMsg *pMsg) {
int32_t
leftLen
=
pMsg
->
contLen
;
char
*
pCont
=
(
char
*
)
pMsg
->
pCont
;
SRpcContext
*
pRpcContext
=
NULL
;
dTrace
(
"dnode read msg disposal"
);
// SMsgDesc *pDesc = pCont;
// pDesc->numOfVnodes = htonl(pDesc->numOfVnodes);
// pCont += sizeof(SMsgDesc);
...
...
@@ -88,7 +93,7 @@ void dnodeRead(SRpcMsg *pMsg) {
while
(
leftLen
>
0
)
{
SMsgHead
*
pHead
=
(
SMsgHead
*
)
pCont
;
pHead
->
vgId
=
1
;
//htonl(pHead->vgId);
pHead
->
vgId
=
1
;
//htonl(pHead->vgId);
pHead
->
contLen
=
pMsg
->
contLen
;
//htonl(pHead->contLen);
void
*
pVnode
=
dnodeGetVnode
(
pHead
->
vgId
);
...
...
@@ -223,43 +228,81 @@ static void dnodeProcessReadResult(SReadMsg *pRead) {
}
static
void
dnodeProcessQueryMsg
(
SReadMsg
*
pMsg
)
{
void
*
pQInfo
=
(
void
*
)
100
;
dTrace
(
"query msg is disposed, qInfo:%p"
,
pQInfo
);
SQueryTableMsg
*
pQueryTableMsg
=
(
SQueryTableMsg
*
)
pMsg
->
pCont
;
SQInfo
*
pQInfo
=
NULL
;
void
*
tsdb
=
dnodeGetVnodeTsdb
(
pMsg
->
pVnode
);
int32_t
code
=
qCreateQueryInfo
(
tsdb
,
pQueryTableMsg
,
&
pQInfo
);
SQueryTableRsp
*
pRsp
=
(
SQueryTableRsp
*
)
rpcMallocCont
(
sizeof
(
SQueryTableRsp
));
pRsp
->
code
=
0
;
pRsp
->
code
=
code
;
pRsp
->
qhandle
=
htobe64
((
uint64_t
)
(
pQInfo
));
SRpcMsg
rpcRsp
=
{
.
handle
=
pMsg
->
rpcMsg
.
handle
,
.
pCont
=
pRsp
,
.
contLen
=
sizeof
(
SQueryTableRsp
),
.
code
=
0
,
.
code
=
code
,
.
msgType
=
0
};
rpcSendResponse
(
&
rpcRsp
);
// do execute query
qTableQuery
(
pQInfo
);
}
static
void
dnodeProcessRetrieveMsg
(
SReadMsg
*
pMsg
)
{
SRetrieveTableMsg
*
pRetrieve
=
pMsg
->
pCont
;
void
*
pQInfo
=
htobe64
(
pRetrieve
->
qhandle
);
dTrace
(
"retrieve msg is disposed, qInfo:%p"
,
pQInfo
);
assert
(
pQInfo
!=
NULL
);
int32_t
contLen
=
100
;
SRetrieveTableRsp
*
pRsp
=
(
SRetrieveTableRsp
*
)
rpcMallocCont
(
contLen
);
pRsp
->
numOfRows
=
0
;
pRsp
->
precision
=
0
;
pRsp
->
offset
=
0
;
pRsp
->
useconds
=
0
;
SRpcMsg
rpcRsp
=
{
.
handle
=
pMsg
->
rpcMsg
.
handle
,
.
pCont
=
pRsp
,
.
contLen
=
contLen
,
.
code
=
0
,
.
msgType
=
0
};
dTrace
(
"QInfo:%p vgId:%d, retrieve msg is received"
,
pQInfo
,
pRetrieve
->
header
.
vgId
);
int32_t
rowSize
=
0
;
int32_t
numOfRows
=
0
;
int32_t
contLen
=
0
;
SRpcMsg
rpcRsp
=
{
0
};
int32_t
code
=
qRetrieveQueryResultInfo
(
pQInfo
,
&
numOfRows
,
&
rowSize
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
contLen
=
sizeof
(
SRetrieveTableRsp
);
SRetrieveTableRsp
*
pRsp
=
(
SRetrieveTableRsp
*
)
rpcMallocCont
(
contLen
);
pRsp
->
numOfRows
=
0
;
pRsp
->
precision
=
0
;
pRsp
->
offset
=
0
;
pRsp
->
useconds
=
0
;
rpcRsp
=
(
SRpcMsg
)
{
.
handle
=
pMsg
->
rpcMsg
.
handle
,
.
pCont
=
pRsp
,
.
contLen
=
contLen
,
.
code
=
code
,
.
msgType
=
0
};
//todo free qinfo
}
else
{
contLen
=
100
;
SRetrieveTableRsp
*
pRsp
=
(
SRetrieveTableRsp
*
)
rpcMallocCont
(
contLen
);
pRsp
->
numOfRows
=
htonl
(
1
);
pRsp
->
precision
=
htons
(
0
);
pRsp
->
offset
=
htobe64
(
0
);
pRsp
->
useconds
=
htobe64
(
0
);
// todo set the data
*
(
int64_t
*
)
pRsp
->
data
=
1000
;
rpcRsp
=
(
SRpcMsg
)
{
.
handle
=
pMsg
->
rpcMsg
.
handle
,
.
pCont
=
pRsp
,
.
contLen
=
contLen
,
.
code
=
code
,
.
msgType
=
0
};
}
rpcSendResponse
(
&
rpcRsp
);
}
src/dnode/src/dnodeWrite.c
浏览文件 @
ddd05d9c
...
...
@@ -275,7 +275,12 @@ static void dnodeProcessSubmitMsg(SWriteMsg *pMsg) {
pRsp
->
numOfRows
=
htonl
(
1
);
pRsp
->
affectedRows
=
htonl
(
1
);
pRsp
->
numOfFailedBlocks
=
0
;
void
*
tsdb
=
dnodeGetVnodeTsdb
(
pMsg
->
pVnode
);
assert
(
tsdb
!=
NULL
);
tsdbInsertData
(
tsdb
,
pMsg
->
pCont
);
SRpcMsg
rpcRsp
=
{
.
handle
=
pMsg
->
rpcMsg
.
handle
,
.
pCont
=
pRsp
,
...
...
@@ -283,6 +288,7 @@ static void dnodeProcessSubmitMsg(SWriteMsg *pMsg) {
.
code
=
0
,
.
msgType
=
0
};
rpcSendResponse
(
&
rpcRsp
);
}
...
...
src/inc/taosmsg.h
浏览文件 @
ddd05d9c
...
...
@@ -188,21 +188,51 @@ extern char *taosMsg[];
#pragma pack(push, 1)
typedef
struct
{
int32_t
vnode
;
int32_t
sid
;
int32_t
sversion
;
uint64_t
uid
;
int16_t
numOfRows
;
char
payLoad
[];
}
SShellSubmitBlock
;
//
typedef struct {
//
int32_t vnode;
//
int32_t sid;
//
int32_t sversion;
//
uint64_t uid;
//
int16_t numOfRows;
//
char payLoad[];
//
} SShellSubmitBlock;
typedef
struct
{
int16_t
import
;
int16_t
vnode
;
int32_t
numOfSid
;
/* total number of sid */
char
blks
[];
/* numOfSid blocks, each blocks for one table */
}
SShellSubmitMsg
;
int32_t
numOfVnodes
;
}
SMsgDesc
;
typedef
struct
SMsgHead
{
int32_t
contLen
;
int32_t
vgId
;
}
SMsgHead
;
//typedef struct {
// SMsgDesc desc;
// SMsgHead header;
// int16_t import;
// int32_t numOfTables; // total number of sid
// char blks[]; // number of data blocks, each table has at least one data block
//} SShellSubmitMsg;
// Submit message for one table
typedef
struct
SSubmitBlk
{
int64_t
uid
;
// table unique id
int32_t
tid
;
// table id
int32_t
padding
;
// TODO just for padding here
int32_t
sversion
;
// data schema version
int32_t
len
;
// data part length, not including the SSubmitBlk head
int16_t
numOfRows
;
// total number of rows in current submit block
char
data
[];
}
SSubmitBlk
;
// Submit message for this TSDB
typedef
struct
SSubmitMsg
{
SMsgHead
header
;
int32_t
length
;
int32_t
compressed
:
2
;
int32_t
numOfBlocks
:
30
;
SSubmitBlk
blocks
[];
}
SSubmitMsg
;
typedef
struct
{
int32_t
index
;
// index of failed block in submit blocks
...
...
@@ -232,15 +262,6 @@ typedef struct {
uint32_t
ip
;
}
SVnodeDesc
;
typedef
struct
{
int32_t
numOfVnodes
;
}
SMsgDesc
;
typedef
struct
{
int32_t
contLen
;
int32_t
vgId
;
}
SMsgHead
;
typedef
struct
{
int32_t
contLen
;
int32_t
vgId
;
...
...
@@ -434,15 +455,11 @@ typedef struct SColumnInfo {
SColumnFilterInfo
*
filters
;
}
SColumnInfo
;
/*
* enable vnode to understand how to group several tables with different tag;
*/
typedef
struct
STableSidExtInfo
{
typedef
struct
STableIdInfo
{
int32_t
sid
;
int64_t
uid
;
TSKEY
key
;
// key for subscription
char
tags
[];
}
STableSidExtInfo
;
TSKEY
key
;
// last accessed ts, for subscription
}
STableIdInfo
;
typedef
struct
STimeWindow
{
TSKEY
skey
;
...
...
@@ -455,10 +472,10 @@ typedef struct STimeWindow {
* the outputCols will be 3 while the numOfCols is 1.
*/
typedef
struct
{
int16_t
vnode
;
int32_t
contLen
;
// msg header
int16_t
vgId
;
int32_t
numOfTables
;
uint64_t
pSidExtInfo
;
// table id & tag info ptr, in windows pointer may
uint64_t
uid
;
STimeWindow
window
;
...
...
@@ -504,19 +521,21 @@ typedef struct {
}
SQueryTableMsg
;
typedef
struct
{
char
code
;
int32_t
code
;
uint64_t
qhandle
;
}
SQueryTableRsp
;
typedef
struct
{
SMsgHead
header
;
uint64_t
qhandle
;
uint16_t
free
;
}
SRetrieveTableMsg
;
typedef
struct
{
typedef
struct
SRetrieveTableRsp
{
int32_t
numOfRows
;
int8_t
completed
;
// all results are returned to client
int16_t
precision
;
int64_t
offset
;
// updated offset value for multi-vnode projection query
int64_t
offset
;
// updated offset value for multi-vnode projection query
int64_t
useconds
;
char
data
[];
}
SRetrieveTableRsp
;
...
...
@@ -670,7 +689,7 @@ typedef struct {
SVnodeDesc
vpeerDesc
[
TSDB_VNODES_SUPPORT
];
int16_t
index
;
// used locally
int32_t
numOfSids
;
int32_t
pSidExtInfoList
[];
// offset value of STable
SidExt
Info
int32_t
pSidExtInfoList
[];
// offset value of STable
Id
Info
}
SVnodeSidList
;
typedef
struct
{
...
...
@@ -692,7 +711,7 @@ typedef struct STableMetaMsg {
int8_t
numOfVpeers
;
SVnodeDesc
vpeerDesc
[
TSDB_VNODES_SUPPORT
];
int32_t
sid
;
int32_t
vg
i
d
;
int32_t
vg
I
d
;
uint64_t
uid
;
SSchema
schema
[];
}
STableMetaMsg
;
...
...
src/mnode/src/mgmtChildTable.c
浏览文件 @
ddd05d9c
...
...
@@ -413,7 +413,7 @@ int32_t mgmtModifyChildTableTagValueByName(SChildTableObj *pTable, char *tagName
int32_t
mgmtGetChildTableMeta
(
SDbObj
*
pDb
,
SChildTableObj
*
pTable
,
STableMetaMsg
*
pMeta
,
bool
usePublicIp
)
{
pMeta
->
uid
=
htobe64
(
pTable
->
uid
);
pMeta
->
sid
=
htonl
(
pTable
->
sid
);
pMeta
->
vg
i
d
=
htonl
(
pTable
->
vgId
);
pMeta
->
vg
I
d
=
htonl
(
pTable
->
vgId
);
pMeta
->
sversion
=
htons
(
pTable
->
superTable
->
sversion
);
pMeta
->
precision
=
pDb
->
cfg
.
precision
;
pMeta
->
numOfTags
=
pTable
->
superTable
->
numOfTags
;
...
...
src/mnode/src/mgmtDnode.c
浏览文件 @
ddd05d9c
...
...
@@ -553,11 +553,14 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
pDnode
->
privateIp
=
htonl
(
pStatus
->
privateIp
);
pDnode
->
publicIp
=
htonl
(
pStatus
->
publicIp
);
pDnode
->
lastReboot
=
htonl
(
pStatus
->
lastReboot
);
pDnode
->
numOfTotalVnodes
=
htons
(
pStatus
->
numOfTotalVnodes
);
pDnode
->
numOfCores
=
htons
(
pStatus
->
numOfCores
);
pDnode
->
diskAvailable
=
pStatus
->
diskAvailable
;
pDnode
->
alternativeRole
=
pStatus
->
alternativeRole
;
if
(
pDnode
->
numOfTotalVnodes
==
0
)
{
pDnode
->
numOfTotalVnodes
=
htons
(
pStatus
->
numOfTotalVnodes
);
}
if
(
pStatus
->
dnodeId
==
0
)
{
mTrace
(
"dnode:%d, first access, privateIp:%s, name:%s, "
,
pDnode
->
dnodeId
,
taosIpStr
(
pDnode
->
privateIp
),
pDnode
->
dnodeName
);
mgmtSetDnodeMaxVnodes
(
pDnode
);
...
...
src/mnode/src/mgmtNormalTable.c
浏览文件 @
ddd05d9c
...
...
@@ -493,7 +493,7 @@ static int32_t mgmtSetSchemaFromNormalTable(SSchema *pSchema, SNormalTableObj *p
int32_t
mgmtGetNormalTableMeta
(
SDbObj
*
pDb
,
SNormalTableObj
*
pTable
,
STableMetaMsg
*
pMeta
,
bool
usePublicIp
)
{
pMeta
->
uid
=
htobe64
(
pTable
->
uid
);
pMeta
->
sid
=
htonl
(
pTable
->
sid
);
pMeta
->
vg
i
d
=
htonl
(
pTable
->
vgId
);
pMeta
->
vg
I
d
=
htonl
(
pTable
->
vgId
);
pMeta
->
sversion
=
htons
(
pTable
->
sversion
);
pMeta
->
precision
=
pDb
->
cfg
.
precision
;
pMeta
->
numOfTags
=
0
;
...
...
src/mnode/src/mgmtSuperTable.c
浏览文件 @
ddd05d9c
...
...
@@ -622,7 +622,7 @@ int32_t mgmtSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) {
int32_t
mgmtGetSuperTableMeta
(
SDbObj
*
pDb
,
SSuperTableObj
*
pTable
,
STableMetaMsg
*
pMeta
,
bool
usePublicIp
)
{
pMeta
->
uid
=
htobe64
(
pTable
->
uid
);
pMeta
->
sid
=
htonl
(
pTable
->
sid
);
pMeta
->
vg
i
d
=
htonl
(
pTable
->
vgId
);
pMeta
->
vg
I
d
=
htonl
(
pTable
->
vgId
);
pMeta
->
sversion
=
htons
(
pTable
->
sversion
);
pMeta
->
precision
=
pDb
->
cfg
.
precision
;
pMeta
->
numOfTags
=
pTable
->
numOfTags
;
...
...
src/query/CMakeLists.txt
浏览文件 @
ddd05d9c
...
...
@@ -11,5 +11,5 @@ INCLUDE_DIRECTORIES(inc)
IF
((
TD_LINUX_64
)
OR
(
TD_LINUX_32 AND TD_ARM
))
AUX_SOURCE_DIRECTORY
(
src SRC
)
ADD_LIBRARY
(
query
${
SRC
}
)
TARGET_LINK_LIBRARIES
(
query tutil m rt
)
TARGET_LINK_LIBRARIES
(
query t
sdb t
util m rt
)
ENDIF
()
\ No newline at end of file
src/query/inc/qextbuffer.h
浏览文件 @
ddd05d9c
...
...
@@ -124,9 +124,8 @@ typedef struct tTagSchema {
typedef
struct
tSidSet
{
int32_t
numOfSids
;
int32_t
numOfSubSet
;
STable
SidExtInfo
**
pSids
;
STable
IdInfo
**
pTableIdList
;
int32_t
*
starterPos
;
// position of each subgroup, generated according to
SColumnModel
*
pColumnModel
;
SColumnOrderInfo
orderIdx
;
}
tSidSet
;
...
...
src/query/inc/queryExecutor.h
浏览文件 @
ddd05d9c
...
...
@@ -25,6 +25,7 @@
#include "taosdef.h"
#include "tref.h"
#include "tsqlfunction.h"
#include "tarray.h"
typedef
struct
SData
{
int32_t
num
;
...
...
@@ -39,7 +40,7 @@ enum {
struct
SColumnFilterElem
;
typedef
bool
(
*
__filter_func_t
)(
struct
SColumnFilterElem
*
pFilter
,
char
*
val1
,
char
*
val2
);
typedef
int
(
*
__block_search_fn_t
)(
char
*
data
,
int
num
,
int64_t
key
,
in
t
order
);
typedef
int
32_t
(
*
__block_search_fn_t
)(
char
*
data
,
int32_t
num
,
int64_t
key
,
int32_
t
order
);
typedef
struct
SSqlGroupbyExpr
{
int16_t
tableIndex
;
...
...
@@ -142,7 +143,7 @@ typedef struct SQuery {
int32_t
pos
;
int64_t
pointsOffset
;
// the number of points offset to save read data
SData
**
sdata
;
int32_t
capacity
;
SSingleColumnFilterInfo
*
pFilterInfo
;
}
SQuery
;
...
...
@@ -152,7 +153,6 @@ typedef struct SQueryCostSummary {
typedef
struct
SQueryRuntimeEnv
{
SResultInfo
*
resultInfo
;
// todo refactor to merge with SWindowResInfo
SQuery
*
pQuery
;
void
*
pTabObj
;
SData
**
pInterpoBuf
;
SQLFunctionCtx
*
pCtx
;
int16_t
numOfRowsPerPage
;
...
...
@@ -171,19 +171,21 @@ typedef struct SQueryRuntimeEnv {
typedef
struct
SQInfo
{
uint64_t
signature
;
void
*
pVnode
;
TSKEY
startTime
;
int64_t
elapsedTime
;
SResultRec
rec
;
int
pointsReturned
;
int
pointsInterpo
;
int
code
;
// error code to returned to client
int32_t
pointsReturned
;
int32_t
pointsInterpo
;
int32_t
code
;
// error code to returned to client
int32_t
killed
;
// denotes if current query is killed
sem_t
dataReady
;
S
HashObj
*
pTable
List
;
// table list
S
Array
*
pTableId
List
;
// table list
SQueryRuntimeEnv
runtimeEnv
;
int32_t
subgroupIdx
;
int32_t
offset
;
/* offset in group result set of subgroup */
tSidSet
*
pSidSet
;
//
tSidSet* pSidSet;
T_REF_DECLARE
()
/*
* the query is executed position on which meter of the whole list.
...
...
@@ -204,13 +206,13 @@ typedef struct SQInfo {
* @param pQInfo
* @return
*/
int32_t
qCreateQueryInfo
(
void
*
p
Read
Msg
,
SQInfo
**
pQInfo
);
int32_t
qCreateQueryInfo
(
void
*
p
Vnode
,
SQueryTableMsg
*
pQueryTable
Msg
,
SQInfo
**
pQInfo
);
/**
* query on single table
* @param pReadMsg
*/
void
qTableQuery
(
void
*
pReadMsg
);
void
qTableQuery
(
SQInfo
*
pQInfo
);
/**
* query on super table
...
...
@@ -218,4 +220,13 @@ void qTableQuery(void* pReadMsg);
*/
void
qSuperTableQuery
(
void
*
pReadMsg
);
/**
* wait for the query completed, and retrieve final results to client
* @param pQInfo
*/
int32_t
qRetrieveQueryResultInfo
(
SQInfo
*
pQInfo
,
int32_t
*
numOfRows
,
int32_t
*
rowsize
);
//int32_t qBuildQueryResult(SQInfo* pQInfo, void* pBuf);
#endif // TDENGINE_QUERYEXECUTOR_H
src/query/src/queryExecutor.c
浏览文件 @
ddd05d9c
此差异已折叠。
点击以展开。
src/rpc/src/rpcMain.c
浏览文件 @
ddd05d9c
...
...
@@ -287,7 +287,7 @@ void rpcClose(void *param) {
(
*
taosCleanUpConn
[
pRpc
->
connType
])(
pRpc
->
udphandle
);
for
(
int
i
=
0
;
i
<
pRpc
->
sessions
;
++
i
)
{
if
(
pRpc
->
connList
[
i
].
user
[
0
])
{
if
(
pRpc
->
connList
&&
pRpc
->
connList
[
i
].
user
[
0
])
{
rpcCloseConn
((
void
*
)(
pRpc
->
connList
+
i
));
}
}
...
...
@@ -495,35 +495,35 @@ static void rpcCloseConn(void *thandle) {
SRpcConn
*
pConn
=
(
SRpcConn
*
)
thandle
;
SRpcInfo
*
pRpc
=
pConn
->
pRpc
;
rpcLockConn
(
pConn
)
;
if
(
pConn
->
user
[
0
]
==
0
)
return
;
if
(
pConn
->
user
[
0
])
{
pConn
->
user
[
0
]
=
0
;
if
(
taosCloseConn
[
pConn
->
connType
])
(
*
taosCloseConn
[
pConn
->
connType
])(
pConn
->
chandle
);
taosTmrStopA
(
&
pConn
->
pTimer
);
taosTmrStopA
(
&
pConn
->
pIdleTimer
);
if
(
pRpc
->
connType
==
TAOS_CONN_SERVER
)
{
char
hashstr
[
40
]
=
{
0
};
sprintf
(
hashstr
,
"%x:%x:%x:%d"
,
pConn
->
peerIp
,
pConn
->
linkUid
,
pConn
->
peerId
,
pConn
->
connType
);
taosDeleteStrHash
(
pRpc
->
hash
,
hashstr
);
rpcFreeMsg
(
pConn
->
pRspMsg
);
// it may have a response msg saved, but not request msg
pConn
->
pRspMsg
=
NULL
;
pConn
->
inType
=
0
;
pConn
->
inTranId
=
0
;
}
else
{
pConn
->
outType
=
0
;
pConn
->
outTranId
=
0
;
pConn
->
pReqMsg
=
NULL
;
}
rpcLockConn
(
pConn
);
taosFreeId
(
pRpc
->
idPool
,
pConn
->
sid
)
;
pConn
->
pContext
=
NULL
;
pConn
->
user
[
0
]
=
0
;
if
(
taosCloseConn
[
pConn
->
connType
])
(
*
taosCloseConn
[
pConn
->
connType
])(
pConn
->
chandle
)
;
tTrace
(
"%s %p, rpc connection is closed"
,
pRpc
->
label
,
pConn
);
taosTmrStopA
(
&
pConn
->
pTimer
);
taosTmrStopA
(
&
pConn
->
pIdleTimer
);
if
(
pRpc
->
connType
==
TAOS_CONN_SERVER
)
{
char
hashstr
[
40
]
=
{
0
};
sprintf
(
hashstr
,
"%x:%x:%x:%d"
,
pConn
->
peerIp
,
pConn
->
linkUid
,
pConn
->
peerId
,
pConn
->
connType
);
taosDeleteStrHash
(
pRpc
->
hash
,
hashstr
);
rpcFreeMsg
(
pConn
->
pRspMsg
);
// it may have a response msg saved, but not request msg
pConn
->
pRspMsg
=
NULL
;
pConn
->
inType
=
0
;
pConn
->
inTranId
=
0
;
}
else
{
pConn
->
outType
=
0
;
pConn
->
outTranId
=
0
;
pConn
->
pReqMsg
=
NULL
;
}
taosFreeId
(
pRpc
->
idPool
,
pConn
->
sid
);
pConn
->
pContext
=
NULL
;
tTrace
(
"%s %p, rpc connection is closed"
,
pRpc
->
label
,
pConn
);
rpcUnlockConn
(
pConn
);
}
...
...
src/util/inc/tlist.h
0 → 100644
浏览文件 @
ddd05d9c
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIST_
#define _TD_LIST_
#ifdef __cplusplus
extern
"C"
{
#endif
typedef
enum
{
TD_LIST_FORWARD
,
TD_LIST_BACKWARD
}
TD_LIST_DIRECTION_T
;
typedef
struct
_list_node
{
struct
_list_node
*
next
;
struct
_list_node
*
prev
;
char
data
[];
}
SListNode
;
typedef
struct
{
struct
_list_node
*
head
;
struct
_list_node
*
tail
;
int
numOfEles
;
int
eleSize
;
}
SList
;
typedef
struct
{
SListNode
*
next
;
TD_LIST_DIRECTION_T
direction
;
}
SListIter
;
#define listHead(l) (l)->head
#define listTail(l) (l)->tail
#define listNEles(l) (l)->numOfEles
#define listEleSize(l) (l)->eleSize
#define isListEmpty(l) ((l)->numOfEles == 0)
#define listNodeFree(n) free(n);
SList
*
tdListNew
(
int
eleSize
);
void
tdListFree
(
SList
*
list
);
void
tdListEmpty
(
SList
*
list
);
void
tdListPrependNode
(
SList
*
list
,
SListNode
*
node
);
void
tdListAppendNode
(
SList
*
list
,
SListNode
*
node
);
int
tdListPrepend
(
SList
*
list
,
void
*
data
);
int
tdListAppend
(
SList
*
list
,
void
*
data
);
SListNode
*
tdListPopHead
(
SList
*
list
);
SListNode
*
tdListPopTail
(
SList
*
list
);
SListNode
*
tdListPopNode
(
SList
*
list
,
SListNode
*
node
);
void
tdListMove
(
SList
*
src
,
SList
*
dst
);
void
tdListNodeGetData
(
SList
*
list
,
SListNode
*
node
,
void
*
target
);
void
tdListInitIter
(
SList
*
list
,
SListIter
*
pIter
,
TD_LIST_DIRECTION_T
direction
);
SListNode
*
tdListNext
(
SListIter
*
pIter
);
#ifdef __cplusplus
}
#endif
#endif
\ No newline at end of file
src/util/src/tlist.c
0 → 100644
浏览文件 @
ddd05d9c
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stdlib.h>
#include <string.h>
#include "tlist.h"
SList
*
tdListNew
(
int
eleSize
)
{
SList
*
list
=
(
SList
*
)
malloc
(
sizeof
(
SList
));
if
(
list
==
NULL
)
return
NULL
;
list
->
eleSize
=
eleSize
;
list
->
numOfEles
=
0
;
list
->
head
=
list
->
tail
=
NULL
;
return
list
;
}
void
tdListEmpty
(
SList
*
list
)
{
SListNode
*
node
=
list
->
head
;
while
(
node
)
{
list
->
head
=
node
->
next
;
free
(
node
);
node
=
list
->
head
;
}
list
->
head
=
list
->
tail
=
0
;
list
->
numOfEles
=
0
;
}
void
tdListFree
(
SList
*
list
)
{
tdListEmpty
(
list
);
free
(
list
);
}
void
tdListPrependNode
(
SList
*
list
,
SListNode
*
node
)
{
if
(
list
->
head
==
NULL
)
{
list
->
head
=
node
;
list
->
tail
=
node
;
}
else
{
node
->
next
=
list
->
head
;
node
->
prev
=
NULL
;
list
->
head
->
prev
=
node
;
list
->
head
=
node
;
}
list
->
numOfEles
++
;
}
void
tdListAppendNode
(
SList
*
list
,
SListNode
*
node
)
{
if
(
list
->
head
==
NULL
)
{
list
->
head
=
node
;
list
->
tail
=
node
;
}
else
{
node
->
prev
=
list
->
tail
;
node
->
next
=
NULL
;
list
->
tail
->
next
=
node
;
list
->
tail
=
node
;
}
list
->
numOfEles
++
;
}
int
tdListPrepend
(
SList
*
list
,
void
*
data
)
{
SListNode
*
node
=
(
SListNode
*
)
malloc
(
sizeof
(
SListNode
)
+
list
->
eleSize
);
if
(
node
==
NULL
)
return
-
1
;
memcpy
((
void
*
)(
node
->
data
),
data
,
list
->
eleSize
);
tdListPrependNode
(
list
,
node
);
return
0
;
}
int
tdListAppend
(
SList
*
list
,
void
*
data
)
{
SListNode
*
node
=
(
SListNode
*
)
malloc
(
sizeof
(
SListNode
)
+
list
->
eleSize
);
if
(
node
==
NULL
)
return
-
1
;
memcpy
((
void
*
)(
node
->
data
),
data
,
list
->
eleSize
);
tdListAppendNode
(
list
,
node
);
return
0
;
}
SListNode
*
tdListPopHead
(
SList
*
list
)
{
if
(
list
->
head
==
NULL
)
return
NULL
;
SListNode
*
node
=
list
->
head
;
if
(
node
->
next
==
NULL
)
{
list
->
head
=
NULL
;
list
->
tail
=
NULL
;
}
else
{
list
->
head
=
node
->
next
;
}
list
->
numOfEles
--
;
return
node
;
}
SListNode
*
tdListPopTail
(
SList
*
list
)
{
if
(
list
->
tail
==
NULL
)
return
NULL
;
SListNode
*
node
=
list
->
tail
;
if
(
node
->
prev
==
NULL
)
{
list
->
head
=
NULL
;
list
->
tail
=
NULL
;
}
else
{
list
->
tail
=
node
->
prev
;
}
list
->
numOfEles
--
;
return
node
;
}
SListNode
*
tdListPopNode
(
SList
*
list
,
SListNode
*
node
)
{
if
(
list
->
head
==
node
)
{
list
->
head
=
node
->
next
;
}
if
(
list
->
tail
==
node
)
{
list
->
tail
=
node
->
prev
;
}
if
(
node
->
prev
!=
NULL
)
{
node
->
prev
->
next
=
node
->
next
;
}
if
(
node
->
next
!=
NULL
)
{
node
->
next
->
prev
=
node
->
prev
;
}
list
->
numOfEles
--
;
return
node
;
}
// Move all node elements from src to dst, the dst is assumed as an empty list
void
tdListMove
(
SList
*
src
,
SList
*
dst
)
{
// assert(dst->eleSize == src->eleSize);
dst
->
numOfEles
=
src
->
numOfEles
;
dst
->
head
=
src
->
head
;
dst
->
tail
=
src
->
tail
;
src
->
numOfEles
=
0
;
src
->
head
=
src
->
tail
=
NULL
;
}
void
tdListNodeGetData
(
SList
*
list
,
SListNode
*
node
,
void
*
target
)
{
memcpy
(
target
,
node
->
data
,
list
->
eleSize
);
}
void
tdListInitIter
(
SList
*
list
,
SListIter
*
pIter
,
TD_LIST_DIRECTION_T
direction
)
{
pIter
->
direction
=
direction
;
if
(
direction
==
TD_LIST_FORWARD
)
{
pIter
->
next
=
list
->
head
;
}
else
{
pIter
->
next
=
list
->
tail
;
}
}
SListNode
*
tdListNext
(
SListIter
*
pIter
)
{
SListNode
*
node
=
pIter
->
next
;
if
(
node
==
NULL
)
return
NULL
;
if
(
pIter
->
direction
==
TD_LIST_FORWARD
)
{
pIter
->
next
=
node
->
next
;
}
else
{
pIter
->
next
=
node
->
prev
;
}
return
node
;
}
\ No newline at end of file
src/util/src/tskiplist.c
浏览文件 @
ddd05d9c
...
...
@@ -15,7 +15,6 @@
#include "os.h"
#include "tlog.h"
// #include "tsdb.h"
#include "tskiplist.h"
#include "tutil.h"
...
...
@@ -395,6 +394,7 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) {
SSkipListNode
*
px
=
pSkipList
->
pHead
;
SSkipListNode
*
forward
[
MAX_SKIP_LIST_LEVEL
]
=
{
0
};
bool
identical
=
false
;
for
(
int32_t
i
=
pSkipList
->
level
-
1
;
i
>=
0
;
--
i
)
{
SSkipListNode
*
p
=
SL_GET_FORWARD_POINTER
(
px
,
i
);
while
(
p
!=
NULL
)
{
...
...
@@ -402,11 +402,16 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) {
char
*
newDatakey
=
SL_GET_NODE_KEY
(
pSkipList
,
pNode
);
// if the forward element is less than the specified key, forward one step
if
(
pSkipList
->
comparFn
(
key
,
newDatakey
)
<
0
)
{
int32_t
ret
=
pSkipList
->
comparFn
(
key
,
newDatakey
);
if
(
ret
<
0
)
{
px
=
p
;
p
=
SL_GET_FORWARD_POINTER
(
px
,
i
);
}
else
{
if
(
identical
==
false
)
{
identical
=
(
ret
==
0
);
}
break
;
}
}
...
...
@@ -418,17 +423,12 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) {
}
// if the skip list does not allowed identical key inserted, the new data will be discarded.
if
(
pSkipList
->
keyInfo
.
dupKey
==
0
&&
forward
[
0
]
!=
pSkipList
->
pHead
)
{
char
*
key
=
SL_GET_NODE_KEY
(
pSkipList
,
forward
[
0
]);
char
*
pNewDataKey
=
SL_GET_NODE_KEY
(
pSkipList
,
pNode
);
if
(
pSkipList
->
comparFn
(
key
,
pNewDataKey
)
==
0
)
{
if
(
pSkipList
->
lock
)
{
pthread_rwlock_unlock
(
pSkipList
->
lock
);
}
return
forward
[
0
];
if
(
pSkipList
->
keyInfo
.
dupKey
==
0
&&
identical
)
{
if
(
pSkipList
->
lock
)
{
pthread_rwlock_unlock
(
pSkipList
->
lock
);
}
return
forward
[
0
];
}
#if SKIP_LIST_RECORD_PERFORMANCE
...
...
src/vnode/tsdb/inc/tsdb.h
浏览文件 @
ddd05d9c
...
...
@@ -58,6 +58,9 @@ int32_t tsdbDropRepo(tsdb_repo_t *repo);
tsdb_repo_t
*
tsdbOpenRepo
(
char
*
tsdbDir
);
int32_t
tsdbCloseRepo
(
tsdb_repo_t
*
repo
);
int32_t
tsdbConfigRepo
(
tsdb_repo_t
*
repo
,
STsdbCfg
*
pCfg
);
int32_t
tsdbTriggerCommit
(
tsdb_repo_t
*
repo
);
int32_t
tsdbLockRepo
(
tsdb_repo_t
*
repo
);
int32_t
tsdbUnLockRepo
(
tsdb_repo_t
*
repo
);
// --------- TSDB TABLE DEFINITION
typedef
struct
{
...
...
@@ -87,15 +90,6 @@ int tsdbCreateTable(tsdb_repo_t *repo, STableCfg *pCfg);
int
tsdbDropTable
(
tsdb_repo_t
*
pRepo
,
STableId
tableId
);
int
tsdbAlterTable
(
tsdb_repo_t
*
repo
,
STableCfg
*
pCfg
);
// Submit message for one table
typedef
struct
{
STableId
tableId
;
int32_t
padding
;
// TODO just for padding here
int32_t
sversion
;
// data schema version
int32_t
len
;
// data part length, not including the SSubmitBlk head
char
data
[];
}
SSubmitBlk
;
typedef
struct
{
int32_t
totalLen
;
int32_t
len
;
...
...
@@ -105,15 +99,10 @@ typedef struct {
int
tsdbInitSubmitBlkIter
(
SSubmitBlk
*
pBlock
,
SSubmitBlkIter
*
pIter
);
SDataRow
tsdbGetSubmitBlkNext
(
SSubmitBlkIter
*
pIter
);
// Submit message for this TSDB
typedef
struct
{
int32_t
length
;
int32_t
compressed
;
SSubmitBlk
blocks
[];
}
SSubmitMsg
;
#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
struct
STsdbRepo
;
// SSubmitMsg Iterator
typedef
struct
{
int32_t
totalLen
;
...
...
@@ -242,7 +231,7 @@ typedef void *tsdbpos_t;
* @param pTableList table sid list
* @return
*/
tsdb_query_handle_t
*
tsdbQueryByTableId
(
STsdbQueryCond
*
pCond
,
SArray
*
idList
,
SArray
*
pColumnInfo
);
tsdb_query_handle_t
*
tsdbQueryByTableId
(
tsdb_repo_t
*
tsdb
,
STsdbQueryCond
*
pCond
,
SArray
*
idList
,
SArray
*
pColumnInfo
);
/**
* move to next block
...
...
src/vnode/tsdb/inc/tsdbCache.h
浏览文件 @
ddd05d9c
...
...
@@ -17,45 +17,50 @@
#include <stdint.h>
// #include "cache.h"
#include "taosdef.h"
#include "tlist.h"
#include "tsdb.h"
#ifdef __cplusplus
extern
"C"
{
#endif
#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16
*1024*
1024
/* 16M */
#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16
* 1024 *
1024
/* 16M */
typedef
struct
{
int64_t
skey
;
// start key
int64_t
ekey
;
// end key
int32_t
numOfRows
;
// numOfRows
}
STableCacheInfo
;
int
blockId
;
int
offset
;
int
remain
;
int
padding
;
char
data
[];
}
STsdbCacheBlock
;
typedef
struct
_tsdb_cache_block
{
char
*
pData
;
STableCacheInfo
*
pTableInfo
;
struct
_tsdb_cache_block
*
prev
;
struct
_tsdb_cache_block
*
next
;
}
STSDBCacheBlock
;
typedef
struct
{
int64_t
index
;
SList
*
memPool
;
}
STsdbCachePool
;
// Use a doublely linked list to implement this
typedef
struct
STSDBCache
{
// Number of blocks the cache is allocated
int32_t
numOfBlocks
;
STSDBCacheBlock
*
cacheList
;
void
*
current
;
}
STsdbCache
;
typedef
struct
{
TSKEY
keyFirst
;
TSKEY
keyLast
;
int64_t
numOfPoints
;
SList
*
list
;
}
SCacheMem
;
// ---- Operation on STSDBCacheBlock
#define TSDB_CACHE_BLOCK_DATA(pBlock) ((pBlock)->pData)
#define TSDB_CACHE_AVAIL_SPACE(pBlock) ((char *)((pBlock)->pTableInfo) - ((pBlock)->pData))
#define TSDB_TABLE_INFO_OF_CACHE(pBlock, tableId) ((pBlock)->pTableInfo)[tableId]
#define TSDB_NEXT_CACHE_BLOCK(pBlock) ((pBlock)->next)
#define TSDB_PREV_CACHE_BLOCK(pBlock) ((pBlock)->prev)
typedef
struct
{
int
maxBytes
;
int
cacheBlockSize
;
int
totalCacheBlocks
;
STsdbCachePool
pool
;
STsdbCacheBlock
*
curBlock
;
SCacheMem
*
mem
;
SCacheMem
*
imem
;
tsdb_repo_t
*
pRepo
;
}
STsdbCache
;
STsdbCache
*
tsdbInitCache
(
int
64_t
maxSize
);
int32_t
tsdbFreeCache
(
STsdbCache
*
pCache
);
void
*
tsdbAllocFromCache
(
STsdbCache
*
pCache
,
int
64_t
bytes
);
STsdbCache
*
tsdbInitCache
(
int
maxBytes
,
int
cacheBlockSize
,
tsdb_repo_t
*
pRepo
);
void
tsdbFreeCache
(
STsdbCache
*
pCache
);
void
*
tsdbAllocFromCache
(
STsdbCache
*
pCache
,
int
bytes
,
TSKEY
key
);
#ifdef __cplusplus
}
...
...
src/vnode/tsdb/inc/tsdbFile.h
浏览文件 @
ddd05d9c
...
...
@@ -18,16 +18,20 @@
#include <stdint.h>
#include "taosdef.h"
#include "tglobalcfg.h"
#ifdef __cplusplus
extern
"C"
{
#endif
#define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile))
#define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3)
typedef
enum
{
TSDB_FILE_TYPE_HEAD
,
// .head file type
TSDB_FILE_TYPE_DATA
,
// .data file type
TSDB_FILE_TYPE_LAST
,
// .last file type
TSDB_FILE_TYPE_M
ETA
// .meta file type
TSDB_FILE_TYPE_HEAD
=
0
,
// .head file type
TSDB_FILE_TYPE_DATA
,
// .data file type
TSDB_FILE_TYPE_LAST
,
// .last file type
TSDB_FILE_TYPE_M
AX
}
TSDB_FILE_TYPE
;
extern
const
char
*
tsdbFileSuffix
[];
...
...
@@ -38,16 +42,15 @@ typedef struct {
}
SFileInfo
;
typedef
struct
{
int
fd
;
int64_t
size
;
// total size of the file
int64_t
tombSize
;
// unused file size
int8_t
type
;
char
fname
[
128
];
int64_t
size
;
// total size of the file
int64_t
tombSize
;
// unused file size
}
SFile
;
typedef
struct
{
int32_t
fileId
;
SFile
fhead
;
SFile
fdata
;
SFile
flast
;
SFile
files
[
TSDB_FILE_TYPE_MAX
];
}
SFileGroup
;
// TSDB file handle
...
...
@@ -56,17 +59,38 @@ typedef struct {
int32_t
keep
;
int32_t
minRowPerFBlock
;
int32_t
maxRowsPerFBlock
;
int32_t
maxTables
;
SFileGroup
fGroup
[];
}
STsdbFileH
;
#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) <= TSDB_FILE_TYPE_META)
/**
* if numOfSubBlocks == -1, then the SCompBlock is a sub-block
* if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to
* the data block offset and length
* if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the
* binary
*/
typedef
struct
{
int64_t
last
:
1
;
// If the block in data file or last file
int64_t
offset
:
63
;
// Offset of data block or sub-block index depending on numOfSubBlocks
int32_t
algorithm
:
8
;
// Compression algorithm
int32_t
numOfPoints
:
24
;
// Number of total points
int32_t
sversion
;
// Schema version
int32_t
len
;
// Data block length or nothing
int16_t
numOfSubBlocks
;
// Number of sub-blocks;
int16_t
numOfCols
;
TSKEY
keyFirst
;
TSKEY
keyLast
;
}
SCompBlock
;
STsdbFileH
*
tsdbInitFile
(
char
*
dataDir
,
int32_t
daysPerFile
,
int32_t
keep
,
int32_t
minRowsPerFBlock
,
int32_t
maxRowsPerFBlock
);
void
tsdbCloseFile
(
STsdbFileH
*
pFileH
);
#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) < TSDB_FILE_TYPE_MAX)
char
*
tsdbGetFileName
(
char
*
dirName
,
char
*
fname
,
TSDB_FILE_TYPE
type
);
STsdbFileH
*
tsdbInitFile
(
char
*
dataDir
,
int32_t
daysPerFile
,
int32_t
keep
,
int32_t
minRowsPerFBlock
,
int32_t
maxRowsPerFBlock
,
int32_t
maxTables
);
void
tsdbCloseFile
(
STsdbFileH
*
pFileH
);
int
tsdbCreateFileGroup
(
char
*
dataDir
,
int
fileId
,
SFileGroup
*
pFGroup
,
int
maxTables
);
void
tsdbGetKeyRangeOfFileId
(
int32_t
daysPerFile
,
int8_t
precision
,
int32_t
fileId
,
TSKEY
*
minKey
,
TSKEY
*
maxKey
);
#ifdef __cplusplus
}
#endif
...
...
src/vnode/tsdb/inc/tsdbMeta.h
浏览文件 @
ddd05d9c
...
...
@@ -33,22 +33,28 @@ extern "C" {
#define IS_CREATE_STABLE(pCfg) ((pCfg)->tagValues != NULL)
typedef
struct
{
TSKEY
keyFirst
;
TSKEY
keyLast
;
int32_t
numOfPoints
;
void
*
pData
;
}
SMemTable
;
// ---------- TSDB TABLE DEFINITION
typedef
struct
STable
{
int8_t
type
;
STableId
tableId
;
int32_t
superUid
;
// Super table UID
int32_t
sversion
;
STSchema
*
schema
;
STSchema
*
tagSchema
;
SDataRow
tagVal
;
union
{
void
*
pData
;
// For TSDB_NORMAL_TABLE and TSDB_CHILD_TABLE, it is the skiplist for cache data
void
*
pIndex
;
// For TSDB_SUPER_TABLE, it is the skiplist index
}
content
;
int8_t
type
;
STableId
tableId
;
int64_t
superUid
;
// Super table UID
int32_t
sversion
;
STSchema
*
schema
;
STSchema
*
tagSchema
;
SDataRow
tagVal
;
SMemTable
*
mem
;
SMemTable
*
imem
;
void
*
pIndex
;
// For TSDB_SUPER_TABLE, it is the skiplist index
void
*
eventHandler
;
// TODO
void
*
streamHandler
;
// TODO
struct
STable
*
next
;
// TODO: remove the next
struct
STable
*
next
;
// TODO: remove the next
}
STable
;
void
*
tsdbEncodeTable
(
STable
*
pTable
,
int
*
contLen
);
...
...
@@ -68,6 +74,8 @@ typedef struct {
void
*
map
;
// table map of (uid ===> table)
SMetaFile
*
mfh
;
// meta file handle
int
maxRowBytes
;
int
maxCols
;
}
STsdbMeta
;
STsdbMeta
*
tsdbInitMeta
(
const
char
*
rootDir
,
int32_t
maxTables
);
...
...
@@ -90,11 +98,14 @@ int32_t tsdbFreeMeta(STsdbMeta *pMeta);
#define TSDB_TABLE_OF_ID(pHandle, id) ((pHandle)->pTables)[id]
#define TSDB_GET_TABLE_OF_NAME(pHandle, name)
/* TODO */
STsdbMeta
*
tsdbGetMeta
(
tsdb_repo_t
*
pRepo
);
int32_t
tsdbCreateTableImpl
(
STsdbMeta
*
pMeta
,
STableCfg
*
pCfg
);
int32_t
tsdbDropTableImpl
(
STsdbMeta
*
pMeta
,
STableId
tableId
);
STable
*
tsdbIsValidTableToInsert
(
STsdbMeta
*
pMeta
,
STableId
tableId
);
int32_t
tsdbInsertRowToTableImpl
(
SSkipListNode
*
pNode
,
STable
*
pTable
);
//
int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable);
STable
*
tsdbGetTableByUid
(
STsdbMeta
*
pMeta
,
int64_t
uid
);
char
*
getTupleKey
(
const
void
*
data
);
#ifdef __cplusplus
}
...
...
src/vnode/tsdb/src/tsdbCache.c
浏览文件 @
ddd05d9c
...
...
@@ -14,24 +14,126 @@
*/
#include <stdlib.h>
#include "tsdb.h"
#include "tsdbCache.h"
STsdbCache
*
tsdbInitCache
(
int64_t
maxSize
)
{
STsdbCache
*
pCacheHandle
=
(
STsdbCache
*
)
malloc
(
sizeof
(
STsdbCache
));
if
(
pCacheHandle
==
NULL
)
{
// TODO : deal with the error
return
NULL
;
static
int
tsdbAllocBlockFromPool
(
STsdbCache
*
pCache
);
static
void
tsdbFreeBlockList
(
SCacheMem
*
mem
);
STsdbCache
*
tsdbInitCache
(
int
maxBytes
,
int
cacheBlockSize
,
tsdb_repo_t
*
pRepo
)
{
STsdbCache
*
pCache
=
(
STsdbCache
*
)
calloc
(
1
,
sizeof
(
STsdbCache
));
if
(
pCache
==
NULL
)
return
NULL
;
if
(
cacheBlockSize
<
0
)
cacheBlockSize
=
TSDB_DEFAULT_CACHE_BLOCK_SIZE
;
pCache
->
maxBytes
=
maxBytes
;
pCache
->
cacheBlockSize
=
cacheBlockSize
;
pCache
->
pRepo
=
pRepo
;
int
nBlocks
=
maxBytes
/
cacheBlockSize
+
1
;
if
(
nBlocks
<=
1
)
nBlocks
=
2
;
pCache
->
totalCacheBlocks
=
nBlocks
;
STsdbCachePool
*
pPool
=
&
(
pCache
->
pool
);
pPool
->
index
=
0
;
pPool
->
memPool
=
tdListNew
(
sizeof
(
STsdbCacheBlock
*
));
if
(
pPool
->
memPool
==
NULL
)
goto
_err
;
for
(
int
i
=
0
;
i
<
nBlocks
;
i
++
)
{
STsdbCacheBlock
*
pBlock
=
(
STsdbCacheBlock
*
)
malloc
(
sizeof
(
STsdbCacheBlock
)
+
cacheBlockSize
);
if
(
pBlock
==
NULL
)
{
goto
_err
;
}
pBlock
->
offset
=
0
;
pBlock
->
remain
=
cacheBlockSize
;
tdListAppend
(
pPool
->
memPool
,
(
void
*
)(
&
pBlock
));
}
return
pCacheHandle
;
pCache
->
mem
=
NULL
;
pCache
->
imem
=
NULL
;
return
pCache
;
_err:
tsdbFreeCache
(
pCache
);
return
NULL
;
}
void
tsdbFreeCache
(
STsdbCache
*
pCache
)
{
tsdbFreeBlockList
(
pCache
->
imem
);
tsdbFreeBlockList
(
pCache
->
mem
);
tsdbFreeBlockList
(
pCache
->
pool
.
memPool
);
free
(
pCache
);
}
int32_t
tsdbFreeCache
(
STsdbCache
*
pHandle
)
{
return
0
;
}
void
*
tsdbAllocFromCache
(
STsdbCache
*
pCache
,
int
bytes
,
TSKEY
key
)
{
if
(
pCache
==
NULL
)
return
NULL
;
if
(
bytes
>
pCache
->
cacheBlockSize
)
return
NULL
;
if
(
pCache
->
curBlock
==
NULL
||
pCache
->
curBlock
->
remain
<
bytes
)
{
if
(
pCache
->
curBlock
!=
NULL
&&
(
pCache
->
mem
->
list
)
>=
pCache
->
totalCacheBlocks
/
2
)
{
tsdbTriggerCommit
(
pCache
->
pRepo
);
}
if
(
tsdbAllocBlockFromPool
(
pCache
)
<
0
)
{
// TODO: deal with the error
}
}
void
*
tsdbAllocFromCache
(
STsdbCache
*
pCache
,
int64_t
bytes
)
{
// TODO: implement here
void
*
ptr
=
malloc
(
bytes
);
if
(
ptr
==
NULL
)
return
NULL
;
void
*
ptr
=
(
void
*
)(
pCache
->
curBlock
->
data
+
pCache
->
curBlock
->
offset
);
pCache
->
curBlock
->
offset
+=
bytes
;
pCache
->
curBlock
->
remain
-=
bytes
;
memset
(
ptr
,
0
,
bytes
);
if
(
key
<
pCache
->
mem
->
keyFirst
)
pCache
->
mem
->
keyFirst
=
key
;
if
(
key
>
pCache
->
mem
->
keyLast
)
pCache
->
mem
->
keyLast
=
key
;
pCache
->
mem
->
numOfPoints
++
;
return
ptr
;
}
static
void
tsdbFreeBlockList
(
SCacheMem
*
mem
)
{
if
(
mem
==
NULL
)
return
;
SList
*
list
=
mem
->
list
;
SListNode
*
node
=
NULL
;
STsdbCacheBlock
*
pBlock
=
NULL
;
while
((
node
=
tdListPopHead
(
list
))
!=
NULL
)
{
tdListNodeGetData
(
list
,
node
,
(
void
*
)(
&
pBlock
));
free
(
pBlock
);
listNodeFree
(
node
);
}
tdListFree
(
list
);
free
(
mem
);
}
static
int
tsdbAllocBlockFromPool
(
STsdbCache
*
pCache
)
{
STsdbCachePool
*
pPool
=
&
(
pCache
->
pool
);
tsdbLockRepo
(
pCache
->
pRepo
);
if
(
listNEles
(
pPool
->
memPool
)
==
0
)
{
tsdbUnLockRepo
(
pCache
->
pRepo
);
return
-
1
;
}
SListNode
*
node
=
tdListPopHead
(
pPool
->
memPool
);
STsdbCacheBlock
*
pBlock
=
NULL
;
tdListNodeGetData
(
pPool
->
memPool
,
node
,
(
void
*
)(
&
pBlock
));
pBlock
->
blockId
=
pPool
->
index
++
;
pBlock
->
offset
=
0
;
pBlock
->
remain
=
pCache
->
cacheBlockSize
;
if
(
pCache
->
mem
==
NULL
)
{
// Create a new one
pCache
->
mem
=
(
SCacheMem
*
)
malloc
(
sizeof
(
SCacheMem
));
if
(
pCache
->
mem
==
NULL
)
return
NULL
;
pCache
->
mem
->
keyFirst
=
INT64_MAX
;
pCache
->
mem
->
keyLast
=
0
;
pCache
->
mem
->
numOfPoints
=
0
;
pCache
->
mem
->
list
=
tdListNew
(
sizeof
(
STsdbCacheBlock
*
));
}
tdListAppendNode
(
pCache
->
mem
->
list
,
node
);
pCache
->
curBlock
=
pBlock
;
tsdbUnLockRepo
(
pCache
->
pRepo
);
return
0
;
}
\ No newline at end of file
src/vnode/tsdb/src/tsdbFile.c
浏览文件 @
ddd05d9c
...
...
@@ -12,82 +12,159 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <dirent.h>
#include <fcntl.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <dirent.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "tsdbFile.h"
#include "tglobalcfg.h"
// int64_t tsMsPerDay[] = {
// 86400000L, // TSDB_PRECISION_MILLI
// 86400000000L, // TSDB_PRECISION_MICRO
// 86400000000000L // TSDB_PRECISION_NANO
// };
#define
tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile))
#define
tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3)
#define
TSDB_FILE_HEAD_SIZE 512
#define
TSDB_FILE_DELIMITER 0xF00AFA0F
typedef
struct
{
int32_t
len
;
int32_t
padding
;
// For padding purpose
int64_t
offset
;
}
SCompHeader
;
typedef
struct
{
int64_t
uid
;
int64_t
last
:
1
;
int64_t
numOfBlocks
:
63
;
int32_t
delimiter
;
}
SCompInfo
;
typedef
struct
{
TSKEY
keyFirst
;
TSKEY
keyLast
;
int32_t
numOfBlocks
;
int32_t
offset
;
}
SCompIdx
;
typedef
struct
{
TSKEY
keyFirst
;
TSKEY
keyLast
;
int64_t
offset
;
int32_t
len
;
int32_t
sversion
;
}
SCompBlock
;
int32_t
delimiter
;
// For recovery usage
int32_t
checksum
;
// TODO: decide if checksum logic in this file or make it one API
int64_t
uid
;
int32_t
padding
;
// For padding purpose
int32_t
numOfBlocks
;
// TODO: make the struct padding
SCompBlock
blocks
[];
}
SCompInfo
;
// TODO: take pre-calculation into account
typedef
struct
{
int64_t
uid
;
}
SBlock
;
int16_t
colId
;
// Column ID
int16_t
len
;
// Column length
int32_t
type
:
8
;
int32_t
offset
:
24
;
}
SCompCol
;
// TODO: Take recover into account
typedef
struct
{
int16_t
colId
;
int16_t
bytes
;
int32_t
nNullPoints
;
int32_t
type
:
8
;
int32_t
offset
:
24
;
int32_t
len
;
// fields for pre-aggregate
// TODO: pre-aggregation should be seperated
int64_t
sum
;
int64_t
max
;
int64_t
min
;
int16_t
maxIdx
;
int16_t
minIdx
;
}
SField
;
int32_t
delimiter
;
// For recovery usage
int32_t
numOfCols
;
// For recovery usage
int64_t
uid
;
// For recovery usage
SCompCol
cols
[];
}
SCompData
;
const
char
*
tsdbFileSuffix
[]
=
{
".head"
,
// TSDB_FILE_TYPE_HEAD
".data"
,
// TSDB_FILE_TYPE_DATA
".last"
,
// TSDB_FILE_TYPE_LAST
".meta"
// TSDB_FILE_TYPE_META
".last"
// TSDB_FILE_TYPE_LAST
};
static
int
tsdbWriteFileHead
(
int
fd
,
SFile
*
pFile
)
{
char
head
[
TSDB_FILE_HEAD_SIZE
]
=
"
\0
"
;
pFile
->
size
+=
TSDB_FILE_HEAD_SIZE
;
// TODO: write version and File statistic to the head
lseek
(
fd
,
0
,
SEEK_SET
);
if
(
write
(
fd
,
head
,
TSDB_FILE_HEAD_SIZE
)
<
0
)
return
-
1
;
return
0
;
}
static
int
tsdbWriteHeadFileIdx
(
int
fd
,
int
maxTables
,
SFile
*
pFile
)
{
int
size
=
sizeof
(
SCompIdx
)
*
maxTables
;
void
*
buf
=
calloc
(
1
,
size
);
if
(
buf
==
NULL
)
return
-
1
;
if
(
lseek
(
fd
,
TSDB_FILE_HEAD_SIZE
,
SEEK_SET
)
<
0
)
{
free
(
buf
);
return
-
1
;
}
if
(
write
(
fd
,
buf
,
size
)
<
0
)
{
free
(
buf
);
return
-
1
;
}
pFile
->
size
+=
size
;
return
0
;
}
static
int
tsdbGetFileName
(
char
*
dataDir
,
int
fileId
,
int8_t
type
,
char
*
fname
)
{
if
(
dataDir
==
NULL
||
fname
==
NULL
||
!
IS_VALID_TSDB_FILE_TYPE
(
type
))
return
-
1
;
sprintf
(
fname
,
"%s/f%d%s"
,
dataDir
,
fileId
,
tsdbFileSuffix
[
type
]);
return
0
;
}
/**
* Create a file and set the SFile object
*/
static
int
tsdbCreateFile
(
char
*
dataDir
,
int
fileId
,
int8_t
type
,
int
maxTables
,
SFile
*
pFile
)
{
memset
((
void
*
)
pFile
,
0
,
sizeof
(
SFile
));
pFile
->
type
=
type
;
tsdbGetFileName
(
dataDir
,
fileId
,
type
,
pFile
->
fname
);
if
(
access
(
pFile
->
fname
,
F_OK
)
==
0
)
{
// File already exists
return
-
1
;
}
int
fd
=
open
(
pFile
->
fname
,
O_WRONLY
|
O_CREAT
,
0755
);
if
(
fd
<
0
)
return
-
1
;
if
(
type
==
TSDB_FILE_TYPE_HEAD
)
{
if
(
tsdbWriteHeadFileIdx
(
fd
,
maxTables
,
pFile
)
<
0
)
{
close
(
fd
);
return
-
1
;
}
}
if
(
tsdbWriteFileHead
(
fd
,
pFile
)
<
0
)
{
close
(
fd
);
return
-
1
;
}
close
(
fd
);
return
0
;
}
static
int
tsdbRemoveFile
(
SFile
*
pFile
)
{
if
(
pFile
==
NULL
)
return
-
1
;
return
remove
(
pFile
->
fname
);
}
// Create a file group with fileId and return a SFileGroup object
int
tsdbCreateFileGroup
(
char
*
dataDir
,
int
fileId
,
SFileGroup
*
pFGroup
,
int
maxTables
)
{
if
(
dataDir
==
NULL
||
pFGroup
==
NULL
)
return
-
1
;
memset
((
void
*
)
pFGroup
,
0
,
sizeof
(
SFileGroup
));
for
(
int
type
=
TSDB_FILE_TYPE_HEAD
;
type
<
TSDB_FILE_TYPE_MAX
;
type
++
)
{
if
(
tsdbCreateFile
(
dataDir
,
fileId
,
type
,
maxTables
,
&
(
pFGroup
->
files
[
type
]))
<
0
)
{
// TODO: deal with the error here, remove the created files
return
-
1
;
}
}
pFGroup
->
fileId
=
fileId
;
return
0
;
}
/**
* Initialize the TSDB file handle
*/
STsdbFileH
*
tsdbInitFile
(
char
*
dataDir
,
int32_t
daysPerFile
,
int32_t
keep
,
int32_t
minRowsPerFBlock
,
int32_t
maxRowsPerFBlock
)
{
int32_t
maxRowsPerFBlock
,
int32_t
maxTables
)
{
STsdbFileH
*
pTsdbFileH
=
(
STsdbFileH
*
)
calloc
(
1
,
sizeof
(
STsdbFileH
)
+
sizeof
(
SFileGroup
)
*
tsdbGetMaxNumOfFiles
(
keep
,
daysPerFile
));
if
(
pTsdbFileH
==
NULL
)
return
NULL
;
...
...
@@ -96,6 +173,7 @@ STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32
pTsdbFileH
->
keep
=
keep
;
pTsdbFileH
->
minRowPerFBlock
=
minRowsPerFBlock
;
pTsdbFileH
->
maxRowsPerFBlock
=
maxRowsPerFBlock
;
pTsdbFileH
->
maxTables
=
maxTables
;
// Open the directory to read information of each file
DIR
*
dir
=
opendir
(
dataDir
);
...
...
@@ -104,8 +182,9 @@ STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32
return
NULL
;
}
struct
dirent
*
dp
;
char
fname
[
256
];
struct
dirent
*
dp
;
while
((
dp
=
readdir
(
dir
))
!=
NULL
)
{
if
(
strncmp
(
dp
->
d_name
,
"."
,
1
)
==
0
||
strncmp
(
dp
->
d_name
,
".."
,
2
)
==
0
)
continue
;
if
(
true
/* check if the file is the .head file */
)
{
...
...
@@ -125,24 +204,7 @@ STsdbFileH *tsdbInitFile(char *dataDir, int32_t daysPerFile, int32_t keep, int32
return
pTsdbFileH
;
}
/**
* Closet the file handle
*/
void
tsdbCloseFile
(
STsdbFileH
*
pFileH
)
{
// TODO
}
char
*
tsdbGetFileName
(
char
*
dirName
,
char
*
fname
,
TSDB_FILE_TYPE
type
)
{
if
(
!
IS_VALID_TSDB_FILE_TYPE
(
type
))
return
NULL
;
char
*
fileName
=
(
char
*
)
malloc
(
strlen
(
dirName
)
+
strlen
(
fname
)
+
strlen
(
tsdbFileSuffix
[
type
])
+
5
);
if
(
fileName
==
NULL
)
return
NULL
;
sprintf
(
fileName
,
"%s/%s%s"
,
dirName
,
fname
,
tsdbFileSuffix
[
type
]);
return
fileName
;
}
static
void
tsdbGetKeyRangeOfFileId
(
int32_t
daysPerFile
,
int8_t
precision
,
int32_t
fileId
,
TSKEY
*
minKey
,
void
tsdbGetKeyRangeOfFileId
(
int32_t
daysPerFile
,
int8_t
precision
,
int32_t
fileId
,
TSKEY
*
minKey
,
TSKEY
*
maxKey
)
{
*
minKey
=
fileId
*
daysPerFile
*
tsMsPerDay
[
precision
];
*
maxKey
=
*
minKey
+
daysPerFile
*
tsMsPerDay
[
precision
]
-
1
;
...
...
src/vnode/tsdb/src/tsdbMain.c
浏览文件 @
ddd05d9c
...
...
@@ -44,6 +44,7 @@
#define TSDB_CFG_FILE_NAME "CONFIG"
#define TSDB_DATA_DIR_NAME "data"
#define TSDB_DEFAULT_FILE_BLOCK_ROW_OPTION 0.7
enum
{
TSDB_REPO_STATE_ACTIVE
,
TSDB_REPO_STATE_CLOSED
,
TSDB_REPO_STATE_CONFIGURING
};
...
...
@@ -58,13 +59,16 @@ typedef struct _tsdb_repo {
// The cache Handle
STsdbCache
*
tsdbCache
;
// The TSDB file handle
STsdbFileH
*
tsdbFileH
;
// Disk tier handle for multi-tier storage
void
*
diskTier
;
// File Store
void
*
tsdbFiles
;
pthread_mutex_t
mutex
;
pthread_mutex_t
tsdbMutex
;
int
commit
;
pthread_t
commitThread
;
// A limiter to monitor the resources used by tsdb
void
*
limiter
;
...
...
@@ -79,6 +83,8 @@ static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo);
static
int
tsdbOpenMetaFile
(
char
*
tsdbDir
);
static
int32_t
tsdbInsertDataToTable
(
tsdb_repo_t
*
repo
,
SSubmitBlk
*
pBlock
);
static
int32_t
tsdbRestoreCfg
(
STsdbRepo
*
pRepo
,
STsdbCfg
*
pCfg
);
static
int32_t
tsdbGetDataDirName
(
STsdbRepo
*
pRepo
,
char
*
fname
);
static
void
*
tsdbCommitToFile
(
void
*
arg
);
#define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid]
#define TSDB_GET_TABLE_BY_NAME(pRepo, name)
...
...
@@ -144,6 +150,7 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO
pRepo
->
rootDir
=
strdup
(
rootDir
);
pRepo
->
config
=
*
pCfg
;
pRepo
->
limiter
=
limiter
;
pthread_mutex_init
(
&
pRepo
->
mutex
,
NULL
);
// Create the environment files and directories
if
(
tsdbSetRepoEnv
(
pRepo
)
<
0
)
{
...
...
@@ -162,7 +169,7 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO
pRepo
->
tsdbMeta
=
pMeta
;
// Initialize cache
STsdbCache
*
pCache
=
tsdbInitCache
(
pCfg
->
maxCacheSize
);
STsdbCache
*
pCache
=
tsdbInitCache
(
pCfg
->
maxCacheSize
,
-
1
,
(
tsdb_repo_t
*
)
pRepo
);
if
(
pCache
==
NULL
)
{
free
(
pRepo
->
rootDir
);
tsdbFreeMeta
(
pRepo
->
tsdbMeta
);
...
...
@@ -171,6 +178,19 @@ tsdb_repo_t *tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO
}
pRepo
->
tsdbCache
=
pCache
;
// Initialize file handle
char
dataDir
[
128
]
=
"
\0
"
;
tsdbGetDataDirName
(
pRepo
,
dataDir
);
pRepo
->
tsdbFileH
=
tsdbInitFile
(
dataDir
,
pCfg
->
daysPerFile
,
pCfg
->
keep
,
pCfg
->
minRowsPerFileBlock
,
pCfg
->
maxRowsPerFileBlock
,
pCfg
->
maxTables
);
if
(
pRepo
->
tsdbFileH
==
NULL
)
{
free
(
pRepo
->
rootDir
);
tsdbFreeCache
(
pRepo
->
tsdbCache
);
tsdbFreeMeta
(
pRepo
->
tsdbMeta
);
free
(
pRepo
);
return
NULL
;
}
pRepo
->
state
=
TSDB_REPO_STATE_ACTIVE
;
return
(
tsdb_repo_t
*
)
pRepo
;
...
...
@@ -230,7 +250,7 @@ tsdb_repo_t *tsdbOpenRepo(char *tsdbDir) {
return
NULL
;
}
pRepo
->
tsdbCache
=
tsdbInitCache
(
pRepo
->
config
.
maxCacheSize
);
pRepo
->
tsdbCache
=
tsdbInitCache
(
pRepo
->
config
.
maxCacheSize
,
-
1
,
(
tsdb_repo_t
*
)
pRepo
);
if
(
pRepo
->
tsdbCache
==
NULL
)
{
tsdbFreeMeta
(
pRepo
->
tsdbMeta
);
free
(
pRepo
->
rootDir
);
...
...
@@ -284,6 +304,45 @@ int32_t tsdbConfigRepo(tsdb_repo_t *repo, STsdbCfg *pCfg) {
return
0
;
}
int32_t
tsdbTriggerCommit
(
tsdb_repo_t
*
repo
)
{
STsdbRepo
*
pRepo
=
(
STsdbRepo
*
)
repo
;
tsdbLockRepo
(
repo
);
if
(
pRepo
->
commit
)
{
tsdbUnLockRepo
(
repo
);
return
-
1
;
}
pRepo
->
commit
=
1
;
// Loop to move pData to iData
for
(
int
i
=
0
;
i
<
pRepo
->
config
.
maxTables
;
i
++
)
{
STable
*
pTable
=
pRepo
->
tsdbMeta
->
tables
[
i
];
if
(
pTable
!=
NULL
&&
pTable
->
mem
!=
NULL
)
{
pTable
->
imem
=
pTable
->
mem
;
pTable
->
mem
=
NULL
;
}
}
// TODO: Loop to move mem to imem
pRepo
->
tsdbCache
->
imem
=
pRepo
->
tsdbCache
->
mem
;
pRepo
->
tsdbCache
->
mem
=
NULL
;
pRepo
->
tsdbCache
->
curBlock
=
NULL
;
// TODO: here should set as detached or use join for memory leak
pthread_create
(
&
(
pRepo
->
commitThread
),
NULL
,
tsdbCommitToFile
,
(
void
*
)
repo
);
tsdbUnLockRepo
(
repo
);
return
0
;
}
int32_t
tsdbLockRepo
(
tsdb_repo_t
*
repo
)
{
STsdbRepo
*
pRepo
=
(
STsdbRepo
*
)
repo
;
return
pthread_mutex_lock
(
repo
);
}
int32_t
tsdbUnLockRepo
(
tsdb_repo_t
*
repo
)
{
STsdbRepo
*
pRepo
=
(
STsdbRepo
*
)
repo
;
return
pthread_mutex_unlock
(
repo
);
}
/**
* Get the TSDB repository information, including some statistics
* @param pRepo the TSDB repository handle
...
...
@@ -440,6 +499,10 @@ SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) {
int
tsdbInitSubmitMsgIter
(
SSubmitMsg
*
pMsg
,
SSubmitMsgIter
*
pIter
)
{
if
(
pMsg
==
NULL
||
pIter
==
NULL
)
return
-
1
;
pMsg
->
length
=
htonl
(
pMsg
->
length
);
pMsg
->
numOfBlocks
=
htonl
(
pMsg
->
numOfBlocks
);
pMsg
->
compressed
=
htonl
(
pMsg
->
compressed
);
pIter
->
totalLen
=
pMsg
->
length
;
pIter
->
len
=
TSDB_SUBMIT_MSG_HEAD_SIZE
;
if
(
pMsg
->
length
<=
TSDB_SUBMIT_MSG_HEAD_SIZE
)
{
...
...
@@ -454,7 +517,15 @@ int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
SSubmitBlk
*
tsdbGetSubmitMsgNext
(
SSubmitMsgIter
*
pIter
)
{
SSubmitBlk
*
pBlock
=
pIter
->
pBlock
;
if
(
pBlock
==
NULL
)
return
NULL
;
pBlock
->
len
=
htonl
(
pBlock
->
len
);
pBlock
->
numOfRows
=
htons
(
pBlock
->
numOfRows
);
pBlock
->
uid
=
htobe64
(
pBlock
->
uid
);
pBlock
->
tid
=
htonl
(
pBlock
->
tid
);
pBlock
->
sversion
=
htonl
(
pBlock
->
sversion
);
pBlock
->
padding
=
htonl
(
pBlock
->
padding
);
pIter
->
len
=
pIter
->
len
+
sizeof
(
SSubmitBlk
)
+
pBlock
->
len
;
if
(
pIter
->
len
>=
pIter
->
totalLen
)
{
pIter
->
pBlock
=
NULL
;
...
...
@@ -465,6 +536,11 @@ SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) {
return
pBlock
;
}
STsdbMeta
*
tsdbGetMeta
(
tsdb_repo_t
*
pRepo
)
{
STsdbRepo
*
tsdb
=
(
STsdbRepo
*
)
pRepo
;
return
tsdb
->
tsdbMeta
;
}
// Check the configuration and set default options
static
int32_t
tsdbCheckAndSetDefaultCfg
(
STsdbCfg
*
pCfg
)
{
// Check precision
...
...
@@ -612,9 +688,6 @@ static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo) {
rmdir
(
dirName
);
char
*
metaFname
=
tsdbGetFileName
(
pRepo
->
rootDir
,
"tsdb"
,
TSDB_FILE_TYPE_META
);
remove
(
metaFname
);
return
0
;
}
...
...
@@ -628,10 +701,21 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
int32_t
level
=
0
;
int32_t
headSize
=
0
;
tSkipListRandNodeInfo
(
pTable
->
content
.
pData
,
&
level
,
&
headSize
);
if
(
pTable
->
mem
==
NULL
)
{
pTable
->
mem
=
(
SMemTable
*
)
calloc
(
1
,
sizeof
(
SMemTable
));
if
(
pTable
->
mem
==
NULL
)
return
-
1
;
pTable
->
mem
->
pData
=
tSkipListCreate
(
5
,
TSDB_DATA_TYPE_TIMESTAMP
,
TYPE_BYTES
[
TSDB_DATA_TYPE_TIMESTAMP
],
0
,
0
,
getTupleKey
);
pTable
->
mem
->
keyFirst
=
INT64_MAX
;
pTable
->
mem
->
keyLast
=
0
;
}
tSkipListRandNodeInfo
(
pTable
->
mem
->
pData
,
&
level
,
&
headSize
);
TSKEY
key
=
dataRowKey
(
row
);
printf
(
"insert:%lld, size:%d
\n
"
,
key
,
pTable
->
mem
->
numOfPoints
);
// Copy row into the memory
SSkipListNode
*
pNode
=
tsdbAllocFromCache
(
pRepo
->
tsdbCache
,
headSize
+
dataRowLen
(
row
));
SSkipListNode
*
pNode
=
tsdbAllocFromCache
(
pRepo
->
tsdbCache
,
headSize
+
dataRowLen
(
row
)
,
key
);
if
(
pNode
==
NULL
)
{
// TODO: deal with allocate failure
}
...
...
@@ -640,7 +724,19 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
dataRowCpy
(
SL_GET_NODE_DATA
(
pNode
),
row
);
// Insert the skiplist node into the data
tsdbInsertRowToTableImpl
(
pNode
,
pTable
);
if
(
pTable
->
mem
==
NULL
)
{
pTable
->
mem
=
(
SMemTable
*
)
calloc
(
1
,
sizeof
(
SMemTable
));
if
(
pTable
->
mem
==
NULL
)
return
-
1
;
pTable
->
mem
->
pData
=
tSkipListCreate
(
5
,
TSDB_DATA_TYPE_TIMESTAMP
,
TYPE_BYTES
[
TSDB_DATA_TYPE_TIMESTAMP
],
0
,
0
,
getTupleKey
);
pTable
->
mem
->
keyFirst
=
INT64_MAX
;
pTable
->
mem
->
keyLast
=
0
;
}
tSkipListPut
(
pTable
->
mem
->
pData
,
pNode
);
if
(
key
>
pTable
->
mem
->
keyLast
)
pTable
->
mem
->
keyLast
=
key
;
if
(
key
<
pTable
->
mem
->
keyFirst
)
pTable
->
mem
->
keyFirst
=
key
;
pTable
->
mem
->
numOfPoints
=
tSkipListGetSize
(
pTable
->
mem
->
pData
);
// pTable->mem->numOfPoints++;
return
0
;
}
...
...
@@ -648,7 +744,8 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable
static
int32_t
tsdbInsertDataToTable
(
tsdb_repo_t
*
repo
,
SSubmitBlk
*
pBlock
)
{
STsdbRepo
*
pRepo
=
(
STsdbRepo
*
)
repo
;
STable
*
pTable
=
tsdbIsValidTableToInsert
(
pRepo
->
tsdbMeta
,
pBlock
->
tableId
);
STableId
tableId
=
{.
uid
=
pBlock
->
uid
,
.
tid
=
pBlock
->
tid
};
STable
*
pTable
=
tsdbIsValidTableToInsert
(
pRepo
->
tsdbMeta
,
tableId
);
if
(
pTable
==
NULL
)
return
-
1
;
SSubmitBlkIter
blkIter
;
...
...
@@ -662,4 +759,106 @@ static int32_t tsdbInsertDataToTable(tsdb_repo_t *repo, SSubmitBlk *pBlock) {
}
return
0
;
}
static
int
tsdbReadRowsFromCache
(
SSkipListIterator
*
pIter
,
TSKEY
maxKey
,
int
maxRowsToRead
,
SDataCol
**
cols
,
STSchema
*
pSchema
)
{
int
numOfRows
=
0
;
do
{
SSkipListNode
*
node
=
tSkipListIterGet
(
pIter
);
if
(
node
==
NULL
)
break
;
SDataRow
row
=
SL_GET_NODE_DATA
(
node
);
if
(
dataRowKey
(
row
)
>
maxKey
)
break
;
// Convert row data to column data
// for (int i = 0; i < schemaNCols(pSchema); i++) {
// STColumn *pCol = schemaColAt(pSchema, i);
// memcpy(cols[i]->data + TYPE_BYTES[colType(pCol)] * numOfRows, dataRowAt(row, pCol->offset),
// TYPE_BYTES[colType(pCol)]);
// }
numOfRows
++
;
if
(
numOfRows
>
maxRowsToRead
)
break
;
}
while
(
tSkipListIterNext
(
pIter
));
return
numOfRows
;
}
// Commit to file
static
void
*
tsdbCommitToFile
(
void
*
arg
)
{
// TODO
STsdbRepo
*
pRepo
=
(
STsdbRepo
*
)
arg
;
STsdbMeta
*
pMeta
=
pRepo
->
tsdbMeta
;
STsdbCache
*
pCache
=
pRepo
->
tsdbCache
;
STsdbCfg
*
pCfg
=
&
(
pRepo
->
config
);
if
(
pCache
->
imem
==
NULL
)
return
;
int
sfid
=
tsdbGetKeyFileId
(
pCache
->
imem
->
keyFirst
,
pCfg
->
daysPerFile
,
pCfg
->
precision
);
int
efid
=
tsdbGetKeyFileId
(
pCache
->
imem
->
keyLast
,
pCfg
->
daysPerFile
,
pCfg
->
precision
);
SSkipListIterator
**
iters
=
(
SSkipListIterator
**
)
calloc
(
pCfg
->
maxTables
,
sizeof
(
SSkipListIterator
*
));
if
(
iters
==
NULL
)
{
// TODO: deal with the error
return
NULL
;
}
int
maxCols
=
pMeta
->
maxCols
;
int
maxBytes
=
pMeta
->
maxRowBytes
;
SDataCol
**
cols
=
(
SDataCol
**
)
malloc
(
sizeof
(
SDataCol
*
)
*
maxCols
);
void
*
buf
=
malloc
((
maxBytes
+
sizeof
(
SDataCol
))
*
pCfg
->
maxRowsPerFileBlock
);
for
(
int
fid
=
sfid
;
fid
<=
efid
;
fid
++
)
{
TSKEY
minKey
=
0
,
maxKey
=
0
;
tsdbGetKeyRangeOfFileId
(
pCfg
->
daysPerFile
,
pCfg
->
precision
,
fid
,
&
minKey
,
&
maxKey
);
for
(
int
tid
=
0
;
tid
<
pCfg
->
maxTables
;
tid
++
)
{
STable
*
pTable
=
pMeta
->
tables
[
tid
];
if
(
pTable
==
NULL
||
pTable
->
imem
==
NULL
)
continue
;
if
(
iters
[
tid
]
==
NULL
)
{
// create table iterator
iters
[
tid
]
=
tSkipListCreateIter
(
pTable
->
imem
->
pData
);
// TODO: deal with the error
if
(
iters
[
tid
]
==
NULL
)
break
;
if
(
!
tSkipListIterNext
(
iters
[
tid
]))
{
// assert(0);
}
}
// Init row data part
cols
[
0
]
=
(
SDataCol
*
)
buf
;
for
(
int
col
=
1
;
col
<
schemaNCols
(
pTable
->
schema
);
col
++
)
{
cols
[
col
]
=
(
SDataCol
*
)((
char
*
)(
cols
[
col
-
1
])
+
sizeof
(
SDataCol
)
+
colBytes
(
schemaColAt
(
pTable
->
schema
,
col
-
1
))
*
pCfg
->
maxRowsPerFileBlock
);
}
// Loop the iterator
int
rowsRead
=
0
;
while
((
rowsRead
=
tsdbReadRowsFromCache
(
iters
[
tid
],
maxKey
,
pCfg
->
maxRowsPerFileBlock
,
cols
,
pTable
->
schema
))
>
0
)
{
// printf("rowsRead:%d-----------\n", rowsRead);
int
k
=
0
;
}
}
}
// Free the iterator
for
(
int
tid
=
0
;
tid
<
pCfg
->
maxTables
;
tid
++
)
{
if
(
iters
[
tid
]
!=
NULL
)
tSkipListDestroyIter
(
iters
[
tid
]);
}
free
(
buf
);
free
(
cols
);
free
(
iters
);
tsdbLockRepo
(
arg
);
tdListMove
(
pCache
->
imem
->
list
,
pCache
->
pool
.
memPool
);
free
(
pCache
->
imem
);
pCache
->
imem
=
NULL
;
pRepo
->
commit
=
0
;
// TODO: free the skiplist
for
(
int
i
=
0
;
i
<
pCfg
->
maxTables
;
i
++
)
{
STable
*
pTable
=
pMeta
->
tables
[
i
];
if
(
pTable
&&
pTable
->
imem
)
{
// Here has memory leak
pTable
->
imem
=
NULL
;
}
}
tsdbUnLockRepo
(
arg
);
return
NULL
;
}
\ No newline at end of file
src/vnode/tsdb/src/tsdbMeta.c
浏览文件 @
ddd05d9c
...
...
@@ -18,7 +18,6 @@ static int tsdbAddTableIntoMap(STsdbMeta *pMeta, STable *pTable);
static
int
tsdbAddTableIntoIndex
(
STsdbMeta
*
pMeta
,
STable
*
pTable
);
static
int
tsdbRemoveTableFromIndex
(
STsdbMeta
*
pMeta
,
STable
*
pTable
);
static
int
tsdbEstimateTableEncodeSize
(
STable
*
pTable
);
static
char
*
getTupleKey
(
const
void
*
data
);
/**
* Encode a TSDB table object as a binary content
...
...
@@ -102,12 +101,9 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
if
(
pTable
==
NULL
)
return
-
1
;
if
(
pTable
->
type
==
TSDB_SUPER_TABLE
)
{
pTable
->
content
.
pIndex
=
pTable
->
pIndex
=
tSkipListCreate
(
TSDB_SUPER_TABLE_SL_LEVEL
,
TSDB_DATA_TYPE_TIMESTAMP
,
sizeof
(
int64_t
),
1
,
0
,
getTupleKey
);
}
else
{
pTable
->
content
.
pData
=
tSkipListCreate
(
TSDB_SUPER_TABLE_SL_LEVEL
,
TSDB_DATA_TYPE_TIMESTAMP
,
TYPE_BYTES
[
TSDB_DATA_TYPE_TIMESTAMP
],
0
,
0
,
getTupleKey
);
}
}
tsdbAddTableToMeta
(
pMeta
,
pTable
,
false
);
...
...
@@ -137,6 +133,8 @@ STsdbMeta *tsdbInitMeta(const char *rootDir, int32_t maxTables) {
pMeta
->
nTables
=
0
;
pMeta
->
superList
=
NULL
;
pMeta
->
tables
=
(
STable
**
)
calloc
(
maxTables
,
sizeof
(
STable
*
));
pMeta
->
maxRowBytes
=
0
;
pMeta
->
maxCols
=
0
;
if
(
pMeta
->
tables
==
NULL
)
{
free
(
pMeta
);
return
NULL
;
...
...
@@ -208,10 +206,10 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) {
super
->
schema
=
tdDupSchema
(
pCfg
->
schema
);
super
->
tagSchema
=
tdDupSchema
(
pCfg
->
tagSchema
);
super
->
tagVal
=
tdDataRowDup
(
pCfg
->
tagValues
);
super
->
content
.
pIndex
=
tSkipListCreate
(
TSDB_SUPER_TABLE_SL_LEVEL
,
TSDB_DATA_TYPE_TIMESTAMP
,
sizeof
(
int64_t
),
1
,
super
->
pIndex
=
tSkipListCreate
(
TSDB_SUPER_TABLE_SL_LEVEL
,
TSDB_DATA_TYPE_TIMESTAMP
,
sizeof
(
int64_t
),
1
,
0
,
getTupleKey
);
// Allow duplicate key, no lock
if
(
super
->
content
.
pIndex
==
NULL
)
{
if
(
super
->
pIndex
==
NULL
)
{
tdFreeSchema
(
super
->
schema
);
tdFreeSchema
(
super
->
tagSchema
);
tdFreeDataRow
(
super
->
tagVal
);
...
...
@@ -223,7 +221,7 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) {
}
}
STable
*
table
=
(
STable
*
)
malloc
(
sizeof
(
STable
));
STable
*
table
=
(
STable
*
)
calloc
(
1
,
sizeof
(
STable
));
if
(
table
==
NULL
)
{
if
(
newSuper
)
tsdbFreeTable
(
super
);
return
-
1
;
...
...
@@ -238,8 +236,11 @@ int32_t tsdbCreateTableImpl(STsdbMeta *pMeta, STableCfg *pCfg) {
table
->
type
=
TSDB_NORMAL_TABLE
;
table
->
superUid
=
-
1
;
table
->
schema
=
tdDupSchema
(
pCfg
->
schema
);
if
(
schemaNCols
(
table
->
schema
)
>
pMeta
->
maxCols
)
pMeta
->
maxCols
=
schemaNCols
(
table
->
schema
);
tdUpdateSchema
(
table
->
schema
);
int
bytes
=
tdMaxRowBytesFromSchema
(
table
->
schema
);
if
(
bytes
>
pMeta
->
maxRowBytes
)
pMeta
->
maxRowBytes
=
bytes
;
}
table
->
content
.
pData
=
tSkipListCreate
(
TSDB_SUPER_TABLE_SL_LEVEL
,
TSDB_DATA_TYPE_TIMESTAMP
,
TYPE_BYTES
[
TSDB_DATA_TYPE_TIMESTAMP
],
0
,
0
,
getTupleKey
);
// Register to meta
if
(
newSuper
)
tsdbAddTableToMeta
(
pMeta
,
super
,
true
);
...
...
@@ -299,10 +300,10 @@ int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) {
return
0
;
}
int32_t
tsdbInsertRowToTableImpl
(
SSkipListNode
*
pNode
,
STable
*
pTable
)
{
tSkipListPut
(
pTable
->
content
.
pData
,
pNode
);
return
0
;
}
//
int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable) {
// tSkipListPut(pTable->mem->
pData, pNode);
//
return 0;
//
}
static
int
tsdbFreeTable
(
STable
*
pTable
)
{
// TODO: finish this function
...
...
@@ -314,10 +315,8 @@ static int tsdbFreeTable(STable *pTable) {
// Free content
if
(
TSDB_TABLE_IS_SUPER_TABLE
(
pTable
))
{
tSkipListDestroy
(
pTable
->
content
.
pIndex
);
}
else
{
tSkipListDestroy
(
pTable
->
content
.
pData
);
}
tSkipListDestroy
(
pTable
->
pIndex
);
}
free
(
pTable
);
return
0
;
...
...
@@ -404,7 +403,7 @@ static int tsdbEstimateTableEncodeSize(STable *pTable) {
return
size
;
}
static
char
*
getTupleKey
(
const
void
*
data
)
{
char
*
getTupleKey
(
const
void
*
data
)
{
SDataRow
row
=
(
SDataRow
)
data
;
return
dataRowAt
(
row
,
TD_DATA_ROW_HEAD_SIZE
);
...
...
src/vnode/tsdb/src/tsdbRead.c
浏览文件 @
ddd05d9c
...
...
@@ -14,5 +14,378 @@
*/
#include "os.h"
#include "tutil.h"
#include "tsdb.h"
#include "tsdbFile.h"
#include "tsdbMeta.h"
#define EXTRA_BYTES 2
#define PRIMARY_TSCOL_REQUIRED(c) (((SColumnInfoEx *)taosArrayGet(c, 0))->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)
#define QUERY_IS_ASC_QUERY(o) (o == TSQL_SO_ASC)
#define QH_GET_NUM_OF_COLS(handle) (taosArrayGetSize((handle)->pColumns))
typedef
struct
SField
{
// todo need the definition
}
SField
;
typedef
struct
SHeaderFileInfo
{
int32_t
fileId
;
}
SHeaderFileInfo
;
typedef
struct
SQueryHandlePos
{
int32_t
fileId
;
int32_t
slot
;
int32_t
pos
;
int32_t
fileIndex
;
}
SQueryHandlePos
;
typedef
struct
SDataBlockLoadInfo
{
int32_t
fileListIndex
;
int32_t
fileId
;
int32_t
slotIdx
;
int32_t
sid
;
SArray
*
pLoadedCols
;
}
SDataBlockLoadInfo
;
typedef
struct
SLoadCompBlockInfo
{
int32_t
sid
;
/* meter sid */
int32_t
fileId
;
int32_t
fileListIndex
;
}
SLoadCompBlockInfo
;
typedef
struct
SQueryFilesInfo
{
SArray
*
pFileInfo
;
int32_t
current
;
// the memory mapped header file, NOTE: only one header file can be mmap.
int32_t
vnodeId
;
int32_t
headerFd
;
// header file fd
int64_t
headerFileSize
;
int32_t
dataFd
;
int32_t
lastFd
;
char
headerFilePath
[
PATH_MAX
];
// current opened header file name
char
dataFilePath
[
PATH_MAX
];
// current opened data file name
char
lastFilePath
[
PATH_MAX
];
// current opened last file path
char
dbFilePathPrefix
[
PATH_MAX
];
}
SQueryFilesInfo
;
typedef
struct
STableQueryRec
{
TSKEY
lastKey
;
STable
*
pTableObj
;
int64_t
offsetInHeaderFile
;
int32_t
numOfBlocks
;
int32_t
start
;
SCompBlock
*
pBlock
;
}
STableQueryRec
;
typedef
struct
{
SCompBlock
*
compBlock
;
SField
*
fields
;
}
SCompBlockFields
;
typedef
struct
STableDataBlockInfoEx
{
SCompBlockFields
pBlock
;
STableQueryRec
*
pMeterDataInfo
;
int32_t
blockIndex
;
int32_t
groupIdx
;
/* number of group is less than the total number of meters */
}
STableDataBlockInfoEx
;
typedef
struct
STsdbQueryHandle
{
struct
STsdbRepo
*
pTsdb
;
SQueryHandlePos
cur
;
// current position
SQueryHandlePos
start
;
// the start position, used for secondary/third iteration
int32_t
unzipBufSize
;
char
*
unzipBuffer
;
char
*
secondaryUnzipBuffer
;
SDataBlockLoadInfo
dataBlockLoadInfo
;
/* record current block load information */
SLoadCompBlockInfo
compBlockLoadInfo
;
/* record current compblock information in SQuery */
SQueryFilesInfo
vnodeFileInfo
;
int16_t
numOfRowsPerPage
;
uint16_t
flag
;
// denotes reversed scan of data or not
int16_t
order
;
STimeWindow
window
;
// the primary query time window that applies to all queries
TSKEY
lastKey
;
int32_t
blockBufferSize
;
SCompBlock
*
pBlock
;
int32_t
numOfBlocks
;
SField
**
pFields
;
SArray
*
pColumns
;
// column list, SColumnInfoEx array list
SArray
*
pTableIdList
;
// table id object list
bool
locateStart
;
int32_t
realNumOfRows
;
bool
loadDataAfterSeek
;
// load data after seek.
STableDataBlockInfoEx
*
pDataBlockInfoEx
;
STableQueryRec
*
pTableQueryInfo
;
int32_t
tableIndex
;
bool
isFirstSlot
;
void
*
qinfo
;
// query info handle, for debug purpose
}
STsdbQueryHandle
;
int32_t
doAllocateBuf
(
STsdbQueryHandle
*
pQueryHandle
,
int32_t
rowsPerFileBlock
)
{
// record the maximum column width among columns of this meter/metric
SColumnInfoEx
*
pColumn
=
taosArrayGet
(
pQueryHandle
->
pColumns
,
0
);
int32_t
maxColWidth
=
pColumn
->
info
.
bytes
;
for
(
int32_t
i
=
1
;
i
<
QH_GET_NUM_OF_COLS
(
pQueryHandle
);
++
i
)
{
int32_t
bytes
=
pColumn
[
i
].
info
.
bytes
;
if
(
bytes
>
maxColWidth
)
{
maxColWidth
=
bytes
;
}
}
// only one unzip buffer required, since we can unzip each column one by one
pQueryHandle
->
unzipBufSize
=
(
size_t
)(
maxColWidth
*
rowsPerFileBlock
+
EXTRA_BYTES
);
// plus extra_bytes
pQueryHandle
->
unzipBuffer
=
(
char
*
)
calloc
(
1
,
pQueryHandle
->
unzipBufSize
);
pQueryHandle
->
secondaryUnzipBuffer
=
(
char
*
)
calloc
(
1
,
pQueryHandle
->
unzipBufSize
);
if
(
pQueryHandle
->
unzipBuffer
==
NULL
||
pQueryHandle
->
secondaryUnzipBuffer
==
NULL
)
{
goto
_error_clean
;
}
return
TSDB_CODE_SUCCESS
;
_error_clean:
tfree
(
pQueryHandle
->
unzipBuffer
);
tfree
(
pQueryHandle
->
secondaryUnzipBuffer
);
return
TSDB_CODE_SERV_OUT_OF_MEMORY
;
}
static
void
initQueryFileInfoFD
(
SQueryFilesInfo
*
pVnodeFilesInfo
)
{
pVnodeFilesInfo
->
current
=
-
1
;
pVnodeFilesInfo
->
headerFileSize
=
-
1
;
pVnodeFilesInfo
->
headerFd
=
FD_INITIALIZER
;
// set the initial value
pVnodeFilesInfo
->
dataFd
=
FD_INITIALIZER
;
pVnodeFilesInfo
->
lastFd
=
FD_INITIALIZER
;
}
static
void
vnodeInitDataBlockLoadInfo
(
SDataBlockLoadInfo
*
pBlockLoadInfo
)
{
pBlockLoadInfo
->
slotIdx
=
-
1
;
pBlockLoadInfo
->
fileId
=
-
1
;
pBlockLoadInfo
->
sid
=
-
1
;
pBlockLoadInfo
->
fileListIndex
=
-
1
;
}
static
void
vnodeInitCompBlockLoadInfo
(
SLoadCompBlockInfo
*
pCompBlockLoadInfo
)
{
pCompBlockLoadInfo
->
sid
=
-
1
;
pCompBlockLoadInfo
->
fileId
=
-
1
;
pCompBlockLoadInfo
->
fileListIndex
=
-
1
;
}
static
int
fileOrderComparFn
(
const
void
*
p1
,
const
void
*
p2
)
{
SHeaderFileInfo
*
pInfo1
=
(
SHeaderFileInfo
*
)
p1
;
SHeaderFileInfo
*
pInfo2
=
(
SHeaderFileInfo
*
)
p2
;
if
(
pInfo1
->
fileId
==
pInfo2
->
fileId
)
{
return
0
;
}
return
(
pInfo1
->
fileId
>
pInfo2
->
fileId
)
?
1
:
-
1
;
}
void
vnodeRecordAllFiles
(
int32_t
vnodeId
,
SQueryFilesInfo
*
pVnodeFilesInfo
)
{
char
suffix
[]
=
".head"
;
pVnodeFilesInfo
->
pFileInfo
=
taosArrayInit
(
4
,
sizeof
(
int32_t
));
struct
dirent
*
pEntry
=
NULL
;
pVnodeFilesInfo
->
vnodeId
=
vnodeId
;
char
*
tsDirectory
=
""
;
sprintf
(
pVnodeFilesInfo
->
dbFilePathPrefix
,
"%s/vnode%d/db/"
,
tsDirectory
,
vnodeId
);
DIR
*
pDir
=
opendir
(
pVnodeFilesInfo
->
dbFilePathPrefix
);
if
(
pDir
==
NULL
)
{
// dError("QInfo:%p failed to open directory:%s, %s", pQInfo, pVnodeFilesInfo->dbFilePathPrefix,
// strerror(errno));
return
;
}
while
((
pEntry
=
readdir
(
pDir
))
!=
NULL
)
{
if
((
pEntry
->
d_name
[
0
]
==
'.'
&&
pEntry
->
d_name
[
1
]
==
'\0'
)
||
(
strcmp
(
pEntry
->
d_name
,
".."
)
==
0
))
{
continue
;
}
if
(
pEntry
->
d_type
&
DT_DIR
)
{
continue
;
}
size_t
len
=
strlen
(
pEntry
->
d_name
);
if
(
strcasecmp
(
&
pEntry
->
d_name
[
len
-
5
],
suffix
)
!=
0
)
{
continue
;
}
int32_t
vid
=
0
;
int32_t
fid
=
0
;
sscanf
(
pEntry
->
d_name
,
"v%df%d"
,
&
vid
,
&
fid
);
if
(
vid
!=
vnodeId
)
{
/* ignore error files */
// dError("QInfo:%p error data file:%s in vid:%d, ignore", pQInfo, pEntry->d_name, vnodeId);
continue
;
}
// int32_t firstFid = pVnode->fileId - pVnode->numOfFiles + 1;
// if (fid > pVnode->fileId || fid < firstFid) {
// dError("QInfo:%p error data file:%s in vid:%d, fid:%d, fid range:%d-%d", pQInfo, pEntry->d_name, vnodeId,
// fid, firstFid, pVnode->fileId);
// continue;
// }
assert
(
fid
>=
0
&&
vid
>=
0
);
taosArrayPush
(
pVnodeFilesInfo
->
pFileInfo
,
&
fid
);
}
closedir
(
pDir
);
// dTrace("QInfo:%p find %d data files in %s to be checked", pQInfo, pVnodeFilesInfo->numOfFiles,
// pVnodeFilesInfo->dbFilePathPrefix);
// order the files information according their names */
size_t
numOfFiles
=
taosArrayGetSize
(
pVnodeFilesInfo
->
pFileInfo
);
qsort
(
pVnodeFilesInfo
->
pFileInfo
->
pData
,
numOfFiles
,
sizeof
(
SHeaderFileInfo
),
fileOrderComparFn
);
}
tsdb_query_handle_t
*
tsdbQueryByTableId
(
tsdb_repo_t
*
tsdb
,
STsdbQueryCond
*
pCond
,
SArray
*
idList
,
SArray
*
pColumnInfo
)
{
// todo 1. filter not exist table
// todo 2. add the reference count for each table that is involved in query
STsdbQueryHandle
*
pQueryHandle
=
calloc
(
1
,
sizeof
(
STsdbQueryHandle
));
pQueryHandle
->
order
=
pCond
->
order
;
pQueryHandle
->
window
=
pCond
->
twindow
;
pQueryHandle
->
pTsdb
=
tsdb
;
pQueryHandle
->
pTableIdList
=
idList
;
pQueryHandle
->
pColumns
=
pColumnInfo
;
pQueryHandle
->
loadDataAfterSeek
=
false
;
pQueryHandle
->
isFirstSlot
=
true
;
pQueryHandle
->
lastKey
=
pQueryHandle
->
window
.
skey
;
// ascending query
// malloc buffer in order to load data from file
int32_t
numOfCols
=
taosArrayGetSize
(
pColumnInfo
);
size_t
bufferCapacity
=
4096
;
pQueryHandle
->
pColumns
=
taosArrayInit
(
numOfCols
,
sizeof
(
SColumnInfoEx
));
for
(
int32_t
i
=
0
;
i
<
numOfCols
;
++
i
)
{
SColumnInfoEx
*
pCol
=
taosArrayGet
(
pColumnInfo
,
i
);
SColumnInfoEx
pDest
=
{{
0
},
0
};
pDest
.
pData
=
calloc
(
1
,
EXTRA_BYTES
+
bufferCapacity
*
pCol
->
info
.
bytes
);
pDest
.
info
=
pCol
->
info
;
taosArrayPush
(
pQueryHandle
->
pColumns
,
&
pDest
);
}
if
(
doAllocateBuf
(
pQueryHandle
,
bufferCapacity
)
!=
TSDB_CODE_SUCCESS
)
{
return
NULL
;
}
initQueryFileInfoFD
(
&
pQueryHandle
->
vnodeFileInfo
);
vnodeInitDataBlockLoadInfo
(
&
pQueryHandle
->
dataBlockLoadInfo
);
vnodeInitCompBlockLoadInfo
(
&
pQueryHandle
->
compBlockLoadInfo
);
int32_t
vnodeId
=
1
;
vnodeRecordAllFiles
(
vnodeId
,
&
pQueryHandle
->
vnodeFileInfo
);
return
(
tsdb_query_handle_t
)
pQueryHandle
;
}
static
int32_t
next
=
1
;
bool
tsdbNextDataBlock
(
tsdb_query_handle_t
*
pQueryHandle
)
{
if
(
next
==
0
)
{
return
false
;
}
else
{
next
=
0
;
return
true
;
}
}
static
int
tsdbReadRowsFromCache
(
SSkipListIterator
*
pIter
,
TSKEY
maxKey
,
int
maxRowsToRead
,
TSKEY
*
skey
,
TSKEY
*
ekey
,
STsdbQueryHandle
*
pHandle
)
{
int
numOfRows
=
0
;
int32_t
numOfCols
=
taosArrayGetSize
(
pHandle
->
pColumns
);
*
skey
=
INT64_MIN
;
while
(
tSkipListIterNext
(
pIter
))
{
SSkipListNode
*
node
=
tSkipListIterGet
(
pIter
);
if
(
node
==
NULL
)
break
;
SDataRow
row
=
SL_GET_NODE_DATA
(
node
);
if
(
dataRowKey
(
row
)
>
maxKey
)
break
;
// Convert row data to column data
if
(
*
skey
==
INT64_MIN
)
{
*
skey
=
dataRowKey
(
row
);
}
*
ekey
=
dataRowKey
(
row
);
int32_t
offset
=
0
;
for
(
int32_t
i
=
0
;
i
<
numOfCols
;
++
i
)
{
SColumnInfoEx
*
pColInfo
=
taosArrayGet
(
pHandle
->
pColumns
,
0
);
memcpy
(
pColInfo
->
pData
+
numOfRows
*
pColInfo
->
info
.
bytes
,
dataRowTuple
(
row
)
+
offset
,
pColInfo
->
info
.
bytes
);
offset
+=
pColInfo
->
info
.
bytes
;
}
numOfRows
++
;
if
(
numOfRows
>
maxRowsToRead
)
break
;
};
return
numOfRows
;
}
// copy data from cache into data block
SDataBlockInfo
tsdbRetrieveDataBlockInfo
(
tsdb_query_handle_t
*
pQueryHandle
)
{
STsdbQueryHandle
*
pHandle
=
(
STsdbQueryHandle
*
)
pQueryHandle
;
STableIdInfo
*
idInfo
=
taosArrayGet
(
pHandle
->
pTableIdList
,
0
);
STableId
tableId
=
{.
uid
=
idInfo
->
uid
,
.
tid
=
idInfo
->
sid
};
STable
*
pTable
=
tsdbIsValidTableToInsert
(
tsdbGetMeta
(
pHandle
->
pTsdb
),
tableId
);
assert
(
pTable
!=
NULL
);
TSKEY
skey
=
0
,
ekey
=
0
;
int32_t
rows
=
0
;
if
(
pTable
->
mem
!=
NULL
)
{
SSkipListIterator
*
iter
=
tSkipListCreateIter
(
pTable
->
mem
->
pData
);
rows
=
tsdbReadRowsFromCache
(
iter
,
INT64_MAX
,
4000
,
&
skey
,
&
ekey
,
pHandle
);
}
SDataBlockInfo
blockInfo
=
{
.
uid
=
tableId
.
uid
,
.
sid
=
tableId
.
tid
,
.
size
=
rows
,
.
window
=
{.
skey
=
skey
,
.
ekey
=
ekey
}
};
return
blockInfo
;
}
// return null for data block in cache
int32_t
tsdbRetrieveDataBlockStatisInfo
(
tsdb_query_handle_t
*
pQueryHandle
,
SDataStatis
**
pBlockStatis
)
{
*
pBlockStatis
=
NULL
;
return
TSDB_CODE_SUCCESS
;
}
SArray
*
tsdbRetrieveDataBlock
(
tsdb_query_handle_t
*
pQueryHandle
,
SArray
*
pIdList
)
{
}
int32_t
tsdbResetQuery
(
tsdb_query_handle_t
*
pQueryHandle
,
STimeWindow
*
window
,
tsdbpos_t
position
,
int16_t
order
)
{}
int32_t
tsdbDataBlockSeek
(
tsdb_query_handle_t
*
pQueryHandle
,
tsdbpos_t
pos
)
{}
tsdbpos_t
tsdbDataBlockTell
(
tsdb_query_handle_t
*
pQueryHandle
)
{
return
NULL
;
}
SArray
*
tsdbRetrieveDataRow
(
tsdb_query_handle_t
*
pQueryHandle
,
SArray
*
pIdList
,
SQueryRowCond
*
pCond
)
{}
tsdb_query_handle_t
*
tsdbQueryFromTagConds
(
STsdbQueryCond
*
pCond
,
int16_t
stableId
,
const
char
*
pTagFilterStr
)
{}
STableIDList
*
tsdbGetTableList
(
tsdb_query_handle_t
*
pQueryHandle
)
{}
STableIDList
*
tsdbQueryTableList
(
int16_t
stableId
,
const
char
*
pTagCond
)
{}
src/vnode/tsdb/tests/tsdbTests.cpp
浏览文件 @
ddd05d9c
#include <gtest/gtest.h>
#include <stdlib.h>
#include <sys/time.h>
#include "tsdb.h"
#include "dataformat.h"
#include "tsdbFile.h"
#include "tsdbMeta.h"
TEST
(
TsdbTest
,
tableEncodeDecode
)
{
double
getCurTime
()
{
struct
timeval
tv
;
gettimeofday
(
&
tv
,
NULL
);
return
tv
.
tv_sec
+
tv
.
tv_usec
*
1E-6
;
}
TEST
(
TsdbTest
,
DISABLED_tableEncodeDecode
)
{
STable
*
pTable
=
(
STable
*
)
malloc
(
sizeof
(
STable
));
pTable
->
type
=
TSDB_NORMAL_TABLE
;
...
...
@@ -39,7 +47,6 @@ TEST(TsdbTest, tableEncodeDecode) {
ASSERT_EQ
(
pTable
->
superUid
,
tTable
->
superUid
);
ASSERT_EQ
(
pTable
->
sversion
,
tTable
->
sversion
);
ASSERT_EQ
(
memcmp
(
pTable
->
schema
,
tTable
->
schema
,
sizeof
(
STSchema
)
+
sizeof
(
STColumn
)
*
nCols
),
0
);
ASSERT_EQ
(
tTable
->
content
.
pData
,
nullptr
);
}
TEST
(
TsdbTest
,
createRepo
)
{
...
...
@@ -71,39 +78,59 @@ TEST(TsdbTest, createRepo) {
tsdbCreateTable
(
pRepo
,
&
tCfg
);
// // 3. Loop to write some simple data
int
nRows
=
10
;
SSubmitMsg
*
pMsg
=
(
SSubmitMsg
*
)
malloc
(
sizeof
(
SSubmitMsg
)
+
sizeof
(
SSubmitBlk
)
+
tdMaxRowBytesFromSchema
(
schema
)
*
nRows
);
SSubmitBlk
*
pBlock
=
pMsg
->
blocks
;
pBlock
->
tableId
=
{.
uid
=
987607499877672L
,
.
tid
=
0
};
pBlock
->
sversion
=
0
;
pBlock
->
len
=
0
;
int
nRows
=
10000000
;
int
rowsPerSubmit
=
100
;
int64_t
start_time
=
1584081000000
;
for
(
int
i
=
0
;
i
<
nRows
;
i
++
)
{
int64_t
ttime
=
start_time
+
1000
*
i
;
SDataRow
row
=
(
SDataRow
)(
pBlock
->
data
+
pBlock
->
len
);
tdInitDataRow
(
row
,
schema
);
for
(
int
j
=
0
;
j
<
schemaNCols
(
schema
);
j
++
)
{
if
(
j
==
0
)
{
// Just for timestamp
tdAppendColVal
(
row
,
(
void
*
)(
&
ttime
),
schemaColAt
(
schema
,
j
));
}
else
{
// For int
int
val
=
10
;
tdAppendColVal
(
row
,
(
void
*
)(
&
val
),
schemaColAt
(
schema
,
j
));
}
SSubmitMsg
*
pMsg
=
(
SSubmitMsg
*
)
malloc
(
sizeof
(
SSubmitMsg
)
+
sizeof
(
SSubmitBlk
)
+
tdMaxRowBytesFromSchema
(
schema
)
*
rowsPerSubmit
);
double
stime
=
getCurTime
();
for
(
int
k
=
0
;
k
<
nRows
/
rowsPerSubmit
;
k
++
)
{
SSubmitBlk
*
pBlock
=
pMsg
->
blocks
;
pBlock
->
tableId
=
{.
uid
=
987607499877672L
,
.
tid
=
0
};
pBlock
->
sversion
=
0
;
pBlock
->
len
=
0
;
for
(
int
i
=
0
;
i
<
rowsPerSubmit
;
i
++
)
{
// start_time += 1000;
start_time
-=
1000
;
SDataRow
row
=
(
SDataRow
)(
pBlock
->
data
+
pBlock
->
len
);
tdInitDataRow
(
row
,
schema
);
for
(
int
j
=
0
;
j
<
schemaNCols
(
schema
);
j
++
)
{
if
(
j
==
0
)
{
// Just for timestamp
tdAppendColVal
(
row
,
(
void
*
)(
&
start_time
),
schemaColAt
(
schema
,
j
));
}
else
{
// For int
int
val
=
10
;
tdAppendColVal
(
row
,
(
void
*
)(
&
val
),
schemaColAt
(
schema
,
j
));
}
}
pBlock
->
len
+=
dataRowLen
(
row
);
}
p
Block
->
len
+=
dataRowLen
(
row
)
;
p
Msg
->
length
=
pMsg
->
length
+
sizeof
(
SSubmitBlk
)
+
pBlock
->
len
;
tsdbInsertData
(
pRepo
,
pMsg
);
}
pMsg
->
length
=
pMsg
->
length
+
sizeof
(
SSubmitBlk
)
+
pBlock
->
len
;
tsdbInsertData
(
pRepo
,
pMsg
);
double
etime
=
getCurTime
();
printf
(
"Spent %f seconds to write %d records
\n
"
,
etime
-
stime
,
nRows
);
// tsdbTriggerCommit(pRepo);
int
k
=
0
;
}
TEST
(
TsdbTest
,
openRepo
)
{
TEST
(
TsdbTest
,
DISABLED_
openRepo
)
{
tsdb_repo_t
*
pRepo
=
tsdbOpenRepo
(
"/home/ubuntu/work/ttest/vnode0"
);
ASSERT_NE
(
pRepo
,
nullptr
);
}
TEST
(
TsdbTest
,
DISABLED_createFileGroup
)
{
SFileGroup
fGroup
;
ASSERT_EQ
(
tsdbCreateFileGroup
(
"/home/ubuntu/work/ttest/vnode0/data"
,
1820
,
&
fGroup
,
1000
),
0
);
int
k
=
0
;
}
\ No newline at end of file
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录