Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
ae3730c3
T
TDengine
项目概览
taosdata
/
TDengine
大约 1 年 前同步成功
通知
1181
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
ae3730c3
编写于
4月 23, 2023
作者:
dengyihao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add ttl to stream state key
上级
f2c36621
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
152 addition
and
61 deletion
+152
-61
include/libs/stream/streamState.h
include/libs/stream/streamState.h
+4
-3
source/libs/stream/src/streamStateRocksdb.c
source/libs/stream/src/streamStateRocksdb.c
+148
-58
未找到文件。
include/libs/stream/streamState.h
浏览文件 @
ae3730c3
...
@@ -40,6 +40,7 @@ typedef struct STdbState {
...
@@ -40,6 +40,7 @@ typedef struct STdbState {
struct
SStreamTask
*
pOwner
;
struct
SStreamTask
*
pOwner
;
void
*
param
;
void
*
param
;
void
*
env
;
void
*
env
;
void
*
compactFactory
;
TDB
*
db
;
TDB
*
db
;
TTB
*
pStateDb
;
TTB
*
pStateDb
;
...
@@ -149,11 +150,11 @@ typedef struct SStateSessionKey {
...
@@ -149,11 +150,11 @@ typedef struct SStateSessionKey {
int64_t
opNum
;
int64_t
opNum
;
}
SStateSessionKey
;
}
SStateSessionKey
;
typedef
struct
s
treamValue
{
typedef
struct
SS
treamValue
{
int64_t
unixTimestamp
;
int64_t
unixTimestamp
;
int32_t
len
;
int32_t
len
;
char
data
[
0
]
;
char
*
data
;
}
s
treamValue
;
}
SS
treamValue
;
int
sessionRangeKeyCmpr
(
const
SSessionKey
*
pWin1
,
const
SSessionKey
*
pWin2
);
int
sessionRangeKeyCmpr
(
const
SSessionKey
*
pWin1
,
const
SSessionKey
*
pWin2
);
int
sessionWinKeyCmpr
(
const
SSessionKey
*
pWin1
,
const
SSessionKey
*
pWin2
);
int
sessionWinKeyCmpr
(
const
SSessionKey
*
pWin1
,
const
SSessionKey
*
pWin2
);
...
...
source/libs/stream/src/streamStateRocksdb.c
浏览文件 @
ae3730c3
...
@@ -289,24 +289,24 @@ int parKeyToString(void* k, char* buf) {
...
@@ -289,24 +289,24 @@ int parKeyToString(void* k, char* buf) {
return
n
;
return
n
;
}
}
int
stremaValueEncode
(
void
*
k
,
char
*
buf
)
{
int
stremaValueEncode
(
void
*
k
,
char
*
buf
)
{
int
len
=
0
;
int
len
=
0
;
s
treamValue
*
key
=
k
;
SS
treamValue
*
key
=
k
;
len
+=
taosEncodeFixedI64
((
void
**
)
&
buf
,
key
->
unixTimestamp
);
len
+=
taosEncodeFixedI64
((
void
**
)
&
buf
,
key
->
unixTimestamp
);
len
+=
taosEncodeFixedI32
((
void
**
)
&
buf
,
key
->
len
);
len
+=
taosEncodeFixedI32
((
void
**
)
&
buf
,
key
->
len
);
len
+=
taosEncodeBinary
((
void
**
)
&
buf
,
key
->
data
,
key
->
len
);
len
+=
taosEncodeBinary
((
void
**
)
&
buf
,
key
->
data
,
key
->
len
);
return
len
;
return
len
;
}
}
int
streamValueDecode
(
void
*
k
,
char
*
buf
)
{
int
streamValueDecode
(
void
*
k
,
char
*
buf
)
{
s
treamValue
*
key
=
k
;
SS
treamValue
*
key
=
k
;
char
*
p
=
buf
;
char
*
p
=
buf
;
p
=
taosDecodeFixedI64
(
p
,
&
key
->
unixTimestamp
);
p
=
taosDecodeFixedI64
(
p
,
&
key
->
unixTimestamp
);
p
=
taosDecodeFixedI32
(
p
,
&
key
->
len
);
p
=
taosDecodeFixedI32
(
p
,
&
key
->
len
);
p
=
taosDecodeBinary
(
p
,
(
void
**
)
&
key
->
data
,
key
->
len
);
p
=
taosDecodeBinary
(
p
,
(
void
**
)
&
key
->
data
,
key
->
len
);
return
p
-
buf
;
return
p
-
buf
;
}
}
int32_t
streamValueToString
(
void
*
k
,
char
*
buf
)
{
int32_t
streamValueToString
(
void
*
k
,
char
*
buf
)
{
s
treamValue
*
key
=
k
;
SS
treamValue
*
key
=
k
;
int
n
=
0
;
int
n
=
0
;
n
+=
sprintf
(
buf
+
n
,
"[unixTimestamp:%"
PRIi64
","
,
key
->
unixTimestamp
);
n
+=
sprintf
(
buf
+
n
,
"[unixTimestamp:%"
PRIi64
","
,
key
->
unixTimestamp
);
n
+=
sprintf
(
buf
+
n
,
"len:%d,"
,
key
->
len
);
n
+=
sprintf
(
buf
+
n
,
"len:%d,"
,
key
->
len
);
n
+=
sprintf
(
buf
+
n
,
"data:%s]"
,
key
->
data
);
n
+=
sprintf
(
buf
+
n
,
"data:%s]"
,
key
->
data
);
...
@@ -315,7 +315,7 @@ int32_t streamValueToString(void* k, char* buf) {
...
@@ -315,7 +315,7 @@ int32_t streamValueToString(void* k, char* buf) {
/*1: stale, 0: no stale*/
/*1: stale, 0: no stale*/
int32_t
streaValueIsStale
(
void
*
k
,
int64_t
ts
)
{
int32_t
streaValueIsStale
(
void
*
k
,
int64_t
ts
)
{
s
treamValue
*
key
=
k
;
SS
treamValue
*
key
=
k
;
if
(
key
->
unixTimestamp
<
ts
)
{
if
(
key
->
unixTimestamp
<
ts
)
{
return
1
;
return
1
;
}
}
...
@@ -325,6 +325,8 @@ int32_t streaValueIsStale(void* k, int64_t ts) {
...
@@ -325,6 +325,8 @@ int32_t streaValueIsStale(void* k, int64_t ts) {
typedef
struct
{
typedef
struct
{
void
*
tableOpt
;
void
*
tableOpt
;
void
*
lru
;
// global or not
void
*
lru
;
// global or not
void
*
filteFactory
;
}
rocksdbCfParam
;
}
rocksdbCfParam
;
const
char
*
cfName
[]
=
{
"default"
,
"state"
,
"fill"
,
"sess"
,
"func"
,
"parname"
,
"partag"
};
const
char
*
cfName
[]
=
{
"default"
,
"state"
,
"fill"
,
"sess"
,
"func"
,
"parname"
,
"partag"
};
...
@@ -334,6 +336,8 @@ typedef int (*ToStringFunc)(void* key, char* buf);
...
@@ -334,6 +336,8 @@ typedef int (*ToStringFunc)(void* key, char* buf);
typedef
const
char
*
(
*
CompareName
)(
void
*
statue
);
typedef
const
char
*
(
*
CompareName
)(
void
*
statue
);
typedef
int
(
*
BackendCmpFunc
)(
void
*
state
,
const
char
*
aBuf
,
size_t
aLen
,
const
char
*
bBuf
,
size_t
bLen
);
typedef
int
(
*
BackendCmpFunc
)(
void
*
state
,
const
char
*
aBuf
,
size_t
aLen
,
const
char
*
bBuf
,
size_t
bLen
);
typedef
void
(
*
DestroyFunc
)(
void
*
state
);
typedef
void
(
*
DestroyFunc
)(
void
*
state
);
typedef
int32_t
(
*
EncodeValueFunc
)(
void
*
value
,
int32_t
vlen
,
int64_t
ttl
,
char
**
dest
);
typedef
int32_t
(
*
DecodeValueFunc
)(
void
*
value
,
int32_t
vlen
,
int64_t
*
ttl
,
char
**
dest
);
const
char
*
compareDefaultName
(
void
*
name
);
const
char
*
compareDefaultName
(
void
*
name
);
const
char
*
compareStateName
(
void
*
name
);
const
char
*
compareStateName
(
void
*
name
);
...
@@ -346,28 +350,57 @@ const char* comparePartagKeyName(void* name);
...
@@ -346,28 +350,57 @@ const char* comparePartagKeyName(void* name);
void
destroyFunc
(
void
*
stata
)
{
return
;
}
void
destroyFunc
(
void
*
stata
)
{
return
;
}
typedef
struct
{
typedef
struct
{
const
char
*
key
;
const
char
*
key
;
int32_t
len
;
int32_t
len
;
int
idx
;
int
idx
;
BackendCmpFunc
cmpFunc
;
BackendCmpFunc
cmpFunc
;
EncodeFunc
enFunc
;
EncodeFunc
enFunc
;
DecodeFunc
deFunc
;
DecodeFunc
deFunc
;
ToStringFunc
toStrFunc
;
ToStringFunc
toStrFunc
;
CompareName
cmpName
;
CompareName
cmpName
;
DestroyFunc
detroyFunc
;
DestroyFunc
detroyFunc
;
EncodeValueFunc
enValueFunc
;
DecodeValueFunc
deValueFunc
;
}
SCfInit
;
}
SCfInit
;
int32_t
encodeValueFunc
(
void
*
value
,
int32_t
vlen
,
int64_t
ttl
,
char
**
dest
)
{
SStreamValue
key
=
{.
unixTimestamp
=
ttl
,
.
len
=
vlen
,
.
data
=
(
char
*
)(
value
)};
char
*
p
=
taosMemoryCalloc
(
1
,
sizeof
(
int64_t
)
+
sizeof
(
int32_t
)
+
key
.
len
);
char
*
buf
=
p
;
int32_t
len
=
0
;
len
+=
taosEncodeFixedI64
((
void
**
)
&
buf
,
key
.
unixTimestamp
);
len
+=
taosEncodeFixedI32
((
void
**
)
&
buf
,
key
.
len
);
len
+=
taosEncodeBinary
((
void
**
)
&
buf
,
(
char
*
)
value
,
vlen
);
*
dest
=
p
;
return
len
;
}
int32_t
decodeValueFunc
(
void
*
value
,
int32_t
vlen
,
int64_t
*
ttl
,
char
**
dest
)
{
SStreamValue
key
=
{
0
};
char
*
p
=
value
;
p
=
taosDecodeFixedI64
(
p
,
&
key
.
unixTimestamp
);
p
=
taosDecodeFixedI32
(
p
,
&
key
.
len
);
p
=
taosDecodeBinary
(
p
,
(
void
**
)
&
(
key
.
data
),
key
.
len
);
*
ttl
=
key
.
unixTimestamp
;
*
dest
=
key
.
data
;
return
key
.
len
;
}
SCfInit
ginitDict
[]
=
{
SCfInit
ginitDict
[]
=
{
{
"default"
,
7
,
0
,
defaultKeyComp
,
defaultKeyEncode
,
defaultKeyDecode
,
defaultKeyToString
,
compareDefaultName
,
{
"default"
,
7
,
0
,
defaultKeyComp
,
defaultKeyEncode
,
defaultKeyDecode
,
defaultKeyToString
,
compareDefaultName
,
destroyFunc
},
destroyFunc
,
encodeValueFunc
,
decodeValueFunc
},
{
"state"
,
5
,
1
,
stateKeyDBComp
,
stateKeyEncode
,
stateKeyDecode
,
stateKeyToString
,
compareStateName
,
destroyFunc
},
{
"state"
,
5
,
1
,
stateKeyDBComp
,
stateKeyEncode
,
stateKeyDecode
,
stateKeyToString
,
compareStateName
,
destroyFunc
,
{
"fill"
,
4
,
2
,
winKeyDBComp
,
winKeyEncode
,
winKeyDecode
,
winKeyToString
,
compareWinKeyName
,
destroyFunc
},
encodeValueFunc
,
decodeValueFunc
},
{
"fill"
,
4
,
2
,
winKeyDBComp
,
winKeyEncode
,
winKeyDecode
,
winKeyToString
,
compareWinKeyName
,
destroyFunc
,
encodeValueFunc
,
decodeValueFunc
},
{
"sess"
,
4
,
3
,
stateSessionKeyDBComp
,
stateSessionKeyEncode
,
stateSessionKeyDecode
,
stateSessionKeyToString
,
{
"sess"
,
4
,
3
,
stateSessionKeyDBComp
,
stateSessionKeyEncode
,
stateSessionKeyDecode
,
stateSessionKeyToString
,
compareSessionKeyName
,
destroyFunc
},
compareSessionKeyName
,
destroyFunc
,
encodeValueFunc
,
decodeValueFunc
},
{
"func"
,
4
,
4
,
tupleKeyDBComp
,
tupleKeyEncode
,
tupleKeyDecode
,
tupleKeyToString
,
compareFuncKeyName
,
destroyFunc
},
{
"func"
,
4
,
4
,
tupleKeyDBComp
,
tupleKeyEncode
,
tupleKeyDecode
,
tupleKeyToString
,
compareFuncKeyName
,
destroyFunc
,
{
"parname"
,
7
,
5
,
parKeyDBComp
,
parKeyEncode
,
parKeyDecode
,
parKeyToString
,
compareParKeyName
,
destroyFunc
},
encodeValueFunc
,
decodeValueFunc
},
{
"partag"
,
6
,
6
,
parKeyDBComp
,
parKeyEncode
,
parKeyDecode
,
parKeyToString
,
comparePartagKeyName
,
destroyFunc
},
{
"parname"
,
7
,
5
,
parKeyDBComp
,
parKeyEncode
,
parKeyDecode
,
parKeyToString
,
compareParKeyName
,
destroyFunc
,
encodeValueFunc
,
decodeValueFunc
},
{
"partag"
,
6
,
6
,
parKeyDBComp
,
parKeyEncode
,
parKeyDecode
,
parKeyToString
,
comparePartagKeyName
,
destroyFunc
,
encodeValueFunc
,
decodeValueFunc
},
};
};
const
char
*
compareDefaultName
(
void
*
name
)
{
return
ginitDict
[
0
].
key
;
}
const
char
*
compareDefaultName
(
void
*
name
)
{
return
ginitDict
[
0
].
key
;
}
...
@@ -378,6 +411,41 @@ const char* compareFuncKeyName(void* name) { return ginitDict[4].key; }
...
@@ -378,6 +411,41 @@ const char* compareFuncKeyName(void* name) { return ginitDict[4].key; }
const
char
*
compareParKeyName
(
void
*
name
)
{
return
ginitDict
[
5
].
key
;
}
const
char
*
compareParKeyName
(
void
*
name
)
{
return
ginitDict
[
5
].
key
;
}
const
char
*
comparePartagKeyName
(
void
*
name
)
{
return
ginitDict
[
6
].
key
;
}
const
char
*
comparePartagKeyName
(
void
*
name
)
{
return
ginitDict
[
6
].
key
;
}
typedef
struct
SCompactFilteFactory
{
void
*
status
;
}
SCompactFilteFactory
;
void
destroyCompactFilteFactory
(
void
*
arg
)
{
if
(
arg
==
NULL
)
return
;
}
const
char
*
compactFilteFactoryName
(
void
*
arg
)
{
SCompactFilteFactory
*
state
=
arg
;
return
"stream_compact_filter"
;
}
void
destroyCompactFilte
(
void
*
arg
)
{
if
(
arg
==
NULL
)
return
;
}
unsigned
char
compactFilte
(
void
*
arg
,
int
level
,
const
char
*
key
,
size_t
klen
,
const
char
*
val
,
size_t
vlen
,
char
**
newval
,
size_t
*
newvlen
,
unsigned
char
*
value_changed
)
{
int64_t
unixTime
=
taosGetTimestampMs
();
SStreamValue
value
;
memset
(
&
value
,
0
,
sizeof
(
value
));
streamValueDecode
(
&
value
,
(
char
*
)
val
);
taosMemoryFree
(
value
.
data
);
if
(
value
.
unixTimestamp
!=
0
&&
value
.
unixTimestamp
<
unixTime
)
{
return
1
;
}
return
0
;
}
const
char
*
compactFilteName
(
void
*
arg
)
{
return
"stream_filte"
;
}
rocksdb_compactionfilter_t
*
compactFilteFactoryCreateFilter
(
void
*
arg
,
rocksdb_compactionfiltercontext_t
*
ctx
)
{
SCompactFilteFactory
*
state
=
arg
;
rocksdb_compactionfilter_t
*
filter
=
rocksdb_compactionfilter_create
(
NULL
,
destroyCompactFilte
,
compactFilte
,
compactFilteName
);
return
filter
;
}
int
streamInitBackend
(
SStreamState
*
pState
,
char
*
path
)
{
int
streamInitBackend
(
SStreamState
*
pState
,
char
*
path
)
{
rocksdb_env_t
*
env
=
rocksdb_create_default_env
();
// rocksdb_envoptions_create();
rocksdb_env_t
*
env
=
rocksdb_create_default_env
();
// rocksdb_envoptions_create();
rocksdb_env_set_low_priority_background_threads
(
env
,
4
);
rocksdb_env_set_low_priority_background_threads
(
env
,
4
);
...
@@ -392,6 +460,10 @@ int streamInitBackend(SStreamState* pState, char* path) {
...
@@ -392,6 +460,10 @@ int streamInitBackend(SStreamState* pState, char* path) {
rocksdb_options_set_create_missing_column_families
(
opts
,
1
);
rocksdb_options_set_create_missing_column_families
(
opts
,
1
);
rocksdb_options_set_write_buffer_size
(
opts
,
128
<<
20
);
rocksdb_options_set_write_buffer_size
(
opts
,
128
<<
20
);
rocksdb_compactionfilterfactory_t
*
factory
=
rocksdb_compactionfilterfactory_create
(
NULL
,
destroyCompactFilteFactory
,
compactFilteFactoryCreateFilter
,
compactFilteFactoryName
);
rocksdb_options_set_compaction_filter_factory
(
opts
,
factory
);
char
*
err
=
NULL
;
char
*
err
=
NULL
;
int
cfLen
=
sizeof
(
ginitDict
)
/
sizeof
(
ginitDict
[
0
]);
int
cfLen
=
sizeof
(
ginitDict
)
/
sizeof
(
ginitDict
[
0
]);
...
@@ -411,8 +483,6 @@ int streamInitBackend(SStreamState* pState, char* path) {
...
@@ -411,8 +483,6 @@ int streamInitBackend(SStreamState* pState, char* path) {
param
[
i
].
tableOpt
=
tableOpt
;
param
[
i
].
tableOpt
=
tableOpt
;
param
[
i
].
lru
=
cache
;
param
[
i
].
lru
=
cache
;
// rocksdb_slicetransform_t* trans = rocksdb_slicetransform_create_fixed_prefix(8);
// rocksdb_options_set_prefix_extractor((rocksdb_options_t*)cfOpt[i], trans);
};
};
rocksdb_comparator_t
**
pCompare
=
taosMemoryCalloc
(
cfLen
,
sizeof
(
rocksdb_comparator_t
**
));
rocksdb_comparator_t
**
pCompare
=
taosMemoryCalloc
(
cfLen
,
sizeof
(
rocksdb_comparator_t
**
));
...
@@ -437,6 +507,7 @@ int streamInitBackend(SStreamState* pState, char* path) {
...
@@ -437,6 +507,7 @@ int streamInitBackend(SStreamState* pState, char* path) {
pState
->
pTdbState
->
dbOpt
=
opts
;
pState
->
pTdbState
->
dbOpt
=
opts
;
pState
->
pTdbState
->
param
=
param
;
pState
->
pTdbState
->
param
=
param
;
pState
->
pTdbState
->
env
=
env
;
pState
->
pTdbState
->
env
=
env
;
pState
->
pTdbState
->
compactFactory
=
factory
;
return
0
;
return
0
;
}
}
void
streamCleanBackend
(
SStreamState
*
pState
)
{
void
streamCleanBackend
(
SStreamState
*
pState
)
{
...
@@ -468,11 +539,13 @@ void streamCleanBackend(SStreamState* pState) {
...
@@ -468,11 +539,13 @@ void streamCleanBackend(SStreamState* pState) {
rocksdb_cache_destroy
(
param
[
i
].
lru
);
rocksdb_cache_destroy
(
param
[
i
].
lru
);
rocksdb_block_based_options_destroy
(
param
[
i
].
tableOpt
);
rocksdb_block_based_options_destroy
(
param
[
i
].
tableOpt
);
// rocksdb_compactionfilterfactory_destroy(param[i].filteFactory);
}
}
taosMemoryFreeClear
(
pState
->
pTdbState
->
cfOpts
);
taosMemoryFreeClear
(
pState
->
pTdbState
->
cfOpts
);
taosMemoryFree
(
pState
->
pTdbState
->
pCompare
);
taosMemoryFree
(
pState
->
pTdbState
->
pCompare
);
taosMemoryFree
(
pState
->
pTdbState
->
param
);
taosMemoryFree
(
pState
->
pTdbState
->
param
);
rocksdb_env_destroy
(
pState
->
pTdbState
->
env
);
rocksdb_env_destroy
(
pState
->
pTdbState
->
env
);
rocksdb_compactionfilterfactory_destroy
(
pState
->
pTdbState
->
compactFactory
);
pState
->
pTdbState
->
rocksdb
=
NULL
;
pState
->
pTdbState
->
rocksdb
=
NULL
;
}
}
...
@@ -514,31 +587,34 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa
...
@@ -514,31 +587,34 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa
return
rocksdb_create_iterator_cf
(
pState
->
pTdbState
->
rocksdb
,
rOpt
,
pState
->
pTdbState
->
pHandle
[
idx
]);
return
rocksdb_create_iterator_cf
(
pState
->
pTdbState
->
rocksdb
,
rOpt
,
pState
->
pTdbState
->
pHandle
[
idx
]);
}
}
#define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \
#define STREAM_STATE_PUT_ROCKSDB(pState, funcname, key, value, vLen) \
do { \
do { \
code = 0; \
code = 0; \
char buf[128] = {0}; \
char buf[128] = {0}; \
char* err = NULL; \
char* err = NULL; \
int i = streamGetInit(funcname); \
int i = streamGetInit(funcname); \
if (i < 0) { \
if (i < 0) { \
qWarn("streamState failed to get cf name: %s", funcname); \
qWarn("streamState failed to get cf name: %s", funcname); \
code = -1; \
code = -1; \
break; \
break; \
} \
} \
char toString[128] = {0}; \
char toString[128] = {0}; \
if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \
if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \
int32_t klen = ginitDict[i].enFunc((void*)key, buf); \
int32_t klen = ginitDict[i].enFunc((void*)key, buf); \
rocksdb_column_family_handle_t* pHandle = pState->pTdbState->pHandle[ginitDict[i].idx]; \
rocksdb_column_family_handle_t* pHandle = pState->pTdbState->pHandle[ginitDict[i].idx]; \
rocksdb_t* db = pState->pTdbState->rocksdb; \
rocksdb_t* db = pState->pTdbState->rocksdb; \
rocksdb_writeoptions_t* opts = pState->pTdbState->writeOpts; \
rocksdb_writeoptions_t* opts = pState->pTdbState->writeOpts; \
rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)value, (size_t)vLen, &err); \
char* ttlV = NULL; \
if (err != NULL) { \
int32_t ttlVLen = ginitDict[i].enValueFunc((char*)value, vLen, 0, &ttlV); \
taosMemoryFree(err); \
rocksdb_put_cf(db, opts, pHandle, (const char*)buf, klen, (const char*)ttlV, (size_t)ttlVLen, &err); \
qDebug("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \
if (err != NULL) { \
code = -1; \
taosMemoryFree(err); \
} else { \
qDebug("streamState str: %s failed to write to %s, err: %s", toString, funcname, err); \
qDebug("streamState str:%s succ to write to %s, valLen:%d", toString, funcname, vLen); \
code = -1; \
} \
} else { \
qDebug("streamState str:%s succ to write to %s, valLen:%d", toString, funcname, vLen); \
} \
taosMemoryFree(ttlV); \
} while (0);
} while (0);
#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \
#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \
...
@@ -565,8 +641,15 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa
...
@@ -565,8 +641,15 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa
if (err != NULL) taosMemoryFree(err); \
if (err != NULL) taosMemoryFree(err); \
code = -1; \
code = -1; \
} else { \
} else { \
if (pVal != NULL) *pVal = val; \
char * p = NULL, *end = NULL; \
if (vLen != NULL) *vLen = len; \
int64_t ttl; \
int32_t vlen = ginitDict[i].deValueFunc(val, len, &ttl, &p); \
if (pVal != NULL) { \
*pVal = p; \
} else { \
taosMemoryFree(p); \
} \
if (vLen != NULL) *vLen = vlen; \
} \
} \
if (err != NULL) { \
if (err != NULL) { \
taosMemoryFree(err); \
taosMemoryFree(err); \
...
@@ -606,7 +689,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa
...
@@ -606,7 +689,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa
int32_t
streamStateFuncPut_rocksdb
(
SStreamState
*
pState
,
const
STupleKey
*
key
,
const
void
*
value
,
int32_t
vLen
)
{
int32_t
streamStateFuncPut_rocksdb
(
SStreamState
*
pState
,
const
STupleKey
*
key
,
const
void
*
value
,
int32_t
vLen
)
{
int
code
=
0
;
int
code
=
0
;
STREAM_STATE_PUT_ROCKSDB
(
pState
,
"func"
,
key
,
value
,
vLen
);
STREAM_STATE_PUT_ROCKSDB
(
pState
,
"func"
,
key
,
(
void
*
)
value
,
vLen
);
return
code
;
return
code
;
}
}
int32_t
streamStateFuncGet_rocksdb
(
SStreamState
*
pState
,
const
STupleKey
*
key
,
void
**
pVal
,
int32_t
*
pVLen
)
{
int32_t
streamStateFuncGet_rocksdb
(
SStreamState
*
pState
,
const
STupleKey
*
key
,
void
**
pVal
,
int32_t
*
pVLen
)
{
...
@@ -624,7 +707,7 @@ int32_t streamStatePut_rocksdb(SStreamState* pState, const SWinKey* key, const v
...
@@ -624,7 +707,7 @@ int32_t streamStatePut_rocksdb(SStreamState* pState, const SWinKey* key, const v
int
code
=
0
;
int
code
=
0
;
SStateKey
sKey
=
{.
key
=
*
key
,
.
opNum
=
pState
->
number
};
SStateKey
sKey
=
{.
key
=
*
key
,
.
opNum
=
pState
->
number
};
STREAM_STATE_PUT_ROCKSDB
(
pState
,
"state"
,
&
sKey
,
value
,
vLen
);
STREAM_STATE_PUT_ROCKSDB
(
pState
,
"state"
,
&
sKey
,
(
void
*
)
value
,
vLen
);
return
code
;
return
code
;
}
}
int32_t
streamStatePutBatch_rocksdb
(
SStreamState
*
pState
,
void
*
pBatch
)
{
int32_t
streamStatePutBatch_rocksdb
(
SStreamState
*
pState
,
void
*
pBatch
)
{
...
@@ -661,8 +744,11 @@ int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_wr
...
@@ -661,8 +744,11 @@ int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_wr
char
buf
[
128
]
=
{
0
};
char
buf
[
128
]
=
{
0
};
int32_t
klen
=
ginitDict
[
i
].
enFunc
((
void
*
)
key
,
buf
);
int32_t
klen
=
ginitDict
[
i
].
enFunc
((
void
*
)
key
,
buf
);
char
*
ttlV
=
NULL
;
int32_t
ttlVLen
=
ginitDict
[
i
].
enValueFunc
(
val
,
vlen
,
0
,
&
ttlV
);
rocksdb_column_family_handle_t
*
pCf
=
pState
->
pTdbState
->
pHandle
[
ginitDict
[
i
].
idx
];
rocksdb_column_family_handle_t
*
pCf
=
pState
->
pTdbState
->
pHandle
[
ginitDict
[
i
].
idx
];
rocksdb_writebatch_put_cf
((
rocksdb_writebatch_t
*
)
pBatch
,
pCf
,
buf
,
(
size_t
)
klen
,
val
,
(
size_t
)
vlen
);
rocksdb_writebatch_put_cf
((
rocksdb_writebatch_t
*
)
pBatch
,
pCf
,
buf
,
(
size_t
)
klen
,
ttlV
,
(
size_t
)
ttlVLen
);
taosMemoryFree
(
ttlV
);
return
0
;
return
0
;
}
}
...
@@ -844,7 +930,10 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) {
...
@@ -844,7 +930,10 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) {
sKeyStr
,
sLen
,
eKeyStr
,
eLen
,
&
err
);
sKeyStr
,
sLen
,
eKeyStr
,
eLen
,
&
err
);
// rocksdb_compact_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->pHandle[0], sKeyStr, sLen, eKeyStr, eLen);
// rocksdb_compact_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->pHandle[0], sKeyStr, sLen, eKeyStr, eLen);
if
(
err
!=
NULL
)
{
if
(
err
!=
NULL
)
{
qWarn
(
"failed to delete range cf(state) err: %s, start: %s, end:%s"
,
err
,
toStringStart
,
toStringEnd
);
qWarn
(
"failed to delete range cf(state) err: %s, "
"start: %s, end:%s"
,
err
,
toStringStart
,
toStringEnd
);
taosMemoryFree
(
err
);
taosMemoryFree
(
err
);
}
}
...
@@ -923,7 +1012,8 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta
...
@@ -923,7 +1012,8 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta
rocksdb_iter_prev
(
pCur
->
iter
);
rocksdb_iter_prev
(
pCur
->
iter
);
if
(
!
rocksdb_iter_valid
(
pCur
->
iter
))
{
if
(
!
rocksdb_iter_valid
(
pCur
->
iter
))
{
// qWarn("streamState failed to seek key prev %s", toString);
// qWarn("streamState failed to seek key prev
// %s", toString);
streamStateFreeCur
(
pCur
);
streamStateFreeCur
(
pCur
);
return
NULL
;
return
NULL
;
}
}
...
@@ -1528,7 +1618,7 @@ int32_t streamStateGetParTag_rocksdb(SStreamState* pState, int64_t groupId, void
...
@@ -1528,7 +1618,7 @@ int32_t streamStateGetParTag_rocksdb(SStreamState* pState, int64_t groupId, void
int32_t
streamStatePutParName_rocksdb
(
SStreamState
*
pState
,
int64_t
groupId
,
const
char
tbname
[
TSDB_TABLE_NAME_LEN
])
{
int32_t
streamStatePutParName_rocksdb
(
SStreamState
*
pState
,
int64_t
groupId
,
const
char
tbname
[
TSDB_TABLE_NAME_LEN
])
{
int
code
=
0
;
int
code
=
0
;
STREAM_STATE_PUT_ROCKSDB
(
pState
,
"parname"
,
&
groupId
,
tbname
,
TSDB_TABLE_NAME_LEN
);
STREAM_STATE_PUT_ROCKSDB
(
pState
,
"parname"
,
&
groupId
,
(
char
*
)
tbname
,
TSDB_TABLE_NAME_LEN
);
return
code
;
return
code
;
}
}
int32_t
streamStateGetParName_rocksdb
(
SStreamState
*
pState
,
int64_t
groupId
,
void
**
pVal
)
{
int32_t
streamStateGetParName_rocksdb
(
SStreamState
*
pState
,
int64_t
groupId
,
void
**
pVal
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录