Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
d04a33f3
T
TDengine
项目概览
taosdata
/
TDengine
1 年多 前同步成功
通知
1185
Star
22016
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
d04a33f3
编写于
6月 14, 2023
作者:
M
Minglei Jin
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix(tsdb/cache): rewrite cache update to fix cpu usage
上级
5612f7ff
变更
1
隐藏空白更改
内联
并排
Showing
1 changed file
with
140 addition
and
87 deletion
+140
-87
source/dnode/vnode/src/tsdb/tsdbCache.c
source/dnode/vnode/src/tsdb/tsdbCache.c
+140
-87
未找到文件。
source/dnode/vnode/src/tsdb/tsdbCache.c
浏览文件 @
d04a33f3
...
@@ -344,6 +344,11 @@ static void tsdbCacheDeleter(const void *key, size_t keyLen, void *value) {
...
@@ -344,6 +344,11 @@ static void tsdbCacheDeleter(const void *key, size_t keyLen, void *value) {
}
}
}
}
typedef
struct
{
int
idx
;
SLastKey
key
;
}
SIdxKey
;
int32_t
tsdbCacheUpdate
(
STsdb
*
pTsdb
,
tb_uid_t
suid
,
tb_uid_t
uid
,
TSDBROW
*
pRow
)
{
int32_t
tsdbCacheUpdate
(
STsdb
*
pTsdb
,
tb_uid_t
suid
,
tb_uid_t
uid
,
TSDBROW
*
pRow
)
{
int32_t
code
=
0
;
int32_t
code
=
0
;
...
@@ -370,113 +375,166 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
...
@@ -370,113 +375,166 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
tsdbRowClose
(
&
iter
);
tsdbRowClose
(
&
iter
);
// 3, build keys & multi get from rocks
// 3, build keys & multi get from rocks
int
num_keys
=
TARRAY_SIZE
(
aColVal
);
int
num_keys
=
TARRAY_SIZE
(
aColVal
);
char
**
keys_list
=
taosMemoryCalloc
(
num_keys
*
2
,
sizeof
(
char
*
));
size_t
*
keys_list_sizes
=
taosMemoryCalloc
(
num_keys
*
2
,
sizeof
(
size_t
));
char
*
key_list
=
taosMemoryMalloc
(
num_keys
*
ROCKS_KEY_LEN
*
2
);
for
(
int
i
=
0
;
i
<
num_keys
;
++
i
)
{
SColVal
*
pColVal
=
(
SColVal
*
)
taosArrayGet
(
aColVal
,
i
);
int16_t
cid
=
pColVal
->
cid
;
memcpy
(
key_list
+
i
*
ROCKS_KEY_LEN
,
&
(
SLastKey
){.
ltype
=
1
,
.
uid
=
uid
,
.
cid
=
cid
},
ROCKS_KEY_LEN
);
memcpy
(
key_list
+
i
*
ROCKS_KEY_LEN
+
num_keys
*
ROCKS_KEY_LEN
,
&
(
SLastKey
){.
ltype
=
0
,
.
uid
=
uid
,
.
cid
=
cid
},
ROCKS_KEY_LEN
);
keys_list
[
i
]
=
key_list
+
i
*
ROCKS_KEY_LEN
;
keys_list
[
num_keys
+
i
]
=
key_list
+
i
*
ROCKS_KEY_LEN
+
num_keys
*
ROCKS_KEY_LEN
;
keys_list_sizes
[
i
]
=
ROCKS_KEY_LEN
;
keys_list_sizes
[
num_keys
+
i
]
=
ROCKS_KEY_LEN
;
}
char
**
values_list
=
taosMemoryCalloc
(
num_keys
*
2
,
sizeof
(
char
*
));
size_t
*
values_list_sizes
=
taosMemoryCalloc
(
num_keys
*
2
,
sizeof
(
size_t
));
char
**
errs
=
taosMemoryCalloc
(
num_keys
*
2
,
sizeof
(
char
*
));
taosThreadMutexLock
(
&
pTsdb
->
rCache
.
rMutex
);
rocksdb_multi_get
(
pTsdb
->
rCache
.
db
,
pTsdb
->
rCache
.
readoptions
,
num_keys
*
2
,
(
const
char
*
const
*
)
keys_list
,
keys_list_sizes
,
values_list
,
values_list_sizes
,
errs
);
for
(
int
i
=
0
;
i
<
num_keys
*
2
;
++
i
)
{
rocksdb_free
(
errs
[
i
]);
}
taosMemoryFree
(
key_list
);
taosMemoryFree
(
keys_list
);
taosMemoryFree
(
keys_list_sizes
);
taosMemoryFree
(
errs
);
TSKEY
keyTs
=
TSDBROW_TS
(
pRow
);
TSKEY
keyTs
=
TSDBROW_TS
(
pRow
);
rocksdb_writebatch_t
*
wb
=
pTsdb
->
rCache
.
writebatch
;
rocksdb_writebatch_t
*
wb
=
pTsdb
->
rCache
.
writebatch
;
SArray
*
oColVal
=
taosArrayInit
(
num_keys
,
sizeof
(
SColVal
));
SArray
*
remainCols
=
NULL
;
SLRUCache
*
pCache
=
pTsdb
->
lruCache
;
taosThreadMutexLock
(
&
pTsdb
->
lruMutex
);
for
(
int
i
=
0
;
i
<
num_keys
;
++
i
)
{
for
(
int
i
=
0
;
i
<
num_keys
;
++
i
)
{
SColVal
*
pColVal
=
(
SColVal
*
)
taosArrayGet
(
aColVal
,
i
);
SColVal
*
pColVal
=
(
SColVal
*
)
taosArrayGet
(
aColVal
,
i
);
int16_t
cid
=
pColVal
->
cid
;
// if (!COL_VAL_IS_NONE(pColVal)) {
SLastKey
*
key
=
&
(
SLastKey
){.
ltype
=
0
,
.
uid
=
uid
,
.
cid
=
cid
};
SLastCol
*
pLastCol
=
tsdbCacheDeserialize
(
values_list
[
i
+
num_keys
]);
size_t
klen
=
ROCKS_KEY_LEN
;
LRUHandle
*
h
=
taosLRUCacheLookup
(
pCache
,
key
,
klen
);
if
(
NULL
==
pLastCol
||
pLastCol
->
ts
<=
keyTs
)
{
if
(
h
)
{
char
*
value
=
NULL
;
SLastCol
*
pLastCol
=
(
SLastCol
*
)
taosLRUCacheValue
(
pCache
,
h
);
size_t
vlen
=
0
;
tsdbCacheSerialize
(
&
(
SLastCol
){.
ts
=
keyTs
,
.
colVal
=
*
pColVal
},
&
value
,
&
vlen
);
SLastKey
key
=
(
SLastKey
){.
ltype
=
0
,
.
uid
=
uid
,
.
cid
=
pColVal
->
cid
};
size_t
klen
=
ROCKS_KEY_LEN
;
rocksdb_writebatch_put
(
wb
,
(
char
*
)
&
key
,
klen
,
value
,
vlen
);
pLastCol
=
(
SLastCol
*
)
value
;
if
(
pLastCol
->
ts
<=
keyTs
)
{
SLastCol
*
pTmpLastCol
=
taosMemoryCalloc
(
1
,
sizeof
(
SLastCol
));
uint8_t
*
pVal
=
NULL
;
*
pTmpLastCol
=
*
pLastCol
;
int
nData
=
pLastCol
->
colVal
.
value
.
nData
;
pLastCol
=
pTmpLastCol
;
if
(
IS_VAR_DATA_TYPE
(
pColVal
->
type
))
{
pVal
=
pLastCol
->
colVal
.
value
.
pData
;
}
pLastCol
->
ts
=
keyTs
;
pLastCol
->
colVal
=
*
pColVal
;
if
(
IS_VAR_DATA_TYPE
(
pColVal
->
type
))
{
if
(
nData
<
pColVal
->
value
.
nData
)
{
taosMemoryFree
(
pVal
);
pLastCol
->
colVal
.
value
.
pData
=
taosMemoryCalloc
(
1
,
pColVal
->
value
.
nData
);
}
else
{
pLastCol
->
colVal
.
value
.
pData
=
pVal
;
}
if
(
pColVal
->
value
.
nData
)
{
memcpy
(
pLastCol
->
colVal
.
value
.
pData
,
pColVal
->
value
.
pData
,
pColVal
->
value
.
nData
);
}
}
reallocVarData
(
&
pLastCol
->
colVal
);
char
*
value
=
NULL
;
size_t
charge
=
sizeof
(
*
pLastCol
);
size_t
vlen
=
0
;
if
(
IS_VAR_DATA_TYPE
(
pLastCol
->
colVal
.
type
))
{
tsdbCacheSerialize
(
pLastCol
,
&
value
,
&
vlen
);
charge
+=
pLastCol
->
colVal
.
value
.
nData
;
// tsdbCacheSerialize(&(SLastCol){.ts = keyTs, .colVal = *pColVal}, &value, &vlen);
rocksdb_writebatch_put
(
wb
,
(
char
*
)
key
,
klen
,
value
,
vlen
);
taosMemoryFree
(
value
);
}
}
LRUStatus
status
=
taosLRUCacheInsert
(
pTsdb
->
lruCache
,
&
key
,
ROCKS_KEY_LEN
,
pLastCol
,
charge
,
tsdbCacheDeleter
,
taosLRUCacheRelease
(
pCache
,
h
,
false
);
NULL
,
TAOS_LRU_PRIORITY_LOW
);
}
else
{
if
(
status
!=
TAOS_LRU_STATUS_OK
)
{
if
(
!
remainCols
)
{
code
=
-
1
;
remainCols
=
taosArrayInit
(
num_keys
*
2
,
sizeof
(
SIdxKey
))
;
}
}
taosArrayPush
(
remainCols
,
&
(
SIdxKey
){
i
,
*
key
});
taosMemoryFree
(
value
);
}
}
if
(
COL_VAL_IS_VALUE
(
pColVal
))
{
if
(
COL_VAL_IS_VALUE
(
pColVal
))
{
SLastCol
*
pLastCol
=
tsdbCacheDeserialize
(
values_list
[
i
]);
key
->
ltype
=
1
;
LRUHandle
*
h
=
taosLRUCacheLookup
(
pCache
,
key
,
klen
);
if
(
NULL
==
pLastCol
||
pLastCol
->
ts
<=
keyTs
)
{
if
(
h
)
{
char
*
value
=
NULL
;
SLastCol
*
pLastCol
=
(
SLastCol
*
)
taosLRUCacheValue
(
pCache
,
h
);
size_t
vlen
=
0
;
tsdbCacheSerialize
(
&
(
SLastCol
){.
ts
=
keyTs
,
.
colVal
=
*
pColVal
},
&
value
,
&
vlen
);
SLastKey
key
=
(
SLastKey
){.
ltype
=
1
,
.
uid
=
uid
,
.
cid
=
pColVal
->
cid
};
rocksdb_writebatch_put
(
wb
,
(
char
*
)
&
key
,
ROCKS_KEY_LEN
,
value
,
vlen
);
pLastCol
=
(
SLastCol
*
)
value
;
if
(
pLastCol
->
ts
<=
keyTs
)
{
SLastCol
*
pTmpLastCol
=
taosMemoryCalloc
(
1
,
sizeof
(
SLastCol
));
uint8_t
*
pVal
=
NULL
;
*
pTmpLastCol
=
*
pLastCol
;
int
nData
=
pLastCol
->
colVal
.
value
.
nData
;
pLastCol
=
pTmpLastCol
;
if
(
IS_VAR_DATA_TYPE
(
pColVal
->
type
))
{
pVal
=
pLastCol
->
colVal
.
value
.
pData
;
}
pLastCol
->
ts
=
keyTs
;
pLastCol
->
colVal
=
*
pColVal
;
if
(
IS_VAR_DATA_TYPE
(
pColVal
->
type
))
{
if
(
nData
<
pColVal
->
value
.
nData
)
{
taosMemoryFree
(
pVal
);
pLastCol
->
colVal
.
value
.
pData
=
taosMemoryCalloc
(
1
,
pColVal
->
value
.
nData
);
}
else
{
pLastCol
->
colVal
.
value
.
pData
=
pVal
;
}
if
(
pColVal
->
value
.
nData
)
{
memcpy
(
pLastCol
->
colVal
.
value
.
pData
,
pColVal
->
value
.
pData
,
pColVal
->
value
.
nData
);
}
}
reallocVarData
(
&
pLastCol
->
colVal
);
char
*
value
=
NULL
;
size_t
charge
=
sizeof
(
*
pLastCol
);
size_t
vlen
=
0
;
if
(
IS_VAR_DATA_TYPE
(
pLastCol
->
colVal
.
type
))
{
tsdbCacheSerialize
(
pLastCol
,
&
value
,
&
vlen
);
charge
+=
pLastCol
->
colVal
.
value
.
nData
;
rocksdb_writebatch_put
(
wb
,
(
char
*
)
key
,
klen
,
value
,
vlen
);
taosMemoryFree
(
value
);
}
}
LRUStatus
status
=
taosLRUCacheInsert
(
pTsdb
->
lruCache
,
&
key
,
ROCKS_KEY_LEN
,
pLastCol
,
charge
,
tsdbCacheDeleter
,
taosLRUCacheRelease
(
pCache
,
h
,
false
);
NULL
,
TAOS_LRU_PRIORITY_LOW
);
}
else
{
if
(
status
!=
TAOS_LRU_STATUS_OK
)
{
if
(
!
remainCols
)
{
code
=
-
1
;
remainCols
=
taosArrayInit
(
num_keys
*
2
,
sizeof
(
SIdxKey
))
;
}
}
taosArrayPush
(
remainCols
,
&
(
SIdxKey
){
i
,
*
key
});
}
}
}
taosMemoryFree
(
value
);
num_keys
=
TARRAY_SIZE
(
remainCols
);
if
(
remainCols
&&
num_keys
>
0
)
{
char
**
keys_list
=
taosMemoryCalloc
(
num_keys
,
sizeof
(
char
*
));
size_t
*
keys_list_sizes
=
taosMemoryCalloc
(
num_keys
,
sizeof
(
size_t
));
for
(
int
i
=
0
;
i
<
num_keys
;
++
i
)
{
SIdxKey
*
idxKey
=
&
((
SIdxKey
*
)
TARRAY_DATA
(
remainCols
))[
i
];
keys_list
[
i
]
=
(
char
*
)
&
idxKey
->
key
;
keys_list_sizes
[
i
]
=
ROCKS_KEY_LEN
;
}
char
**
values_list
=
taosMemoryCalloc
(
num_keys
,
sizeof
(
char
*
));
size_t
*
values_list_sizes
=
taosMemoryCalloc
(
num_keys
,
sizeof
(
size_t
));
char
**
errs
=
taosMemoryCalloc
(
num_keys
,
sizeof
(
char
*
));
rocksdb_multi_get
(
pTsdb
->
rCache
.
db
,
pTsdb
->
rCache
.
readoptions
,
num_keys
,
(
const
char
*
const
*
)
keys_list
,
keys_list_sizes
,
values_list
,
values_list_sizes
,
errs
);
for
(
int
i
=
0
;
i
<
num_keys
;
++
i
)
{
rocksdb_free
(
errs
[
i
]);
}
taosMemoryFree
(
errs
);
taosMemoryFree
(
keys_list
);
taosMemoryFree
(
keys_list_sizes
);
taosMemoryFree
(
values_list_sizes
);
for
(
int
i
=
0
;
i
<
num_keys
;
++
i
)
{
SIdxKey
*
idxKey
=
&
((
SIdxKey
*
)
TARRAY_DATA
(
remainCols
))[
i
];
SColVal
*
pColVal
=
(
SColVal
*
)
taosArrayGet
(
aColVal
,
idxKey
->
idx
);
SLastCol
*
pLastCol
=
tsdbCacheDeserialize
(
values_list
[
i
]);
if
(
idxKey
->
key
.
ltype
==
0
)
{
if
(
NULL
==
pLastCol
||
pLastCol
->
ts
<=
keyTs
)
{
char
*
value
=
NULL
;
size_t
vlen
=
0
;
tsdbCacheSerialize
(
&
(
SLastCol
){.
ts
=
keyTs
,
.
colVal
=
*
pColVal
},
&
value
,
&
vlen
);
// SLastKey key = (SLastKey){.ltype = 0, .uid = uid, .cid = pColVal->cid};
rocksdb_writebatch_put
(
wb
,
(
char
*
)
&
idxKey
->
key
,
ROCKS_KEY_LEN
,
value
,
vlen
);
taosMemoryFree
(
value
);
}
}
else
{
if
(
COL_VAL_IS_VALUE
(
pColVal
))
{
SLastCol
*
pLastCol
=
tsdbCacheDeserialize
(
values_list
[
i
]);
if
(
NULL
==
pLastCol
||
pLastCol
->
ts
<=
keyTs
)
{
char
*
value
=
NULL
;
size_t
vlen
=
0
;
tsdbCacheSerialize
(
&
(
SLastCol
){.
ts
=
keyTs
,
.
colVal
=
*
pColVal
},
&
value
,
&
vlen
);
// SLastKey key = (SLastKey){.ltype = 1, .uid = uid, .cid = pColVal->cid};
rocksdb_writebatch_put
(
wb
,
(
char
*
)
&
idxKey
->
key
,
ROCKS_KEY_LEN
,
value
,
vlen
);
taosMemoryFree
(
value
);
}
}
}
}
rocksdb_free
(
values_list
[
i
]);
}
}
//}
taosMemoryFree
(
values_list
);
rocksdb_free
(
values_list
[
i
]);
taosArrayDestroy
(
remainCols
);
rocksdb_free
(
values_list
[
i
+
num_keys
]);
}
}
taosMemoryFree
(
values_list
);
taosMemoryFree
(
values_list_sizes
);
rocksMayWrite
(
pTsdb
,
true
,
false
,
false
);
rocksMayWrite
(
pTsdb
,
true
,
false
,
false
);
taosThreadMutexUnlock
(
&
pTsdb
->
rCache
.
r
Mutex
);
taosThreadMutexUnlock
(
&
pTsdb
->
lru
Mutex
);
_exit:
_exit:
taosArrayDestroy
(
aColVal
);
taosArrayDestroy
(
aColVal
);
...
@@ -651,11 +709,6 @@ static SLastCol *tsdbCacheLoadCol(STsdb *pTsdb, SCacheRowsReader *pr, int16_t sl
...
@@ -651,11 +709,6 @@ static SLastCol *tsdbCacheLoadCol(STsdb *pTsdb, SCacheRowsReader *pr, int16_t sl
return
pLastCol
;
return
pLastCol
;
}
}
typedef
struct
{
int
idx
;
SLastKey
key
;
}
SIdxKey
;
static
int32_t
tsdbCacheLoadFromRaw
(
STsdb
*
pTsdb
,
tb_uid_t
uid
,
SArray
*
pLastArray
,
SArray
*
remainCols
,
static
int32_t
tsdbCacheLoadFromRaw
(
STsdb
*
pTsdb
,
tb_uid_t
uid
,
SArray
*
pLastArray
,
SArray
*
remainCols
,
SCacheRowsReader
*
pr
,
int8_t
ltype
)
{
SCacheRowsReader
*
pr
,
int8_t
ltype
)
{
int32_t
code
=
0
;
int32_t
code
=
0
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录