Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
0cd12fd3
T
TDengine
项目概览
taosdata
/
TDengine
12 个月 前同步成功
通知
1180
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
0cd12fd3
编写于
11月 22, 2021
作者:
dengyihao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
update builde node compile
上级
59cc5e42
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
153 addition
and
9 deletion
+153
-9
source/libs/index/inc/index_fst.h
source/libs/index/inc/index_fst.h
+3
-3
source/libs/index/inc/index_fst_counting_writer.h
source/libs/index/inc/index_fst_counting_writer.h
+42
-0
source/libs/index/inc/index_fst_node.h
source/libs/index/inc/index_fst_node.h
+6
-0
source/libs/index/inc/index_fst_registry.h
source/libs/index/inc/index_fst_registry.h
+2
-0
source/libs/index/src/index_fst.c
source/libs/index/src/index_fst.c
+35
-4
source/libs/index/src/index_fst_counting_writer.c
source/libs/index/src/index_fst_counting_writer.c
+43
-0
source/libs/index/src/index_fst_node.c
source/libs/index/src/index_fst_node.c
+22
-0
source/libs/index/src/index_fst_registry.c
source/libs/index/src/index_fst_registry.c
+0
-2
未找到文件。
source/libs/index/inc/index_fst.h
浏览文件 @
0cd12fd3
...
...
@@ -49,7 +49,7 @@ typedef struct FstUnFinishedNodes {
#define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack)
FstUnFinishedNodes
*
F
stUnFinishedNodesCreate
();
FstUnFinishedNodes
*
f
stUnFinishedNodesCreate
();
void
fstUnFinishedNodesPushEmpty
(
FstUnFinishedNodes
*
nodes
,
bool
isFinal
);
FstBuilderNode
*
fstUnFinishedNodesPopRoot
(
FstUnFinishedNodes
*
nodes
);
FstBuilderNode
*
fstUnFinishedNodesPopFreeze
(
FstUnFinishedNodes
*
nodes
,
CompiledAddr
addr
);
...
...
@@ -62,9 +62,9 @@ uint64_t FstUnFinishedNodesFindCommPreifxAndSetOutput(FstUnFinishedNodes *node,
typedef
struct
FstBuilder
{
FstCountingWriter
wtr
;
// The FST raw data is written directly to `wtr`.
FstCountingWriter
*
wrt
;
// The FST raw data is written directly to `wtr`.
FstUnFinishedNodes
*
unfinished
;
// The stack of unfinished nodes
FstRegistry
registry
;
// A map of finished nodes.
FstRegistry
*
registry
;
// A map of finished nodes.
SArray
*
last
;
// The last word added
CompiledAddr
lastAddr
;
// The address of the last compiled node
uint64_t
len
;
// num of keys added
...
...
source/libs/index/inc/index_fst_counting_writer.h
0 → 100644
浏览文件 @
0cd12fd3
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __INDEX_FST_COUNTING_WRITER_H__
#define __INDEX_FST_COUNTING_WRITER_H__
typedef
uint32_t
CheckSummer
;
typedef
struct
FstCountingWriter
{
void
*
wrt
;
// wrap any writer that counts and checksum bytes written
uint64_t
count
;
CheckSummer
summer
;
}
FstCountingWriter
;
uint64_t
fstCountingWriterWrite
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
bufLen
);
int
FstCountingWriterFlush
(
FstCountingWriter
*
write
);
FstCountingWriter
*
fstCountingWriterCreate
(
void
*
wtr
);
#define FST_WRITER_COUNT(writer) (writer->count)
#define FST_WRITER_INTER_WRITER(writer) (writer->wtr)
#define FST_WRITE_CHECK_SUMMER(writer) (writer->summer)
#endif
source/libs/index/inc/index_fst_node.h
浏览文件 @
0cd12fd3
...
...
@@ -17,7 +17,11 @@
#define __INDEX_FST_NODE_H__
#include "index_fst_util.h"
#include "index_fst_counting_writer.h"
#define FST_BUILDER_NODE_IS_FINAL(bn) (bn->isFinal)
#define FST_BUILDER_NODE_TRANS_ISEMPTY(bn) (taosArrayGetSize(bn->trans) == 0)
#define FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn) (bn->finalOutput == 0)
typedef
struct
FstTransition
{
uint8_t
inp
;
//The byte input associated with this transition.
...
...
@@ -37,4 +41,6 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src);
void
fstBuilderNodeCloneFrom
(
FstBuilderNode
*
dst
,
FstBuilderNode
*
src
);
bool
fstBuilderNodeCompileTo
(
FstBuilderNode
*
b
,
FstCountingWriter
*
wrt
,
CompiledAddr
lastAddr
,
CompiledAddr
startAddr
);
#endif
source/libs/index/inc/index_fst_registry.h
浏览文件 @
0cd12fd3
...
...
@@ -24,6 +24,8 @@ typedef struct FstRegistryCell {
FstBuilderNode
*
node
;
}
FstRegistryCell
;
#define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS)
#define FST_REGISTRY_CELL_INSERT(cell, tAddr) do {cell->addr = tAddr;} while(0)
//typedef struct FstRegistryCache {
...
...
source/libs/index/src/index_fst.c
浏览文件 @
0cd12fd3
...
...
@@ -279,12 +279,43 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) {
FstBuilder
*
b
=
malloc
(
sizeof
(
FstBuilder
));
if
(
NULL
==
b
)
{
return
b
;
}
FstCountingWriter
wtr
=
{.
wtr
=
w
,
.
count
=
0
,
.
summer
=
0
};
b
->
wtr
=
wtr
;
b
->
unfinished
=
malloc
(
sizeof
(
FstUnFinishedNodes
));
b
->
wrt
=
fstCountingWriterCreate
(
w
);
b
->
unfinished
=
fstUnFinishedNodesCreate
();
b
->
registry
=
fstRegistryCreate
(
10000
,
2
)
;
b
->
last
=
NULL
;
b
->
lastAddr
=
NONE_ADDRESS
;
b
->
len
=
0
;
return
b
;
}
void
fstBuilderCheckLastKey
(
FstBuilder
*
b
,
FstSlice
bs
,
bool
ckDupe
)
{
}
CompiledAddr
fstBuilderCompile
(
FstBuilder
*
b
,
FstBuilderNode
*
bn
)
{
if
(
FST_BUILDER_NODE_IS_FINAL
(
bn
)
&&
FST_BUILDER_NODE_TRANS_ISEMPTY
(
bn
)
&&
FST_BUILDER_NODE_FINALOUTPUT_ISZERO
(
bn
))
{
return
EMPTY_ADDRESS
;
}
FstRegistryEntry
*
entry
=
fstRegistryGetEntry
(
b
->
registry
,
bn
);
if
(
entry
->
state
==
FOUND
)
{
CompiledAddr
ret
=
entry
->
addr
;
tfree
(
entry
);
return
ret
;
}
CompiledAddr
startAddr
=
(
CompiledAddr
)(
FST_WRITER_COUNT
(
b
->
wrt
));
fstBuilderNodeCompileTo
(
bn
,
b
->
wrt
,
b
->
lastAddr
,
startAddr
);
b
->
lastAddr
=
(
CompiledAddr
)(
FST_WRITER_COUNT
(
b
->
wrt
))
-
1
;
if
(
entry
->
state
==
NOTFOUND
)
{
FST_REGISTRY_CELL_INSERT
(
entry
->
cell
,
b
->
lastAddr
);
}
free
(
entry
);
return
b
->
lastAddr
;
}
FstSlice
fstNodeAsSlice
(
FstNode
*
node
)
{
FstSlice
*
slice
=
&
node
->
data
;
FstSlice
s
=
fstSliceCopy
(
slice
,
slice
->
end
,
slice
->
dLen
-
1
);
...
...
source/libs/index/src/index_fst_counting_writer.c
0 → 100644
浏览文件 @
0cd12fd3
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
FstCountingWriter
*
fstCountingWriterCreate
(
void
*
wrt
)
{
FstCountingWriter
*
cw
=
calloc
(
1
,
sizeof
(
FstCountingWriter
));
if
(
cw
==
NULL
)
{
return
NULL
;
}
cw
->
wrt
=
wrt
;
return
cw
;
}
void
FstCountingWriterDestroy
(
FstCountingWriter
*
cw
)
{
// free wrt object: close fd or free mem
free
(
cw
);
}
uint64_t
fstCountingWriterWrite
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
bufLen
)
{
if
(
write
==
NULL
)
{
return
0
;
}
// update checksum
// write data to file/socket or mem
write
->
count
+=
bufLen
;
return
bufLen
;
}
int
FstCountingWriterFlush
(
FstCountingWriter
*
write
)
{
//write->wtr->flush
return
1
;
}
source/libs/index/src/index_fst_node.c
浏览文件 @
0cd12fd3
...
...
@@ -54,3 +54,25 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) {
src
->
trans
=
NULL
;
}
bool
fstBuilderNodeCompileTo
(
FstBuilderNode
*
b
,
FstCountingWriter
*
wrt
,
CompiledAddr
lastAddr
,
CompiledAddr
startAddr
)
{
size_t
sz
=
taosArrayGetSize
(
b
->
trans
);
assert
(
sz
<
256
);
if
(
FST_BUILDER_NODE_IS_FINAL
(
b
)
&&
FST_BUILDER_NODE_TRANS_ISEMPTY
(
b
)
&&
FST_BUILDER_NODE_FINALOUTPUT_ISZERO
(
b
))
{
return
true
;
}
else
if
(
sz
!=
1
||
b
->
isFinal
)
{
// AnyTrans->Compile(w, addr, node);
}
else
{
FstTransition
*
tran
=
taosArrayGet
(
b
->
trans
,
0
);
if
(
tran
->
addr
==
lastAddr
&&
tran
->
out
==
0
)
{
//OneTransNext::compile(w, lastAddr, tran->inp);
return
true
;
}
else
{
//OneTrans::Compile(w, lastAddr, *tran);
return
true
;
}
}
return
true
;
}
source/libs/index/src/index_fst_registry.c
浏览文件 @
0cd12fd3
...
...
@@ -64,8 +64,6 @@ static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) {
s
-=
1
;
}
}
#define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS)
#define FST_REGISTRY_CELL_INSERT(cell, addr) do {cell->addr = addr;} while(0)
FstRegistry
*
fstRegistryCreate
(
uint64_t
tableSize
,
uint64_t
mruSize
)
{
FstRegistry
*
registry
=
malloc
(
sizeof
(
FstRegistry
));
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录