Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
10b70175
TDengine
项目概览
taosdata
/
TDengine
1 年多 前同步成功
通知
1185
Star
22016
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
10b70175
编写于
12月 21, 2021
作者:
dengyihao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
update index TFile write
上级
f7be7909
变更
20
展开全部
隐藏空白更改
内联
并排
Showing
20 changed file
with
816 addition
and
1006 deletion
+816
-1006
source/libs/index/inc/indexInt.h
source/libs/index/inc/indexInt.h
+9
-9
source/libs/index/inc/index_cache.h
source/libs/index/inc/index_cache.h
+5
-6
source/libs/index/inc/index_fst.h
source/libs/index/inc/index_fst.h
+109
-111
source/libs/index/inc/index_fst_automation.h
source/libs/index/inc/index_fst_automation.h
+17
-17
source/libs/index/inc/index_fst_counting_writer.h
source/libs/index/inc/index_fst_counting_writer.h
+18
-18
source/libs/index/inc/index_fst_node.h
source/libs/index/inc/index_fst_node.h
+6
-6
source/libs/index/inc/index_fst_registry.h
source/libs/index/inc/index_fst_registry.h
+10
-8
source/libs/index/inc/index_fst_util.h
source/libs/index/inc/index_fst_util.h
+11
-11
source/libs/index/inc/index_tfile.h
source/libs/index/inc/index_tfile.h
+26
-26
source/libs/index/inc/index_util.h
source/libs/index/inc/index_util.h
+12
-12
source/libs/index/src/index.c
source/libs/index/src/index.c
+88
-67
source/libs/index/src/index_cache.c
source/libs/index/src/index_cache.c
+25
-29
source/libs/index/src/index_fst.c
source/libs/index/src/index_fst.c
+240
-225
source/libs/index/src/index_fst_automation.c
source/libs/index/src/index_fst_automation.c
+59
-54
source/libs/index/src/index_fst_common.c
source/libs/index/src/index_fst_common.c
+16
-256
source/libs/index/src/index_fst_counting_writer.c
source/libs/index/src/index_fst_counting_writer.c
+21
-19
source/libs/index/src/index_fst_node.c
source/libs/index/src/index_fst_node.c
+20
-36
source/libs/index/src/index_fst_registry.c
source/libs/index/src/index_fst_registry.c
+24
-22
source/libs/index/src/index_fst_util.c
source/libs/index/src/index_fst_util.c
+20
-18
source/libs/index/src/index_tfile.c
source/libs/index/src/index_tfile.c
+80
-56
未找到文件。
source/libs/index/inc/indexInt.h
浏览文件 @
10b70175
...
...
@@ -42,11 +42,11 @@ typedef struct SIndexStat {
struct
SIndex
{
#ifdef USE_LUCENE
index_t
*
index
;
index_t
*
index
;
#endif
void
*
cache
;
void
*
tindex
;
SHashObj
*
colObj
;
// < field name, field id>
void
*
cache
;
void
*
tindex
;
SHashObj
*
colObj
;
// < field name, field id>
int64_t
suid
;
// current super table id, -1 is normal table
int
colId
;
// field id allocated to cache
...
...
@@ -58,7 +58,7 @@ struct SIndex {
struct
SIndexOpts
{
#ifdef USE_LUCENE
void
*
opts
;
void
*
opts
;
#endif
#ifdef USE_INVERTED_INDEX
...
...
@@ -69,7 +69,7 @@ struct SIndexOpts {
struct
SIndexMultiTermQuery
{
EIndexOperatorType
opera
;
SArray
*
query
;
SArray
*
query
;
};
// field and key;
...
...
@@ -77,14 +77,14 @@ typedef struct SIndexTerm {
int64_t
suid
;
SIndexOperOnColumn
operType
;
// oper type, add/del/update
uint8_t
colType
;
// term data type, str/interger/json
char
*
colName
;
char
*
colName
;
int32_t
nColName
;
char
*
colVal
;
char
*
colVal
;
int32_t
nColVal
;
}
SIndexTerm
;
typedef
struct
SIndexTermQuery
{
SIndexTerm
*
term
;
SIndexTerm
*
term
;
EIndexQueryType
qType
;
}
SIndexTermQuery
;
...
...
source/libs/index/inc/index_cache.h
浏览文件 @
10b70175
...
...
@@ -34,19 +34,18 @@ extern "C" {
typedef
struct
IndexCache
{
T_REF_DECLARE
()
SSkipList
*
skiplist
;
SSkipList
*
skiplist
;
}
IndexCache
;
//
IndexCache
*
indexCacheCreate
();
IndexCache
*
indexCacheCreate
();
void
indexCacheDestroy
(
void
*
cache
);
void
indexCacheDestroy
(
void
*
cache
);
int
indexCachePut
(
void
*
cache
,
SIndexTerm
*
term
,
int16_t
colId
,
int32_t
version
,
uint64_t
uid
);
int
indexCachePut
(
void
*
cache
,
SIndexTerm
*
term
,
int16_t
colId
,
int32_t
version
,
uint64_t
uid
);
// int indexCacheGet(void *cache, uint64_t *rst);
int
indexCacheSearch
(
void
*
cache
,
SIndexTermQuery
*
query
,
int16_t
colId
,
int32_t
version
,
SArray
*
result
,
STermValueType
*
s
);
int
indexCacheSearch
(
void
*
cache
,
SIndexTermQuery
*
query
,
int16_t
colId
,
int32_t
version
,
SArray
*
result
,
STermValueType
*
s
);
#ifdef __cplusplus
}
...
...
source/libs/index/inc/index_fst.h
浏览文件 @
10b70175
...
...
@@ -40,10 +40,10 @@ typedef struct FstBoundWithData {
}
FstBoundWithData
;
typedef
struct
FstStreamBuilder
{
Fst
*
fst
;
AutomationCtx
*
aut
;
FstBoundWithData
*
min
;
FstBoundWithData
*
max
;
Fst
*
fst
;
AutomationCtx
*
aut
;
FstBoundWithData
*
min
;
FstBoundWithData
*
max
;
}
FstStreamBuilder
,
FstStreamWithStateBuilder
;
typedef
struct
FstRange
{
...
...
@@ -55,10 +55,10 @@ typedef enum { GE, GT, LE, LT } RangeType;
typedef
enum
{
OneTransNext
,
OneTrans
,
AnyTrans
,
EmptyFinal
}
State
;
typedef
enum
{
Ordered
,
OutOfOrdered
,
DuplicateKey
}
OrderType
;
FstBoundWithData
*
fstBoundStateCreate
(
FstBound
type
,
FstSlice
*
data
);
bool
fstBoundWithDataExceededBy
(
FstBoundWithData
*
bound
,
FstSlice
*
slice
);
bool
fstBoundWithDataIsEmpty
(
FstBoundWithData
*
bound
);
bool
fstBoundWithDataIsIncluded
(
FstBoundWithData
*
bound
);
FstBoundWithData
*
fstBoundStateCreate
(
FstBound
type
,
FstSlice
*
data
);
bool
fstBoundWithDataExceededBy
(
FstBoundWithData
*
bound
,
FstSlice
*
slice
);
bool
fstBoundWithDataIsEmpty
(
FstBoundWithData
*
bound
);
bool
fstBoundWithDataIsIncluded
(
FstBoundWithData
*
bound
);
typedef
struct
FstOutput
{
bool
null
;
...
...
@@ -71,46 +71,46 @@ typedef struct FstOutput {
* TODO: simple function name
*/
typedef
struct
FstUnFinishedNodes
{
SArray
*
stack
;
// <FstBuilderNodeUnfinished> } FstUnFinishedNodes;
SArray
*
stack
;
// <FstBuilderNodeUnfinished> } FstUnFinishedNodes;
}
FstUnFinishedNodes
;
#define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack)
FstUnFinishedNodes
*
fstUnFinishedNodesCreate
();
void
fstUnFinishedNodesDestroy
(
FstUnFinishedNodes
*
node
);
void
fstUnFinishedNodesPushEmpty
(
FstUnFinishedNodes
*
nodes
,
bool
isFinal
);
void
fstUnFinishedNodesSetRootOutput
(
FstUnFinishedNodes
*
node
,
Output
out
);
void
fstUnFinishedNodesTopLastFreeze
(
FstUnFinishedNodes
*
node
,
CompiledAddr
addr
);
void
fstUnFinishedNodesAddSuffix
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
,
Output
out
);
uint64_t
fstUnFinishedNodesFindCommPrefix
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
);
FstBuilderNode
*
fstUnFinishedNodesPopRoot
(
FstUnFinishedNodes
*
nodes
);
FstBuilderNode
*
fstUnFinishedNodesPopFreeze
(
FstUnFinishedNodes
*
nodes
,
CompiledAddr
addr
);
FstBuilderNode
*
fstUnFinishedNodesPopEmpty
(
FstUnFinishedNodes
*
nodes
);
FstUnFinishedNodes
*
fstUnFinishedNodesCreate
();
void
fstUnFinishedNodesDestroy
(
FstUnFinishedNodes
*
node
);
void
fstUnFinishedNodesPushEmpty
(
FstUnFinishedNodes
*
nodes
,
bool
isFinal
);
void
fstUnFinishedNodesSetRootOutput
(
FstUnFinishedNodes
*
node
,
Output
out
);
void
fstUnFinishedNodesTopLastFreeze
(
FstUnFinishedNodes
*
node
,
CompiledAddr
addr
);
void
fstUnFinishedNodesAddSuffix
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
,
Output
out
);
uint64_t
fstUnFinishedNodesFindCommPrefix
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
);
FstBuilderNode
*
fstUnFinishedNodesPopRoot
(
FstUnFinishedNodes
*
nodes
);
FstBuilderNode
*
fstUnFinishedNodesPopFreeze
(
FstUnFinishedNodes
*
nodes
,
CompiledAddr
addr
);
FstBuilderNode
*
fstUnFinishedNodesPopEmpty
(
FstUnFinishedNodes
*
nodes
);
uint64_t
fstUnFinishedNodesFindCommPrefixAndSetOutput
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
,
Output
in
,
Output
*
out
);
uint64_t
fstUnFinishedNodesFindCommPrefixAndSetOutput
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
,
Output
in
,
Output
*
out
);
typedef
struct
FstBuilder
{
FstCountingWriter
*
wrt
;
// The FST raw data is written directly to `wtr`.
FstUnFinishedNodes
*
unfinished
;
// The stack of unfinished nodes
FstRegistry
*
registry
;
// A map of finished nodes.
FstCountingWriter
*
wrt
;
// The FST raw data is written directly to `wtr`.
FstUnFinishedNodes
*
unfinished
;
// The stack of unfinished nodes
FstRegistry
*
registry
;
// A map of finished nodes.
FstSlice
last
;
// The last word added
CompiledAddr
lastAddr
;
// The address of the last compiled node
uint64_t
len
;
// num of keys added
}
FstBuilder
;
FstBuilder
*
fstBuilderCreate
(
void
*
w
,
FstType
ty
);
FstBuilder
*
fstBuilderCreate
(
void
*
w
,
FstType
ty
);
void
fstBuilderDestroy
(
FstBuilder
*
b
);
void
fstBuilderInsertOutput
(
FstBuilder
*
b
,
FstSlice
bs
,
Output
in
);
bool
fstBuilderInsert
(
FstBuilder
*
b
,
FstSlice
bs
,
Output
in
);
void
fstBuilderCompileFrom
(
FstBuilder
*
b
,
uint64_t
istate
);
void
*
fstBuilerIntoInner
(
FstBuilder
*
b
);
void
fstBuilderFinish
(
FstBuilder
*
b
);
OrderType
fstBuilderCheckLastKey
(
FstBuilder
*
b
,
FstSlice
bs
,
bool
ckDup
);
CompiledAddr
fstBuilderCompile
(
FstBuilder
*
b
,
FstBuilderNode
*
bn
);
void
fstBuilderDestroy
(
FstBuilder
*
b
);
void
fstBuilderInsertOutput
(
FstBuilder
*
b
,
FstSlice
bs
,
Output
in
);
bool
fstBuilderInsert
(
FstBuilder
*
b
,
FstSlice
bs
,
Output
in
);
void
fstBuilderCompileFrom
(
FstBuilder
*
b
,
uint64_t
istate
);
void
*
fstBuilerIntoInner
(
FstBuilder
*
b
);
void
fstBuilderFinish
(
FstBuilder
*
b
);
OrderType
fstBuilderCheckLastKey
(
FstBuilder
*
b
,
FstSlice
bs
,
bool
ckDup
);
CompiledAddr
fstBuilderCompile
(
FstBuilder
*
b
,
FstBuilderNode
*
bn
);
typedef
struct
FstTransitions
{
FstNode
*
node
;
FstNode
*
node
;
FstRange
range
;
}
FstTransitions
;
...
...
@@ -121,56 +121,55 @@ typedef struct FstState {
uint8_t
val
;
}
FstState
;
FstState
fstStateCreateFrom
(
FstSlice
*
data
,
CompiledAddr
addr
);
FstState
fstStateCreateFrom
(
FstSlice
*
data
,
CompiledAddr
addr
);
FstState
fstStateCreate
(
State
state
);
// compile
void
fstStateCompileForOneTransNext
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
uint8_t
inp
);
void
fstStateCompileForOneTrans
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
FstTransition
*
trn
);
void
fstStateCompileForAnyTrans
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
FstBuilderNode
*
node
);
void
fstStateCompileForOneTransNext
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
uint8_t
inp
);
void
fstStateCompileForOneTrans
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
FstTransition
*
trn
);
void
fstStateCompileForAnyTrans
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
FstBuilderNode
*
node
);
// set_comm_input
void
fstStateSetCommInput
(
FstState
*
state
,
uint8_t
inp
);
void
fstStateSetCommInput
(
FstState
*
state
,
uint8_t
inp
);
// comm_input
uint8_t
fstStateCommInput
(
FstState
*
state
,
bool
*
null
);
uint8_t
fstStateCommInput
(
FstState
*
state
,
bool
*
null
);
// input_len
uint64_t
fstStateInputLen
(
FstState
*
state
);
uint64_t
fstStateInputLen
(
FstState
*
state
);
// end_addr
uint64_t
fstStateEndAddrForOneTransNext
(
FstState
*
state
,
FstSlice
*
data
);
uint64_t
fstStateEndAddrForOneTrans
(
FstState
*
state
,
FstSlice
*
data
,
PackSizes
sizes
);
uint64_t
fstStateEndAddrForAnyTrans
(
FstState
*
state
,
uint64_t
version
,
FstSlice
*
date
,
PackSizes
sizes
,
uint64_t
nTrans
);
uint64_t
fstStateEndAddrForOneTransNext
(
FstState
*
state
,
FstSlice
*
data
);
uint64_t
fstStateEndAddrForOneTrans
(
FstState
*
state
,
FstSlice
*
data
,
PackSizes
sizes
);
uint64_t
fstStateEndAddrForAnyTrans
(
FstState
*
state
,
uint64_t
version
,
FstSlice
*
date
,
PackSizes
sizes
,
uint64_t
nTrans
);
// input
uint8_t
fstStateInput
(
FstState
*
state
,
FstNode
*
node
);
uint8_t
fstStateInputForAnyTrans
(
FstState
*
state
,
FstNode
*
node
,
uint64_t
i
);
uint8_t
fstStateInput
(
FstState
*
state
,
FstNode
*
node
);
uint8_t
fstStateInputForAnyTrans
(
FstState
*
state
,
FstNode
*
node
,
uint64_t
i
);
// trans_addr
CompiledAddr
fstStateTransAddr
(
FstState
*
state
,
FstNode
*
node
);
CompiledAddr
fstStateTransAddrForAnyTrans
(
FstState
*
state
,
FstNode
*
node
,
uint64_t
i
);
CompiledAddr
fstStateTransAddr
(
FstState
*
state
,
FstNode
*
node
);
CompiledAddr
fstStateTransAddrForAnyTrans
(
FstState
*
state
,
FstNode
*
node
,
uint64_t
i
);
// sizes
PackSizes
fstStateSizes
(
FstState
*
state
,
FstSlice
*
data
);
PackSizes
fstStateSizes
(
FstState
*
state
,
FstSlice
*
data
);
// Output
Output
fstStateOutput
(
FstState
*
state
,
FstNode
*
node
);
Output
fstStateOutputForAnyTrans
(
FstState
*
state
,
FstNode
*
node
,
uint64_t
i
);
Output
fstStateOutput
(
FstState
*
state
,
FstNode
*
node
);
Output
fstStateOutputForAnyTrans
(
FstState
*
state
,
FstNode
*
node
,
uint64_t
i
);
// anyTrans specify function
void
fstStateSetFinalState
(
FstState
*
state
,
bool
yes
);
bool
fstStateIsFinalState
(
FstState
*
state
);
void
fstStateSetStateNtrans
(
FstState
*
state
,
uint8_t
n
);
void
fstStateSetFinalState
(
FstState
*
state
,
bool
yes
);
bool
fstStateIsFinalState
(
FstState
*
state
);
void
fstStateSetStateNtrans
(
FstState
*
state
,
uint8_t
n
);
// state_ntrans
uint8_t
fstStateStateNtrans
(
FstState
*
state
,
bool
*
null
);
uint64_t
fstStateTotalTransSize
(
FstState
*
state
,
uint64_t
version
,
PackSizes
size
,
uint64_t
nTrans
);
uint64_t
fstStateTransIndexSize
(
FstState
*
state
,
uint64_t
version
,
uint64_t
nTrans
);
uint64_t
fstStateNtransLen
(
FstState
*
state
);
uint64_t
fstStateNtrans
(
FstState
*
state
,
FstSlice
*
slice
);
Output
fstStateFinalOutput
(
FstState
*
state
,
uint64_t
version
,
FstSlice
*
date
,
PackSizes
sizes
,
uint64_t
nTrans
);
uint64_t
fstStateFindInput
(
FstState
*
state
,
FstNode
*
node
,
uint8_t
b
,
bool
*
null
);
uint8_t
fstStateStateNtrans
(
FstState
*
state
,
bool
*
null
);
uint64_t
fstStateTotalTransSize
(
FstState
*
state
,
uint64_t
version
,
PackSizes
size
,
uint64_t
nTrans
);
uint64_t
fstStateTransIndexSize
(
FstState
*
state
,
uint64_t
version
,
uint64_t
nTrans
);
uint64_t
fstStateNtransLen
(
FstState
*
state
);
uint64_t
fstStateNtrans
(
FstState
*
state
,
FstSlice
*
slice
);
Output
fstStateFinalOutput
(
FstState
*
state
,
uint64_t
version
,
FstSlice
*
date
,
PackSizes
sizes
,
uint64_t
nTrans
);
uint64_t
fstStateFindInput
(
FstState
*
state
,
FstNode
*
node
,
uint8_t
b
,
bool
*
null
);
#define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext)
#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans)
...
...
@@ -187,13 +186,13 @@ typedef struct FstLastTransition {
* TODO: simple function name
*/
typedef
struct
FstBuilderNodeUnfinished
{
FstBuilderNode
*
node
;
FstLastTransition
*
last
;
FstBuilderNode
*
node
;
FstLastTransition
*
last
;
}
FstBuilderNodeUnfinished
;
void
fstBuilderNodeUnfinishedLastCompiled
(
FstBuilderNodeUnfinished
*
node
,
CompiledAddr
addr
);
void
fstBuilderNodeUnfinishedLastCompiled
(
FstBuilderNodeUnfinished
*
node
,
CompiledAddr
addr
);
void
fstBuilderNodeUnfinishedAddOutputPrefix
(
FstBuilderNodeUnfinished
*
node
,
Output
out
);
void
fstBuilderNodeUnfinishedAddOutputPrefix
(
FstBuilderNodeUnfinished
*
node
,
Output
out
);
/*
* FstNode and helper function
...
...
@@ -224,18 +223,18 @@ typedef struct FstNode {
// Return the address of this node.
#define FST_NODE_ADDR(node) node->start
FstNode
*
fstNodeCreate
(
int64_t
version
,
CompiledAddr
addr
,
FstSlice
*
data
);
void
fstNodeDestroy
(
FstNode
*
fstNode
);
FstNode
*
fstNodeCreate
(
int64_t
version
,
CompiledAddr
addr
,
FstSlice
*
data
);
void
fstNodeDestroy
(
FstNode
*
fstNode
);
FstTransitions
fstNodeTransitionIter
(
FstNode
*
node
);
FstTransitions
*
fstNodeTransitions
(
FstNode
*
node
);
bool
fstNodeGetTransitionAt
(
FstNode
*
node
,
uint64_t
i
,
FstTransition
*
res
);
bool
fstNodeGetTransitionAddrAt
(
FstNode
*
node
,
uint64_t
i
,
CompiledAddr
*
res
);
bool
fstNodeFindInput
(
FstNode
*
node
,
uint8_t
b
,
uint64_t
*
res
);
FstTransitions
fstNodeTransitionIter
(
FstNode
*
node
);
FstTransitions
*
fstNodeTransitions
(
FstNode
*
node
);
bool
fstNodeGetTransitionAt
(
FstNode
*
node
,
uint64_t
i
,
FstTransition
*
res
);
bool
fstNodeGetTransitionAddrAt
(
FstNode
*
node
,
uint64_t
i
,
CompiledAddr
*
res
);
bool
fstNodeFindInput
(
FstNode
*
node
,
uint8_t
b
,
uint64_t
*
res
);
bool
fstNodeCompile
(
FstNode
*
node
,
void
*
w
,
CompiledAddr
lastAddr
,
CompiledAddr
addr
,
FstBuilderNode
*
builderNode
);
bool
fstNodeCompile
(
FstNode
*
node
,
void
*
w
,
CompiledAddr
lastAddr
,
CompiledAddr
addr
,
FstBuilderNode
*
builderNode
);
FstSlice
fstNodeAsSlice
(
FstNode
*
node
);
FstSlice
fstNodeAsSlice
(
FstNode
*
node
);
// ops
...
...
@@ -244,8 +243,8 @@ typedef struct FstIndexedValue {
uint64_t
value
;
}
FstIndexedValue
;
FstLastTransition
*
fstLastTransitionCreate
(
uint8_t
inp
,
Output
out
);
void
fstLastTransitionDestroy
(
FstLastTransition
*
trn
);
FstLastTransition
*
fstLastTransitionCreate
(
uint8_t
inp
,
Output
out
);
void
fstLastTransitionDestroy
(
FstLastTransition
*
trn
);
typedef
struct
FstMeta
{
uint64_t
version
;
...
...
@@ -256,75 +255,74 @@ typedef struct FstMeta {
}
FstMeta
;
typedef
struct
Fst
{
FstMeta
*
meta
;
FstSlice
*
data
;
//
FstNode
*
root
;
//
FstMeta
*
meta
;
FstSlice
*
data
;
//
FstNode
*
root
;
//
}
Fst
;
// refactor simple function
Fst
*
fstCreate
(
FstSlice
*
data
);
void
fstDestroy
(
Fst
*
fst
);
Fst
*
fstCreate
(
FstSlice
*
data
);
void
fstDestroy
(
Fst
*
fst
);
bool
fstGet
(
Fst
*
fst
,
FstSlice
*
b
,
Output
*
out
);
FstNode
*
fstGetNode
(
Fst
*
fst
,
CompiledAddr
);
FstNode
*
fstGetRoot
(
Fst
*
fst
);
FstType
fstGetType
(
Fst
*
fst
);
CompiledAddr
fstGetRootAddr
(
Fst
*
fst
);
Output
fstEmptyFinalOutput
(
Fst
*
fst
,
bool
*
null
);
FstStreamBuilder
*
fstSearch
(
Fst
*
fst
,
AutomationCtx
*
ctx
);
bool
fstGet
(
Fst
*
fst
,
FstSlice
*
b
,
Output
*
out
);
FstNode
*
fstGetNode
(
Fst
*
fst
,
CompiledAddr
);
FstNode
*
fstGetRoot
(
Fst
*
fst
);
FstType
fstGetType
(
Fst
*
fst
);
CompiledAddr
fstGetRootAddr
(
Fst
*
fst
);
Output
fstEmptyFinalOutput
(
Fst
*
fst
,
bool
*
null
);
FstStreamBuilder
*
fstSearch
(
Fst
*
fst
,
AutomationCtx
*
ctx
);
FstStreamWithStateBuilder
*
fstSearchWithState
(
Fst
*
fst
,
AutomationCtx
*
ctx
);
FstStreamWithStateBuilder
*
fstSearchWithState
(
Fst
*
fst
,
AutomationCtx
*
ctx
);
// into stream to expand later
StreamWithState
*
streamBuilderIntoStream
(
FstStreamBuilder
*
sb
);
StreamWithState
*
streamBuilderIntoStream
(
FstStreamBuilder
*
sb
);
bool
fstVerify
(
Fst
*
fst
);
bool
fstVerify
(
Fst
*
fst
);
// refactor this function
bool
fstBuilderNodeCompileTo
(
FstBuilderNode
*
b
,
FstCountingWriter
*
wrt
,
CompiledAddr
lastAddr
,
CompiledAddr
startAddr
);
bool
fstBuilderNodeCompileTo
(
FstBuilderNode
*
b
,
FstCountingWriter
*
wrt
,
CompiledAddr
lastAddr
,
CompiledAddr
startAddr
);
typedef
struct
StreamState
{
FstNode
*
node
;
FstNode
*
node
;
uint64_t
trans
;
FstOutput
out
;
void
*
autState
;
void
*
autState
;
}
StreamState
;
void
streamStateDestroy
(
void
*
s
);
void
streamStateDestroy
(
void
*
s
);
typedef
struct
StreamWithState
{
Fst
*
fst
;
AutomationCtx
*
aut
;
SArray
*
inp
;
Fst
*
fst
;
AutomationCtx
*
aut
;
SArray
*
inp
;
FstOutput
emptyOutput
;
SArray
*
stack
;
// <StreamState>
FstBoundWithData
*
endAt
;
SArray
*
stack
;
// <StreamState>
FstBoundWithData
*
endAt
;
}
StreamWithState
;
typedef
struct
StreamWithStateResult
{
FstSlice
data
;
FstOutput
out
;
void
*
state
;
void
*
state
;
}
StreamWithStateResult
;
StreamWithStateResult
*
swsResultCreate
(
FstSlice
*
data
,
FstOutput
fOut
,
void
*
state
);
void
swsResultDestroy
(
StreamWithStateResult
*
result
);
StreamWithStateResult
*
swsResultCreate
(
FstSlice
*
data
,
FstOutput
fOut
,
void
*
state
);
void
swsResultDestroy
(
StreamWithStateResult
*
result
);
typedef
void
*
(
*
StreamCallback
)(
void
*
);
StreamWithState
*
streamWithStateCreate
(
Fst
*
fst
,
AutomationCtx
*
automation
,
FstBoundWithData
*
min
,
FstBoundWithData
*
max
);
typedef
void
*
(
*
StreamCallback
)(
void
*
);
StreamWithState
*
streamWithStateCreate
(
Fst
*
fst
,
AutomationCtx
*
automation
,
FstBoundWithData
*
min
,
FstBoundWithData
*
max
);
void
streamWithStateDestroy
(
StreamWithState
*
sws
);
void
streamWithStateDestroy
(
StreamWithState
*
sws
);
bool
streamWithStateSeekMin
(
StreamWithState
*
sws
,
FstBoundWithData
*
min
);
bool
streamWithStateSeekMin
(
StreamWithState
*
sws
,
FstBoundWithData
*
min
);
StreamWithStateResult
*
streamWithStateNextWith
(
StreamWithState
*
sws
,
StreamCallback
callback
);
StreamWithStateResult
*
streamWithStateNextWith
(
StreamWithState
*
sws
,
StreamCallback
callback
);
FstStreamBuilder
*
fstStreamBuilderCreate
(
Fst
*
fst
,
AutomationCtx
*
aut
);
FstStreamBuilder
*
fstStreamBuilderCreate
(
Fst
*
fst
,
AutomationCtx
*
aut
);
// set up bound range
// refator, simple code by marco
FstStreamBuilder
*
fstStreamBuilderRange
(
FstStreamBuilder
*
b
,
FstSlice
*
val
,
RangeType
type
);
FstStreamBuilder
*
fstStreamBuilderRange
(
FstStreamBuilder
*
b
,
FstSlice
*
val
,
RangeType
type
);
#ifdef __cplusplus
}
...
...
source/libs/index/inc/index_fst_automation.h
浏览文件 @
10b70175
...
...
@@ -26,18 +26,18 @@ typedef struct AutomationCtx AutomationCtx;
typedef
enum
AutomationType
{
AUTOMATION_PREFIX
,
AUTMMATION_MATCH
}
AutomationType
;
typedef
struct
StartWith
{
AutomationCtx
*
autoSelf
;
AutomationCtx
*
autoSelf
;
}
StartWith
;
typedef
struct
Complement
{
AutomationCtx
*
autoSelf
;
AutomationCtx
*
autoSelf
;
}
Complement
;
// automation
typedef
struct
AutomationCtx
{
AutomationType
type
;
void
*
stdata
;
char
*
data
;
void
*
stdata
;
char
*
data
;
}
AutomationCtx
;
typedef
enum
ValueType
{
FST_INT
,
FST_CHAR
,
FST_ARRAY
}
ValueType
;
...
...
@@ -48,27 +48,27 @@ typedef struct StartWithStateValue {
ValueType
type
;
union
{
int
val
;
char
*
ptr
;
SArray
*
arr
;
char
*
ptr
;
SArray
*
arr
;
// add more type
};
}
StartWithStateValue
;
StartWithStateValue
*
startWithStateValueCreate
(
StartWithStateKind
kind
,
ValueType
ty
,
void
*
val
);
StartWithStateValue
*
startWithStateValueDump
(
StartWithStateValue
*
sv
);
void
startWithStateValueDestroy
(
void
*
sv
);
StartWithStateValue
*
startWithStateValueCreate
(
StartWithStateKind
kind
,
ValueType
ty
,
void
*
val
);
StartWithStateValue
*
startWithStateValueDump
(
StartWithStateValue
*
sv
);
void
startWithStateValueDestroy
(
void
*
sv
);
typedef
struct
AutomationFunc
{
void
*
(
*
start
)(
AutomationCtx
*
ctx
);
bool
(
*
isMatch
)(
AutomationCtx
*
ctx
,
void
*
);
bool
(
*
canMatch
)(
AutomationCtx
*
ctx
,
void
*
data
);
bool
(
*
willAlwaysMatch
)(
AutomationCtx
*
ctx
,
void
*
state
);
void
*
(
*
accept
)(
AutomationCtx
*
ctx
,
void
*
state
,
uint8_t
byte
);
void
*
(
*
acceptEof
)(
AutomationCtx
*
ct
,
void
*
state
);
void
*
(
*
start
)(
AutomationCtx
*
ctx
);
bool
(
*
isMatch
)(
AutomationCtx
*
ctx
,
void
*
);
bool
(
*
canMatch
)(
AutomationCtx
*
ctx
,
void
*
data
);
bool
(
*
willAlwaysMatch
)(
AutomationCtx
*
ctx
,
void
*
state
);
void
*
(
*
accept
)(
AutomationCtx
*
ctx
,
void
*
state
,
uint8_t
byte
);
void
*
(
*
acceptEof
)(
AutomationCtx
*
ct
,
void
*
state
);
}
AutomationFunc
;
AutomationCtx
*
automCtxCreate
(
void
*
data
,
AutomationType
atype
);
void
automCtxDestroy
(
AutomationCtx
*
ctx
);
AutomationCtx
*
automCtxCreate
(
void
*
data
,
AutomationType
atype
);
void
automCtxDestroy
(
AutomationCtx
*
ctx
);
extern
AutomationFunc
automFuncs
[];
#ifdef __cplusplus
...
...
source/libs/index/inc/index_fst_counting_writer.h
浏览文件 @
10b70175
...
...
@@ -28,9 +28,9 @@ static char tmpFile[] = "./index";
typedef
enum
WriterType
{
TMemory
,
TFile
}
WriterType
;
typedef
struct
WriterCtx
{
int
(
*
write
)(
struct
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
int
(
*
read
)(
struct
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
int
(
*
flush
)(
struct
WriterCtx
*
ctx
);
int
(
*
write
)(
struct
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
int
(
*
read
)(
struct
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
int
(
*
flush
)(
struct
WriterCtx
*
ctx
);
WriterType
type
;
union
{
struct
{
...
...
@@ -39,41 +39,41 @@ typedef struct WriterCtx {
}
file
;
struct
{
int32_t
capa
;
char
*
buf
;
char
*
buf
;
}
mem
;
};
int32_t
offset
;
int32_t
limit
;
}
WriterCtx
;
static
int
writeCtxDoWrite
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
static
int
writeCtxDoRead
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
static
int
writeCtxDoFlush
(
WriterCtx
*
ctx
);
static
int
writeCtxDoWrite
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
static
int
writeCtxDoRead
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
static
int
writeCtxDoFlush
(
WriterCtx
*
ctx
);
WriterCtx
*
writerCtxCreate
(
WriterType
type
,
const
char
*
path
,
bool
readOnly
,
int32_t
capacity
);
void
writerCtxDestroy
(
WriterCtx
*
w
);
WriterCtx
*
writerCtxCreate
(
WriterType
type
,
const
char
*
path
,
bool
readOnly
,
int32_t
capacity
);
void
writerCtxDestroy
(
WriterCtx
*
w
);
typedef
uint32_t
CheckSummer
;
typedef
struct
FstCountingWriter
{
void
*
wrt
;
// wrap any writer that counts and checksum bytes written
void
*
wrt
;
// wrap any writer that counts and checksum bytes written
uint64_t
count
;
CheckSummer
summer
;
}
FstCountingWriter
;
int
fstCountingWriterWrite
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
);
int
fstCountingWriterWrite
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
);
int
fstCountingWriterRead
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
);
int
fstCountingWriterRead
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
);
int
fstCountingWriterFlush
(
FstCountingWriter
*
write
);
int
fstCountingWriterFlush
(
FstCountingWriter
*
write
);
uint32_t
fstCountingWriterMaskedCheckSum
(
FstCountingWriter
*
write
);
uint32_t
fstCountingWriterMaskedCheckSum
(
FstCountingWriter
*
write
);
FstCountingWriter
*
fstCountingWriterCreate
(
void
*
wtr
);
void
fstCountingWriterDestroy
(
FstCountingWriter
*
w
);
FstCountingWriter
*
fstCountingWriterCreate
(
void
*
wtr
);
void
fstCountingWriterDestroy
(
FstCountingWriter
*
w
);
void
fstCountingWriterPackUintIn
(
FstCountingWriter
*
writer
,
uint64_t
n
,
uint8_t
nBytes
);
uint8_t
fstCountingWriterPackUint
(
FstCountingWriter
*
writer
,
uint64_t
n
);
void
fstCountingWriterPackUintIn
(
FstCountingWriter
*
writer
,
uint64_t
n
,
uint8_t
nBytes
);
uint8_t
fstCountingWriterPackUint
(
FstCountingWriter
*
writer
,
uint64_t
n
);
#define FST_WRITER_COUNT(writer) (writer->count)
#define FST_WRITER_INTER_WRITER(writer) (writer->wtr)
...
...
source/libs/index/inc/index_fst_node.h
浏览文件 @
10b70175
...
...
@@ -36,20 +36,20 @@ typedef struct FstTransition {
typedef
struct
FstBuilderNode
{
bool
isFinal
;
Output
finalOutput
;
SArray
*
trans
;
// <FstTransition>
SArray
*
trans
;
// <FstTransition>
}
FstBuilderNode
;
FstBuilderNode
*
fstBuilderNodeDefault
();
FstBuilderNode
*
fstBuilderNodeDefault
();
FstBuilderNode
*
fstBuilderNodeClone
(
FstBuilderNode
*
src
);
FstBuilderNode
*
fstBuilderNodeClone
(
FstBuilderNode
*
src
);
void
fstBuilderNodeCloneFrom
(
FstBuilderNode
*
dst
,
FstBuilderNode
*
src
);
void
fstBuilderNodeCloneFrom
(
FstBuilderNode
*
dst
,
FstBuilderNode
*
src
);
// bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt,
// CompiledAddr lastAddr, CompiledAddr startAddr);
bool
fstBuilderNodeEqual
(
FstBuilderNode
*
n1
,
FstBuilderNode
*
n2
);
bool
fstBuilderNodeEqual
(
FstBuilderNode
*
n1
,
FstBuilderNode
*
n2
);
void
fstBuilderNodeDestroy
(
FstBuilderNode
*
node
);
void
fstBuilderNodeDestroy
(
FstBuilderNode
*
node
);
#ifdef __cplusplus
}
...
...
source/libs/index/inc/index_fst_registry.h
浏览文件 @
10b70175
...
...
@@ -25,12 +25,14 @@ extern "C" {
typedef
struct
FstRegistryCell
{
CompiledAddr
addr
;
FstBuilderNode
*
node
;
FstBuilderNode
*
node
;
}
FstRegistryCell
;
#define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS)
#define FST_REGISTRY_CELL_INSERT(cell, tAddr) \
do { cell->addr = tAddr; } while (0)
do { \
cell->addr = tAddr; \
} while (0)
// typedef struct FstRegistryCache {
// SArray *cells;
...
...
@@ -43,22 +45,22 @@ typedef enum { FOUND, NOTFOUND, REJECTED } FstRegistryEntryState;
typedef
struct
FstRegistryEntry
{
FstRegistryEntryState
state
;
CompiledAddr
addr
;
FstRegistryCell
*
cell
;
FstRegistryCell
*
cell
;
}
FstRegistryEntry
;
// Registry relation function
typedef
struct
FstRegistry
{
SArray
*
table
;
//<FstRegistryCell>
SArray
*
table
;
//<FstRegistryCell>
uint64_t
tableSize
;
// num of rows
uint64_t
mruSize
;
// num of columns
}
FstRegistry
;
//
FstRegistry
*
fstRegistryCreate
(
uint64_t
tableSize
,
uint64_t
mruSize
);
void
fstRegistryDestroy
(
FstRegistry
*
registry
);
FstRegistry
*
fstRegistryCreate
(
uint64_t
tableSize
,
uint64_t
mruSize
);
void
fstRegistryDestroy
(
FstRegistry
*
registry
);
FstRegistryEntry
*
fstRegistryGetEntry
(
FstRegistry
*
registry
,
FstBuilderNode
*
bNode
);
void
fstRegistryEntryDestroy
(
FstRegistryEntry
*
entry
);
FstRegistryEntry
*
fstRegistryGetEntry
(
FstRegistry
*
registry
,
FstBuilderNode
*
bNode
);
void
fstRegistryEntryDestroy
(
FstRegistryEntry
*
entry
);
#ifdef __cplusplus
}
...
...
source/libs/index/inc/index_fst_util.h
浏览文件 @
10b70175
...
...
@@ -69,29 +69,29 @@ extern const uint64_t TRANS_INDEX_THRESHOLD;
// uint8_t commonIdx(uint8_t v, uint8_t max);
uint8_t
packSize
(
uint64_t
n
);
uint64_t
unpackUint64
(
uint8_t
*
ch
,
uint8_t
sz
);
uint64_t
unpackUint64
(
uint8_t
*
ch
,
uint8_t
sz
);
uint8_t
packDeltaSize
(
CompiledAddr
nodeAddr
,
CompiledAddr
transAddr
);
CompiledAddr
unpackDelta
(
char
*
data
,
uint64_t
len
,
uint64_t
nodeAddr
);
CompiledAddr
unpackDelta
(
char
*
data
,
uint64_t
len
,
uint64_t
nodeAddr
);
typedef
struct
FstString
{
uint8_t
*
data
;
uint8_t
*
data
;
uint32_t
len
;
int32_t
ref
;
}
FstString
;
typedef
struct
FstSlice
{
FstString
*
str
;
FstString
*
str
;
int32_t
start
;
int32_t
end
;
}
FstSlice
;
FstSlice
fstSliceCreate
(
uint8_t
*
data
,
uint64_t
len
);
FstSlice
fstSliceCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
);
FstSlice
fstSliceDeepCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
);
bool
fstSliceIsEmpty
(
FstSlice
*
s
);
int
fstSliceCompare
(
FstSlice
*
s1
,
FstSlice
*
s2
);
void
fstSliceDestroy
(
FstSlice
*
s
);
uint8_t
*
fstSliceData
(
FstSlice
*
s
,
int32_t
*
sz
);
FstSlice
fstSliceCreate
(
uint8_t
*
data
,
uint64_t
len
);
FstSlice
fstSliceCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
);
FstSlice
fstSliceDeepCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
);
bool
fstSliceIsEmpty
(
FstSlice
*
s
);
int
fstSliceCompare
(
FstSlice
*
s1
,
FstSlice
*
s2
);
void
fstSliceDestroy
(
FstSlice
*
s
);
uint8_t
*
fstSliceData
(
FstSlice
*
s
,
int32_t
*
sz
);
#define FST_SLICE_LEN(s) (s->end - s->start + 1)
...
...
source/libs/index/inc/index_tfile.h
浏览文件 @
10b70175
...
...
@@ -37,77 +37,77 @@ typedef struct TFileHeader {
uint8_t
colType
;
}
TFileHeader
;
#define TFILE_HEADER_SIZE (sizeof(TF
ILE_HEADER_SIZE) + sizeof(uint32_t));
#define TFILE_HEADER_SIZE (sizeof(TF
ileHeader) + sizeof(uint32_t))
#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t))
typedef
struct
TFileCacheKey
{
uint64_t
suid
;
uint8_t
colType
;
int32_t
version
;
char
*
colName
;
char
*
colName
;
int32_t
nColName
;
}
TFileCacheKey
;
// table cache
// refactor to LRU cache later
typedef
struct
TFileCache
{
SHashObj
*
tableCache
;
SHashObj
*
tableCache
;
int16_t
capacity
;
// add more param
}
TFileCache
;
typedef
struct
TFileWriter
{
FstBuilder
*
fb
;
WriterCtx
*
ctx
;
FstBuilder
*
fb
;
WriterCtx
*
ctx
;
TFileHeader
header
;
}
TFileWriter
;
typedef
struct
TFileReader
{
T_REF_DECLARE
()
Fst
*
fst
;
WriterCtx
*
ctx
;
Fst
*
fst
;
WriterCtx
*
ctx
;
TFileHeader
header
;
}
TFileReader
;
typedef
struct
IndexTFile
{
char
*
path
;
TFileCache
*
cache
;
TFileWriter
*
tw
;
char
*
path
;
TFileCache
*
cache
;
TFileWriter
*
tw
;
}
IndexTFile
;
typedef
struct
TFileWriterOpt
{
uint64_t
suid
;
int8_t
colType
;
char
*
colName
;
char
*
colName
;
int32_t
nColName
;
int32_t
version
;
}
TFileWriterOpt
;
typedef
struct
TFileReaderOpt
{
uint64_t
suid
;
char
*
colName
;
char
*
colName
;
int32_t
nColName
;
}
TFileReaderOpt
;
// tfile cache, manage tindex reader
TFileCache
*
tfileCacheCreate
(
const
char
*
path
);
void
tfileCacheDestroy
(
TFileCache
*
tcache
);
TFileReader
*
tfileCacheGet
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
);
void
tfileCachePut
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
,
TFileReader
*
reader
);
TFileCache
*
tfileCacheCreate
(
const
char
*
path
);
void
tfileCacheDestroy
(
TFileCache
*
tcache
);
TFileReader
*
tfileCacheGet
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
);
void
tfileCachePut
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
,
TFileReader
*
reader
);
TFileReader
*
tfileReaderCreate
(
WriterCtx
*
ctx
);
void
tfileReaderDestroy
(
TFileReader
*
reader
);
int
tfileReaderSearch
(
TFileReader
*
reader
,
SIndexTermQuery
*
query
,
SArray
*
result
);
TFileReader
*
tfileReaderCreate
(
WriterCtx
*
ctx
);
void
tfileReaderDestroy
(
TFileReader
*
reader
);
int
tfileReaderSearch
(
TFileReader
*
reader
,
SIndexTermQuery
*
query
,
SArray
*
result
);
TFileWriter
*
tfileWriterCreate
(
WriterCtx
*
ctx
,
TFileHeader
*
header
);
void
tfileWriterDestroy
(
TFileWriter
*
tw
);
int
tfileWriterPut
(
TFileWriter
*
tw
,
const
char
*
key
,
int32_t
nKey
,
const
char
*
val
,
int32_t
nVal
);
int
tfileWriterFinish
(
TFileWriter
*
tw
);
TFileWriter
*
tfileWriterCreate
(
WriterCtx
*
ctx
,
TFileHeader
*
header
);
void
tfileWriterDestroy
(
TFileWriter
*
tw
);
int
tfileWriterPut
(
TFileWriter
*
tw
,
const
char
*
key
,
int32_t
nKey
,
const
char
*
val
,
int32_t
nVal
);
int
tfileWriterFinish
(
TFileWriter
*
tw
);
//
IndexTFile
*
indexTFileCreate
(
const
char
*
path
);
int
indexTFilePut
(
void
*
tfile
,
SIndexTerm
*
term
,
uint64_t
uid
);
int
indexTFileSearch
(
void
*
tfile
,
SIndexTermQuery
*
query
,
SArray
*
result
);
IndexTFile
*
indexTFileCreate
(
const
char
*
path
);
int
indexTFilePut
(
void
*
tfile
,
SIndexTerm
*
term
,
uint64_t
uid
);
int
indexTFileSearch
(
void
*
tfile
,
SIndexTermQuery
*
query
,
SArray
*
result
);
#ifdef __cplusplus
}
...
...
source/libs/index/inc/index_util.h
浏览文件 @
10b70175
...
...
@@ -19,29 +19,29 @@
extern
"C"
{
#endif
#define SERIALIZE_MEM_TO_BUF(buf, key, mem)
\
do {
\
memcpy((void
*)buf, (void
*)(&key->mem), sizeof(key->mem)); \
buf += sizeof(key->mem);
\
#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \
do { \
memcpy((void
*)buf, (void
*)(&key->mem), sizeof(key->mem)); \
buf += sizeof(key->mem); \
} while (0)
#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \
do { \
memcpy((void
*)buf, (void *)key->mem, len);
\
memcpy((void
*)buf, (void*)key->mem, len);
\
buf += len; \
} while (0)
#define SERIALIZE_VAR_TO_BUF(buf, var, type)
\
do {
\
type c = var;
\
assert(sizeof(var) == sizeof(type));
\
memcpy((void
*)buf, (void
*)&c, sizeof(c)); \
buf += sizeof(c);
\
#define SERIALIZE_VAR_TO_BUF(buf, var, type) \
do { \
type c = var; \
assert(sizeof(var) == sizeof(type)); \
memcpy((void
*)buf, (void
*)&c, sizeof(c)); \
buf += sizeof(c); \
} while (0)
#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \
do { \
memcpy((void
*)buf, (void *)var, len);
\
memcpy((void
*)buf, (void*)var, len);
\
buf += len; \
} while (0)
...
...
source/libs/index/src/index.c
浏览文件 @
10b70175
...
...
@@ -23,9 +23,9 @@
#include "lucene++/Lucene_c.h"
#endif
static
int
uidCompare
(
const
void
*
a
,
const
void
*
b
)
{
uint64_t
u1
=
*
(
uint64_t
*
)
a
;
uint64_t
u2
=
*
(
uint64_t
*
)
b
;
static
int
uidCompare
(
const
void
*
a
,
const
void
*
b
)
{
uint64_t
u1
=
*
(
uint64_t
*
)
a
;
uint64_t
u2
=
*
(
uint64_t
*
)
b
;
if
(
u1
==
u2
)
{
return
0
;
}
else
{
...
...
@@ -40,24 +40,26 @@ typedef struct SIdxColInfo {
static
pthread_once_t
isInit
=
PTHREAD_ONCE_INIT
;
static
void
indexInit
();
static
int
indexTermSearch
(
SIndex
*
sIdx
,
SIndexTermQuery
*
term
,
SArray
**
result
);
static
int
indexMergeCacheIntoTindex
(
SIndex
*
sIdx
);
static
int
indexTermSearch
(
SIndex
*
sIdx
,
SIndexTermQuery
*
term
,
SArray
**
result
);
static
int
indexMergeCacheIntoTindex
(
SIndex
*
sIdx
);
static
void
indexInterResultsDestroy
(
SArray
*
results
);
static
int
indexMergeFinalResults
(
SArray
*
interResults
,
EIndexOperatorType
oType
,
SArray
*
finalResult
);
static
void
indexInterResultsDestroy
(
SArray
*
results
);
static
int
indexMergeFinalResults
(
SArray
*
interResults
,
EIndexOperatorType
oType
,
SArray
*
finalResult
);
int
indexOpen
(
SIndexOpts
*
opts
,
const
char
*
path
,
SIndex
**
index
)
{
int
indexOpen
(
SIndexOpts
*
opts
,
const
char
*
path
,
SIndex
**
index
)
{
pthread_once
(
&
isInit
,
indexInit
);
SIndex
*
sIdx
=
calloc
(
1
,
sizeof
(
SIndex
));
if
(
sIdx
==
NULL
)
{
return
-
1
;
}
SIndex
*
sIdx
=
calloc
(
1
,
sizeof
(
SIndex
));
if
(
sIdx
==
NULL
)
{
return
-
1
;
}
#ifdef USE_LUCENE
index_t
*
index
=
index_open
(
path
);
index_t
*
index
=
index_open
(
path
);
sIdx
->
index
=
index
;
#endif
#ifdef USE_INVERTED_INDEX
sIdx
->
cache
=
(
void
*
)
indexCacheCreate
();
sIdx
->
cache
=
(
void
*
)
indexCacheCreate
();
sIdx
->
tindex
=
NULL
;
sIdx
->
colObj
=
taosHashInit
(
8
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BINARY
),
true
,
HASH_ENTRY_LOCK
);
sIdx
->
colId
=
1
;
...
...
@@ -72,7 +74,7 @@ int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) {
return
-
1
;
}
void
indexClose
(
SIndex
*
sIdx
)
{
void
indexClose
(
SIndex
*
sIdx
)
{
#ifdef USE_LUCENE
index_close
(
sIdex
->
index
);
sIdx
->
index
=
NULL
;
...
...
@@ -87,16 +89,16 @@ void indexClose(SIndex *sIdx) {
return
;
}
int
indexPut
(
SIndex
*
index
,
SIndexMultiTerm
*
fVals
,
uint64_t
uid
)
{
int
indexPut
(
SIndex
*
index
,
SIndexMultiTerm
*
fVals
,
uint64_t
uid
)
{
#ifdef USE_LUCENE
index_document_t
*
doc
=
index_document_create
();
index_document_t
*
doc
=
index_document_create
();
char
buf
[
16
]
=
{
0
};
sprintf
(
buf
,
"%d"
,
uid
);
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
fVals
);
i
++
)
{
SIndexTerm
*
p
=
taosArrayGetP
(
fVals
,
i
);
index_document_add
(
doc
,
(
const
char
*
)(
p
->
key
),
p
->
nKey
,
(
const
char
*
)(
p
->
val
),
p
->
nVal
,
1
);
SIndexTerm
*
p
=
taosArrayGetP
(
fVals
,
i
);
index_document_add
(
doc
,
(
const
char
*
)(
p
->
key
),
p
->
nKey
,
(
const
char
*
)(
p
->
val
),
p
->
nVal
,
1
);
}
index_document_add
(
doc
,
NULL
,
0
,
buf
,
strlen
(
buf
),
0
);
...
...
@@ -109,8 +111,8 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) {
// TODO(yihao): reduce the lock range
pthread_mutex_lock
(
&
index
->
mtx
);
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
fVals
);
i
++
)
{
SIndexTerm
*
p
=
taosArrayGetP
(
fVals
,
i
);
SIdxColInfo
*
fi
=
taosHashGet
(
index
->
colObj
,
p
->
colName
,
p
->
nColName
);
SIndexTerm
*
p
=
taosArrayGetP
(
fVals
,
i
);
SIdxColInfo
*
fi
=
taosHashGet
(
index
->
colObj
,
p
->
colName
,
p
->
nColName
);
if
(
fi
==
NULL
)
{
SIdxColInfo
tfi
=
{.
colId
=
index
->
colId
};
index
->
cVersion
++
;
...
...
@@ -123,30 +125,32 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) {
pthread_mutex_unlock
(
&
index
->
mtx
);
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
fVals
);
i
++
)
{
SIndexTerm
*
p
=
taosArrayGetP
(
fVals
,
i
);
SIdxColInfo
*
fi
=
taosHashGet
(
index
->
colObj
,
p
->
colName
,
p
->
nColName
);
SIndexTerm
*
p
=
taosArrayGetP
(
fVals
,
i
);
SIdxColInfo
*
fi
=
taosHashGet
(
index
->
colObj
,
p
->
colName
,
p
->
nColName
);
assert
(
fi
!=
NULL
);
int32_t
colId
=
fi
->
colId
;
int32_t
version
=
index
->
cVersion
;
int
ret
=
indexCachePut
(
index
->
cache
,
p
,
colId
,
version
,
uid
);
if
(
ret
!=
0
)
{
return
ret
;
}
if
(
ret
!=
0
)
{
return
ret
;
}
}
#endif
return
0
;
}
int
indexSearch
(
SIndex
*
index
,
SIndexMultiTermQuery
*
multiQuerys
,
SArray
*
result
)
{
int
indexSearch
(
SIndex
*
index
,
SIndexMultiTermQuery
*
multiQuerys
,
SArray
*
result
)
{
#ifdef USE_LUCENE
EIndexOperatorType
opera
=
multiQuerys
->
opera
;
int
nQuery
=
taosArrayGetSize
(
multiQuerys
->
query
);
char
**
fields
=
malloc
(
sizeof
(
char
*
)
*
nQuery
);
char
**
keys
=
malloc
(
sizeof
(
char
*
)
*
nQuery
);
int
*
types
=
malloc
(
sizeof
(
int
)
*
nQuery
);
char
**
fields
=
malloc
(
sizeof
(
char
*
)
*
nQuery
);
char
**
keys
=
malloc
(
sizeof
(
char
*
)
*
nQuery
);
int
*
types
=
malloc
(
sizeof
(
int
)
*
nQuery
);
for
(
int
i
=
0
;
i
<
nQuery
;
i
++
)
{
SIndexTermQuery
*
p
=
taosArrayGet
(
multiQuerys
->
query
,
i
);
SIndexTerm
*
term
=
p
->
field_value
;
SIndexTermQuery
*
p
=
taosArrayGet
(
multiQuerys
->
query
,
i
);
SIndexTerm
*
term
=
p
->
field_value
;
fields
[
i
]
=
calloc
(
1
,
term
->
nKey
+
1
);
keys
[
i
]
=
calloc
(
1
,
term
->
nVal
+
1
);
...
...
@@ -155,11 +159,13 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
memcpy
(
keys
[
i
],
term
->
val
,
term
->
nVal
);
types
[
i
]
=
(
int
)(
p
->
type
);
}
int
*
tResult
=
NULL
;
int
*
tResult
=
NULL
;
int
tsz
=
0
;
index_multi_search
(
index
->
index
,
(
const
char
**
)
fields
,
(
const
char
**
)
keys
,
types
,
nQuery
,
opera
,
&
tResult
,
&
tsz
);
index_multi_search
(
index
->
index
,
(
const
char
**
)
fields
,
(
const
char
**
)
keys
,
types
,
nQuery
,
opera
,
&
tResult
,
&
tsz
);
for
(
int
i
=
0
;
i
<
tsz
;
i
++
)
{
taosArrayPush
(
result
,
&
tResult
[
i
]);
}
for
(
int
i
=
0
;
i
<
tsz
;
i
++
)
{
taosArrayPush
(
result
,
&
tResult
[
i
]);
}
for
(
int
i
=
0
;
i
<
nQuery
;
i
++
)
{
free
(
fields
[
i
]);
...
...
@@ -173,13 +179,13 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
#ifdef USE_INVERTED_INDEX
EIndexOperatorType
opera
=
multiQuerys
->
opera
;
// relation of querys
SArray
*
interResults
=
taosArrayInit
(
4
,
POINTER_BYTES
);
SArray
*
interResults
=
taosArrayInit
(
4
,
POINTER_BYTES
);
int
nQuery
=
taosArrayGetSize
(
multiQuerys
->
query
);
for
(
size_t
i
=
0
;
i
<
nQuery
;
i
++
)
{
SIndexTermQuery
*
qTerm
=
taosArrayGet
(
multiQuerys
->
query
,
i
);
SArray
*
tResult
=
NULL
;
SIndexTermQuery
*
qTerm
=
taosArrayGet
(
multiQuerys
->
query
,
i
);
SArray
*
tResult
=
NULL
;
indexTermSearch
(
index
,
qTerm
,
&
tResult
);
taosArrayPush
(
interResults
,
(
void
*
)
&
tResult
);
taosArrayPush
(
interResults
,
(
void
*
)
&
tResult
);
}
indexMergeFinalResults
(
interResults
,
opera
,
result
);
indexInterResultsDestroy
(
interResults
);
...
...
@@ -188,24 +194,24 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
return
1
;
}
int
indexDelete
(
SIndex
*
index
,
SIndexMultiTermQuery
*
query
)
{
int
indexDelete
(
SIndex
*
index
,
SIndexMultiTermQuery
*
query
)
{
#ifdef USE_INVERTED_INDEX
#endif
return
1
;
}
int
indexRebuild
(
SIndex
*
index
,
SIndexOpts
*
opts
){
int
indexRebuild
(
SIndex
*
index
,
SIndexOpts
*
opts
){
#ifdef USE_INVERTED_INDEX
#endif
}
SIndexOpts
*
indexOptsCreate
()
{
SIndexOpts
*
indexOptsCreate
()
{
#ifdef USE_LUCENE
#endif
return
NULL
;
}
void
indexOptsDestroy
(
SIndexOpts
*
opts
){
void
indexOptsDestroy
(
SIndexOpts
*
opts
){
#ifdef USE_LUCENE
#endif
}
/*
...
...
@@ -213,60 +219,71 @@ void indexOptsDestroy(SIndexOpts *opts){
*
*/
SIndexMultiTermQuery
*
indexMultiTermQueryCreate
(
EIndexOperatorType
opera
)
{
SIndexMultiTermQuery
*
p
=
(
SIndexMultiTermQuery
*
)
malloc
(
sizeof
(
SIndexMultiTermQuery
));
if
(
p
==
NULL
)
{
return
NULL
;
}
SIndexMultiTermQuery
*
indexMultiTermQueryCreate
(
EIndexOperatorType
opera
)
{
SIndexMultiTermQuery
*
p
=
(
SIndexMultiTermQuery
*
)
malloc
(
sizeof
(
SIndexMultiTermQuery
));
if
(
p
==
NULL
)
{
return
NULL
;
}
p
->
opera
=
opera
;
p
->
query
=
taosArrayInit
(
4
,
sizeof
(
SIndexTermQuery
));
return
p
;
}
void
indexMultiTermQueryDestroy
(
SIndexMultiTermQuery
*
pQuery
)
{
void
indexMultiTermQueryDestroy
(
SIndexMultiTermQuery
*
pQuery
)
{
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
pQuery
->
query
);
i
++
)
{
SIndexTermQuery
*
p
=
(
SIndexTermQuery
*
)
taosArrayGet
(
pQuery
->
query
,
i
);
SIndexTermQuery
*
p
=
(
SIndexTermQuery
*
)
taosArrayGet
(
pQuery
->
query
,
i
);
indexTermDestroy
(
p
->
term
);
}
taosArrayDestroy
(
pQuery
->
query
);
free
(
pQuery
);
};
int
indexMultiTermQueryAdd
(
SIndexMultiTermQuery
*
pQuery
,
SIndexTerm
*
term
,
EIndexQueryType
qType
)
{
int
indexMultiTermQueryAdd
(
SIndexMultiTermQuery
*
pQuery
,
SIndexTerm
*
term
,
EIndexQueryType
qType
)
{
SIndexTermQuery
q
=
{.
qType
=
qType
,
.
term
=
term
};
taosArrayPush
(
pQuery
->
query
,
&
q
);
return
0
;
}
SIndexTerm
*
indexTermCreate
(
int64_t
suid
,
SIndexOperOnColumn
oper
,
uint8_t
colType
,
const
char
*
colName
,
int32_t
nColName
,
const
char
*
colVal
,
int32_t
nColVal
)
{
SIndexTerm
*
t
=
(
SIndexTerm
*
)
calloc
(
1
,
(
sizeof
(
SIndexTerm
)));
if
(
t
==
NULL
)
{
return
NULL
;
}
SIndexTerm
*
indexTermCreate
(
int64_t
suid
,
SIndexOperOnColumn
oper
,
uint8_t
colType
,
const
char
*
colName
,
int32_t
nColName
,
const
char
*
colVal
,
int32_t
nColVal
)
{
SIndexTerm
*
t
=
(
SIndexTerm
*
)
calloc
(
1
,
(
sizeof
(
SIndexTerm
)));
if
(
t
==
NULL
)
{
return
NULL
;
}
t
->
suid
=
suid
;
t
->
operType
=
oper
;
t
->
colType
=
colType
;
t
->
colName
=
(
char
*
)
calloc
(
1
,
nColName
+
1
);
t
->
colName
=
(
char
*
)
calloc
(
1
,
nColName
+
1
);
memcpy
(
t
->
colName
,
colName
,
nColName
);
t
->
nColName
=
nColName
;
t
->
colVal
=
(
char
*
)
calloc
(
1
,
nColVal
+
1
);
t
->
colVal
=
(
char
*
)
calloc
(
1
,
nColVal
+
1
);
memcpy
(
t
->
colVal
,
colVal
,
nColVal
);
t
->
nColVal
=
nColVal
;
return
t
;
}
void
indexTermDestroy
(
SIndexTerm
*
p
)
{
void
indexTermDestroy
(
SIndexTerm
*
p
)
{
free
(
p
->
colName
);
free
(
p
->
colVal
);
free
(
p
);
}
SIndexMultiTerm
*
indexMultiTermCreate
()
{
return
taosArrayInit
(
4
,
sizeof
(
SIndexTerm
*
));
}
SIndexMultiTerm
*
indexMultiTermCreate
()
{
return
taosArrayInit
(
4
,
sizeof
(
SIndexTerm
*
));
}
int
indexMultiTermAdd
(
SIndexMultiTerm
*
terms
,
SIndexTerm
*
term
)
{
int
indexMultiTermAdd
(
SIndexMultiTerm
*
terms
,
SIndexTerm
*
term
)
{
taosArrayPush
(
terms
,
&
term
);
return
0
;
}
void
indexMultiTermDestroy
(
SIndexMultiTerm
*
terms
)
{
void
indexMultiTermDestroy
(
SIndexMultiTerm
*
terms
)
{
for
(
int32_t
i
=
0
;
i
<
taosArrayGetSize
(
terms
);
i
++
)
{
SIndexTerm
*
p
=
taosArrayGetP
(
terms
,
i
);
SIndexTerm
*
p
=
taosArrayGetP
(
terms
,
i
);
indexTermDestroy
(
p
);
}
taosArrayDestroy
(
terms
);
...
...
@@ -275,13 +292,13 @@ void indexMultiTermDestroy(SIndexMultiTerm *terms) {
void
indexInit
()
{
// do nothing
}
static
int
indexTermSearch
(
SIndex
*
sIdx
,
SIndexTermQuery
*
query
,
SArray
**
result
)
{
static
int
indexTermSearch
(
SIndex
*
sIdx
,
SIndexTermQuery
*
query
,
SArray
**
result
)
{
int32_t
version
=
-
1
;
int16_t
colId
=
-
1
;
SIdxColInfo
*
colInfo
=
NULL
;
SIdxColInfo
*
colInfo
=
NULL
;
SIndexTerm
*
term
=
query
->
term
;
const
char
*
colName
=
term
->
colName
;
SIndexTerm
*
term
=
query
->
term
;
const
char
*
colName
=
term
->
colName
;
int32_t
nColName
=
term
->
nColName
;
pthread_mutex_lock
(
&
sIdx
->
mtx
);
...
...
@@ -314,19 +331,21 @@ static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result
}
return
0
;
}
static
void
indexInterResultsDestroy
(
SArray
*
results
)
{
if
(
results
==
NULL
)
{
return
;
}
static
void
indexInterResultsDestroy
(
SArray
*
results
)
{
if
(
results
==
NULL
)
{
return
;
}
size_t
sz
=
taosArrayGetSize
(
results
);
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
SArray
*
p
=
taosArrayGetP
(
results
,
i
);
SArray
*
p
=
taosArrayGetP
(
results
,
i
);
taosArrayDestroy
(
p
);
}
taosArrayDestroy
(
results
);
}
static
int
indexMergeFinalResults
(
SArray
*
interResults
,
EIndexOperatorType
oType
,
SArray
*
fResults
)
{
static
int
indexMergeFinalResults
(
SArray
*
interResults
,
EIndexOperatorType
oType
,
SArray
*
fResults
)
{
// refactor, merge interResults into fResults by oType
SArray
*
first
=
taosArrayGetP
(
interResults
,
0
);
SArray
*
first
=
taosArrayGetP
(
interResults
,
0
);
taosArraySort
(
first
,
uidCompare
);
taosArrayRemoveDuplicate
(
first
,
uidCompare
,
NULL
);
...
...
@@ -344,8 +363,10 @@ static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType
}
return
0
;
}
static
int
indexMergeCacheIntoTindex
(
SIndex
*
sIdx
)
{
if
(
sIdx
==
NULL
)
{
return
-
1
;
}
static
int
indexMergeCacheIntoTindex
(
SIndex
*
sIdx
)
{
if
(
sIdx
==
NULL
)
{
return
-
1
;
}
indexWarn
(
"suid %"
PRIu64
" merge cache into tindex"
,
sIdx
->
suid
);
return
0
;
}
source/libs/index/src/index_cache.c
浏览文件 @
10b70175
...
...
@@ -20,14 +20,15 @@
#define MAX_INDEX_KEY_LEN 256 // test only, change later
// ref index_cache.h:22
#define CACHE_KEY_LEN(p) \
(sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + \
sizeof(p->operType))
#define CACHE_KEY_LEN(p) \
(sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType))
static
char
*
getIndexKey
(
const
void
*
pData
)
{
return
NULL
;
}
static
int32_t
compareKey
(
const
void
*
l
,
const
void
*
r
)
{
char
*
lp
=
(
char
*
)
l
;
char
*
rp
=
(
char
*
)
r
;
static
char
*
getIndexKey
(
const
void
*
pData
)
{
return
NULL
;
}
static
int32_t
compareKey
(
const
void
*
l
,
const
void
*
r
)
{
char
*
lp
=
(
char
*
)
l
;
char
*
rp
=
(
char
*
)
r
;
// skip total len, not compare
int32_t
ll
,
rl
;
// len
...
...
@@ -95,33 +96,33 @@ static int32_t compareKey(const void *l, const void *r) {
return
0
;
}
IndexCache
*
indexCacheCreate
()
{
IndexCache
*
cache
=
calloc
(
1
,
sizeof
(
IndexCache
));
IndexCache
*
indexCacheCreate
()
{
IndexCache
*
cache
=
calloc
(
1
,
sizeof
(
IndexCache
));
if
(
cache
==
NULL
)
{
indexError
(
"failed to create index cache"
);
return
NULL
;
}
cache
->
skiplist
=
tSkipListCreate
(
MAX_SKIP_LIST_LEVEL
,
TSDB_DATA_TYPE_BINARY
,
MAX_INDEX_KEY_LEN
,
compareKey
,
SL_ALLOW_DUP_KEY
,
getIndexKey
);
cache
->
skiplist
=
tSkipListCreate
(
MAX_SKIP_LIST_LEVEL
,
TSDB_DATA_TYPE_BINARY
,
MAX_INDEX_KEY_LEN
,
compareKey
,
SL_ALLOW_DUP_KEY
,
getIndexKey
);
return
cache
;
}
void
indexCacheDestroy
(
void
*
cache
)
{
IndexCache
*
pCache
=
cache
;
void
indexCacheDestroy
(
void
*
cache
)
{
IndexCache
*
pCache
=
cache
;
if
(
pCache
==
NULL
)
{
return
;
}
tSkipListDestroy
(
pCache
->
skiplist
);
free
(
pCache
);
}
int
indexCachePut
(
void
*
cache
,
SIndexTerm
*
term
,
int16_t
colId
,
int32_t
version
,
uint64_t
uid
)
{
int
indexCachePut
(
void
*
cache
,
SIndexTerm
*
term
,
int16_t
colId
,
int32_t
version
,
uint64_t
uid
)
{
if
(
cache
==
NULL
)
{
return
-
1
;
}
IndexCache
*
pCache
=
cache
;
IndexCache
*
pCache
=
cache
;
// encode data
int32_t
total
=
CACHE_KEY_LEN
(
term
);
char
*
buf
=
calloc
(
1
,
total
);
char
*
p
=
buf
;
char
*
buf
=
calloc
(
1
,
total
);
char
*
p
=
buf
;
SERIALIZE_VAR_TO_BUF
(
p
,
total
,
int32_t
);
SERIALIZE_VAR_TO_BUF
(
p
,
colId
,
int16_t
);
...
...
@@ -135,28 +136,23 @@ int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version,
SERIALIZE_MEM_TO_BUF
(
p
,
term
,
operType
);
tSkipListPut
(
pCache
->
skiplist
,
(
void
*
)
buf
);
tSkipListPut
(
pCache
->
skiplist
,
(
void
*
)
buf
);
return
0
;
// encode end
}
int
indexCacheDel
(
void
*
cache
,
int32_t
fieldId
,
const
char
*
fieldValue
,
int32_t
fvlen
,
uint64_t
uid
,
int8_t
operType
)
{
IndexCache
*
pCache
=
cache
;
int
indexCacheDel
(
void
*
cache
,
int32_t
fieldId
,
const
char
*
fieldValue
,
int32_t
fvlen
,
uint64_t
uid
,
int8_t
operType
)
{
IndexCache
*
pCache
=
cache
;
return
0
;
}
int
indexCacheSearch
(
void
*
cache
,
SIndexTermQuery
*
query
,
int16_t
colId
,
int32_t
version
,
SArray
*
result
,
STermValueType
*
s
)
{
int
indexCacheSearch
(
void
*
cache
,
SIndexTermQuery
*
query
,
int16_t
colId
,
int32_t
version
,
SArray
*
result
,
STermValueType
*
s
)
{
if
(
cache
==
NULL
)
{
return
-
1
;
}
IndexCache
*
pCache
=
cache
;
SIndexTerm
*
term
=
query
->
term
;
IndexCache
*
pCache
=
cache
;
SIndexTerm
*
term
=
query
->
term
;
EIndexQueryType
qtype
=
query
->
qType
;
int32_t
keyLen
=
CACHE_KEY_LEN
(
term
);
char
*
buf
=
calloc
(
1
,
keyLen
);
char
*
buf
=
calloc
(
1
,
keyLen
);
if
(
qtype
==
QUERY_TERM
)
{
//
}
else
if
(
qtype
==
QUERY_PREFIX
)
{
...
...
source/libs/index/src/index_fst.c
浏览文件 @
10b70175
此差异已折叠。
点击以展开。
source/libs/index/src/index_fst_automation.c
浏览文件 @
10b70175
...
...
@@ -15,19 +15,17 @@
#include "index_fst_automation.h"
StartWithStateValue
*
startWithStateValueCreate
(
StartWithStateKind
kind
,
ValueType
ty
,
void
*
val
)
{
StartWithStateValue
*
nsv
=
calloc
(
1
,
sizeof
(
StartWithStateValue
));
if
(
nsv
==
NULL
)
{
return
NULL
;
}
StartWithStateValue
*
startWithStateValueCreate
(
StartWithStateKind
kind
,
ValueType
ty
,
void
*
val
)
{
StartWithStateValue
*
nsv
=
calloc
(
1
,
sizeof
(
StartWithStateValue
));
if
(
nsv
==
NULL
)
{
return
NULL
;
}
nsv
->
kind
=
kind
;
nsv
->
type
=
ty
;
if
(
ty
==
FST_INT
)
{
nsv
->
val
=
*
(
int
*
)
val
;
nsv
->
val
=
*
(
int
*
)
val
;
}
else
if
(
ty
==
FST_CHAR
)
{
size_t
len
=
strlen
((
char
*
)
val
);
nsv
->
ptr
=
(
char
*
)
calloc
(
1
,
len
+
1
);
size_t
len
=
strlen
((
char
*
)
val
);
nsv
->
ptr
=
(
char
*
)
calloc
(
1
,
len
+
1
);
memcpy
(
nsv
->
ptr
,
val
,
len
);
}
else
if
(
ty
==
FST_ARRAY
)
{
// TODO,
...
...
@@ -35,11 +33,9 @@ StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueTyp
}
return
nsv
;
}
void
startWithStateValueDestroy
(
void
*
val
)
{
StartWithStateValue
*
sv
=
(
StartWithStateValue
*
)
val
;
if
(
sv
==
NULL
)
{
return
;
}
void
startWithStateValueDestroy
(
void
*
val
)
{
StartWithStateValue
*
sv
=
(
StartWithStateValue
*
)
val
;
if
(
sv
==
NULL
)
{
return
;
}
if
(
sv
->
type
==
FST_INT
)
{
//
...
...
@@ -50,11 +46,9 @@ void startWithStateValueDestroy(void *val) {
}
free
(
sv
);
}
StartWithStateValue
*
startWithStateValueDump
(
StartWithStateValue
*
sv
)
{
StartWithStateValue
*
nsv
=
calloc
(
1
,
sizeof
(
StartWithStateValue
));
if
(
nsv
==
NULL
)
{
return
NULL
;
}
StartWithStateValue
*
startWithStateValueDump
(
StartWithStateValue
*
sv
)
{
StartWithStateValue
*
nsv
=
calloc
(
1
,
sizeof
(
StartWithStateValue
));
if
(
nsv
==
NULL
)
{
return
NULL
;
}
nsv
->
kind
=
sv
->
kind
;
nsv
->
type
=
sv
->
type
;
...
...
@@ -62,41 +56,40 @@ StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) {
nsv
->
val
=
sv
->
val
;
}
else
if
(
nsv
->
type
==
FST_CHAR
)
{
size_t
len
=
strlen
(
sv
->
ptr
);
nsv
->
ptr
=
(
char
*
)
calloc
(
1
,
len
+
1
);
nsv
->
ptr
=
(
char
*
)
calloc
(
1
,
len
+
1
);
memcpy
(
nsv
->
ptr
,
sv
->
ptr
,
len
);
}
else
if
(
nsv
->
type
==
FST_ARRAY
)
{
//
}
return
nsv
;
}
// prefix query, impl later
static
void
*
prefixStart
(
AutomationCtx
*
ctx
)
{
StartWithStateValue
*
data
=
(
StartWithStateValue
*
)(
ctx
->
stdata
);
static
void
*
prefixStart
(
AutomationCtx
*
ctx
)
{
StartWithStateValue
*
data
=
(
StartWithStateValue
*
)(
ctx
->
stdata
);
return
startWithStateValueDump
(
data
);
};
static
bool
prefixIsMatch
(
AutomationCtx
*
ctx
,
void
*
sv
)
{
StartWithStateValue
*
ssv
=
(
StartWithStateValue
*
)
sv
;
static
bool
prefixIsMatch
(
AutomationCtx
*
ctx
,
void
*
sv
)
{
StartWithStateValue
*
ssv
=
(
StartWithStateValue
*
)
sv
;
return
ssv
->
val
==
strlen
(
ctx
->
data
);
}
static
bool
prefixCanMatch
(
AutomationCtx
*
ctx
,
void
*
sv
)
{
StartWithStateValue
*
ssv
=
(
StartWithStateValue
*
)
sv
;
static
bool
prefixCanMatch
(
AutomationCtx
*
ctx
,
void
*
sv
)
{
StartWithStateValue
*
ssv
=
(
StartWithStateValue
*
)
sv
;
return
ssv
->
val
>=
0
;
}
static
bool
prefixWillAlwaysMatch
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
true
;
}
static
void
*
prefixAccept
(
AutomationCtx
*
ctx
,
void
*
state
,
uint8_t
byte
)
{
StartWithStateValue
*
ssv
=
(
StartWithStateValue
*
)
state
;
if
(
ssv
==
NULL
||
ctx
==
NULL
)
{
return
NULL
;
}
static
bool
prefixWillAlwaysMatch
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
true
;
}
static
void
*
prefixAccept
(
AutomationCtx
*
ctx
,
void
*
state
,
uint8_t
byte
)
{
StartWithStateValue
*
ssv
=
(
StartWithStateValue
*
)
state
;
if
(
ssv
==
NULL
||
ctx
==
NULL
)
{
return
NULL
;
}
char
*
data
=
ctx
->
data
;
if
(
ssv
->
kind
==
Done
)
{
return
startWithStateValueCreate
(
Done
,
FST_INT
,
&
ssv
->
val
);
}
char
*
data
=
ctx
->
data
;
if
(
ssv
->
kind
==
Done
)
{
return
startWithStateValueCreate
(
Done
,
FST_INT
,
&
ssv
->
val
);
}
if
((
strlen
(
data
)
>
ssv
->
val
)
&&
data
[
ssv
->
val
]
==
byte
)
{
int
val
=
ssv
->
val
+
1
;
StartWithStateValue
*
nsv
=
startWithStateValueCreate
(
Running
,
FST_INT
,
&
val
);
StartWithStateValue
*
nsv
=
startWithStateValueCreate
(
Running
,
FST_INT
,
&
val
);
if
(
prefixIsMatch
(
ctx
,
nsv
))
{
nsv
->
kind
=
Done
;
}
else
{
...
...
@@ -106,18 +99,32 @@ static void *prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) {
}
return
NULL
;
}
static
void
*
prefixAcceptEof
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
NULL
;
}
static
void
*
prefixAcceptEof
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
NULL
;
}
// pattern query, impl later
static
void
*
patternStart
(
AutomationCtx
*
ctx
)
{
return
NULL
;
}
static
bool
patternIsMatch
(
AutomationCtx
*
ctx
,
void
*
data
)
{
return
true
;
}
static
bool
patternCanMatch
(
AutomationCtx
*
ctx
,
void
*
data
)
{
return
true
;
}
static
bool
patternWillAlwaysMatch
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
true
;
}
static
void
*
patternStart
(
AutomationCtx
*
ctx
)
{
return
NULL
;
}
static
bool
patternIsMatch
(
AutomationCtx
*
ctx
,
void
*
data
)
{
return
true
;
}
static
bool
patternCanMatch
(
AutomationCtx
*
ctx
,
void
*
data
)
{
return
true
;
}
static
bool
patternWillAlwaysMatch
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
true
;
}
static
void
*
patternAccept
(
AutomationCtx
*
ctx
,
void
*
state
,
uint8_t
byte
)
{
return
NULL
;
}
static
void
*
patternAccept
(
AutomationCtx
*
ctx
,
void
*
state
,
uint8_t
byte
)
{
return
NULL
;
}
static
void
*
patternAcceptEof
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
NULL
;
}
static
void
*
patternAcceptEof
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
NULL
;
}
AutomationFunc
automFuncs
[]
=
{
{
prefixStart
,
prefixIsMatch
,
prefixCanMatch
,
prefixWillAlwaysMatch
,
prefixAccept
,
prefixAcceptEof
},
...
...
@@ -125,34 +132,32 @@ AutomationFunc automFuncs[] = {
// add more search type
};
AutomationCtx
*
automCtxCreate
(
void
*
data
,
AutomationType
atype
)
{
AutomationCtx
*
ctx
=
calloc
(
1
,
sizeof
(
AutomationCtx
));
if
(
ctx
==
NULL
)
{
return
NULL
;
}
AutomationCtx
*
automCtxCreate
(
void
*
data
,
AutomationType
atype
)
{
AutomationCtx
*
ctx
=
calloc
(
1
,
sizeof
(
AutomationCtx
));
if
(
ctx
==
NULL
)
{
return
NULL
;
}
StartWithStateValue
*
sv
=
NULL
;
StartWithStateValue
*
sv
=
NULL
;
if
(
atype
==
AUTOMATION_PREFIX
)
{
int
val
=
0
;
sv
=
startWithStateValueCreate
(
Running
,
FST_INT
,
&
val
);
ctx
->
stdata
=
(
void
*
)
sv
;
ctx
->
stdata
=
(
void
*
)
sv
;
}
else
if
(
atype
==
AUTMMATION_MATCH
)
{
}
else
{
// add more search type
}
char
*
src
=
(
char
*
)
data
;
char
*
src
=
(
char
*
)
data
;
size_t
len
=
strlen
(
src
);
char
*
dst
=
(
char
*
)
malloc
(
len
*
sizeof
(
char
)
+
1
);
char
*
dst
=
(
char
*
)
malloc
(
len
*
sizeof
(
char
)
+
1
);
memcpy
(
dst
,
src
,
len
);
dst
[
len
]
=
0
;
ctx
->
data
=
dst
;
ctx
->
type
=
atype
;
ctx
->
stdata
=
(
void
*
)
sv
;
ctx
->
stdata
=
(
void
*
)
sv
;
return
ctx
;
}
void
automCtxDestroy
(
AutomationCtx
*
ctx
)
{
void
automCtxDestroy
(
AutomationCtx
*
ctx
)
{
startWithStateValueDestroy
(
ctx
->
stdata
);
free
(
ctx
->
data
);
free
(
ctx
);
...
...
source/libs/index/src/index_fst_common.c
浏览文件 @
10b70175
...
...
@@ -274,260 +274,20 @@ const uint8_t COMMON_INPUTS[] = {
};
const
char
COMMON_INPUTS_INV
[]
=
{
't'
,
'e'
,
'/'
,
'o'
,
'a'
,
's'
,
'r'
,
'i'
,
'p'
,
'c'
,
'n'
,
'w'
,
'.'
,
'h'
,
'l'
,
'm'
,
'-'
,
'd'
,
'u'
,
'0'
,
'1'
,
'2'
,
'g'
,
'='
,
':'
,
'b'
,
'f'
,
'3'
,
'y'
,
'5'
,
'&'
,
'_'
,
'4'
,
'v'
,
'9'
,
'6'
,
'7'
,
'8'
,
'k'
,
'%'
,
'?'
,
'x'
,
'C'
,
'D'
,
'A'
,
'S'
,
'F'
,
'I'
,
'B'
,
'E'
,
'j'
,
'P'
,
'T'
,
'z'
,
'R'
,
'N'
,
'M'
,
'+'
,
'L'
,
'O'
,
'q'
,
'H'
,
'G'
,
'W'
,
'U'
,
'V'
,
','
,
'Y'
,
'K'
,
'J'
,
'Z'
,
'X'
,
'Q'
,
';'
,
')'
,
'('
,
'~'
,
'['
,
']'
,
'$'
,
'!'
,
'\''
,
'*'
,
'@'
,
'\x00'
,
'\x01'
,
'\x02'
,
'\x03'
,
'\x04'
,
'\x05'
,
'\x06'
,
'\x07'
,
'\x08'
,
'\t'
,
'\n'
,
'\x0b'
,
'\x0c'
,
'\r'
,
'\x0e'
,
'\x0f'
,
'\x10'
,
'\x11'
,
'\x12'
,
'\x13'
,
'\x14'
,
'\x15'
,
'\x16'
,
'\x17'
,
'\x18'
,
'\x19'
,
'\x1a'
,
'\x1b'
,
'\x1c'
,
'\x1d'
,
'\x1e'
,
'\x1f'
,
' '
,
'"'
,
'#'
,
'<'
,
'>'
,
'\\'
,
'^'
,
'`'
,
'{'
,
'|'
,
'}'
,
'\x7f'
,
'\x80'
,
'\x81'
,
'\x82'
,
'\x83'
,
'\x84'
,
'\x85'
,
'\x86'
,
'\x87'
,
'\x88'
,
'\x89'
,
'\x8a'
,
'\x8b'
,
'\x8c'
,
'\x8d'
,
'\x8e'
,
'\x8f'
,
'\x90'
,
'\x91'
,
'\x92'
,
'\x93'
,
'\x94'
,
'\x95'
,
'\x96'
,
'\x97'
,
'\x98'
,
'\x99'
,
'\x9a'
,
'\x9b'
,
'\x9c'
,
'\x9d'
,
'\x9e'
,
'\x9f'
,
'\xa0'
,
'\xa1'
,
'\xa2'
,
'\xa3'
,
'\xa4'
,
'\xa5'
,
'\xa6'
,
'\xa7'
,
'\xa8'
,
'\xa9'
,
'\xaa'
,
'\xab'
,
'\xac'
,
'\xad'
,
'\xae'
,
'\xaf'
,
'\xb0'
,
'\xb1'
,
'\xb2'
,
'\xb3'
,
'\xb4'
,
'\xb5'
,
'\xb6'
,
'\xb7'
,
'\xb8'
,
'\xb9'
,
'\xba'
,
'\xbb'
,
'\xbc'
,
'\xbd'
,
'\xbe'
,
'\xbf'
,
'\xc0'
,
'\xc1'
,
'\xc2'
,
'\xc3'
,
'\xc4'
,
'\xc5'
,
'\xc6'
,
'\xc7'
,
'\xc8'
,
'\xc9'
,
'\xca'
,
'\xcb'
,
'\xcc'
,
'\xcd'
,
'\xce'
,
'\xcf'
,
'\xd0'
,
'\xd1'
,
'\xd2'
,
'\xd3'
,
'\xd4'
,
'\xd5'
,
'\xd6'
,
'\xd7'
,
'\xd8'
,
'\xd9'
,
'\xda'
,
'\xdb'
,
'\xdc'
,
'\xdd'
,
'\xde'
,
'\xdf'
,
'\xe0'
,
'\xe1'
,
'\xe2'
,
'\xe3'
,
'\xe4'
,
'\xe5'
,
'\xe6'
,
'\xe7'
,
'\xe8'
,
'\xe9'
,
'\xea'
,
'\xeb'
,
'\xec'
,
'\xed'
,
'\xee'
,
'\xef'
,
'\xf0'
,
'\xf1'
,
'\xf2'
,
'\xf3'
,
'\xf4'
,
'\xf5'
,
'\xf6'
,
'\xf7'
,
'\xf8'
,
'\xf9'
,
'\xfa'
,
'\xfb'
,
'\xfc'
,
'\xfd'
,
'\xfe'
,
'\xff'
,
't'
,
'e'
,
'/'
,
'o'
,
'a'
,
's'
,
'r'
,
'i'
,
'p'
,
'c'
,
'n'
,
'w'
,
'.'
,
'h'
,
'l'
,
'm'
,
'-'
,
'd'
,
'u'
,
'0'
,
'1'
,
'2'
,
'g'
,
'='
,
':'
,
'b'
,
'f'
,
'3'
,
'y'
,
'5'
,
'&'
,
'_'
,
'4'
,
'v'
,
'9'
,
'6'
,
'7'
,
'8'
,
'k'
,
'%'
,
'?'
,
'x'
,
'C'
,
'D'
,
'A'
,
'S'
,
'F'
,
'I'
,
'B'
,
'E'
,
'j'
,
'P'
,
'T'
,
'z'
,
'R'
,
'N'
,
'M'
,
'+'
,
'L'
,
'O'
,
'q'
,
'H'
,
'G'
,
'W'
,
'U'
,
'V'
,
','
,
'Y'
,
'K'
,
'J'
,
'Z'
,
'X'
,
'Q'
,
';'
,
')'
,
'('
,
'~'
,
'['
,
']'
,
'$'
,
'!'
,
'\''
,
'*'
,
'@'
,
'\x00'
,
'\x01'
,
'\x02'
,
'\x03'
,
'\x04'
,
'\x05'
,
'\x06'
,
'\x07'
,
'\x08'
,
'\t'
,
'\n'
,
'\x0b'
,
'\x0c'
,
'\r'
,
'\x0e'
,
'\x0f'
,
'\x10'
,
'\x11'
,
'\x12'
,
'\x13'
,
'\x14'
,
'\x15'
,
'\x16'
,
'\x17'
,
'\x18'
,
'\x19'
,
'\x1a'
,
'\x1b'
,
'\x1c'
,
'\x1d'
,
'\x1e'
,
'\x1f'
,
' '
,
'"'
,
'#'
,
'<'
,
'>'
,
'\\'
,
'^'
,
'`'
,
'{'
,
'|'
,
'}'
,
'\x7f'
,
'\x80'
,
'\x81'
,
'\x82'
,
'\x83'
,
'\x84'
,
'\x85'
,
'\x86'
,
'\x87'
,
'\x88'
,
'\x89'
,
'\x8a'
,
'\x8b'
,
'\x8c'
,
'\x8d'
,
'\x8e'
,
'\x8f'
,
'\x90'
,
'\x91'
,
'\x92'
,
'\x93'
,
'\x94'
,
'\x95'
,
'\x96'
,
'\x97'
,
'\x98'
,
'\x99'
,
'\x9a'
,
'\x9b'
,
'\x9c'
,
'\x9d'
,
'\x9e'
,
'\x9f'
,
'\xa0'
,
'\xa1'
,
'\xa2'
,
'\xa3'
,
'\xa4'
,
'\xa5'
,
'\xa6'
,
'\xa7'
,
'\xa8'
,
'\xa9'
,
'\xaa'
,
'\xab'
,
'\xac'
,
'\xad'
,
'\xae'
,
'\xaf'
,
'\xb0'
,
'\xb1'
,
'\xb2'
,
'\xb3'
,
'\xb4'
,
'\xb5'
,
'\xb6'
,
'\xb7'
,
'\xb8'
,
'\xb9'
,
'\xba'
,
'\xbb'
,
'\xbc'
,
'\xbd'
,
'\xbe'
,
'\xbf'
,
'\xc0'
,
'\xc1'
,
'\xc2'
,
'\xc3'
,
'\xc4'
,
'\xc5'
,
'\xc6'
,
'\xc7'
,
'\xc8'
,
'\xc9'
,
'\xca'
,
'\xcb'
,
'\xcc'
,
'\xcd'
,
'\xce'
,
'\xcf'
,
'\xd0'
,
'\xd1'
,
'\xd2'
,
'\xd3'
,
'\xd4'
,
'\xd5'
,
'\xd6'
,
'\xd7'
,
'\xd8'
,
'\xd9'
,
'\xda'
,
'\xdb'
,
'\xdc'
,
'\xdd'
,
'\xde'
,
'\xdf'
,
'\xe0'
,
'\xe1'
,
'\xe2'
,
'\xe3'
,
'\xe4'
,
'\xe5'
,
'\xe6'
,
'\xe7'
,
'\xe8'
,
'\xe9'
,
'\xea'
,
'\xeb'
,
'\xec'
,
'\xed'
,
'\xee'
,
'\xef'
,
'\xf0'
,
'\xf1'
,
'\xf2'
,
'\xf3'
,
'\xf4'
,
'\xf5'
,
'\xf6'
,
'\xf7'
,
'\xf8'
,
'\xf9'
,
'\xfa'
,
'\xfb'
,
'\xfc'
,
'\xfd'
,
'\xfe'
,
'\xff'
,
};
source/libs/index/src/index_fst_counting_writer.c
浏览文件 @
10b70175
...
...
@@ -17,7 +17,7 @@
#include "index_fst_util.h"
#include "tutil.h"
static
int
writeCtxDoWrite
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
)
{
static
int
writeCtxDoWrite
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
)
{
if
(
ctx
->
offset
+
len
>
ctx
->
limit
)
{
return
-
1
;
}
if
(
ctx
->
type
==
TFile
)
{
...
...
@@ -28,7 +28,7 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) {
ctx
->
offset
+=
len
;
return
len
;
}
static
int
writeCtxDoRead
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
)
{
static
int
writeCtxDoRead
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
)
{
int
nRead
=
0
;
if
(
ctx
->
type
==
TFile
)
{
nRead
=
tfRead
(
ctx
->
file
.
fd
,
buf
,
len
);
...
...
@@ -39,7 +39,7 @@ static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) {
return
nRead
;
}
static
int
writeCtxDoFlush
(
WriterCtx
*
ctx
)
{
static
int
writeCtxDoFlush
(
WriterCtx
*
ctx
)
{
if
(
ctx
->
type
==
TFile
)
{
// tfFsync(ctx->fd);
// tfFlush(ctx->file.fd);
...
...
@@ -49,8 +49,8 @@ static int writeCtxDoFlush(WriterCtx *ctx) {
return
1
;
}
WriterCtx
*
writerCtxCreate
(
WriterType
type
,
const
char
*
path
,
bool
readOnly
,
int32_t
capacity
)
{
WriterCtx
*
ctx
=
calloc
(
1
,
sizeof
(
WriterCtx
));
WriterCtx
*
writerCtxCreate
(
WriterType
type
,
const
char
*
path
,
bool
readOnly
,
int32_t
capacity
)
{
WriterCtx
*
ctx
=
calloc
(
1
,
sizeof
(
WriterCtx
));
if
(
ctx
==
NULL
)
{
return
NULL
;
}
ctx
->
type
=
type
;
...
...
@@ -82,7 +82,7 @@ END:
if
(
ctx
->
type
==
TMemory
)
{
free
(
ctx
->
mem
.
buf
);
}
free
(
ctx
);
}
void
writerCtxDestroy
(
WriterCtx
*
ctx
)
{
void
writerCtxDestroy
(
WriterCtx
*
ctx
)
{
if
(
ctx
->
type
==
TMemory
)
{
free
(
ctx
->
mem
.
buf
);
}
else
{
...
...
@@ -91,51 +91,53 @@ void writerCtxDestroy(WriterCtx *ctx) {
free
(
ctx
);
}
FstCountingWriter
*
fstCountingWriterCreate
(
void
*
wrt
)
{
FstCountingWriter
*
cw
=
calloc
(
1
,
sizeof
(
FstCountingWriter
));
FstCountingWriter
*
fstCountingWriterCreate
(
void
*
wrt
)
{
FstCountingWriter
*
cw
=
calloc
(
1
,
sizeof
(
FstCountingWriter
));
if
(
cw
==
NULL
)
{
return
NULL
;
}
cw
->
wrt
=
wrt
;
//(void *)(writerCtxCreate(TFile, readOnly));
return
cw
;
}
void
fstCountingWriterDestroy
(
FstCountingWriter
*
cw
)
{
void
fstCountingWriterDestroy
(
FstCountingWriter
*
cw
)
{
// free wrt object: close fd or free mem
fstCountingWriterFlush
(
cw
);
// writerCtxDestroy((WriterCtx *)(cw->wrt));
free
(
cw
);
}
int
fstCountingWriterWrite
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
)
{
int
fstCountingWriterWrite
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
)
{
if
(
write
==
NULL
)
{
return
0
;
}
// update checksum
// write data to file/socket or mem
WriterCtx
*
ctx
=
write
->
wrt
;
WriterCtx
*
ctx
=
write
->
wrt
;
int
nWrite
=
ctx
->
write
(
ctx
,
buf
,
len
);
assert
(
nWrite
==
len
);
write
->
count
+=
len
;
return
len
;
}
int
fstCountingWriterRead
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
)
{
int
fstCountingWriterRead
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
)
{
if
(
write
==
NULL
)
{
return
0
;
}
WriterCtx
*
ctx
=
write
->
wrt
;
WriterCtx
*
ctx
=
write
->
wrt
;
int
nRead
=
ctx
->
read
(
ctx
,
buf
,
len
);
// assert(nRead == len);
return
nRead
;
}
uint32_t
fstCountingWriterMaskedCheckSum
(
FstCountingWriter
*
write
)
{
return
0
;
}
int
fstCountingWriterFlush
(
FstCountingWriter
*
write
)
{
WriterCtx
*
ctx
=
write
->
wrt
;
uint32_t
fstCountingWriterMaskedCheckSum
(
FstCountingWriter
*
write
)
{
return
0
;
}
int
fstCountingWriterFlush
(
FstCountingWriter
*
write
)
{
WriterCtx
*
ctx
=
write
->
wrt
;
ctx
->
flush
(
ctx
);
// write->wtr->flush
return
1
;
}
void
fstCountingWriterPackUintIn
(
FstCountingWriter
*
writer
,
uint64_t
n
,
uint8_t
nBytes
)
{
void
fstCountingWriterPackUintIn
(
FstCountingWriter
*
writer
,
uint64_t
n
,
uint8_t
nBytes
)
{
assert
(
1
<=
nBytes
&&
nBytes
<=
8
);
uint8_t
*
buf
=
calloc
(
8
,
sizeof
(
uint8_t
));
uint8_t
*
buf
=
calloc
(
8
,
sizeof
(
uint8_t
));
for
(
uint8_t
i
=
0
;
i
<
nBytes
;
i
++
)
{
buf
[
i
]
=
(
uint8_t
)
n
;
n
=
n
>>
8
;
...
...
@@ -145,7 +147,7 @@ void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t
return
;
}
uint8_t
fstCountingWriterPackUint
(
FstCountingWriter
*
writer
,
uint64_t
n
)
{
uint8_t
fstCountingWriterPackUint
(
FstCountingWriter
*
writer
,
uint64_t
n
)
{
uint8_t
nBytes
=
packSize
(
n
);
fstCountingWriterPackUintIn
(
writer
,
n
,
nBytes
);
return
nBytes
;
...
...
source/libs/index/src/index_fst_node.c
浏览文件 @
10b70175
...
...
@@ -14,60 +14,46 @@
*/
#include "index_fst_node.h"
FstBuilderNode
*
fstBuilderNodeDefault
()
{
FstBuilderNode
*
bn
=
malloc
(
sizeof
(
FstBuilderNode
));
FstBuilderNode
*
fstBuilderNodeDefault
()
{
FstBuilderNode
*
bn
=
malloc
(
sizeof
(
FstBuilderNode
));
bn
->
isFinal
=
false
;
bn
->
finalOutput
=
0
;
bn
->
trans
=
taosArrayInit
(
16
,
sizeof
(
FstTransition
));
return
bn
;
}
void
fstBuilderNodeDestroy
(
FstBuilderNode
*
node
)
{
if
(
node
==
NULL
)
{
return
;
}
void
fstBuilderNodeDestroy
(
FstBuilderNode
*
node
)
{
if
(
node
==
NULL
)
{
return
;
}
taosArrayDestroy
(
node
->
trans
);
free
(
node
);
}
bool
fstBuilderNodeEqual
(
FstBuilderNode
*
n1
,
FstBuilderNode
*
n2
)
{
if
(
n1
==
n2
)
{
return
true
;
}
if
(
n1
==
NULL
||
n2
==
NULL
)
{
return
false
;
}
bool
fstBuilderNodeEqual
(
FstBuilderNode
*
n1
,
FstBuilderNode
*
n2
)
{
if
(
n1
==
n2
)
{
return
true
;
}
if
(
n1
==
NULL
||
n2
==
NULL
)
{
return
false
;
}
if
(
n1
->
isFinal
!=
n2
->
isFinal
||
n1
->
finalOutput
!=
n2
->
finalOutput
)
{
return
false
;
}
if
(
n1
->
isFinal
!=
n2
->
isFinal
||
n1
->
finalOutput
!=
n2
->
finalOutput
)
{
return
false
;
}
size_t
s1
=
n1
->
trans
?
taosArrayGetSize
(
n1
->
trans
)
:
0
;
size_t
s2
=
n2
->
trans
?
taosArrayGetSize
(
n2
->
trans
)
:
0
;
if
(
s1
!=
s2
)
{
return
false
;
}
if
(
s1
!=
s2
)
{
return
false
;
}
for
(
size_t
i
=
0
;
i
<
s1
;
i
++
)
{
FstTransition
*
t1
=
taosArrayGet
(
n1
->
trans
,
i
);
FstTransition
*
t2
=
taosArrayGet
(
n2
->
trans
,
i
);
if
(
t1
->
inp
!=
t2
->
inp
||
t1
->
out
!=
t2
->
out
||
t1
->
addr
!=
t2
->
addr
)
{
return
false
;
}
FstTransition
*
t1
=
taosArrayGet
(
n1
->
trans
,
i
);
FstTransition
*
t2
=
taosArrayGet
(
n2
->
trans
,
i
);
if
(
t1
->
inp
!=
t2
->
inp
||
t1
->
out
!=
t2
->
out
||
t1
->
addr
!=
t2
->
addr
)
{
return
false
;
}
}
return
true
;
}
FstBuilderNode
*
fstBuilderNodeClone
(
FstBuilderNode
*
src
)
{
FstBuilderNode
*
node
=
malloc
(
sizeof
(
FstBuilderNode
));
if
(
node
==
NULL
)
{
return
NULL
;
}
FstBuilderNode
*
fstBuilderNodeClone
(
FstBuilderNode
*
src
)
{
FstBuilderNode
*
node
=
malloc
(
sizeof
(
FstBuilderNode
));
if
(
node
==
NULL
)
{
return
NULL
;
}
//
size_t
sz
=
taosArrayGetSize
(
src
->
trans
);
SArray
*
trans
=
taosArrayInit
(
sz
,
sizeof
(
FstTransition
));
SArray
*
trans
=
taosArrayInit
(
sz
,
sizeof
(
FstTransition
));
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
FstTransition
*
tran
=
taosArrayGet
(
src
->
trans
,
i
);
FstTransition
*
tran
=
taosArrayGet
(
src
->
trans
,
i
);
taosArrayPush
(
trans
,
tran
);
}
...
...
@@ -77,10 +63,8 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) {
return
node
;
}
// not destroy src, User's bussiness
void
fstBuilderNodeCloneFrom
(
FstBuilderNode
*
dst
,
FstBuilderNode
*
src
)
{
if
(
dst
==
NULL
||
src
==
NULL
)
{
return
;
}
void
fstBuilderNodeCloneFrom
(
FstBuilderNode
*
dst
,
FstBuilderNode
*
src
)
{
if
(
dst
==
NULL
||
src
==
NULL
)
{
return
;
}
dst
->
isFinal
=
src
->
isFinal
;
dst
->
finalOutput
=
src
->
finalOutput
;
...
...
@@ -90,7 +74,7 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) {
size_t
sz
=
taosArrayGetSize
(
src
->
trans
);
dst
->
trans
=
taosArrayInit
(
sz
,
sizeof
(
FstTransition
));
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
FstTransition
*
trn
=
taosArrayGet
(
src
->
trans
,
i
);
FstTransition
*
trn
=
taosArrayGet
(
src
->
trans
,
i
);
taosArrayPush
(
dst
->
trans
,
trn
);
}
}
...
...
source/libs/index/src/index_fst_registry.c
浏览文件 @
10b70175
...
...
@@ -15,7 +15,7 @@
#include "index_fst_registry.h"
uint64_t
fstRegistryHash
(
FstRegistry
*
registry
,
FstBuilderNode
*
bNode
)
{
uint64_t
fstRegistryHash
(
FstRegistry
*
registry
,
FstBuilderNode
*
bNode
)
{
// TODO(yihaoDeng): refactor later
const
uint64_t
FNV_PRIME
=
1099511628211
;
uint64_t
h
=
14695981039346656037u
;
...
...
@@ -25,19 +25,19 @@ uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) {
uint32_t
sz
=
(
uint32_t
)
taosArrayGetSize
(
bNode
->
trans
);
for
(
uint32_t
i
=
0
;
i
<
sz
;
i
++
)
{
FstTransition
*
trn
=
taosArrayGet
(
bNode
->
trans
,
i
);
FstTransition
*
trn
=
taosArrayGet
(
bNode
->
trans
,
i
);
h
=
(
h
^
(
uint64_t
)(
trn
->
inp
))
*
FNV_PRIME
;
h
=
(
h
^
(
uint64_t
)(
trn
->
out
))
*
FNV_PRIME
;
h
=
(
h
^
(
uint64_t
)(
trn
->
addr
))
*
FNV_PRIME
;
}
return
h
%
(
registry
->
tableSize
);
}
static
void
fstRegistryCellSwap
(
SArray
*
arr
,
uint32_t
a
,
uint32_t
b
)
{
static
void
fstRegistryCellSwap
(
SArray
*
arr
,
uint32_t
a
,
uint32_t
b
)
{
size_t
sz
=
taosArrayGetSize
(
arr
);
if
(
a
>=
sz
||
b
>=
sz
)
{
return
;
}
FstRegistryCell
*
cell1
=
(
FstRegistryCell
*
)
taosArrayGet
(
arr
,
a
);
FstRegistryCell
*
cell2
=
(
FstRegistryCell
*
)
taosArrayGet
(
arr
,
b
);
FstRegistryCell
*
cell1
=
(
FstRegistryCell
*
)
taosArrayGet
(
arr
,
a
);
FstRegistryCell
*
cell2
=
(
FstRegistryCell
*
)
taosArrayGet
(
arr
,
b
);
FstRegistryCell
t
=
{.
addr
=
cell1
->
addr
,
.
node
=
cell1
->
node
};
...
...
@@ -49,7 +49,7 @@ static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) {
return
;
}
static
void
fstRegistryCellPromote
(
SArray
*
arr
,
uint32_t
start
,
uint32_t
end
)
{
static
void
fstRegistryCellPromote
(
SArray
*
arr
,
uint32_t
start
,
uint32_t
end
)
{
size_t
sz
=
taosArrayGetSize
(
arr
);
if
(
start
>=
sz
&&
end
>=
sz
)
{
return
;
}
...
...
@@ -63,12 +63,12 @@ static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) {
}
}
FstRegistry
*
fstRegistryCreate
(
uint64_t
tableSize
,
uint64_t
mruSize
)
{
FstRegistry
*
registry
=
malloc
(
sizeof
(
FstRegistry
));
FstRegistry
*
fstRegistryCreate
(
uint64_t
tableSize
,
uint64_t
mruSize
)
{
FstRegistry
*
registry
=
malloc
(
sizeof
(
FstRegistry
));
if
(
registry
==
NULL
)
{
return
NULL
;
}
uint64_t
nCells
=
tableSize
*
mruSize
;
SArray
*
tb
=
(
SArray
*
)
taosArrayInit
(
nCells
,
sizeof
(
FstRegistryCell
));
SArray
*
tb
=
(
SArray
*
)
taosArrayInit
(
nCells
,
sizeof
(
FstRegistryCell
));
if
(
NULL
==
tb
)
{
free
(
registry
);
return
NULL
;
...
...
@@ -85,28 +85,28 @@ FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) {
return
registry
;
}
void
fstRegistryDestroy
(
FstRegistry
*
registry
)
{
void
fstRegistryDestroy
(
FstRegistry
*
registry
)
{
if
(
registry
==
NULL
)
{
return
;
}
SArray
*
tb
=
registry
->
table
;
SArray
*
tb
=
registry
->
table
;
size_t
sz
=
taosArrayGetSize
(
tb
);
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
FstRegistryCell
*
cell
=
taosArrayGet
(
tb
,
i
);
FstRegistryCell
*
cell
=
taosArrayGet
(
tb
,
i
);
fstBuilderNodeDestroy
(
cell
->
node
);
}
taosArrayDestroy
(
tb
);
free
(
registry
);
}
FstRegistryEntry
*
fstRegistryGetEntry
(
FstRegistry
*
registry
,
FstBuilderNode
*
bNode
)
{
FstRegistryEntry
*
fstRegistryGetEntry
(
FstRegistry
*
registry
,
FstBuilderNode
*
bNode
)
{
if
(
taosArrayGetSize
(
registry
->
table
)
<=
0
)
{
return
NULL
;
}
uint64_t
bucket
=
fstRegistryHash
(
registry
,
bNode
);
uint64_t
start
=
registry
->
mruSize
*
bucket
;
uint64_t
end
=
start
+
registry
->
mruSize
;
FstRegistryEntry
*
entry
=
malloc
(
sizeof
(
FstRegistryEntry
));
FstRegistryEntry
*
entry
=
malloc
(
sizeof
(
FstRegistryEntry
));
if
(
end
-
start
==
1
)
{
FstRegistryCell
*
cell
=
taosArrayGet
(
registry
->
table
,
start
);
FstRegistryCell
*
cell
=
taosArrayGet
(
registry
->
table
,
start
);
// cell->isNode &&
if
(
cell
->
addr
!=
NONE_ADDRESS
&&
fstBuilderNodeEqual
(
cell
->
node
,
bNode
))
{
entry
->
state
=
FOUND
;
...
...
@@ -118,13 +118,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
entry
->
cell
=
cell
;
// copy or not
}
}
else
if
(
end
-
start
==
2
)
{
FstRegistryCell
*
cell1
=
taosArrayGet
(
registry
->
table
,
start
);
FstRegistryCell
*
cell1
=
taosArrayGet
(
registry
->
table
,
start
);
if
(
cell1
->
addr
!=
NONE_ADDRESS
&&
fstBuilderNodeEqual
(
cell1
->
node
,
bNode
))
{
entry
->
state
=
FOUND
;
entry
->
addr
=
cell1
->
addr
;
return
entry
;
}
FstRegistryCell
*
cell2
=
taosArrayGet
(
registry
->
table
,
start
+
1
);
FstRegistryCell
*
cell2
=
taosArrayGet
(
registry
->
table
,
start
+
1
);
if
(
cell2
->
addr
!=
NONE_ADDRESS
&&
fstBuilderNodeEqual
(
cell2
->
node
,
bNode
))
{
entry
->
state
=
FOUND
;
entry
->
addr
=
cell2
->
addr
;
...
...
@@ -136,13 +136,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
fstBuilderNodeCloneFrom
(
cell2
->
node
,
bNode
);
fstRegistryCellSwap
(
registry
->
table
,
start
,
start
+
1
);
FstRegistryCell
*
cCell
=
taosArrayGet
(
registry
->
table
,
start
);
FstRegistryCell
*
cCell
=
taosArrayGet
(
registry
->
table
,
start
);
entry
->
state
=
NOTFOUND
;
entry
->
cell
=
cCell
;
}
else
{
uint32_t
i
=
start
;
for
(;
i
<
end
;
i
++
)
{
FstRegistryCell
*
cell
=
(
FstRegistryCell
*
)
taosArrayGet
(
registry
->
table
,
i
);
FstRegistryCell
*
cell
=
(
FstRegistryCell
*
)
taosArrayGet
(
registry
->
table
,
i
);
if
(
cell
->
addr
!=
NONE_ADDRESS
&&
fstBuilderNodeEqual
(
cell
->
node
,
bNode
))
{
entry
->
state
=
FOUND
;
entry
->
addr
=
cell
->
addr
;
...
...
@@ -152,16 +152,18 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
}
if
(
i
>=
end
)
{
uint64_t
last
=
end
-
1
;
FstRegistryCell
*
cell
=
(
FstRegistryCell
*
)
taosArrayGet
(
registry
->
table
,
last
);
FstRegistryCell
*
cell
=
(
FstRegistryCell
*
)
taosArrayGet
(
registry
->
table
,
last
);
// clone from bNode, refactor later
fstBuilderNodeCloneFrom
(
cell
->
node
,
bNode
);
fstRegistryCellPromote
(
registry
->
table
,
last
,
start
);
FstRegistryCell
*
cCell
=
taosArrayGet
(
registry
->
table
,
start
);
FstRegistryCell
*
cCell
=
taosArrayGet
(
registry
->
table
,
start
);
entry
->
state
=
NOTFOUND
;
entry
->
cell
=
cCell
;
}
}
return
entry
;
}
void
fstRegistryEntryDestroy
(
FstRegistryEntry
*
entry
)
{
free
(
entry
);
}
void
fstRegistryEntryDestroy
(
FstRegistryEntry
*
entry
)
{
free
(
entry
);
}
source/libs/index/src/index_fst_util.c
浏览文件 @
10b70175
...
...
@@ -61,7 +61,7 @@ uint8_t packSize(uint64_t n) {
}
}
uint64_t
unpackUint64
(
uint8_t
*
ch
,
uint8_t
sz
)
{
uint64_t
unpackUint64
(
uint8_t
*
ch
,
uint8_t
sz
)
{
uint64_t
n
=
0
;
for
(
uint8_t
i
=
0
;
i
<
sz
;
i
++
)
{
//
...
...
@@ -76,7 +76,7 @@ uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr) {
return
packSize
(
nodeAddr
-
transAddr
);
}
}
CompiledAddr
unpackDelta
(
char
*
data
,
uint64_t
len
,
uint64_t
nodeAddr
)
{
CompiledAddr
unpackDelta
(
char
*
data
,
uint64_t
len
,
uint64_t
nodeAddr
)
{
uint64_t
delta
=
unpackUint64
(
data
,
len
);
// delta_add = u64_to_usize
if
(
delta
==
EMPTY_ADDRESS
)
{
...
...
@@ -89,8 +89,8 @@ CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) {
// fst slice func
//
FstSlice
fstSliceCreate
(
uint8_t
*
data
,
uint64_t
len
)
{
FstString
*
str
=
(
FstString
*
)
malloc
(
sizeof
(
FstString
));
FstSlice
fstSliceCreate
(
uint8_t
*
data
,
uint64_t
len
)
{
FstString
*
str
=
(
FstString
*
)
malloc
(
sizeof
(
FstString
));
str
->
ref
=
1
;
str
->
len
=
len
;
str
->
data
=
malloc
(
len
*
sizeof
(
uint8_t
));
...
...
@@ -100,8 +100,8 @@ FstSlice fstSliceCreate(uint8_t *data, uint64_t len) {
return
s
;
}
// just shallow copy
FstSlice
fstSliceCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
)
{
FstString
*
str
=
s
->
str
;
FstSlice
fstSliceCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
)
{
FstString
*
str
=
s
->
str
;
str
->
ref
++
;
// uint8_t *buf = fstSliceData(s, &alen);
// start = buf + start - (buf - s->start);
...
...
@@ -110,16 +110,16 @@ FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) {
FstSlice
t
=
{.
str
=
str
,
.
start
=
start
+
s
->
start
,
.
end
=
end
+
s
->
start
};
return
t
;
}
FstSlice
fstSliceDeepCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
)
{
FstSlice
fstSliceDeepCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
)
{
int32_t
tlen
=
end
-
start
+
1
;
int32_t
slen
;
uint8_t
*
data
=
fstSliceData
(
s
,
&
slen
);
uint8_t
*
data
=
fstSliceData
(
s
,
&
slen
);
assert
(
tlen
<=
slen
);
uint8_t
*
buf
=
malloc
(
sizeof
(
uint8_t
)
*
tlen
);
uint8_t
*
buf
=
malloc
(
sizeof
(
uint8_t
)
*
tlen
);
memcpy
(
buf
,
data
+
start
,
tlen
);
FstString
*
str
=
malloc
(
sizeof
(
FstString
));
FstString
*
str
=
malloc
(
sizeof
(
FstString
));
str
->
data
=
buf
;
str
->
len
=
tlen
;
str
->
ref
=
1
;
...
...
@@ -130,15 +130,17 @@ FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) {
ans
.
end
=
tlen
-
1
;
return
ans
;
}
bool
fstSliceIsEmpty
(
FstSlice
*
s
)
{
return
s
->
str
==
NULL
||
s
->
str
->
len
==
0
||
s
->
start
<
0
||
s
->
end
<
0
;
}
bool
fstSliceIsEmpty
(
FstSlice
*
s
)
{
return
s
->
str
==
NULL
||
s
->
str
->
len
==
0
||
s
->
start
<
0
||
s
->
end
<
0
;
}
uint8_t
*
fstSliceData
(
FstSlice
*
s
,
int32_t
*
size
)
{
FstString
*
str
=
s
->
str
;
uint8_t
*
fstSliceData
(
FstSlice
*
s
,
int32_t
*
size
)
{
FstString
*
str
=
s
->
str
;
if
(
size
!=
NULL
)
{
*
size
=
s
->
end
-
s
->
start
+
1
;
}
return
str
->
data
+
s
->
start
;
}
void
fstSliceDestroy
(
FstSlice
*
s
)
{
FstString
*
str
=
s
->
str
;
void
fstSliceDestroy
(
FstSlice
*
s
)
{
FstString
*
str
=
s
->
str
;
str
->
ref
--
;
if
(
str
->
ref
<=
0
)
{
free
(
str
->
data
);
...
...
@@ -147,10 +149,10 @@ void fstSliceDestroy(FstSlice *s) {
}
}
int
fstSliceCompare
(
FstSlice
*
a
,
FstSlice
*
b
)
{
int
fstSliceCompare
(
FstSlice
*
a
,
FstSlice
*
b
)
{
int32_t
alen
,
blen
;
uint8_t
*
aBuf
=
fstSliceData
(
a
,
&
alen
);
uint8_t
*
bBuf
=
fstSliceData
(
b
,
&
blen
);
uint8_t
*
aBuf
=
fstSliceData
(
a
,
&
alen
);
uint8_t
*
bBuf
=
fstSliceData
(
b
,
&
blen
);
uint32_t
i
,
j
;
for
(
i
=
0
,
j
=
0
;
i
<
alen
&&
j
<
blen
;
i
++
,
j
++
)
{
...
...
source/libs/index/src/index_tfile.c
浏览文件 @
10b70175
...
...
@@ -22,16 +22,30 @@
#include "index_util.h"
#include "taosdef.h"
static
FORCE_INLINE
int
tfileWriteHeader
(
TFileWriter
*
writer
)
{}
static
FORCE_INLINE
int
tfileReadLoadHeader
(
TFileReader
*
reader
)
{
static
FORCE_INLINE
int
tfileWriteHeader
(
TFileWriter
*
writer
)
{
char
buf
[
TFILE_HEADER_SIZE
]
=
{
0
};
char
*
p
=
buf
;
TFileHeader
*
header
=
&
writer
->
header
;
SERIALIZE_MEM_TO_BUF
(
p
,
header
,
suid
);
SERIALIZE_MEM_TO_BUF
(
p
,
header
,
version
);
SERIALIZE_VAR_TO_BUF
(
p
,
strlen
(
header
->
colName
),
int32_t
);
SERIALIZE_STR_MEM_TO_BUF
(
p
,
header
,
colName
,
strlen
(
header
->
colName
));
SERIALIZE_MEM_TO_BUF
(
p
,
header
,
colType
);
int
offset
=
p
-
buf
;
int
nwrite
=
writer
->
ctx
->
write
(
writer
->
ctx
,
buf
,
offset
);
if
(
offset
!=
nwrite
)
{
return
-
1
;
}
}
static
FORCE_INLINE
int
tfileReadLoadHeader
(
TFileReader
*
reader
)
{
// TODO simple tfile header later
char
buf
[
TFILE_HADER_PRE_SIZE
];
char
*
p
=
buf
;
char
*
p
=
buf
;
int64_t
nread
=
reader
->
ctx
->
read
(
reader
->
ctx
,
buf
,
TFILE_HADER_PRE_SIZE
);
assert
(
nread
==
TFILE_HADER_PRE_SIZE
);
TFileHeader
*
header
=
&
reader
->
header
;
TFileHeader
*
header
=
&
reader
->
header
;
memcpy
(
&
header
->
suid
,
p
,
sizeof
(
header
->
suid
));
p
+=
sizeof
(
header
->
suid
);
...
...
@@ -46,28 +60,29 @@ static FORCE_INLINE int tfileReadLoadHeader(TFileReader *reader) {
nread
=
reader
->
ctx
->
read
(
reader
->
ctx
,
&
header
->
colType
,
sizeof
(
header
->
colType
));
return
0
;
};
static
int
tfileGetFileList
(
const
char
*
path
,
SArray
*
result
)
{
DIR
*
dir
=
opendir
(
path
);
}
static
int
tfileGetFileList
(
const
char
*
path
,
SArray
*
result
)
{
DIR
*
dir
=
opendir
(
path
);
if
(
NULL
==
dir
)
{
return
-
1
;
}
struct
dirent
*
entry
;
struct
dirent
*
entry
;
while
((
entry
=
readdir
(
dir
))
!=
NULL
)
{
size_t
len
=
strlen
(
entry
->
d_name
);
char
*
buf
=
calloc
(
1
,
len
+
1
);
char
*
buf
=
calloc
(
1
,
len
+
1
);
memcpy
(
buf
,
entry
->
d_name
,
len
);
taosArrayPush
(
result
,
&
buf
);
}
closedir
(
dir
);
return
0
;
}
static
void
tfileDestroyFileName
(
void
*
elem
)
{
char
*
p
=
*
(
char
**
)
elem
;
static
void
tfileDestroyFileName
(
void
*
elem
)
{
char
*
p
=
*
(
char
**
)
elem
;
free
(
p
);
}
static
int
tfileCompare
(
const
void
*
a
,
const
void
*
b
)
{
const
char
*
aName
=
*
(
char
**
)
a
;
const
char
*
bName
=
*
(
char
**
)
b
;
static
int
tfileCompare
(
const
void
*
a
,
const
void
*
b
)
{
const
char
*
aName
=
*
(
char
**
)
a
;
const
char
*
bName
=
*
(
char
**
)
b
;
size_t
aLen
=
strlen
(
aName
);
size_t
bLen
=
strlen
(
bName
);
...
...
@@ -75,14 +90,14 @@ static int tfileCompare(const void *a, const void *b) {
return
strncmp
(
aName
,
bName
,
aLen
>
bLen
?
aLen
:
bLen
);
}
// tfile name suid-colId-version.tindex
static
int
tfileParseFileName
(
const
char
*
filename
,
uint64_t
*
suid
,
int
*
colId
,
int
*
version
)
{
static
int
tfileParseFileName
(
const
char
*
filename
,
uint64_t
*
suid
,
int
*
colId
,
int
*
version
)
{
if
(
3
==
sscanf
(
filename
,
"%"
PRIu64
"-%d-%d.tindex"
,
suid
,
colId
,
version
))
{
// read suid & colid & version success
return
0
;
}
return
-
1
;
}
static
void
tfileSerialCacheKey
(
TFileCacheKey
*
key
,
char
*
buf
)
{
static
void
tfileSerialCacheKey
(
TFileCacheKey
*
key
,
char
*
buf
)
{
SERIALIZE_MEM_TO_BUF
(
buf
,
key
,
suid
);
SERIALIZE_VAR_TO_BUF
(
buf
,
'_'
,
char
);
SERIALIZE_MEM_TO_BUF
(
buf
,
key
,
colType
);
...
...
@@ -92,30 +107,30 @@ static void tfileSerialCacheKey(TFileCacheKey *key, char *buf) {
SERIALIZE_STR_MEM_TO_BUF
(
buf
,
key
,
colName
,
key
->
nColName
);
}
TFileCache
*
tfileCacheCreate
(
const
char
*
path
)
{
TFileCache
*
tcache
=
calloc
(
1
,
sizeof
(
TFileCache
));
TFileCache
*
tfileCacheCreate
(
const
char
*
path
)
{
TFileCache
*
tcache
=
calloc
(
1
,
sizeof
(
TFileCache
));
if
(
tcache
==
NULL
)
{
return
NULL
;
}
tcache
->
tableCache
=
taosHashInit
(
8
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BINARY
),
true
,
HASH_ENTRY_LOCK
);
tcache
->
capacity
=
64
;
SArray
*
files
=
taosArrayInit
(
4
,
sizeof
(
void
*
));
SArray
*
files
=
taosArrayInit
(
4
,
sizeof
(
void
*
));
tfileGetFileList
(
path
,
files
);
taosArraySort
(
files
,
tfileCompare
);
uint64_t
suid
;
int32_t
colId
,
version
;
for
(
size_t
i
=
0
;
i
<
taosArrayGetSize
(
files
);
i
++
)
{
char
*
file
=
taosArrayGetP
(
files
,
i
);
if
(
0
!=
tfileParseFileName
(
file
,
&
suid
,
(
int
*
)
&
colId
,
(
int
*
)
&
version
))
{
char
*
file
=
taosArrayGetP
(
files
,
i
);
if
(
0
!=
tfileParseFileName
(
file
,
&
suid
,
(
int
*
)
&
colId
,
(
int
*
)
&
version
))
{
indexInfo
(
"try parse invalid file: %s, skip it"
,
file
);
continue
;
}
WriterCtx
*
wc
=
writerCtxCreate
(
TFile
,
file
,
true
,
1024
*
64
);
WriterCtx
*
wc
=
writerCtxCreate
(
TFile
,
file
,
true
,
1024
*
64
);
if
(
wc
==
NULL
)
{
indexError
(
"failed to open index: %s"
,
file
);
goto
End
;
}
TFileReader
*
reader
=
tfileReaderCreate
(
wc
);
TFileReader
*
reader
=
tfileReaderCreate
(
wc
);
if
(
0
!=
tfileReadLoadHeader
(
reader
))
{
tfileReaderDestroy
(
reader
);
indexError
(
"failed to load index header, index Id: %s"
,
file
);
...
...
@@ -123,12 +138,16 @@ TFileCache *tfileCacheCreate(const char *path) {
}
// loader fst and validate it
TFileHeader
*
header
=
&
reader
->
header
;
TFileCacheKey
key
=
{
.
suid
=
header
->
suid
,
.
version
=
header
->
version
,
.
colName
=
header
->
colName
,
.
nColName
=
strlen
(
header
->
colName
),
.
colType
=
header
->
colType
};
TFileHeader
*
header
=
&
reader
->
header
;
TFileCacheKey
key
=
{.
suid
=
header
->
suid
,
.
version
=
header
->
version
,
.
colName
=
header
->
colName
,
.
nColName
=
strlen
(
header
->
colName
),
.
colType
=
header
->
colType
};
char
buf
[
128
]
=
{
0
};
tfileSerialCacheKey
(
&
key
,
buf
);
taosHashPut
(
tcache
->
tableCache
,
buf
,
strlen
(
buf
),
&
reader
,
sizeof
(
void
*
));
taosHashPut
(
tcache
->
tableCache
,
buf
,
strlen
(
buf
),
&
reader
,
sizeof
(
void
*
));
}
taosArrayDestroyEx
(
files
,
tfileDestroyFileName
);
return
tcache
;
...
...
@@ -137,13 +156,13 @@ End:
taosArrayDestroyEx
(
files
,
tfileDestroyFileName
);
return
NULL
;
}
void
tfileCacheDestroy
(
TFileCache
*
tcache
)
{
void
tfileCacheDestroy
(
TFileCache
*
tcache
)
{
if
(
tcache
==
NULL
)
{
return
;
}
// free table cache
TFileReader
**
reader
=
taosHashIterate
(
tcache
->
tableCache
,
NULL
);
TFileReader
**
reader
=
taosHashIterate
(
tcache
->
tableCache
,
NULL
);
while
(
reader
)
{
TFileReader
*
p
=
*
reader
;
TFileReader
*
p
=
*
reader
;
indexInfo
(
"drop table cache suid: %"
PRIu64
", colName: %s, colType: %d"
,
p
->
header
.
suid
,
p
->
header
.
colName
,
p
->
header
.
colType
);
tfileReaderDestroy
(
p
);
...
...
@@ -153,36 +172,36 @@ void tfileCacheDestroy(TFileCache *tcache) {
free
(
tcache
);
}
TFileReader
*
tfileCacheGet
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
)
{
TFileReader
*
tfileCacheGet
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
)
{
char
buf
[
128
]
=
{
0
};
tfileSerialCacheKey
(
key
,
buf
);
TFileReader
*
reader
=
taosHashGet
(
tcache
->
tableCache
,
buf
,
strlen
(
buf
));
TFileReader
*
reader
=
taosHashGet
(
tcache
->
tableCache
,
buf
,
strlen
(
buf
));
return
reader
;
}
void
tfileCachePut
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
,
TFileReader
*
reader
)
{
void
tfileCachePut
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
,
TFileReader
*
reader
)
{
char
buf
[
128
]
=
{
0
};
tfileSerialCacheKey
(
key
,
buf
);
taosHashPut
(
tcache
->
tableCache
,
buf
,
strlen
(
buf
),
&
reader
,
sizeof
(
void
*
));
taosHashPut
(
tcache
->
tableCache
,
buf
,
strlen
(
buf
),
&
reader
,
sizeof
(
void
*
));
return
;
}
TFileReader
*
tfileReaderCreate
(
WriterCtx
*
ctx
)
{
TFileReader
*
reader
=
calloc
(
1
,
sizeof
(
TFileReader
));
TFileReader
*
tfileReaderCreate
(
WriterCtx
*
ctx
)
{
TFileReader
*
reader
=
calloc
(
1
,
sizeof
(
TFileReader
));
if
(
reader
==
NULL
)
{
return
NULL
;
}
// T_REF_INC(reader);
reader
->
ctx
=
ctx
;
return
reader
;
}
void
tfileReaderDestroy
(
TFileReader
*
reader
)
{
void
tfileReaderDestroy
(
TFileReader
*
reader
)
{
if
(
reader
==
NULL
)
{
return
;
}
// T_REF_INC(reader);
writerCtxDestroy
(
reader
->
ctx
);
free
(
reader
);
}
int
tfileReaderSearch
(
TFileReader
*
reader
,
SIndexTermQuery
*
query
,
SArray
*
result
)
{
SIndexTerm
*
term
=
query
->
term
;
int
tfileReaderSearch
(
TFileReader
*
reader
,
SIndexTermQuery
*
query
,
SArray
*
result
)
{
SIndexTerm
*
term
=
query
->
term
;
// refactor to callback later
if
(
query
->
qType
==
QUERY_TERM
)
{
uint64_t
offset
;
...
...
@@ -200,52 +219,57 @@ int tfileReaderSearch(TFileReader *reader, SIndexTermQuery *query, SArray *resul
return
0
;
}
TFileWriter
*
tfileWriterCreate
(
WriterCtx
*
ctx
,
TFileHeader
*
header
)
{
TFileWriter
*
tfileWriterCreate
(
WriterCtx
*
ctx
,
TFileHeader
*
header
)
{
// char pathBuf[128] = {0};
// sprintf(pathBuf, "%s/% " PRIu64 "-%d-%d.tindex", path, suid, colId, version);
// TFileHeader header = {.suid = suid, .version = version, .colName = {0}, colType = colType};
// memcpy(header.colName, );
char
buf
[
TFILE_HADER_PRE_SIZE
];
int
len
=
TFILE_HADER_PRE_SIZE
;
if
(
len
!=
ctx
->
write
(
ctx
,
buf
,
len
))
{
indexError
(
"index: %"
PRIu64
" failed to write header info"
,
header
->
suid
);
return
NULL
;
}
TFileWriter
*
tw
=
calloc
(
1
,
sizeof
(
TFileWriter
));
//
char buf[TFILE_HADER_PRE_SIZE];
//
int len = TFILE_HADER_PRE_SIZE;
//
if (len != ctx->write(ctx, buf, len)) {
//
indexError("index: %" PRIu64 " failed to write header info", header->suid);
//
return NULL;
//
}
TFileWriter
*
tw
=
calloc
(
1
,
sizeof
(
TFileWriter
));
if
(
tw
==
NULL
)
{
indexError
(
"index: %"
PRIu64
" failed to alloc TFilerWriter"
,
header
->
suid
);
return
NULL
;
}
tw
->
ctx
=
ctx
;
tw
->
header
=
*
header
;
return
tw
;
}
void
tfileWriterDestroy
(
TFileWriter
*
tw
)
{
void
tfileWriterDestroy
(
TFileWriter
*
tw
)
{
if
(
tw
==
NULL
)
{
return
;
}
writerCtxDestroy
(
tw
->
ctx
);
free
(
tw
);
}
IndexTFile
*
indexTFileCreate
(
const
char
*
path
)
{
IndexTFile
*
tfile
=
calloc
(
1
,
sizeof
(
IndexTFile
));
IndexTFile
*
indexTFileCreate
(
const
char
*
path
)
{
IndexTFile
*
tfile
=
calloc
(
1
,
sizeof
(
IndexTFile
));
if
(
tfile
==
NULL
)
{
return
NULL
;
}
tfile
->
cache
=
tfileCacheCreate
(
path
);
return
tfile
;
}
void
IndexTFileDestroy
(
IndexTFile
*
tfile
)
{
free
(
tfile
);
}
void
IndexTFileDestroy
(
IndexTFile
*
tfile
)
{
free
(
tfile
);
}
int
indexTFileSearch
(
void
*
tfile
,
SIndexTermQuery
*
query
,
SArray
*
result
)
{
int
indexTFileSearch
(
void
*
tfile
,
SIndexTermQuery
*
query
,
SArray
*
result
)
{
if
(
tfile
==
NULL
)
{
return
-
1
;
}
IndexTFile
*
pTfile
=
(
IndexTFile
*
)
tfile
;
IndexTFile
*
pTfile
=
(
IndexTFile
*
)
tfile
;
SIndexTerm
*
term
=
query
->
term
;
SIndexTerm
*
term
=
query
->
term
;
TFileCacheKey
key
=
{.
suid
=
term
->
suid
,
.
colType
=
term
->
colType
,
.
version
=
0
,
.
colName
=
term
->
colName
,
.
nColName
=
term
->
nColName
};
TFileReader
*
reader
=
tfileCacheGet
(
pTfile
->
cache
,
&
key
);
TFileReader
*
reader
=
tfileCacheGet
(
pTfile
->
cache
,
&
key
);
return
tfileReaderSearch
(
reader
,
query
,
result
);
}
int
indexTFilePut
(
void
*
tfile
,
SIndexTerm
*
term
,
uint64_t
uid
)
{
int
indexTFilePut
(
void
*
tfile
,
SIndexTerm
*
term
,
uint64_t
uid
)
{
TFileWriterOpt
wOpt
=
{.
suid
=
term
->
suid
,
.
colType
=
term
->
colType
,
.
colName
=
term
->
colName
,
.
nColName
=
term
->
nColName
,
.
version
=
1
};
return
0
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录