Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
d96963a2
T
TDengine
项目概览
taosdata
/
TDengine
11 个月 前同步成功
通知
1179
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d96963a2
编写于
12月 22, 2021
作者:
S
Shengliang Guan
浏览文件
操作
浏览文件
下载
差异文件
Merge remote-tracking branch 'origin/3.0' into feature/dnode3
上级
5b08678f
cd8e91c9
变更
45
隐藏空白更改
内联
并排
Showing
45 changed file
with
2002 addition
and
1454 deletion
+2002
-1454
include/common/taosmsg.h
include/common/taosmsg.h
+26
-1
include/dnode/qnode/qnode.h
include/dnode/qnode/qnode.h
+1
-43
include/libs/index/index.h
include/libs/index/index.h
+22
-17
include/libs/parser/parser.h
include/libs/parser/parser.h
+1
-0
include/libs/qcom/query.h
include/libs/qcom/query.h
+9
-0
include/libs/qworker/qworker.h
include/libs/qworker/qworker.h
+62
-0
include/libs/scheduler/scheduler.h
include/libs/scheduler/scheduler.h
+1
-0
include/util/thash.h
include/util/thash.h
+1
-2
source/libs/CMakeLists.txt
source/libs/CMakeLists.txt
+1
-0
source/libs/executor/src/tfilter.c
source/libs/executor/src/tfilter.c
+1
-1
source/libs/index/inc/indexInt.h
source/libs/index/inc/indexInt.h
+9
-9
source/libs/index/inc/index_cache.h
source/libs/index/inc/index_cache.h
+5
-6
source/libs/index/inc/index_fst.h
source/libs/index/inc/index_fst.h
+109
-111
source/libs/index/inc/index_fst_automation.h
source/libs/index/inc/index_fst_automation.h
+17
-17
source/libs/index/inc/index_fst_counting_writer.h
source/libs/index/inc/index_fst_counting_writer.h
+20
-18
source/libs/index/inc/index_fst_node.h
source/libs/index/inc/index_fst_node.h
+6
-6
source/libs/index/inc/index_fst_registry.h
source/libs/index/inc/index_fst_registry.h
+7
-7
source/libs/index/inc/index_fst_util.h
source/libs/index/inc/index_fst_util.h
+11
-11
source/libs/index/inc/index_tfile.h
source/libs/index/inc/index_tfile.h
+44
-35
source/libs/index/inc/index_util.h
source/libs/index/inc/index_util.h
+12
-12
source/libs/index/src/index.c
source/libs/index/src/index.c
+73
-60
source/libs/index/src/index_cache.c
source/libs/index/src/index_cache.c
+40
-40
source/libs/index/src/index_fst.c
source/libs/index/src/index_fst.c
+280
-352
source/libs/index/src/index_fst_automation.c
source/libs/index/src/index_fst_automation.c
+59
-54
source/libs/index/src/index_fst_common.c
source/libs/index/src/index_fst_common.c
+16
-256
source/libs/index/src/index_fst_counting_writer.c
source/libs/index/src/index_fst_counting_writer.c
+40
-38
source/libs/index/src/index_fst_node.c
source/libs/index/src/index_fst_node.c
+20
-36
source/libs/index/src/index_fst_registry.c
source/libs/index/src/index_fst_registry.c
+29
-37
source/libs/index/src/index_fst_util.c
source/libs/index/src/index_fst_util.c
+22
-21
source/libs/index/src/index_tfile.c
source/libs/index/src/index_tfile.c
+338
-116
source/libs/parser/inc/dataBlockMgt.h
source/libs/parser/inc/dataBlockMgt.h
+2
-1
source/libs/parser/src/dataBlockMgt.c
source/libs/parser/src/dataBlockMgt.c
+8
-30
source/libs/parser/src/insertParser.c
source/libs/parser/src/insertParser.c
+100
-67
source/libs/parser/test/CMakeLists.txt
source/libs/parser/test/CMakeLists.txt
+0
-2
source/libs/parser/test/insertParserTest.cpp
source/libs/parser/test/insertParserTest.cpp
+182
-0
source/libs/parser/test/mockCatalog.cpp
source/libs/parser/test/mockCatalog.cpp
+3
-3
source/libs/parser/test/mockCatalog.h
source/libs/parser/test/mockCatalog.h
+0
-4
source/libs/parser/test/mockCatalogService.cpp
source/libs/parser/test/mockCatalogService.cpp
+3
-2
source/libs/planner/src/physicalPlan.c
source/libs/planner/src/physicalPlan.c
+1
-1
source/libs/qworker/CMakeLists.txt
source/libs/qworker/CMakeLists.txt
+12
-0
source/libs/qworker/inc/qworkerInt.h
source/libs/qworker/inc/qworkerInt.h
+68
-0
source/libs/qworker/src/qworker.c
source/libs/qworker/src/qworker.c
+298
-0
source/libs/scheduler/inc/schedulerInt.h
source/libs/scheduler/inc/schedulerInt.h
+2
-10
source/libs/scheduler/src/scheduler.c
source/libs/scheduler/src/scheduler.c
+32
-26
source/util/src/thash.c
source/util/src/thash.c
+9
-2
未找到文件。
include/common/taosmsg.h
浏览文件 @
d96963a2
...
...
@@ -51,7 +51,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" )
TAOS_DEFINE_MESSAGE_TYPE
(
TSDB_MSG_TYPE_MQ_CONNECT
,
"mq-connect"
)
TAOS_DEFINE_MESSAGE_TYPE
(
TSDB_MSG_TYPE_MQ_DISCONNECT
,
"mq-disconnect"
)
TAOS_DEFINE_MESSAGE_TYPE
(
TSDB_MSG_TYPE_MQ_SET_CUR
,
"mq-set-cur"
)
TAOS_DEFINE_MESSAGE_TYPE
(
TSDB_MSG_TYPE_R
SP_READY
,
"rsp
-ready"
)
TAOS_DEFINE_MESSAGE_TYPE
(
TSDB_MSG_TYPE_R
ES_READY
,
"res
-ready"
)
// message from client to mnode
TAOS_DEFINE_MESSAGE_TYPE
(
TSDB_MSG_TYPE_CONNECT
,
"connect"
)
TAOS_DEFINE_MESSAGE_TYPE
(
TSDB_MSG_TYPE_CREATE_ACCT
,
"create-acct"
)
...
...
@@ -1081,6 +1081,7 @@ typedef struct {
}
SUpdateTagValRsp
;
typedef
struct
SSchedulerQueryMsg
{
uint64_t
schedulerId
;
uint64_t
queryId
;
uint64_t
taskId
;
uint32_t
contentLen
;
...
...
@@ -1088,15 +1089,39 @@ typedef struct SSchedulerQueryMsg {
}
SSchedulerQueryMsg
;
typedef
struct
SSchedulerReadyMsg
{
uint64_t
schedulerId
;
uint64_t
queryId
;
uint64_t
taskId
;
}
SSchedulerReadyMsg
;
typedef
struct
SSchedulerFetchMsg
{
uint64_t
schedulerId
;
uint64_t
queryId
;
uint64_t
taskId
;
}
SSchedulerFetchMsg
;
typedef
struct
SSchedulerStatusMsg
{
uint64_t
schedulerId
;
}
SSchedulerStatusMsg
;
typedef
struct
STaskStatus
{
uint64_t
queryId
;
uint64_t
taskId
;
int8_t
status
;
}
STaskStatus
;
typedef
struct
SSchedulerStatusRsp
{
uint32_t
num
;
STaskStatus
status
[];
}
SSchedulerStatusRsp
;
typedef
struct
SSchedulerCancelMsg
{
uint64_t
schedulerId
;
uint64_t
queryId
;
uint64_t
taskId
;
}
SSchedulerCancelMsg
;
#pragma pack(pop)
...
...
include/dnode/qnode/qnode.h
浏览文件 @
d96963a2
...
...
@@ -19,7 +19,7 @@
#ifdef __cplusplus
extern
"C"
{
#endif
#include "trpc.h"
typedef
struct
{
uint64_t
numOfStartTask
;
...
...
@@ -32,48 +32,6 @@ typedef struct {
uint64_t
numOfErrors
;
}
SQnodeStat
;
/* start Task msg */
typedef
struct
{
uint32_t
schedulerIp
;
uint16_t
schedulerPort
;
int64_t
taskId
;
int64_t
queryId
;
uint32_t
srcIp
;
uint16_t
srcPort
;
}
SQnodeStartTaskMsg
;
/* stop Task msg */
typedef
struct
{
int64_t
taskId
;
}
SQnodeStopTaskMsg
;
/* start/stop Task msg response */
typedef
struct
{
int64_t
taskId
;
int32_t
code
;
}
SQnodeTaskRespMsg
;
/* Task status msg */
typedef
struct
{
int64_t
taskId
;
int32_t
status
;
int64_t
queryId
;
}
SQnodeTaskStatusMsg
;
/* Qnode/Scheduler heartbeat msg */
typedef
struct
{
int32_t
status
;
int32_t
load
;
}
SQnodeHeartbeatMsg
;
/* Qnode sent/received msg */
typedef
struct
{
int8_t
msgType
;
int32_t
msgLen
;
char
msg
[];
}
SQnodeMsg
;
/**
* Start one Qnode in Dnode.
...
...
include/libs/index/index.h
浏览文件 @
d96963a2
...
...
@@ -44,41 +44,46 @@ typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX =
* @param: oper
*
*/
SIndexMultiTermQuery
*
indexMultiTermQueryCreate
(
EIndexOperatorType
oper
);
void
indexMultiTermQueryDestroy
(
SIndexMultiTermQuery
*
pQuery
);
int
indexMultiTermQueryAdd
(
SIndexMultiTermQuery
*
pQuery
,
SIndexTerm
*
term
,
EIndexQueryType
type
);
SIndexMultiTermQuery
*
indexMultiTermQueryCreate
(
EIndexOperatorType
oper
);
void
indexMultiTermQueryDestroy
(
SIndexMultiTermQuery
*
pQuery
);
int
indexMultiTermQueryAdd
(
SIndexMultiTermQuery
*
pQuery
,
SIndexTerm
*
term
,
EIndexQueryType
type
);
/*
* @param:
* @param:
*/
int
indexOpen
(
SIndexOpts
*
opt
,
const
char
*
path
,
SIndex
**
index
);
void
indexClose
(
SIndex
*
index
);
int
indexPut
(
SIndex
*
index
,
SIndexMultiTerm
*
terms
,
uint64_t
uid
);
int
indexDelete
(
SIndex
*
index
,
SIndexMultiTermQuery
*
query
);
int
indexSearch
(
SIndex
*
index
,
SIndexMultiTermQuery
*
query
,
SArray
*
result
);
int
indexRebuild
(
SIndex
*
index
,
SIndexOpts
*
opt
);
int
indexOpen
(
SIndexOpts
*
opt
,
const
char
*
path
,
SIndex
**
index
);
void
indexClose
(
SIndex
*
index
);
int
indexPut
(
SIndex
*
index
,
SIndexMultiTerm
*
terms
,
uint64_t
uid
);
int
indexDelete
(
SIndex
*
index
,
SIndexMultiTermQuery
*
query
);
int
indexSearch
(
SIndex
*
index
,
SIndexMultiTermQuery
*
query
,
SArray
*
result
);
int
indexRebuild
(
SIndex
*
index
,
SIndexOpts
*
opt
);
/*
* @param
* @param
*/
SIndexMultiTerm
*
indexMultiTermCreate
();
int
indexMultiTermAdd
(
SIndexMultiTerm
*
terms
,
SIndexTerm
*
term
);
void
indexMultiTermDestroy
(
SIndexMultiTerm
*
terms
);
SIndexMultiTerm
*
indexMultiTermCreate
();
int
indexMultiTermAdd
(
SIndexMultiTerm
*
terms
,
SIndexTerm
*
term
);
void
indexMultiTermDestroy
(
SIndexMultiTerm
*
terms
);
/*
* @param:
* @param:
*/
SIndexOpts
*
indexOptsCreate
();
void
indexOptsDestroy
(
SIndexOpts
*
opts
);
SIndexOpts
*
indexOptsCreate
();
void
indexOptsDestroy
(
SIndexOpts
*
opts
);
/*
* @param:
* @param:
*/
SIndexTerm
*
indexTermCreate
(
int64_t
suid
,
SIndexOperOnColumn
operType
,
uint8_t
colType
,
const
char
*
colName
,
int32_t
nColName
,
const
char
*
colVal
,
int32_t
nColVal
);
void
indexTermDestroy
(
SIndexTerm
*
p
);
SIndexTerm
*
indexTermCreate
(
int64_t
suid
,
SIndexOperOnColumn
operType
,
uint8_t
colType
,
const
char
*
colName
,
int32_t
nColName
,
const
char
*
colVal
,
int32_t
nColVal
);
void
indexTermDestroy
(
SIndexTerm
*
p
);
#ifdef __cplusplus
}
...
...
include/libs/parser/parser.h
浏览文件 @
d96963a2
...
...
@@ -136,6 +136,7 @@ typedef struct SParseContext {
const
char
*
pDbname
;
void
*
pRpc
;
const
char
*
pClusterId
;
struct
SCatalog
*
pCatalog
;
const
SEpSet
*
pEpSet
;
int64_t
id
;
// query id, generated by uuid generator
int8_t
schemaAttached
;
// denote if submit block is built with table schema or not
...
...
include/libs/qcom/query.h
浏览文件 @
d96963a2
...
...
@@ -24,6 +24,15 @@ extern "C" {
#include "thash.h"
#include "tlog.h"
enum
{
JOB_TASK_STATUS_NOT_START
=
1
,
JOB_TASK_STATUS_EXECUTING
,
JOB_TASK_STATUS_SUCCEED
,
JOB_TASK_STATUS_FAILED
,
JOB_TASK_STATUS_CANCELLING
,
JOB_TASK_STATUS_CANCELLED
};
typedef
struct
STableComInfo
{
uint8_t
numOfTags
;
// the number of tags in schema
uint8_t
precision
;
// the number of precision
...
...
include/libs/qworker/qworker.h
0 → 100644
浏览文件 @
d96963a2
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_QWORKER_H_
#define _TD_QWORKER_H_
#ifdef __cplusplus
extern
"C"
{
#endif
#include "trpc.h"
typedef
struct
SQWorkerCfg
{
uint32_t
maxSchedulerNum
;
uint32_t
maxResCacheNum
;
uint32_t
maxSchTaskNum
;
}
SQWorkerCfg
;
typedef
struct
{
uint64_t
numOfStartTask
;
uint64_t
numOfStopTask
;
uint64_t
numOfRecvedFetch
;
uint64_t
numOfSentHb
;
uint64_t
numOfSentFetch
;
uint64_t
numOfTaskInQueue
;
uint64_t
numOfFetchInQueue
;
uint64_t
numOfErrors
;
}
SQWorkerStat
;
int32_t
qWorkerInit
(
SQWorkerCfg
*
cfg
,
void
**
qWorkerMgmt
);
int32_t
qWorkerProcessQueryMsg
(
void
*
qWorkerMgmt
,
SSchedulerQueryMsg
*
msg
,
SRpcMsg
*
rsp
);
int32_t
qWorkerProcessReadyMsg
(
void
*
qWorkerMgmt
,
SSchedulerReadyMsg
*
msg
,
SRpcMsg
*
rsp
);
int32_t
qWorkerProcessStatusMsg
(
void
*
qWorkerMgmt
,
SSchedulerStatusMsg
*
msg
,
SRpcMsg
*
rsp
);
int32_t
qWorkerProcessFetchMsg
(
void
*
qWorkerMgmt
,
SSchedulerFetchMsg
*
msg
,
SRpcMsg
*
rsp
);
int32_t
qWorkerProcessCancelMsg
(
void
*
qWorkerMgmt
,
SSchedulerCancelMsg
*
msg
,
SRpcMsg
*
rsp
);
void
qWorkerDestroy
(
void
**
qWorkerMgmt
);
#ifdef __cplusplus
}
#endif
#endif
/*_TD_QWORKER_H_*/
include/libs/scheduler/scheduler.h
浏览文件 @
d96963a2
...
...
@@ -25,6 +25,7 @@ extern "C" {
typedef
struct
SSchedulerCfg
{
int32_t
clusterType
;
int32_t
maxJobNum
;
}
SSchedulerCfg
;
typedef
struct
SQueryProfileSummary
{
...
...
include/util/thash.h
浏览文件 @
d96963a2
...
...
@@ -193,11 +193,10 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p);
/**
* Get the corresponding key information for a given data in hash table
* @param pHashObj
* @param data
* @return
*/
int32_t
taosHashGetKey
(
SHashObj
*
pHashObj
,
void
*
data
,
void
**
key
,
size_t
*
keyLen
);
int32_t
taosHashGetKey
(
void
*
data
,
void
**
key
,
size_t
*
keyLen
);
#ifdef __cplusplus
}
...
...
source/libs/CMakeLists.txt
浏览文件 @
d96963a2
...
...
@@ -11,3 +11,4 @@ add_subdirectory(executor)
add_subdirectory
(
planner
)
add_subdirectory
(
function
)
add_subdirectory
(
qcom
)
add_subdirectory
(
qworker
)
source/libs/executor/src/tfilter.c
浏览文件 @
d96963a2
...
...
@@ -1175,7 +1175,7 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
void
*
key
=
NULL
;
len
=
0
;
taosHashGetKey
(
(
SHashObj
*
)
data
,
p
,
&
key
,
&
len
);
taosHashGetKey
(
p
,
&
key
,
&
len
);
void
*
fdata
=
NULL
;
if
(
IS_VAR_DATA_TYPE
(
type
))
{
...
...
source/libs/index/inc/indexInt.h
浏览文件 @
d96963a2
...
...
@@ -42,11 +42,11 @@ typedef struct SIndexStat {
struct
SIndex
{
#ifdef USE_LUCENE
index_t
*
index
;
index_t
*
index
;
#endif
void
*
cache
;
void
*
tindex
;
SHashObj
*
colObj
;
// < field name, field id>
void
*
cache
;
void
*
tindex
;
SHashObj
*
colObj
;
// < field name, field id>
int64_t
suid
;
// current super table id, -1 is normal table
int
colId
;
// field id allocated to cache
...
...
@@ -58,7 +58,7 @@ struct SIndex {
struct
SIndexOpts
{
#ifdef USE_LUCENE
void
*
opts
;
void
*
opts
;
#endif
#ifdef USE_INVERTED_INDEX
...
...
@@ -69,7 +69,7 @@ struct SIndexOpts {
struct
SIndexMultiTermQuery
{
EIndexOperatorType
opera
;
SArray
*
query
;
SArray
*
query
;
};
// field and key;
...
...
@@ -77,14 +77,14 @@ typedef struct SIndexTerm {
int64_t
suid
;
SIndexOperOnColumn
operType
;
// oper type, add/del/update
uint8_t
colType
;
// term data type, str/interger/json
char
*
colName
;
char
*
colName
;
int32_t
nColName
;
char
*
colVal
;
char
*
colVal
;
int32_t
nColVal
;
}
SIndexTerm
;
typedef
struct
SIndexTermQuery
{
SIndexTerm
*
term
;
SIndexTerm
*
term
;
EIndexQueryType
qType
;
}
SIndexTermQuery
;
...
...
source/libs/index/inc/index_cache.h
浏览文件 @
d96963a2
...
...
@@ -34,19 +34,18 @@ extern "C" {
typedef
struct
IndexCache
{
T_REF_DECLARE
()
SSkipList
*
skiplist
;
SSkipList
*
skiplist
;
}
IndexCache
;
//
IndexCache
*
indexCacheCreate
();
IndexCache
*
indexCacheCreate
();
void
indexCacheDestroy
(
void
*
cache
);
void
indexCacheDestroy
(
void
*
cache
);
int
indexCachePut
(
void
*
cache
,
SIndexTerm
*
term
,
int16_t
colId
,
int32_t
version
,
uint64_t
uid
);
int
indexCachePut
(
void
*
cache
,
SIndexTerm
*
term
,
int16_t
colId
,
int32_t
version
,
uint64_t
uid
);
// int indexCacheGet(void *cache, uint64_t *rst);
int
indexCacheSearch
(
void
*
cache
,
SIndexTermQuery
*
query
,
int16_t
colId
,
int32_t
version
,
SArray
*
result
,
STermValueType
*
s
);
int
indexCacheSearch
(
void
*
cache
,
SIndexTermQuery
*
query
,
int16_t
colId
,
int32_t
version
,
SArray
*
result
,
STermValueType
*
s
);
#ifdef __cplusplus
}
...
...
source/libs/index/inc/index_fst.h
浏览文件 @
d96963a2
...
...
@@ -40,10 +40,10 @@ typedef struct FstBoundWithData {
}
FstBoundWithData
;
typedef
struct
FstStreamBuilder
{
Fst
*
fst
;
AutomationCtx
*
aut
;
FstBoundWithData
*
min
;
FstBoundWithData
*
max
;
Fst
*
fst
;
AutomationCtx
*
aut
;
FstBoundWithData
*
min
;
FstBoundWithData
*
max
;
}
FstStreamBuilder
,
FstStreamWithStateBuilder
;
typedef
struct
FstRange
{
...
...
@@ -55,10 +55,10 @@ typedef enum { GE, GT, LE, LT } RangeType;
typedef
enum
{
OneTransNext
,
OneTrans
,
AnyTrans
,
EmptyFinal
}
State
;
typedef
enum
{
Ordered
,
OutOfOrdered
,
DuplicateKey
}
OrderType
;
FstBoundWithData
*
fstBoundStateCreate
(
FstBound
type
,
FstSlice
*
data
);
bool
fstBoundWithDataExceededBy
(
FstBoundWithData
*
bound
,
FstSlice
*
slice
);
bool
fstBoundWithDataIsEmpty
(
FstBoundWithData
*
bound
);
bool
fstBoundWithDataIsIncluded
(
FstBoundWithData
*
bound
);
FstBoundWithData
*
fstBoundStateCreate
(
FstBound
type
,
FstSlice
*
data
);
bool
fstBoundWithDataExceededBy
(
FstBoundWithData
*
bound
,
FstSlice
*
slice
);
bool
fstBoundWithDataIsEmpty
(
FstBoundWithData
*
bound
);
bool
fstBoundWithDataIsIncluded
(
FstBoundWithData
*
bound
);
typedef
struct
FstOutput
{
bool
null
;
...
...
@@ -71,46 +71,46 @@ typedef struct FstOutput {
* TODO: simple function name
*/
typedef
struct
FstUnFinishedNodes
{
SArray
*
stack
;
// <FstBuilderNodeUnfinished> } FstUnFinishedNodes;
SArray
*
stack
;
// <FstBuilderNodeUnfinished> } FstUnFinishedNodes;
}
FstUnFinishedNodes
;
#define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack)
FstUnFinishedNodes
*
fstUnFinishedNodesCreate
();
void
fstUnFinishedNodesDestroy
(
FstUnFinishedNodes
*
node
);
void
fstUnFinishedNodesPushEmpty
(
FstUnFinishedNodes
*
nodes
,
bool
isFinal
);
void
fstUnFinishedNodesSetRootOutput
(
FstUnFinishedNodes
*
node
,
Output
out
);
void
fstUnFinishedNodesTopLastFreeze
(
FstUnFinishedNodes
*
node
,
CompiledAddr
addr
);
void
fstUnFinishedNodesAddSuffix
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
,
Output
out
);
uint64_t
fstUnFinishedNodesFindCommPrefix
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
);
FstBuilderNode
*
fstUnFinishedNodesPopRoot
(
FstUnFinishedNodes
*
nodes
);
FstBuilderNode
*
fstUnFinishedNodesPopFreeze
(
FstUnFinishedNodes
*
nodes
,
CompiledAddr
addr
);
FstBuilderNode
*
fstUnFinishedNodesPopEmpty
(
FstUnFinishedNodes
*
nodes
);
FstUnFinishedNodes
*
fstUnFinishedNodesCreate
();
void
fstUnFinishedNodesDestroy
(
FstUnFinishedNodes
*
node
);
void
fstUnFinishedNodesPushEmpty
(
FstUnFinishedNodes
*
nodes
,
bool
isFinal
);
void
fstUnFinishedNodesSetRootOutput
(
FstUnFinishedNodes
*
node
,
Output
out
);
void
fstUnFinishedNodesTopLastFreeze
(
FstUnFinishedNodes
*
node
,
CompiledAddr
addr
);
void
fstUnFinishedNodesAddSuffix
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
,
Output
out
);
uint64_t
fstUnFinishedNodesFindCommPrefix
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
);
FstBuilderNode
*
fstUnFinishedNodesPopRoot
(
FstUnFinishedNodes
*
nodes
);
FstBuilderNode
*
fstUnFinishedNodesPopFreeze
(
FstUnFinishedNodes
*
nodes
,
CompiledAddr
addr
);
FstBuilderNode
*
fstUnFinishedNodesPopEmpty
(
FstUnFinishedNodes
*
nodes
);
uint64_t
fstUnFinishedNodesFindCommPrefixAndSetOutput
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
,
Output
in
,
Output
*
out
);
uint64_t
fstUnFinishedNodesFindCommPrefixAndSetOutput
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
,
Output
in
,
Output
*
out
);
typedef
struct
FstBuilder
{
FstCountingWriter
*
wrt
;
// The FST raw data is written directly to `wtr`.
FstUnFinishedNodes
*
unfinished
;
// The stack of unfinished nodes
FstRegistry
*
registry
;
// A map of finished nodes.
FstCountingWriter
*
wrt
;
// The FST raw data is written directly to `wtr`.
FstUnFinishedNodes
*
unfinished
;
// The stack of unfinished nodes
FstRegistry
*
registry
;
// A map of finished nodes.
FstSlice
last
;
// The last word added
CompiledAddr
lastAddr
;
// The address of the last compiled node
uint64_t
len
;
// num of keys added
}
FstBuilder
;
FstBuilder
*
fstBuilderCreate
(
void
*
w
,
FstType
ty
);
FstBuilder
*
fstBuilderCreate
(
void
*
w
,
FstType
ty
);
void
fstBuilderDestroy
(
FstBuilder
*
b
);
void
fstBuilderInsertOutput
(
FstBuilder
*
b
,
FstSlice
bs
,
Output
in
);
bool
fstBuilderInsert
(
FstBuilder
*
b
,
FstSlice
bs
,
Output
in
);
void
fstBuilderCompileFrom
(
FstBuilder
*
b
,
uint64_t
istate
);
void
*
fstBuilerIntoInner
(
FstBuilder
*
b
);
void
fstBuilderFinish
(
FstBuilder
*
b
);
OrderType
fstBuilderCheckLastKey
(
FstBuilder
*
b
,
FstSlice
bs
,
bool
ckDup
);
CompiledAddr
fstBuilderCompile
(
FstBuilder
*
b
,
FstBuilderNode
*
bn
);
void
fstBuilderDestroy
(
FstBuilder
*
b
);
void
fstBuilderInsertOutput
(
FstBuilder
*
b
,
FstSlice
bs
,
Output
in
);
bool
fstBuilderInsert
(
FstBuilder
*
b
,
FstSlice
bs
,
Output
in
);
void
fstBuilderCompileFrom
(
FstBuilder
*
b
,
uint64_t
istate
);
void
*
fstBuilerIntoInner
(
FstBuilder
*
b
);
void
fstBuilderFinish
(
FstBuilder
*
b
);
OrderType
fstBuilderCheckLastKey
(
FstBuilder
*
b
,
FstSlice
bs
,
bool
ckDup
);
CompiledAddr
fstBuilderCompile
(
FstBuilder
*
b
,
FstBuilderNode
*
bn
);
typedef
struct
FstTransitions
{
FstNode
*
node
;
FstNode
*
node
;
FstRange
range
;
}
FstTransitions
;
...
...
@@ -121,56 +121,55 @@ typedef struct FstState {
uint8_t
val
;
}
FstState
;
FstState
fstStateCreateFrom
(
FstSlice
*
data
,
CompiledAddr
addr
);
FstState
fstStateCreateFrom
(
FstSlice
*
data
,
CompiledAddr
addr
);
FstState
fstStateCreate
(
State
state
);
// compile
void
fstStateCompileForOneTransNext
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
uint8_t
inp
);
void
fstStateCompileForOneTrans
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
FstTransition
*
trn
);
void
fstStateCompileForAnyTrans
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
FstBuilderNode
*
node
);
void
fstStateCompileForOneTransNext
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
uint8_t
inp
);
void
fstStateCompileForOneTrans
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
FstTransition
*
trn
);
void
fstStateCompileForAnyTrans
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
FstBuilderNode
*
node
);
// set_comm_input
void
fstStateSetCommInput
(
FstState
*
state
,
uint8_t
inp
);
void
fstStateSetCommInput
(
FstState
*
state
,
uint8_t
inp
);
// comm_input
uint8_t
fstStateCommInput
(
FstState
*
state
,
bool
*
null
);
uint8_t
fstStateCommInput
(
FstState
*
state
,
bool
*
null
);
// input_len
uint64_t
fstStateInputLen
(
FstState
*
state
);
uint64_t
fstStateInputLen
(
FstState
*
state
);
// end_addr
uint64_t
fstStateEndAddrForOneTransNext
(
FstState
*
state
,
FstSlice
*
data
);
uint64_t
fstStateEndAddrForOneTrans
(
FstState
*
state
,
FstSlice
*
data
,
PackSizes
sizes
);
uint64_t
fstStateEndAddrForAnyTrans
(
FstState
*
state
,
uint64_t
version
,
FstSlice
*
date
,
PackSizes
sizes
,
uint64_t
nTrans
);
uint64_t
fstStateEndAddrForOneTransNext
(
FstState
*
state
,
FstSlice
*
data
);
uint64_t
fstStateEndAddrForOneTrans
(
FstState
*
state
,
FstSlice
*
data
,
PackSizes
sizes
);
uint64_t
fstStateEndAddrForAnyTrans
(
FstState
*
state
,
uint64_t
version
,
FstSlice
*
date
,
PackSizes
sizes
,
uint64_t
nTrans
);
// input
uint8_t
fstStateInput
(
FstState
*
state
,
FstNode
*
node
);
uint8_t
fstStateInputForAnyTrans
(
FstState
*
state
,
FstNode
*
node
,
uint64_t
i
);
uint8_t
fstStateInput
(
FstState
*
state
,
FstNode
*
node
);
uint8_t
fstStateInputForAnyTrans
(
FstState
*
state
,
FstNode
*
node
,
uint64_t
i
);
// trans_addr
CompiledAddr
fstStateTransAddr
(
FstState
*
state
,
FstNode
*
node
);
CompiledAddr
fstStateTransAddrForAnyTrans
(
FstState
*
state
,
FstNode
*
node
,
uint64_t
i
);
CompiledAddr
fstStateTransAddr
(
FstState
*
state
,
FstNode
*
node
);
CompiledAddr
fstStateTransAddrForAnyTrans
(
FstState
*
state
,
FstNode
*
node
,
uint64_t
i
);
// sizes
PackSizes
fstStateSizes
(
FstState
*
state
,
FstSlice
*
data
);
PackSizes
fstStateSizes
(
FstState
*
state
,
FstSlice
*
data
);
// Output
Output
fstStateOutput
(
FstState
*
state
,
FstNode
*
node
);
Output
fstStateOutputForAnyTrans
(
FstState
*
state
,
FstNode
*
node
,
uint64_t
i
);
Output
fstStateOutput
(
FstState
*
state
,
FstNode
*
node
);
Output
fstStateOutputForAnyTrans
(
FstState
*
state
,
FstNode
*
node
,
uint64_t
i
);
// anyTrans specify function
void
fstStateSetFinalState
(
FstState
*
state
,
bool
yes
);
bool
fstStateIsFinalState
(
FstState
*
state
);
void
fstStateSetStateNtrans
(
FstState
*
state
,
uint8_t
n
);
void
fstStateSetFinalState
(
FstState
*
state
,
bool
yes
);
bool
fstStateIsFinalState
(
FstState
*
state
);
void
fstStateSetStateNtrans
(
FstState
*
state
,
uint8_t
n
);
// state_ntrans
uint8_t
fstStateStateNtrans
(
FstState
*
state
,
bool
*
null
);
uint64_t
fstStateTotalTransSize
(
FstState
*
state
,
uint64_t
version
,
PackSizes
size
,
uint64_t
nTrans
);
uint64_t
fstStateTransIndexSize
(
FstState
*
state
,
uint64_t
version
,
uint64_t
nTrans
);
uint64_t
fstStateNtransLen
(
FstState
*
state
);
uint64_t
fstStateNtrans
(
FstState
*
state
,
FstSlice
*
slice
);
Output
fstStateFinalOutput
(
FstState
*
state
,
uint64_t
version
,
FstSlice
*
date
,
PackSizes
sizes
,
uint64_t
nTrans
);
uint64_t
fstStateFindInput
(
FstState
*
state
,
FstNode
*
node
,
uint8_t
b
,
bool
*
null
);
uint8_t
fstStateStateNtrans
(
FstState
*
state
,
bool
*
null
);
uint64_t
fstStateTotalTransSize
(
FstState
*
state
,
uint64_t
version
,
PackSizes
size
,
uint64_t
nTrans
);
uint64_t
fstStateTransIndexSize
(
FstState
*
state
,
uint64_t
version
,
uint64_t
nTrans
);
uint64_t
fstStateNtransLen
(
FstState
*
state
);
uint64_t
fstStateNtrans
(
FstState
*
state
,
FstSlice
*
slice
);
Output
fstStateFinalOutput
(
FstState
*
state
,
uint64_t
version
,
FstSlice
*
date
,
PackSizes
sizes
,
uint64_t
nTrans
);
uint64_t
fstStateFindInput
(
FstState
*
state
,
FstNode
*
node
,
uint8_t
b
,
bool
*
null
);
#define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext)
#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans)
...
...
@@ -187,13 +186,13 @@ typedef struct FstLastTransition {
* TODO: simple function name
*/
typedef
struct
FstBuilderNodeUnfinished
{
FstBuilderNode
*
node
;
FstLastTransition
*
last
;
FstBuilderNode
*
node
;
FstLastTransition
*
last
;
}
FstBuilderNodeUnfinished
;
void
fstBuilderNodeUnfinishedLastCompiled
(
FstBuilderNodeUnfinished
*
node
,
CompiledAddr
addr
);
void
fstBuilderNodeUnfinishedLastCompiled
(
FstBuilderNodeUnfinished
*
node
,
CompiledAddr
addr
);
void
fstBuilderNodeUnfinishedAddOutputPrefix
(
FstBuilderNodeUnfinished
*
node
,
Output
out
);
void
fstBuilderNodeUnfinishedAddOutputPrefix
(
FstBuilderNodeUnfinished
*
node
,
Output
out
);
/*
* FstNode and helper function
...
...
@@ -224,18 +223,18 @@ typedef struct FstNode {
// Return the address of this node.
#define FST_NODE_ADDR(node) node->start
FstNode
*
fstNodeCreate
(
int64_t
version
,
CompiledAddr
addr
,
FstSlice
*
data
);
void
fstNodeDestroy
(
FstNode
*
fstNode
);
FstNode
*
fstNodeCreate
(
int64_t
version
,
CompiledAddr
addr
,
FstSlice
*
data
);
void
fstNodeDestroy
(
FstNode
*
fstNode
);
FstTransitions
fstNodeTransitionIter
(
FstNode
*
node
);
FstTransitions
*
fstNodeTransitions
(
FstNode
*
node
);
bool
fstNodeGetTransitionAt
(
FstNode
*
node
,
uint64_t
i
,
FstTransition
*
res
);
bool
fstNodeGetTransitionAddrAt
(
FstNode
*
node
,
uint64_t
i
,
CompiledAddr
*
res
);
bool
fstNodeFindInput
(
FstNode
*
node
,
uint8_t
b
,
uint64_t
*
res
);
FstTransitions
fstNodeTransitionIter
(
FstNode
*
node
);
FstTransitions
*
fstNodeTransitions
(
FstNode
*
node
);
bool
fstNodeGetTransitionAt
(
FstNode
*
node
,
uint64_t
i
,
FstTransition
*
res
);
bool
fstNodeGetTransitionAddrAt
(
FstNode
*
node
,
uint64_t
i
,
CompiledAddr
*
res
);
bool
fstNodeFindInput
(
FstNode
*
node
,
uint8_t
b
,
uint64_t
*
res
);
bool
fstNodeCompile
(
FstNode
*
node
,
void
*
w
,
CompiledAddr
lastAddr
,
CompiledAddr
addr
,
FstBuilderNode
*
builderNode
);
bool
fstNodeCompile
(
FstNode
*
node
,
void
*
w
,
CompiledAddr
lastAddr
,
CompiledAddr
addr
,
FstBuilderNode
*
builderNode
);
FstSlice
fstNodeAsSlice
(
FstNode
*
node
);
FstSlice
fstNodeAsSlice
(
FstNode
*
node
);
// ops
...
...
@@ -244,8 +243,8 @@ typedef struct FstIndexedValue {
uint64_t
value
;
}
FstIndexedValue
;
FstLastTransition
*
fstLastTransitionCreate
(
uint8_t
inp
,
Output
out
);
void
fstLastTransitionDestroy
(
FstLastTransition
*
trn
);
FstLastTransition
*
fstLastTransitionCreate
(
uint8_t
inp
,
Output
out
);
void
fstLastTransitionDestroy
(
FstLastTransition
*
trn
);
typedef
struct
FstMeta
{
uint64_t
version
;
...
...
@@ -256,75 +255,74 @@ typedef struct FstMeta {
}
FstMeta
;
typedef
struct
Fst
{
FstMeta
*
meta
;
FstSlice
*
data
;
//
FstNode
*
root
;
//
FstMeta
*
meta
;
FstSlice
*
data
;
//
FstNode
*
root
;
//
}
Fst
;
// refactor simple function
Fst
*
fstCreate
(
FstSlice
*
data
);
void
fstDestroy
(
Fst
*
fst
);
Fst
*
fstCreate
(
FstSlice
*
data
);
void
fstDestroy
(
Fst
*
fst
);
bool
fstGet
(
Fst
*
fst
,
FstSlice
*
b
,
Output
*
out
);
FstNode
*
fstGetNode
(
Fst
*
fst
,
CompiledAddr
);
FstNode
*
fstGetRoot
(
Fst
*
fst
);
FstType
fstGetType
(
Fst
*
fst
);
CompiledAddr
fstGetRootAddr
(
Fst
*
fst
);
Output
fstEmptyFinalOutput
(
Fst
*
fst
,
bool
*
null
);
FstStreamBuilder
*
fstSearch
(
Fst
*
fst
,
AutomationCtx
*
ctx
);
bool
fstGet
(
Fst
*
fst
,
FstSlice
*
b
,
Output
*
out
);
FstNode
*
fstGetNode
(
Fst
*
fst
,
CompiledAddr
);
FstNode
*
fstGetRoot
(
Fst
*
fst
);
FstType
fstGetType
(
Fst
*
fst
);
CompiledAddr
fstGetRootAddr
(
Fst
*
fst
);
Output
fstEmptyFinalOutput
(
Fst
*
fst
,
bool
*
null
);
FstStreamBuilder
*
fstSearch
(
Fst
*
fst
,
AutomationCtx
*
ctx
);
FstStreamWithStateBuilder
*
fstSearchWithState
(
Fst
*
fst
,
AutomationCtx
*
ctx
);
FstStreamWithStateBuilder
*
fstSearchWithState
(
Fst
*
fst
,
AutomationCtx
*
ctx
);
// into stream to expand later
StreamWithState
*
streamBuilderIntoStream
(
FstStreamBuilder
*
sb
);
StreamWithState
*
streamBuilderIntoStream
(
FstStreamBuilder
*
sb
);
bool
fstVerify
(
Fst
*
fst
);
bool
fstVerify
(
Fst
*
fst
);
// refactor this function
bool
fstBuilderNodeCompileTo
(
FstBuilderNode
*
b
,
FstCountingWriter
*
wrt
,
CompiledAddr
lastAddr
,
CompiledAddr
startAddr
);
bool
fstBuilderNodeCompileTo
(
FstBuilderNode
*
b
,
FstCountingWriter
*
wrt
,
CompiledAddr
lastAddr
,
CompiledAddr
startAddr
);
typedef
struct
StreamState
{
FstNode
*
node
;
FstNode
*
node
;
uint64_t
trans
;
FstOutput
out
;
void
*
autState
;
void
*
autState
;
}
StreamState
;
void
streamStateDestroy
(
void
*
s
);
void
streamStateDestroy
(
void
*
s
);
typedef
struct
StreamWithState
{
Fst
*
fst
;
AutomationCtx
*
aut
;
SArray
*
inp
;
Fst
*
fst
;
AutomationCtx
*
aut
;
SArray
*
inp
;
FstOutput
emptyOutput
;
SArray
*
stack
;
// <StreamState>
FstBoundWithData
*
endAt
;
SArray
*
stack
;
// <StreamState>
FstBoundWithData
*
endAt
;
}
StreamWithState
;
typedef
struct
StreamWithStateResult
{
FstSlice
data
;
FstOutput
out
;
void
*
state
;
void
*
state
;
}
StreamWithStateResult
;
StreamWithStateResult
*
swsResultCreate
(
FstSlice
*
data
,
FstOutput
fOut
,
void
*
state
);
void
swsResultDestroy
(
StreamWithStateResult
*
result
);
StreamWithStateResult
*
swsResultCreate
(
FstSlice
*
data
,
FstOutput
fOut
,
void
*
state
);
void
swsResultDestroy
(
StreamWithStateResult
*
result
);
typedef
void
*
(
*
StreamCallback
)(
void
*
);
StreamWithState
*
streamWithStateCreate
(
Fst
*
fst
,
AutomationCtx
*
automation
,
FstBoundWithData
*
min
,
FstBoundWithData
*
max
);
typedef
void
*
(
*
StreamCallback
)(
void
*
);
StreamWithState
*
streamWithStateCreate
(
Fst
*
fst
,
AutomationCtx
*
automation
,
FstBoundWithData
*
min
,
FstBoundWithData
*
max
);
void
streamWithStateDestroy
(
StreamWithState
*
sws
);
void
streamWithStateDestroy
(
StreamWithState
*
sws
);
bool
streamWithStateSeekMin
(
StreamWithState
*
sws
,
FstBoundWithData
*
min
);
bool
streamWithStateSeekMin
(
StreamWithState
*
sws
,
FstBoundWithData
*
min
);
StreamWithStateResult
*
streamWithStateNextWith
(
StreamWithState
*
sws
,
StreamCallback
callback
);
StreamWithStateResult
*
streamWithStateNextWith
(
StreamWithState
*
sws
,
StreamCallback
callback
);
FstStreamBuilder
*
fstStreamBuilderCreate
(
Fst
*
fst
,
AutomationCtx
*
aut
);
FstStreamBuilder
*
fstStreamBuilderCreate
(
Fst
*
fst
,
AutomationCtx
*
aut
);
// set up bound range
// refator, simple code by marco
FstStreamBuilder
*
fstStreamBuilderRange
(
FstStreamBuilder
*
b
,
FstSlice
*
val
,
RangeType
type
);
FstStreamBuilder
*
fstStreamBuilderRange
(
FstStreamBuilder
*
b
,
FstSlice
*
val
,
RangeType
type
);
#ifdef __cplusplus
}
...
...
source/libs/index/inc/index_fst_automation.h
浏览文件 @
d96963a2
...
...
@@ -26,18 +26,18 @@ typedef struct AutomationCtx AutomationCtx;
typedef
enum
AutomationType
{
AUTOMATION_PREFIX
,
AUTMMATION_MATCH
}
AutomationType
;
typedef
struct
StartWith
{
AutomationCtx
*
autoSelf
;
AutomationCtx
*
autoSelf
;
}
StartWith
;
typedef
struct
Complement
{
AutomationCtx
*
autoSelf
;
AutomationCtx
*
autoSelf
;
}
Complement
;
// automation
typedef
struct
AutomationCtx
{
AutomationType
type
;
void
*
stdata
;
char
*
data
;
void
*
stdata
;
char
*
data
;
}
AutomationCtx
;
typedef
enum
ValueType
{
FST_INT
,
FST_CHAR
,
FST_ARRAY
}
ValueType
;
...
...
@@ -48,27 +48,27 @@ typedef struct StartWithStateValue {
ValueType
type
;
union
{
int
val
;
char
*
ptr
;
SArray
*
arr
;
char
*
ptr
;
SArray
*
arr
;
// add more type
};
}
StartWithStateValue
;
StartWithStateValue
*
startWithStateValueCreate
(
StartWithStateKind
kind
,
ValueType
ty
,
void
*
val
);
StartWithStateValue
*
startWithStateValueDump
(
StartWithStateValue
*
sv
);
void
startWithStateValueDestroy
(
void
*
sv
);
StartWithStateValue
*
startWithStateValueCreate
(
StartWithStateKind
kind
,
ValueType
ty
,
void
*
val
);
StartWithStateValue
*
startWithStateValueDump
(
StartWithStateValue
*
sv
);
void
startWithStateValueDestroy
(
void
*
sv
);
typedef
struct
AutomationFunc
{
void
*
(
*
start
)(
AutomationCtx
*
ctx
);
bool
(
*
isMatch
)(
AutomationCtx
*
ctx
,
void
*
);
bool
(
*
canMatch
)(
AutomationCtx
*
ctx
,
void
*
data
);
bool
(
*
willAlwaysMatch
)(
AutomationCtx
*
ctx
,
void
*
state
);
void
*
(
*
accept
)(
AutomationCtx
*
ctx
,
void
*
state
,
uint8_t
byte
);
void
*
(
*
acceptEof
)(
AutomationCtx
*
ct
,
void
*
state
);
void
*
(
*
start
)(
AutomationCtx
*
ctx
);
bool
(
*
isMatch
)(
AutomationCtx
*
ctx
,
void
*
);
bool
(
*
canMatch
)(
AutomationCtx
*
ctx
,
void
*
data
);
bool
(
*
willAlwaysMatch
)(
AutomationCtx
*
ctx
,
void
*
state
);
void
*
(
*
accept
)(
AutomationCtx
*
ctx
,
void
*
state
,
uint8_t
byte
);
void
*
(
*
acceptEof
)(
AutomationCtx
*
ct
,
void
*
state
);
}
AutomationFunc
;
AutomationCtx
*
automCtxCreate
(
void
*
data
,
AutomationType
atype
);
void
automCtxDestroy
(
AutomationCtx
*
ctx
);
AutomationCtx
*
automCtxCreate
(
void
*
data
,
AutomationType
atype
);
void
automCtxDestroy
(
AutomationCtx
*
ctx
);
extern
AutomationFunc
automFuncs
[];
#ifdef __cplusplus
...
...
source/libs/index/inc/index_fst_counting_writer.h
浏览文件 @
d96963a2
...
...
@@ -28,9 +28,10 @@ static char tmpFile[] = "./index";
typedef
enum
WriterType
{
TMemory
,
TFile
}
WriterType
;
typedef
struct
WriterCtx
{
int
(
*
write
)(
struct
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
int
(
*
read
)(
struct
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
int
(
*
flush
)(
struct
WriterCtx
*
ctx
);
int
(
*
write
)(
struct
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
int
(
*
read
)(
struct
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
int
(
*
flush
)(
struct
WriterCtx
*
ctx
);
int
(
*
readFrom
)(
struct
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
,
int32_t
offset
);
WriterType
type
;
union
{
struct
{
...
...
@@ -39,41 +40,42 @@ typedef struct WriterCtx {
}
file
;
struct
{
int32_t
capa
;
char
*
buf
;
char
*
buf
;
}
mem
;
};
int32_t
offset
;
int32_t
limit
;
}
WriterCtx
;
static
int
writeCtxDoWrite
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
static
int
writeCtxDoRead
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
static
int
writeCtxDoFlush
(
WriterCtx
*
ctx
);
static
int
writeCtxDoWrite
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
static
int
writeCtxDoRead
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
);
static
int
writeCtxDoReadFrom
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
,
int32_t
offset
);
static
int
writeCtxDoFlush
(
WriterCtx
*
ctx
);
WriterCtx
*
writerCtxCreate
(
WriterType
type
,
const
char
*
path
,
bool
readOnly
,
int32_t
capacity
);
void
writerCtxDestroy
(
WriterCtx
*
w
);
WriterCtx
*
writerCtxCreate
(
WriterType
type
,
const
char
*
path
,
bool
readOnly
,
int32_t
capacity
);
void
writerCtxDestroy
(
WriterCtx
*
w
);
typedef
uint32_t
CheckSummer
;
typedef
struct
FstCountingWriter
{
void
*
wrt
;
// wrap any writer that counts and checksum bytes written
void
*
wrt
;
// wrap any writer that counts and checksum bytes written
uint64_t
count
;
CheckSummer
summer
;
}
FstCountingWriter
;
int
fstCountingWriterWrite
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
);
int
fstCountingWriterWrite
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
);
int
fstCountingWriterRead
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
);
int
fstCountingWriterRead
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
);
int
fstCountingWriterFlush
(
FstCountingWriter
*
write
);
int
fstCountingWriterFlush
(
FstCountingWriter
*
write
);
uint32_t
fstCountingWriterMaskedCheckSum
(
FstCountingWriter
*
write
);
uint32_t
fstCountingWriterMaskedCheckSum
(
FstCountingWriter
*
write
);
FstCountingWriter
*
fstCountingWriterCreate
(
void
*
wtr
);
void
fstCountingWriterDestroy
(
FstCountingWriter
*
w
);
FstCountingWriter
*
fstCountingWriterCreate
(
void
*
wtr
);
void
fstCountingWriterDestroy
(
FstCountingWriter
*
w
);
void
fstCountingWriterPackUintIn
(
FstCountingWriter
*
writer
,
uint64_t
n
,
uint8_t
nBytes
);
uint8_t
fstCountingWriterPackUint
(
FstCountingWriter
*
writer
,
uint64_t
n
);
void
fstCountingWriterPackUintIn
(
FstCountingWriter
*
writer
,
uint64_t
n
,
uint8_t
nBytes
);
uint8_t
fstCountingWriterPackUint
(
FstCountingWriter
*
writer
,
uint64_t
n
);
#define FST_WRITER_COUNT(writer) (writer->count)
#define FST_WRITER_INTER_WRITER(writer) (writer->wtr)
...
...
source/libs/index/inc/index_fst_node.h
浏览文件 @
d96963a2
...
...
@@ -36,20 +36,20 @@ typedef struct FstTransition {
typedef
struct
FstBuilderNode
{
bool
isFinal
;
Output
finalOutput
;
SArray
*
trans
;
// <FstTransition>
SArray
*
trans
;
// <FstTransition>
}
FstBuilderNode
;
FstBuilderNode
*
fstBuilderNodeDefault
();
FstBuilderNode
*
fstBuilderNodeDefault
();
FstBuilderNode
*
fstBuilderNodeClone
(
FstBuilderNode
*
src
);
FstBuilderNode
*
fstBuilderNodeClone
(
FstBuilderNode
*
src
);
void
fstBuilderNodeCloneFrom
(
FstBuilderNode
*
dst
,
FstBuilderNode
*
src
);
void
fstBuilderNodeCloneFrom
(
FstBuilderNode
*
dst
,
FstBuilderNode
*
src
);
// bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt,
// CompiledAddr lastAddr, CompiledAddr startAddr);
bool
fstBuilderNodeEqual
(
FstBuilderNode
*
n1
,
FstBuilderNode
*
n2
);
bool
fstBuilderNodeEqual
(
FstBuilderNode
*
n1
,
FstBuilderNode
*
n2
);
void
fstBuilderNodeDestroy
(
FstBuilderNode
*
node
);
void
fstBuilderNodeDestroy
(
FstBuilderNode
*
node
);
#ifdef __cplusplus
}
...
...
source/libs/index/inc/index_fst_registry.h
浏览文件 @
d96963a2
...
...
@@ -25,7 +25,7 @@ extern "C" {
typedef
struct
FstRegistryCell
{
CompiledAddr
addr
;
FstBuilderNode
*
node
;
FstBuilderNode
*
node
;
}
FstRegistryCell
;
#define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS)
...
...
@@ -45,22 +45,22 @@ typedef enum { FOUND, NOTFOUND, REJECTED } FstRegistryEntryState;
typedef
struct
FstRegistryEntry
{
FstRegistryEntryState
state
;
CompiledAddr
addr
;
FstRegistryCell
*
cell
;
FstRegistryCell
*
cell
;
}
FstRegistryEntry
;
// Registry relation function
typedef
struct
FstRegistry
{
SArray
*
table
;
//<FstRegistryCell>
SArray
*
table
;
//<FstRegistryCell>
uint64_t
tableSize
;
// num of rows
uint64_t
mruSize
;
// num of columns
}
FstRegistry
;
//
FstRegistry
*
fstRegistryCreate
(
uint64_t
tableSize
,
uint64_t
mruSize
);
void
fstRegistryDestroy
(
FstRegistry
*
registry
);
FstRegistry
*
fstRegistryCreate
(
uint64_t
tableSize
,
uint64_t
mruSize
);
void
fstRegistryDestroy
(
FstRegistry
*
registry
);
FstRegistryEntry
*
fstRegistryGetEntry
(
FstRegistry
*
registry
,
FstBuilderNode
*
bNode
);
void
fstRegistryEntryDestroy
(
FstRegistryEntry
*
entry
);
FstRegistryEntry
*
fstRegistryGetEntry
(
FstRegistry
*
registry
,
FstBuilderNode
*
bNode
);
void
fstRegistryEntryDestroy
(
FstRegistryEntry
*
entry
);
#ifdef __cplusplus
}
...
...
source/libs/index/inc/index_fst_util.h
浏览文件 @
d96963a2
...
...
@@ -69,29 +69,29 @@ extern const uint64_t TRANS_INDEX_THRESHOLD;
// uint8_t commonIdx(uint8_t v, uint8_t max);
uint8_t
packSize
(
uint64_t
n
);
uint64_t
unpackUint64
(
uint8_t
*
ch
,
uint8_t
sz
);
uint64_t
unpackUint64
(
uint8_t
*
ch
,
uint8_t
sz
);
uint8_t
packDeltaSize
(
CompiledAddr
nodeAddr
,
CompiledAddr
transAddr
);
CompiledAddr
unpackDelta
(
char
*
data
,
uint64_t
len
,
uint64_t
nodeAddr
);
CompiledAddr
unpackDelta
(
char
*
data
,
uint64_t
len
,
uint64_t
nodeAddr
);
typedef
struct
FstString
{
uint8_t
*
data
;
uint8_t
*
data
;
uint32_t
len
;
int32_t
ref
;
}
FstString
;
typedef
struct
FstSlice
{
FstString
*
str
;
FstString
*
str
;
int32_t
start
;
int32_t
end
;
}
FstSlice
;
FstSlice
fstSliceCreate
(
uint8_t
*
data
,
uint64_t
len
);
FstSlice
fstSliceCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
);
FstSlice
fstSliceDeepCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
);
bool
fstSliceIsEmpty
(
FstSlice
*
s
);
int
fstSliceCompare
(
FstSlice
*
s1
,
FstSlice
*
s2
);
void
fstSliceDestroy
(
FstSlice
*
s
);
uint8_t
*
fstSliceData
(
FstSlice
*
s
,
int32_t
*
sz
);
FstSlice
fstSliceCreate
(
uint8_t
*
data
,
uint64_t
len
);
FstSlice
fstSliceCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
);
FstSlice
fstSliceDeepCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
);
bool
fstSliceIsEmpty
(
FstSlice
*
s
);
int
fstSliceCompare
(
FstSlice
*
s1
,
FstSlice
*
s2
);
void
fstSliceDestroy
(
FstSlice
*
s
);
uint8_t
*
fstSliceData
(
FstSlice
*
s
,
int32_t
*
sz
);
#define FST_SLICE_LEN(s) (s->end - s->start + 1)
...
...
source/libs/index/inc/index_tfile.h
浏览文件 @
d96963a2
...
...
@@ -26,84 +26,93 @@
extern
"C"
{
#endif
// tfile header
// |<---suid--->|<---version--->|<--
colLen-->|<-colName->|<---type-
->|
// |<-uint64_t->|<---int32_t--->|<--
int32_t->|<-colLen-->|<-uint8_t
->|
// tfile header
content
// |<---suid--->|<---version--->|<--
-----colName------>|<---type-->|<--fstOffset
->|
// |<-uint64_t->|<---int32_t--->|<--
TSDB_COL_NAME_LEN-->|<-uint8_t->|<---int32_t-
->|
typedef
struct
TFileReadHeader
{
#pragma pack(push, 1)
typedef
struct
TFileHeader
{
uint64_t
suid
;
int32_t
version
;
char
colName
[
128
];
//
char
colName
[
TSDB_COL_NAME_LEN
];
//
uint8_t
colType
;
}
TFileReadHeader
;
int32_t
fstOffset
;
}
TFileHeader
;
#pragma pack(pop)
#define TFILE_HEADER_SIZE (sizeof(TFILE_HEADER_SIZE) + sizeof(uint32_t));
#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t))
#define TFILE_HEADER_SIZE (sizeof(TFileHeader))
#define TFILE_HEADER_NO_FST (TFILE_HEADER_SIZE - sizeof(int32_t))
//#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t))
typedef
struct
TFileCacheKey
{
uint64_t
suid
;
uint8_t
colType
;
int32_t
version
;
c
onst
char
*
colName
;
int32_t
nColName
;
uint64_t
suid
;
uint8_t
colType
;
int32_t
version
;
c
har
*
colName
;
int32_t
nColName
;
}
TFileCacheKey
;
// table cache
// refactor to LRU cache later
typedef
struct
TFileCache
{
SHashObj
*
tableCache
;
SHashObj
*
tableCache
;
int16_t
capacity
;
// add more param
}
TFileCache
;
typedef
struct
TFileWriter
{
FstBuilder
*
fb
;
WriterCtx
*
ctx
;
FstBuilder
*
fb
;
WriterCtx
*
ctx
;
TFileHeader
header
;
uint32_t
offset
;
}
TFileWriter
;
typedef
struct
TFileReader
{
T_REF_DECLARE
()
Fst
*
fst
;
WriterCtx
*
ctx
;
TFile
Read
Header
header
;
Fst
*
fst
;
WriterCtx
*
ctx
;
TFileHeader
header
;
}
TFileReader
;
typedef
struct
IndexTFile
{
char
*
path
;
TFileCache
*
cache
;
TFileWriter
*
tw
;
char
*
path
;
TFileCache
*
cache
;
TFileWriter
*
tw
;
}
IndexTFile
;
typedef
struct
TFileWriterOpt
{
uint64_t
suid
;
int8_t
colType
;
char
*
colName
;
char
*
colName
;
int32_t
nColName
;
int32_t
version
;
}
TFileWriterOpt
;
typedef
struct
TFileReaderOpt
{
uint64_t
suid
;
char
*
colName
;
char
*
colName
;
int32_t
nColName
;
}
TFileReaderOpt
;
// tfile cache, manage tindex reader
TFileCache
*
tfileCacheCreate
(
const
char
*
path
);
void
tfileCacheDestroy
(
TFileCache
*
tcache
);
TFileReader
*
tfileCacheGet
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
);
void
tfileCachePut
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
,
TFileReader
*
reader
);
TFileCache
*
tfileCacheCreate
(
const
char
*
path
);
void
tfileCacheDestroy
(
TFileCache
*
tcache
);
TFileReader
*
tfileCacheGet
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
);
void
tfileCachePut
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
,
TFileReader
*
reader
);
TFileReader
*
tfileReaderCreate
();
void
TFileReaderDestroy
(
TFileReader
*
reader
);
TFileReader
*
tfileReaderCreate
(
WriterCtx
*
ctx
);
void
tfileReaderDestroy
(
TFileReader
*
reader
);
int
tfileReaderSearch
(
TFileReader
*
reader
,
SIndexTermQuery
*
query
,
SArray
*
result
);
TFileWriter
*
tfileWriterCreate
(
const
char
*
suid
,
const
char
*
colName
);
void
tfileWriterDestroy
(
TFileWriter
*
tw
);
TFileWriter
*
tfileWriterCreate
(
WriterCtx
*
ctx
,
TFileHeader
*
header
);
void
tfileWriterDestroy
(
TFileWriter
*
tw
);
int
tfileWriterPut
(
TFileWriter
*
tw
,
void
*
data
);
int
tfileWriterFinish
(
TFileWriter
*
tw
);
//
IndexTFile
*
indexTFileCreate
(
const
char
*
path
);
int
indexTFilePut
(
void
*
tfile
,
SIndexTerm
*
term
,
uint64_t
uid
);
int
indexTFileSearch
(
void
*
tfile
,
SIndexTermQuery
*
query
,
SArray
*
result
);
IndexTFile
*
indexTFileCreate
(
const
char
*
path
);
int
indexTFilePut
(
void
*
tfile
,
SIndexTerm
*
term
,
uint64_t
uid
);
int
indexTFileSearch
(
void
*
tfile
,
SIndexTermQuery
*
query
,
SArray
*
result
);
#ifdef __cplusplus
}
...
...
source/libs/index/inc/index_util.h
浏览文件 @
d96963a2
...
...
@@ -19,29 +19,29 @@
extern
"C"
{
#endif
#define SERIALIZE_MEM_TO_BUF(buf, key, mem)
\
do {
\
memcpy((void
*)buf, (void
*)(&key->mem), sizeof(key->mem)); \
buf += sizeof(key->mem);
\
#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \
do { \
memcpy((void
*)buf, (void
*)(&key->mem), sizeof(key->mem)); \
buf += sizeof(key->mem); \
} while (0)
#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \
do { \
memcpy((void
*)buf, (void *)key->mem, len);
\
memcpy((void
*)buf, (void*)key->mem, len);
\
buf += len; \
} while (0)
#define SERIALIZE_VAR_TO_BUF(buf, var, type)
\
do {
\
type c = var;
\
assert(sizeof(var) == sizeof(type));
\
memcpy((void
*)buf, (void
*)&c, sizeof(c)); \
buf += sizeof(c);
\
#define SERIALIZE_VAR_TO_BUF(buf, var, type) \
do { \
type c = var; \
assert(sizeof(var) == sizeof(type)); \
memcpy((void
*)buf, (void
*)&c, sizeof(c)); \
buf += sizeof(c); \
} while (0)
#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \
do { \
memcpy((void
*)buf, (void *)var, len);
\
memcpy((void
*)buf, (void*)var, len);
\
buf += len; \
} while (0)
...
...
source/libs/index/src/index.c
浏览文件 @
d96963a2
...
...
@@ -23,9 +23,9 @@
#include "lucene++/Lucene_c.h"
#endif
static
int
uidCompare
(
const
void
*
a
,
const
void
*
b
)
{
uint64_t
u1
=
*
(
uint64_t
*
)
a
;
uint64_t
u2
=
*
(
uint64_t
*
)
b
;
static
int
uidCompare
(
const
void
*
a
,
const
void
*
b
)
{
uint64_t
u1
=
*
(
uint64_t
*
)
a
;
uint64_t
u2
=
*
(
uint64_t
*
)
b
;
if
(
u1
==
u2
)
{
return
0
;
}
else
{
...
...
@@ -40,25 +40,26 @@ typedef struct SIdxColInfo {
static
pthread_once_t
isInit
=
PTHREAD_ONCE_INIT
;
static
void
indexInit
();
static
int
indexTermSearch
(
SIndex
*
sIdx
,
SIndexTermQuery
*
term
,
SArray
**
result
);
static
int
indexMergeCacheIntoTindex
(
SIndex
*
sIdx
);
static
int
indexTermSearch
(
SIndex
*
sIdx
,
SIndexTermQuery
*
term
,
SArray
**
result
);
static
int
indexMergeCacheIntoTindex
(
SIndex
*
sIdx
);
static
void
indexInterResultsDestroy
(
SArray
*
results
);
static
int
indexMergeFinalResults
(
SArray
*
interResults
,
EIndexOperatorType
oType
,
SArray
*
finalResult
);
static
void
indexInterResultsDestroy
(
SArray
*
results
);
static
int
indexMergeFinalResults
(
SArray
*
interResults
,
EIndexOperatorType
oType
,
SArray
*
finalResult
);
int
indexOpen
(
SIndexOpts
*
opts
,
const
char
*
path
,
SIndex
**
index
)
{
int
indexOpen
(
SIndexOpts
*
opts
,
const
char
*
path
,
SIndex
**
index
)
{
pthread_once
(
&
isInit
,
indexInit
);
SIndex
*
sIdx
=
calloc
(
1
,
sizeof
(
SIndex
));
SIndex
*
sIdx
=
calloc
(
1
,
sizeof
(
SIndex
));
if
(
sIdx
==
NULL
)
{
return
-
1
;
}
#ifdef USE_LUCENE
index_t
*
index
=
index_open
(
path
);
index_t
*
index
=
index_open
(
path
);
sIdx
->
index
=
index
;
#endif
sIdx
->
cache
=
(
void
*
)
indexCacheCreate
();
#ifdef USE_INVERTED_INDEX
sIdx
->
cache
=
(
void
*
)
indexCacheCreate
();
sIdx
->
tindex
=
NULL
;
sIdx
->
colObj
=
taosHashInit
(
8
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BINARY
),
true
,
HASH_ENTRY_LOCK
);
sIdx
->
colId
=
1
;
...
...
@@ -67,9 +68,13 @@ int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) {
*
index
=
sIdx
;
return
0
;
#endif
*
index
=
NULL
;
return
-
1
;
}
void
indexClose
(
SIndex
*
sIdx
)
{
void
indexClose
(
SIndex
*
sIdx
)
{
#ifdef USE_LUCENE
index_close
(
sIdex
->
index
);
sIdx
->
index
=
NULL
;
...
...
@@ -84,16 +89,16 @@ void indexClose(SIndex *sIdx) {
return
;
}
int
indexPut
(
SIndex
*
index
,
SIndexMultiTerm
*
fVals
,
uint64_t
uid
)
{
int
indexPut
(
SIndex
*
index
,
SIndexMultiTerm
*
fVals
,
uint64_t
uid
)
{
#ifdef USE_LUCENE
index_document_t
*
doc
=
index_document_create
();
index_document_t
*
doc
=
index_document_create
();
char
buf
[
16
]
=
{
0
};
sprintf
(
buf
,
"%d"
,
uid
);
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
fVals
);
i
++
)
{
SIndexTerm
*
p
=
taosArrayGetP
(
fVals
,
i
);
index_document_add
(
doc
,
(
const
char
*
)(
p
->
key
),
p
->
nKey
,
(
const
char
*
)(
p
->
val
),
p
->
nVal
,
1
);
SIndexTerm
*
p
=
taosArrayGetP
(
fVals
,
i
);
index_document_add
(
doc
,
(
const
char
*
)(
p
->
key
),
p
->
nKey
,
(
const
char
*
)(
p
->
val
),
p
->
nVal
,
1
);
}
index_document_add
(
doc
,
NULL
,
0
,
buf
,
strlen
(
buf
),
0
);
...
...
@@ -106,8 +111,8 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) {
// TODO(yihao): reduce the lock range
pthread_mutex_lock
(
&
index
->
mtx
);
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
fVals
);
i
++
)
{
SIndexTerm
*
p
=
taosArrayGetP
(
fVals
,
i
);
SIdxColInfo
*
fi
=
taosHashGet
(
index
->
colObj
,
p
->
colName
,
p
->
nColName
);
SIndexTerm
*
p
=
taosArrayGetP
(
fVals
,
i
);
SIdxColInfo
*
fi
=
taosHashGet
(
index
->
colObj
,
p
->
colName
,
p
->
nColName
);
if
(
fi
==
NULL
)
{
SIdxColInfo
tfi
=
{.
colId
=
index
->
colId
};
index
->
cVersion
++
;
...
...
@@ -120,8 +125,8 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) {
pthread_mutex_unlock
(
&
index
->
mtx
);
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
fVals
);
i
++
)
{
SIndexTerm
*
p
=
taosArrayGetP
(
fVals
,
i
);
SIdxColInfo
*
fi
=
taosHashGet
(
index
->
colObj
,
p
->
colName
,
p
->
nColName
);
SIndexTerm
*
p
=
taosArrayGetP
(
fVals
,
i
);
SIdxColInfo
*
fi
=
taosHashGet
(
index
->
colObj
,
p
->
colName
,
p
->
nColName
);
assert
(
fi
!=
NULL
);
int32_t
colId
=
fi
->
colId
;
int32_t
version
=
index
->
cVersion
;
...
...
@@ -134,18 +139,18 @@ int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) {
return
0
;
}
int
indexSearch
(
SIndex
*
index
,
SIndexMultiTermQuery
*
multiQuerys
,
SArray
*
result
)
{
int
indexSearch
(
SIndex
*
index
,
SIndexMultiTermQuery
*
multiQuerys
,
SArray
*
result
)
{
#ifdef USE_LUCENE
EIndexOperatorType
opera
=
multiQuerys
->
opera
;
int
nQuery
=
taosArrayGetSize
(
multiQuerys
->
query
);
char
**
fields
=
malloc
(
sizeof
(
char
*
)
*
nQuery
);
char
**
keys
=
malloc
(
sizeof
(
char
*
)
*
nQuery
);
int
*
types
=
malloc
(
sizeof
(
int
)
*
nQuery
);
char
**
fields
=
malloc
(
sizeof
(
char
*
)
*
nQuery
);
char
**
keys
=
malloc
(
sizeof
(
char
*
)
*
nQuery
);
int
*
types
=
malloc
(
sizeof
(
int
)
*
nQuery
);
for
(
int
i
=
0
;
i
<
nQuery
;
i
++
)
{
SIndexTermQuery
*
p
=
taosArrayGet
(
multiQuerys
->
query
,
i
);
SIndexTerm
*
term
=
p
->
field_value
;
SIndexTermQuery
*
p
=
taosArrayGet
(
multiQuerys
->
query
,
i
);
SIndexTerm
*
term
=
p
->
field_value
;
fields
[
i
]
=
calloc
(
1
,
term
->
nKey
+
1
);
keys
[
i
]
=
calloc
(
1
,
term
->
nVal
+
1
);
...
...
@@ -154,9 +159,9 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
memcpy
(
keys
[
i
],
term
->
val
,
term
->
nVal
);
types
[
i
]
=
(
int
)(
p
->
type
);
}
int
*
tResult
=
NULL
;
int
*
tResult
=
NULL
;
int
tsz
=
0
;
index_multi_search
(
index
->
index
,
(
const
char
**
)
fields
,
(
const
char
**
)
keys
,
types
,
nQuery
,
opera
,
&
tResult
,
&
tsz
);
index_multi_search
(
index
->
index
,
(
const
char
**
)
fields
,
(
const
char
**
)
keys
,
types
,
nQuery
,
opera
,
&
tResult
,
&
tsz
);
for
(
int
i
=
0
;
i
<
tsz
;
i
++
)
{
taosArrayPush
(
result
,
&
tResult
[
i
]);
...
...
@@ -174,13 +179,13 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
#ifdef USE_INVERTED_INDEX
EIndexOperatorType
opera
=
multiQuerys
->
opera
;
// relation of querys
SArray
*
interResults
=
taosArrayInit
(
4
,
POINTER_BYTES
);
SArray
*
interResults
=
taosArrayInit
(
4
,
POINTER_BYTES
);
int
nQuery
=
taosArrayGetSize
(
multiQuerys
->
query
);
for
(
size_t
i
=
0
;
i
<
nQuery
;
i
++
)
{
SIndexTermQuery
*
qTerm
=
taosArrayGet
(
multiQuerys
->
query
,
i
);
SArray
*
tResult
=
NULL
;
SIndexTermQuery
*
qTerm
=
taosArrayGet
(
multiQuerys
->
query
,
i
);
SArray
*
tResult
=
NULL
;
indexTermSearch
(
index
,
qTerm
,
&
tResult
);
taosArrayPush
(
interResults
,
(
void
*
)
&
tResult
);
taosArrayPush
(
interResults
,
(
void
*
)
&
tResult
);
}
indexMergeFinalResults
(
interResults
,
opera
,
result
);
indexInterResultsDestroy
(
interResults
);
...
...
@@ -189,24 +194,24 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
return
1
;
}
int
indexDelete
(
SIndex
*
index
,
SIndexMultiTermQuery
*
query
)
{
int
indexDelete
(
SIndex
*
index
,
SIndexMultiTermQuery
*
query
)
{
#ifdef USE_INVERTED_INDEX
#endif
return
1
;
}
int
indexRebuild
(
SIndex
*
index
,
SIndexOpts
*
opts
){
int
indexRebuild
(
SIndex
*
index
,
SIndexOpts
*
opts
){
#ifdef USE_INVERTED_INDEX
#endif
}
SIndexOpts
*
indexOptsCreate
()
{
SIndexOpts
*
indexOptsCreate
()
{
#ifdef USE_LUCENE
#endif
return
NULL
;
}
void
indexOptsDestroy
(
SIndexOpts
*
opts
){
void
indexOptsDestroy
(
SIndexOpts
*
opts
){
#ifdef USE_LUCENE
#endif
}
/*
...
...
@@ -214,8 +219,8 @@ void indexOptsDestroy(SIndexOpts *opts){
*
*/
SIndexMultiTermQuery
*
indexMultiTermQueryCreate
(
EIndexOperatorType
opera
)
{
SIndexMultiTermQuery
*
p
=
(
SIndexMultiTermQuery
*
)
malloc
(
sizeof
(
SIndexMultiTermQuery
));
SIndexMultiTermQuery
*
indexMultiTermQueryCreate
(
EIndexOperatorType
opera
)
{
SIndexMultiTermQuery
*
p
=
(
SIndexMultiTermQuery
*
)
malloc
(
sizeof
(
SIndexMultiTermQuery
));
if
(
p
==
NULL
)
{
return
NULL
;
}
...
...
@@ -223,23 +228,28 @@ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) {
p
->
query
=
taosArrayInit
(
4
,
sizeof
(
SIndexTermQuery
));
return
p
;
}
void
indexMultiTermQueryDestroy
(
SIndexMultiTermQuery
*
pQuery
)
{
void
indexMultiTermQueryDestroy
(
SIndexMultiTermQuery
*
pQuery
)
{
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
pQuery
->
query
);
i
++
)
{
SIndexTermQuery
*
p
=
(
SIndexTermQuery
*
)
taosArrayGet
(
pQuery
->
query
,
i
);
SIndexTermQuery
*
p
=
(
SIndexTermQuery
*
)
taosArrayGet
(
pQuery
->
query
,
i
);
indexTermDestroy
(
p
->
term
);
}
taosArrayDestroy
(
pQuery
->
query
);
free
(
pQuery
);
};
int
indexMultiTermQueryAdd
(
SIndexMultiTermQuery
*
pQuery
,
SIndexTerm
*
term
,
EIndexQueryType
qType
)
{
int
indexMultiTermQueryAdd
(
SIndexMultiTermQuery
*
pQuery
,
SIndexTerm
*
term
,
EIndexQueryType
qType
)
{
SIndexTermQuery
q
=
{.
qType
=
qType
,
.
term
=
term
};
taosArrayPush
(
pQuery
->
query
,
&
q
);
return
0
;
}
SIndexTerm
*
indexTermCreate
(
int64_t
suid
,
SIndexOperOnColumn
oper
,
uint8_t
colType
,
const
char
*
colName
,
int32_t
nColName
,
const
char
*
colVal
,
int32_t
nColVal
)
{
SIndexTerm
*
t
=
(
SIndexTerm
*
)
calloc
(
1
,
(
sizeof
(
SIndexTerm
)));
SIndexTerm
*
indexTermCreate
(
int64_t
suid
,
SIndexOperOnColumn
oper
,
uint8_t
colType
,
const
char
*
colName
,
int32_t
nColName
,
const
char
*
colVal
,
int32_t
nColVal
)
{
SIndexTerm
*
t
=
(
SIndexTerm
*
)
calloc
(
1
,
(
sizeof
(
SIndexTerm
)));
if
(
t
==
NULL
)
{
return
NULL
;
}
...
...
@@ -248,30 +258,32 @@ SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy
t
->
operType
=
oper
;
t
->
colType
=
colType
;
t
->
colName
=
(
char
*
)
calloc
(
1
,
nColName
+
1
);
t
->
colName
=
(
char
*
)
calloc
(
1
,
nColName
+
1
);
memcpy
(
t
->
colName
,
colName
,
nColName
);
t
->
nColName
=
nColName
;
t
->
colVal
=
(
char
*
)
calloc
(
1
,
nColVal
+
1
);
t
->
colVal
=
(
char
*
)
calloc
(
1
,
nColVal
+
1
);
memcpy
(
t
->
colVal
,
colVal
,
nColVal
);
t
->
nColVal
=
nColVal
;
return
t
;
}
void
indexTermDestroy
(
SIndexTerm
*
p
)
{
void
indexTermDestroy
(
SIndexTerm
*
p
)
{
free
(
p
->
colName
);
free
(
p
->
colVal
);
free
(
p
);
}
SIndexMultiTerm
*
indexMultiTermCreate
()
{
return
taosArrayInit
(
4
,
sizeof
(
SIndexTerm
*
));
}
SIndexMultiTerm
*
indexMultiTermCreate
()
{
return
taosArrayInit
(
4
,
sizeof
(
SIndexTerm
*
));
}
int
indexMultiTermAdd
(
SIndexMultiTerm
*
terms
,
SIndexTerm
*
term
)
{
int
indexMultiTermAdd
(
SIndexMultiTerm
*
terms
,
SIndexTerm
*
term
)
{
taosArrayPush
(
terms
,
&
term
);
return
0
;
}
void
indexMultiTermDestroy
(
SIndexMultiTerm
*
terms
)
{
void
indexMultiTermDestroy
(
SIndexMultiTerm
*
terms
)
{
for
(
int32_t
i
=
0
;
i
<
taosArrayGetSize
(
terms
);
i
++
)
{
SIndexTerm
*
p
=
taosArrayGetP
(
terms
,
i
);
SIndexTerm
*
p
=
taosArrayGetP
(
terms
,
i
);
indexTermDestroy
(
p
);
}
taosArrayDestroy
(
terms
);
...
...
@@ -280,13 +292,13 @@ void indexMultiTermDestroy(SIndexMultiTerm *terms) {
void
indexInit
()
{
// do nothing
}
static
int
indexTermSearch
(
SIndex
*
sIdx
,
SIndexTermQuery
*
query
,
SArray
**
result
)
{
static
int
indexTermSearch
(
SIndex
*
sIdx
,
SIndexTermQuery
*
query
,
SArray
**
result
)
{
int32_t
version
=
-
1
;
int16_t
colId
=
-
1
;
SIdxColInfo
*
colInfo
=
NULL
;
SIdxColInfo
*
colInfo
=
NULL
;
SIndexTerm
*
term
=
query
->
term
;
const
char
*
colName
=
term
->
colName
;
SIndexTerm
*
term
=
query
->
term
;
const
char
*
colName
=
term
->
colName
;
int32_t
nColName
=
term
->
nColName
;
pthread_mutex_lock
(
&
sIdx
->
mtx
);
...
...
@@ -319,23 +331,24 @@ static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result
}
return
0
;
}
static
void
indexInterResultsDestroy
(
SArray
*
results
)
{
static
void
indexInterResultsDestroy
(
SArray
*
results
)
{
if
(
results
==
NULL
)
{
return
;
}
size_t
sz
=
taosArrayGetSize
(
results
);
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
SArray
*
p
=
taosArrayGetP
(
results
,
i
);
SArray
*
p
=
taosArrayGetP
(
results
,
i
);
taosArrayDestroy
(
p
);
}
taosArrayDestroy
(
results
);
}
static
int
indexMergeFinalResults
(
SArray
*
interResults
,
EIndexOperatorType
oType
,
SArray
*
fResults
)
{
static
int
indexMergeFinalResults
(
SArray
*
interResults
,
EIndexOperatorType
oType
,
SArray
*
fResults
)
{
// refactor, merge interResults into fResults by oType
SArray
*
first
=
taosArrayGetP
(
interResults
,
0
);
SArray
*
first
=
taosArrayGetP
(
interResults
,
0
);
taosArraySort
(
first
,
uidCompare
);
taosArrayRemoveDuplicate
(
first
,
uidCompare
,
NULL
);
if
(
oType
==
MUST
)
{
// just one column index, enhance later
taosArrayAddAll
(
fResults
,
first
);
...
...
@@ -350,7 +363,7 @@ static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType
}
return
0
;
}
static
int
indexMergeCacheIntoTindex
(
SIndex
*
sIdx
)
{
static
int
indexMergeCacheIntoTindex
(
SIndex
*
sIdx
)
{
if
(
sIdx
==
NULL
)
{
return
-
1
;
}
...
...
source/libs/index/src/index_cache.c
浏览文件 @
d96963a2
...
...
@@ -20,14 +20,15 @@
#define MAX_INDEX_KEY_LEN 256 // test only, change later
// ref index_cache.h:22
#define CACHE_KEY_LEN(p) \
(sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + \
sizeof(p->operType))
#define CACHE_KEY_LEN(p) \
(sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType))
static
char
*
getIndexKey
(
const
void
*
pData
)
{
return
NULL
;
}
static
int32_t
compareKey
(
const
void
*
l
,
const
void
*
r
)
{
char
*
lp
=
(
char
*
)
l
;
char
*
rp
=
(
char
*
)
r
;
static
char
*
getIndexKey
(
const
void
*
pData
)
{
return
NULL
;
}
static
int32_t
compareKey
(
const
void
*
l
,
const
void
*
r
)
{
char
*
lp
=
(
char
*
)
l
;
char
*
rp
=
(
char
*
)
r
;
// skip total len, not compare
int32_t
ll
,
rl
;
// len
...
...
@@ -40,9 +41,7 @@ static int32_t compareKey(const void *l, const void *r) {
int16_t
lf
,
rf
;
// field id
memcpy
(
&
lf
,
lp
,
sizeof
(
lf
));
memcpy
(
&
rf
,
rp
,
sizeof
(
rf
));
if
(
lf
!=
rf
)
{
return
lf
<
rf
?
-
1
:
1
;
}
if
(
lf
!=
rf
)
{
return
lf
<
rf
?
-
1
:
1
;
}
lp
+=
sizeof
(
lf
);
rp
+=
sizeof
(
rf
);
...
...
@@ -89,41 +88,41 @@ static int32_t compareKey(const void *l, const void *r) {
int32_t
lv
,
rv
;
memcpy
(
&
lv
,
lp
,
sizeof
(
lv
));
memcpy
(
&
rv
,
rp
,
sizeof
(
rv
));
if
(
lv
!=
rv
)
{
return
lv
>
rv
?
-
1
:
1
;
}
if
(
lv
!=
rv
)
{
return
lv
>
rv
?
-
1
:
1
;
}
lp
+=
sizeof
(
lv
);
rp
+=
sizeof
(
rv
);
// not care item type
return
0
;
}
IndexCache
*
indexCacheCreate
()
{
IndexCache
*
cache
=
calloc
(
1
,
sizeof
(
IndexCache
));
cache
->
skiplist
=
tSkipListCreate
(
MAX_SKIP_LIST_LEVEL
,
TSDB_DATA_TYPE_BINARY
,
MAX_INDEX_KEY_LEN
,
compareKey
,
SL_ALLOW_DUP_KEY
,
getIndexKey
);
IndexCache
*
indexCacheCreate
()
{
IndexCache
*
cache
=
calloc
(
1
,
sizeof
(
IndexCache
));
if
(
cache
==
NULL
)
{
indexError
(
"failed to create index cache"
);
return
NULL
;
}
cache
->
skiplist
=
tSkipListCreate
(
MAX_SKIP_LIST_LEVEL
,
TSDB_DATA_TYPE_BINARY
,
MAX_INDEX_KEY_LEN
,
compareKey
,
SL_ALLOW_DUP_KEY
,
getIndexKey
);
return
cache
;
}
void
indexCacheDestroy
(
void
*
cache
)
{
IndexCache
*
pCache
=
cache
;
if
(
pCache
==
NULL
)
{
return
;
}
void
indexCacheDestroy
(
void
*
cache
)
{
IndexCache
*
pCache
=
cache
;
if
(
pCache
==
NULL
)
{
return
;
}
tSkipListDestroy
(
pCache
->
skiplist
);
free
(
pCache
);
}
int
indexCachePut
(
void
*
cache
,
SIndexTerm
*
term
,
int16_t
colId
,
int32_t
version
,
uint64_t
uid
)
{
if
(
cache
==
NULL
)
{
return
-
1
;
}
int
indexCachePut
(
void
*
cache
,
SIndexTerm
*
term
,
int16_t
colId
,
int32_t
version
,
uint64_t
uid
)
{
if
(
cache
==
NULL
)
{
return
-
1
;
}
IndexCache
*
pCache
=
cache
;
IndexCache
*
pCache
=
cache
;
// encode data
int32_t
total
=
CACHE_KEY_LEN
(
term
);
char
*
buf
=
calloc
(
1
,
total
);
char
*
p
=
buf
;
char
*
buf
=
calloc
(
1
,
total
);
char
*
p
=
buf
;
SERIALIZE_VAR_TO_BUF
(
p
,
total
,
int32_t
);
SERIALIZE_VAR_TO_BUF
(
p
,
colId
,
int16_t
);
...
...
@@ -137,30 +136,31 @@ int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version,
SERIALIZE_MEM_TO_BUF
(
p
,
term
,
operType
);
tSkipListPut
(
pCache
->
skiplist
,
(
void
*
)
buf
);
tSkipListPut
(
pCache
->
skiplist
,
(
void
*
)
buf
);
return
0
;
// encode end
}
int
indexCacheDel
(
void
*
cache
,
int32_t
fieldId
,
const
char
*
fieldValue
,
int32_t
fvlen
,
uint64_t
uid
,
int8_t
operType
)
{
IndexCache
*
pCache
=
cache
;
int
indexCacheDel
(
void
*
cache
,
int32_t
fieldId
,
const
char
*
fieldValue
,
int32_t
fvlen
,
uint64_t
uid
,
int8_t
operType
)
{
IndexCache
*
pCache
=
cache
;
return
0
;
}
int
indexCacheSearch
(
void
*
cache
,
SIndexTermQuery
*
query
,
int16_t
colId
,
int32_t
version
,
SArray
*
result
,
STermValueType
*
s
)
{
if
(
cache
==
NULL
)
{
return
-
1
;
}
IndexCache
*
pCache
=
cache
;
SIndexTerm
*
term
=
query
->
term
;
int
indexCacheSearch
(
void
*
cache
,
SIndexTermQuery
*
query
,
int16_t
colId
,
int32_t
version
,
SArray
*
result
,
STermValueType
*
s
)
{
if
(
cache
==
NULL
)
{
return
-
1
;
}
IndexCache
*
pCache
=
cache
;
SIndexTerm
*
term
=
query
->
term
;
EIndexQueryType
qtype
=
query
->
qType
;
int32_t
keyLen
=
CACHE_KEY_LEN
(
term
);
char
*
buf
=
calloc
(
1
,
keyLen
);
char
*
buf
=
calloc
(
1
,
keyLen
);
if
(
qtype
==
QUERY_TERM
)
{
//
}
else
if
(
qtype
==
QUERY_PREFIX
)
{
//
}
else
if
(
qtype
==
QUERY_SUFFIX
)
{
//
}
else
if
(
qtype
==
QUERY_REGEX
)
{
//
}
return
0
;
...
...
source/libs/index/src/index_fst.c
浏览文件 @
d96963a2
...
...
@@ -19,43 +19,39 @@
#include "tchecksum.h"
#include "tcoding.h"
static
void
fstPackDeltaIn
(
FstCountingWriter
*
wrt
,
CompiledAddr
nodeAddr
,
CompiledAddr
transAddr
,
uint8_t
nBytes
)
{
static
void
fstPackDeltaIn
(
FstCountingWriter
*
wrt
,
CompiledAddr
nodeAddr
,
CompiledAddr
transAddr
,
uint8_t
nBytes
)
{
CompiledAddr
deltaAddr
=
(
transAddr
==
EMPTY_ADDRESS
)
?
EMPTY_ADDRESS
:
nodeAddr
-
transAddr
;
fstCountingWriterPackUintIn
(
wrt
,
deltaAddr
,
nBytes
);
}
static
uint8_t
fstPackDetla
(
FstCountingWriter
*
wrt
,
CompiledAddr
nodeAddr
,
CompiledAddr
transAddr
)
{
static
uint8_t
fstPackDetla
(
FstCountingWriter
*
wrt
,
CompiledAddr
nodeAddr
,
CompiledAddr
transAddr
)
{
uint8_t
nBytes
=
packDeltaSize
(
nodeAddr
,
transAddr
);
fstPackDeltaIn
(
wrt
,
nodeAddr
,
transAddr
,
nBytes
);
return
nBytes
;
}
FstUnFinishedNodes
*
fstUnFinishedNodesCreate
()
{
FstUnFinishedNodes
*
nodes
=
malloc
(
sizeof
(
FstUnFinishedNodes
));
if
(
nodes
==
NULL
)
{
return
NULL
;
}
FstUnFinishedNodes
*
fstUnFinishedNodesCreate
()
{
FstUnFinishedNodes
*
nodes
=
malloc
(
sizeof
(
FstUnFinishedNodes
));
if
(
nodes
==
NULL
)
{
return
NULL
;
}
nodes
->
stack
=
(
SArray
*
)
taosArrayInit
(
64
,
sizeof
(
FstBuilderNodeUnfinished
));
nodes
->
stack
=
(
SArray
*
)
taosArrayInit
(
64
,
sizeof
(
FstBuilderNodeUnfinished
));
fstUnFinishedNodesPushEmpty
(
nodes
,
false
);
return
nodes
;
}
void
unFinishedNodeDestroyElem
(
void
*
elem
)
{
FstBuilderNodeUnfinished
*
b
=
(
FstBuilderNodeUnfinished
*
)
elem
;
void
unFinishedNodeDestroyElem
(
void
*
elem
)
{
FstBuilderNodeUnfinished
*
b
=
(
FstBuilderNodeUnfinished
*
)
elem
;
fstBuilderNodeDestroy
(
b
->
node
);
free
(
b
->
last
);
b
->
last
=
NULL
;
}
void
fstUnFinishedNodesDestroy
(
FstUnFinishedNodes
*
nodes
)
{
if
(
nodes
==
NULL
)
{
return
;
}
void
fstUnFinishedNodesDestroy
(
FstUnFinishedNodes
*
nodes
)
{
if
(
nodes
==
NULL
)
{
return
;
}
taosArrayDestroyEx
(
nodes
->
stack
,
unFinishedNodeDestroyElem
);
free
(
nodes
);
}
void
fstUnFinishedNodesPushEmpty
(
FstUnFinishedNodes
*
nodes
,
bool
isFinal
)
{
FstBuilderNode
*
node
=
malloc
(
sizeof
(
FstBuilderNode
));
void
fstUnFinishedNodesPushEmpty
(
FstUnFinishedNodes
*
nodes
,
bool
isFinal
)
{
FstBuilderNode
*
node
=
malloc
(
sizeof
(
FstBuilderNode
));
node
->
isFinal
=
isFinal
;
node
->
finalOutput
=
0
;
node
->
trans
=
taosArrayInit
(
16
,
sizeof
(
FstTransition
));
...
...
@@ -63,56 +59,53 @@ void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) {
FstBuilderNodeUnfinished
un
=
{.
node
=
node
,
.
last
=
NULL
};
taosArrayPush
(
nodes
->
stack
,
&
un
);
}
FstBuilderNode
*
fstUnFinishedNodesPopRoot
(
FstUnFinishedNodes
*
nodes
)
{
FstBuilderNode
*
fstUnFinishedNodesPopRoot
(
FstUnFinishedNodes
*
nodes
)
{
assert
(
taosArrayGetSize
(
nodes
->
stack
)
==
1
);
FstBuilderNodeUnfinished
*
un
=
taosArrayPop
(
nodes
->
stack
);
FstBuilderNodeUnfinished
*
un
=
taosArrayPop
(
nodes
->
stack
);
assert
(
un
->
last
==
NULL
);
return
un
->
node
;
}
FstBuilderNode
*
fstUnFinishedNodesPopFreeze
(
FstUnFinishedNodes
*
nodes
,
CompiledAddr
addr
)
{
FstBuilderNodeUnfinished
*
un
=
taosArrayPop
(
nodes
->
stack
);
FstBuilderNode
*
fstUnFinishedNodesPopFreeze
(
FstUnFinishedNodes
*
nodes
,
CompiledAddr
addr
)
{
FstBuilderNodeUnfinished
*
un
=
taosArrayPop
(
nodes
->
stack
);
fstBuilderNodeUnfinishedLastCompiled
(
un
,
addr
);
// free(un->last); // TODO add func FstLastTransitionFree()
// un->last = NULL;
return
un
->
node
;
}
FstBuilderNode
*
fstUnFinishedNodesPopEmpty
(
FstUnFinishedNodes
*
nodes
)
{
FstBuilderNodeUnfinished
*
un
=
taosArrayPop
(
nodes
->
stack
);
FstBuilderNode
*
fstUnFinishedNodesPopEmpty
(
FstUnFinishedNodes
*
nodes
)
{
FstBuilderNodeUnfinished
*
un
=
taosArrayPop
(
nodes
->
stack
);
assert
(
un
->
last
==
NULL
);
return
un
->
node
;
}
void
fstUnFinishedNodesSetRootOutput
(
FstUnFinishedNodes
*
nodes
,
Output
out
)
{
FstBuilderNodeUnfinished
*
un
=
taosArrayGet
(
nodes
->
stack
,
0
);
void
fstUnFinishedNodesSetRootOutput
(
FstUnFinishedNodes
*
nodes
,
Output
out
)
{
FstBuilderNodeUnfinished
*
un
=
taosArrayGet
(
nodes
->
stack
,
0
);
un
->
node
->
isFinal
=
true
;
un
->
node
->
finalOutput
=
out
;
// un->node->trans = NULL;
}
void
fstUnFinishedNodesTopLastFreeze
(
FstUnFinishedNodes
*
nodes
,
CompiledAddr
addr
)
{
size_t
sz
=
taosArrayGetSize
(
nodes
->
stack
)
-
1
;
FstBuilderNodeUnfinished
*
un
=
taosArrayGet
(
nodes
->
stack
,
sz
);
void
fstUnFinishedNodesTopLastFreeze
(
FstUnFinishedNodes
*
nodes
,
CompiledAddr
addr
)
{
FstBuilderNodeUnfinished
*
un
=
taosArrayGet
(
nodes
->
stack
,
taosArrayGetSize
(
nodes
->
stack
)
-
1
);
fstBuilderNodeUnfinishedLastCompiled
(
un
,
addr
);
}
void
fstUnFinishedNodesAddSuffix
(
FstUnFinishedNodes
*
nodes
,
FstSlice
bs
,
Output
out
)
{
FstSlice
*
s
=
&
bs
;
if
(
fstSliceIsEmpty
(
s
))
{
return
;
}
void
fstUnFinishedNodesAddSuffix
(
FstUnFinishedNodes
*
nodes
,
FstSlice
bs
,
Output
out
)
{
FstSlice
*
s
=
&
bs
;
if
(
fstSliceIsEmpty
(
s
))
{
return
;
}
size_t
sz
=
taosArrayGetSize
(
nodes
->
stack
)
-
1
;
FstBuilderNodeUnfinished
*
un
=
taosArrayGet
(
nodes
->
stack
,
sz
);
FstBuilderNodeUnfinished
*
un
=
taosArrayGet
(
nodes
->
stack
,
sz
);
assert
(
un
->
last
==
NULL
);
// FstLastTransition *trn = malloc(sizeof(FstLastTransition));
// trn->inp = s->data[s->start];
// trn->out = out;
int32_t
len
=
0
;
uint8_t
*
data
=
fstSliceData
(
s
,
&
len
);
uint8_t
*
data
=
fstSliceData
(
s
,
&
len
);
un
->
last
=
fstLastTransitionCreate
(
data
[
0
],
out
);
for
(
uint64_t
i
=
1
;
i
<
len
;
i
++
)
{
FstBuilderNode
*
n
=
malloc
(
sizeof
(
FstBuilderNode
));
FstBuilderNode
*
n
=
malloc
(
sizeof
(
FstBuilderNode
));
n
->
isFinal
=
false
;
n
->
finalOutput
=
0
;
n
->
trans
=
taosArrayInit
(
16
,
sizeof
(
FstTransition
));
...
...
@@ -120,7 +113,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output
// FstLastTransition *trn = malloc(sizeof(FstLastTransition));
// trn->inp = s->data[i];
// trn->out = out;
FstLastTransition
*
trn
=
fstLastTransitionCreate
(
data
[
i
],
0
);
FstLastTransition
*
trn
=
fstLastTransitionCreate
(
data
[
i
],
0
);
FstBuilderNodeUnfinished
un
=
{.
node
=
n
,
.
last
=
trn
};
taosArrayPush
(
nodes
->
stack
,
&
un
);
...
...
@@ -128,15 +121,15 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output
fstUnFinishedNodesPushEmpty
(
nodes
,
true
);
}
uint64_t
fstUnFinishedNodesFindCommPrefix
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
)
{
FstSlice
*
s
=
&
bs
;
uint64_t
fstUnFinishedNodesFindCommPrefix
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
)
{
FstSlice
*
s
=
&
bs
;
size_t
ssz
=
taosArrayGetSize
(
node
->
stack
);
// stack size
uint64_t
count
=
0
;
int32_t
lsz
;
// data len
uint8_t
*
data
=
fstSliceData
(
s
,
&
lsz
);
uint8_t
*
data
=
fstSliceData
(
s
,
&
lsz
);
for
(
size_t
i
=
0
;
i
<
ssz
&&
i
<
lsz
;
i
++
)
{
FstBuilderNodeUnfinished
*
un
=
taosArrayGet
(
node
->
stack
,
i
);
FstBuilderNodeUnfinished
*
un
=
taosArrayGet
(
node
->
stack
,
i
);
if
(
un
->
last
->
inp
==
data
[
i
])
{
count
++
;
}
else
{
...
...
@@ -145,19 +138,19 @@ uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs)
}
return
count
;
}
uint64_t
fstUnFinishedNodesFindCommPrefixAndSetOutput
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
,
Output
in
,
Output
*
out
)
{
FstSlice
*
s
=
&
bs
;
uint64_t
fstUnFinishedNodesFindCommPrefixAndSetOutput
(
FstUnFinishedNodes
*
node
,
FstSlice
bs
,
Output
in
,
Output
*
out
)
{
FstSlice
*
s
=
&
bs
;
size_t
lsz
=
(
size_t
)(
s
->
end
-
s
->
start
+
1
);
// data len
size_t
ssz
=
taosArrayGetSize
(
node
->
stack
);
// stack size
*
out
=
in
;
uint64_t
i
=
0
;
for
(
i
=
0
;
i
<
lsz
&&
i
<
ssz
;
i
++
)
{
FstBuilderNodeUnfinished
*
un
=
taosArrayGet
(
node
->
stack
,
i
);
FstBuilderNodeUnfinished
*
un
=
taosArrayGet
(
node
->
stack
,
i
);
FstLastTransition
*
t
=
un
->
last
;
FstLastTransition
*
t
=
un
->
last
;
uint64_t
addPrefix
=
0
;
uint8_t
*
data
=
fstSliceData
(
s
,
NULL
);
uint8_t
*
data
=
fstSliceData
(
s
,
NULL
);
if
(
t
&&
t
->
inp
==
data
[
i
])
{
uint64_t
commPrefix
=
MIN
(
t
->
out
,
*
out
);
uint64_t
tAddPrefix
=
t
->
out
-
commPrefix
;
...
...
@@ -169,7 +162,7 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node,
}
if
(
addPrefix
!=
0
)
{
if
(
i
+
1
<
ssz
)
{
FstBuilderNodeUnfinished
*
unf
=
taosArrayGet
(
node
->
stack
,
i
+
1
);
FstBuilderNodeUnfinished
*
unf
=
taosArrayGet
(
node
->
stack
,
i
+
1
);
fstBuilderNodeUnfinishedAddOutputPrefix
(
unf
,
addPrefix
);
}
}
...
...
@@ -177,13 +170,11 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node,
return
i
;
}
FstState
fstStateCreateFrom
(
FstSlice
*
slice
,
CompiledAddr
addr
)
{
FstState
fstStateCreateFrom
(
FstSlice
*
slice
,
CompiledAddr
addr
)
{
FstState
fs
=
{.
state
=
EmptyFinal
,
.
val
=
0
};
if
(
addr
==
EMPTY_ADDRESS
)
{
return
fs
;
}
if
(
addr
==
EMPTY_ADDRESS
)
{
return
fs
;
}
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
uint8_t
v
=
data
[
addr
];
uint8_t
t
=
(
v
&
0
b11000000
)
>>
6
;
if
(
t
==
0
b11
)
{
...
...
@@ -197,17 +188,19 @@ FstState fstStateCreateFrom(FstSlice *slice, CompiledAddr addr) {
return
fs
;
}
static
FstState
fstStateDict
[]
=
{{.
state
=
OneTransNext
,
.
val
=
0
b11000000
},
{.
state
=
OneTrans
,
.
val
=
0
b10000000
},
{.
state
=
AnyTrans
,
.
val
=
0
b00000000
},
{.
state
=
EmptyFinal
,
.
val
=
0
b00000000
}};
static
FstState
fstStateDict
[]
=
{{.
state
=
OneTransNext
,
.
val
=
0
b11000000
},
{.
state
=
OneTrans
,
.
val
=
0
b10000000
},
{.
state
=
AnyTrans
,
.
val
=
0
b00000000
},
{.
state
=
EmptyFinal
,
.
val
=
0
b00000000
}};
// debug
static
const
char
*
fstStateStr
[]
=
{
"ONE_TRANS_NEXT"
,
"ONE_TRANS"
,
"ANY_TRANS"
,
"EMPTY_FINAL"
};
static
const
char
*
fstStateStr
[]
=
{
"ONE_TRANS_NEXT"
,
"ONE_TRANS"
,
"ANY_TRANS"
,
"EMPTY_FINAL"
};
FstState
fstStateCreate
(
State
state
)
{
uint8_t
idx
=
(
uint8_t
)
state
;
return
fstStateDict
[
idx
];
}
// compile
void
fstStateCompileForOneTransNext
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
uint8_t
inp
)
{
void
fstStateCompileForOneTransNext
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
uint8_t
inp
)
{
FstState
s
=
fstStateCreate
(
OneTransNext
);
fstStateSetCommInput
(
&
s
,
inp
);
...
...
@@ -221,7 +214,7 @@ void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uin
// w->write_all(&[s.val])
return
;
}
void
fstStateCompileForOneTrans
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
FstTransition
*
trn
)
{
void
fstStateCompileForOneTrans
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
FstTransition
*
trn
)
{
Output
out
=
trn
->
out
;
uint8_t
outPackSize
=
(
out
==
0
?
0
:
fstCountingWriterPackUint
(
w
,
out
));
uint8_t
transPackSize
=
fstPackDetla
(
w
,
addr
,
trn
->
addr
);
...
...
@@ -229,20 +222,18 @@ void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTran
FST_SET_OUTPUT_PACK_SIZE
(
packSizes
,
outPackSize
);
FST_SET_TRANSITION_PACK_SIZE
(
packSizes
,
transPackSize
);
fstCountingWriterWrite
(
w
,
(
char
*
)
&
packSizes
,
sizeof
(
packSizes
));
fstCountingWriterWrite
(
w
,
(
char
*
)
&
packSizes
,
sizeof
(
packSizes
));
FstState
st
=
fstStateCreate
(
OneTrans
);
fstStateSetCommInput
(
&
st
,
trn
->
inp
);
bool
null
=
false
;
uint8_t
inp
=
fstStateCommInput
(
&
st
,
&
null
);
if
(
null
==
true
)
{
fstCountingWriterWrite
(
w
,
(
char
*
)
&
trn
->
inp
,
sizeof
(
trn
->
inp
));
}
fstCountingWriterWrite
(
w
,
(
char
*
)(
&
(
st
.
val
)),
sizeof
(
st
.
val
));
if
(
null
==
true
)
{
fstCountingWriterWrite
(
w
,
(
char
*
)
&
trn
->
inp
,
sizeof
(
trn
->
inp
));
}
fstCountingWriterWrite
(
w
,
(
char
*
)(
&
(
st
.
val
)),
sizeof
(
st
.
val
));
return
;
}
void
fstStateCompileForAnyTrans
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
FstBuilderNode
*
node
)
{
void
fstStateCompileForAnyTrans
(
FstCountingWriter
*
w
,
CompiledAddr
addr
,
FstBuilderNode
*
node
)
{
size_t
sz
=
taosArrayGetSize
(
node
->
trans
);
assert
(
sz
<=
256
);
...
...
@@ -252,7 +243,7 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil
// finalOutput.is_zero()
bool
anyOuts
=
(
node
->
finalOutput
!=
0
);
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
FstTransition
*
t
=
taosArrayGet
(
node
->
trans
,
i
);
FstTransition
*
t
=
taosArrayGet
(
node
->
trans
,
i
);
tSize
=
MAX
(
tSize
,
packDeltaSize
(
addr
,
t
->
addr
));
oSize
=
MAX
(
oSize
,
packSize
(
t
->
out
));
anyOuts
=
anyOuts
||
(
t
->
out
!=
0
);
...
...
@@ -272,21 +263,19 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil
fstStateSetStateNtrans
(
&
st
,
(
uint8_t
)
sz
);
if
(
anyOuts
)
{
if
(
FST_BUILDER_NODE_IS_FINAL
(
node
))
{
fstCountingWriterPackUintIn
(
w
,
node
->
finalOutput
,
oSize
);
}
if
(
FST_BUILDER_NODE_IS_FINAL
(
node
))
{
fstCountingWriterPackUintIn
(
w
,
node
->
finalOutput
,
oSize
);
}
for
(
int32_t
i
=
sz
-
1
;
i
>=
0
;
i
--
)
{
FstTransition
*
t
=
taosArrayGet
(
node
->
trans
,
i
);
FstTransition
*
t
=
taosArrayGet
(
node
->
trans
,
i
);
fstCountingWriterPackUintIn
(
w
,
t
->
out
,
oSize
);
}
}
for
(
int32_t
i
=
sz
-
1
;
i
>=
0
;
i
--
)
{
FstTransition
*
t
=
taosArrayGet
(
node
->
trans
,
i
);
FstTransition
*
t
=
taosArrayGet
(
node
->
trans
,
i
);
fstPackDeltaIn
(
w
,
addr
,
t
->
addr
,
tSize
);
}
for
(
int32_t
i
=
sz
-
1
;
i
>=
0
;
i
--
)
{
FstTransition
*
t
=
taosArrayGet
(
node
->
trans
,
i
);
fstCountingWriterWrite
(
w
,
(
char
*
)
&
t
->
inp
,
1
);
FstTransition
*
t
=
taosArrayGet
(
node
->
trans
,
i
);
fstCountingWriterWrite
(
w
,
(
char
*
)
&
t
->
inp
,
1
);
// fstPackDeltaIn(w, addr, t->addr, tSize);
}
if
(
sz
>
TRANS_INDEX_THRESHOLD
)
{
...
...
@@ -294,20 +283,20 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil
// at that index. (Except when there are 256 transitions.) Namely,
// any value greater than or equal to the number of transitions in
// this node indicates an absent transition.
uint8_t
*
index
=
(
uint8_t
*
)
malloc
(
sizeof
(
uint8_t
)
*
256
);
uint8_t
*
index
=
(
uint8_t
*
)
malloc
(
sizeof
(
uint8_t
)
*
256
);
memset
(
index
,
255
,
sizeof
(
uint8_t
)
*
256
);
/// for (uint8_t i = 0; i < 256; i++) {
// index[i] = 255;
///}
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
FstTransition
*
t
=
taosArrayGet
(
node
->
trans
,
i
);
FstTransition
*
t
=
taosArrayGet
(
node
->
trans
,
i
);
index
[
t
->
inp
]
=
i
;
// fstPackDeltaIn(w, addr, t->addr, tSize);
}
fstCountingWriterWrite
(
w
,
(
char
*
)
index
,
256
);
fstCountingWriterWrite
(
w
,
(
char
*
)
index
,
256
);
free
(
index
);
}
fstCountingWriterWrite
(
w
,
(
char
*
)
&
packSizes
,
1
);
fstCountingWriterWrite
(
w
,
(
char
*
)
&
packSizes
,
1
);
bool
null
=
false
;
fstStateStateNtrans
(
&
st
,
&
null
);
if
(
null
==
true
)
{
...
...
@@ -316,17 +305,17 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil
// encoded in the state byte.
uint8_t
v
=
1
;
if
(
sz
==
256
)
{
fstCountingWriterWrite
(
w
,
(
char
*
)
&
v
,
1
);
fstCountingWriterWrite
(
w
,
(
char
*
)
&
v
,
1
);
}
else
{
fstCountingWriterWrite
(
w
,
(
char
*
)
&
sz
,
1
);
fstCountingWriterWrite
(
w
,
(
char
*
)
&
sz
,
1
);
}
}
fstCountingWriterWrite
(
w
,
(
char
*
)(
&
(
st
.
val
)),
1
);
fstCountingWriterWrite
(
w
,
(
char
*
)(
&
(
st
.
val
)),
1
);
return
;
}
// set_comm_input
void
fstStateSetCommInput
(
FstState
*
s
,
uint8_t
inp
)
{
void
fstStateSetCommInput
(
FstState
*
s
,
uint8_t
inp
)
{
assert
(
s
->
state
==
OneTransNext
||
s
->
state
==
OneTrans
);
uint8_t
val
;
...
...
@@ -335,7 +324,7 @@ void fstStateSetCommInput(FstState *s, uint8_t inp) {
}
// comm_input
uint8_t
fstStateCommInput
(
FstState
*
s
,
bool
*
null
)
{
uint8_t
fstStateCommInput
(
FstState
*
s
,
bool
*
null
)
{
assert
(
s
->
state
==
OneTransNext
||
s
->
state
==
OneTrans
);
uint8_t
v
=
s
->
val
&
0
b00111111
;
if
(
v
==
0
)
{
...
...
@@ -348,7 +337,7 @@ uint8_t fstStateCommInput(FstState *s, bool *null) {
// input_len
uint64_t
fstStateInputLen
(
FstState
*
s
)
{
uint64_t
fstStateInputLen
(
FstState
*
s
)
{
assert
(
s
->
state
==
OneTransNext
||
s
->
state
==
OneTrans
);
bool
null
=
false
;
fstStateCommInput
(
s
,
&
null
);
...
...
@@ -356,17 +345,16 @@ uint64_t fstStateInputLen(FstState *s) {
}
// end_addr
uint64_t
fstStateEndAddrForOneTransNext
(
FstState
*
s
,
FstSlice
*
data
)
{
uint64_t
fstStateEndAddrForOneTransNext
(
FstState
*
s
,
FstSlice
*
data
)
{
assert
(
s
->
state
==
OneTransNext
);
return
FST_SLICE_LEN
(
data
)
-
1
-
fstStateInputLen
(
s
);
}
uint64_t
fstStateEndAddrForOneTrans
(
FstState
*
s
,
FstSlice
*
data
,
PackSizes
sizes
)
{
uint64_t
fstStateEndAddrForOneTrans
(
FstState
*
s
,
FstSlice
*
data
,
PackSizes
sizes
)
{
assert
(
s
->
state
==
OneTrans
);
return
FST_SLICE_LEN
(
data
)
-
1
-
fstStateInputLen
(
s
)
-
1
// pack size
-
FST_GET_TRANSITION_PACK_SIZE
(
sizes
)
-
FST_GET_OUTPUT_PACK_SIZE
(
sizes
);
}
uint64_t
fstStateEndAddrForAnyTrans
(
FstState
*
state
,
uint64_t
version
,
FstSlice
*
date
,
PackSizes
sizes
,
uint64_t
nTrans
)
{
uint64_t
fstStateEndAddrForAnyTrans
(
FstState
*
state
,
uint64_t
version
,
FstSlice
*
date
,
PackSizes
sizes
,
uint64_t
nTrans
)
{
uint8_t
oSizes
=
FST_GET_OUTPUT_PACK_SIZE
(
sizes
);
uint8_t
finalOsize
=
!
fstStateIsFinalState
(
state
)
?
0
:
oSizes
;
return
FST_SLICE_LEN
(
date
)
-
1
-
fstStateNtransLen
(
state
)
-
1
// pack size
...
...
@@ -374,29 +362,29 @@ uint64_t fstStateEndAddrForAnyTrans(
-
finalOsize
;
// final output
}
// input
uint8_t
fstStateInput
(
FstState
*
s
,
FstNode
*
node
)
{
uint8_t
fstStateInput
(
FstState
*
s
,
FstNode
*
node
)
{
assert
(
s
->
state
==
OneTransNext
||
s
->
state
==
OneTrans
);
FstSlice
*
slice
=
&
node
->
data
;
FstSlice
*
slice
=
&
node
->
data
;
bool
null
=
false
;
uint8_t
inp
=
fstStateCommInput
(
s
,
&
null
);
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
return
null
==
false
?
inp
:
data
[
-
1
];
}
uint8_t
fstStateInputForAnyTrans
(
FstState
*
s
,
FstNode
*
node
,
uint64_t
i
)
{
uint8_t
fstStateInputForAnyTrans
(
FstState
*
s
,
FstNode
*
node
,
uint64_t
i
)
{
assert
(
s
->
state
==
AnyTrans
);
FstSlice
*
slice
=
&
node
->
data
;
FstSlice
*
slice
=
&
node
->
data
;
uint64_t
at
=
node
->
start
-
fstStateNtransLen
(
s
)
-
1
// pack size
-
fstStateTransIndexSize
(
s
,
node
->
version
,
node
->
nTrans
)
-
i
-
1
;
// the output size
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
return
data
[
at
];
}
// trans_addr
CompiledAddr
fstStateTransAddr
(
FstState
*
s
,
FstNode
*
node
)
{
CompiledAddr
fstStateTransAddr
(
FstState
*
s
,
FstNode
*
node
)
{
assert
(
s
->
state
==
OneTransNext
||
s
->
state
==
OneTrans
);
FstSlice
*
slice
=
&
node
->
data
;
FstSlice
*
slice
=
&
node
->
data
;
if
(
s
->
state
==
OneTransNext
)
{
return
(
CompiledAddr
)(
node
->
end
)
-
1
;
}
else
{
...
...
@@ -406,23 +394,23 @@ CompiledAddr fstStateTransAddr(FstState *s, FstNode *node) {
-
tSizes
;
// refactor error logic
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
return
unpackDelta
(
data
+
i
,
tSizes
,
node
->
end
);
}
}
CompiledAddr
fstStateTransAddrForAnyTrans
(
FstState
*
s
,
FstNode
*
node
,
uint64_t
i
)
{
CompiledAddr
fstStateTransAddrForAnyTrans
(
FstState
*
s
,
FstNode
*
node
,
uint64_t
i
)
{
assert
(
s
->
state
==
AnyTrans
);
FstSlice
*
slice
=
&
node
->
data
;
FstSlice
*
slice
=
&
node
->
data
;
uint8_t
tSizes
=
FST_GET_TRANSITION_PACK_SIZE
(
node
->
sizes
);
uint64_t
at
=
node
->
start
-
fstStateNtransLen
(
s
)
-
1
-
fstStateTransIndexSize
(
s
,
node
->
version
,
node
->
nTrans
)
-
node
->
nTrans
-
(
i
*
tSizes
)
-
tSizes
;
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
uint64_t
at
=
node
->
start
-
fstStateNtransLen
(
s
)
-
1
-
fstStateTransIndexSize
(
s
,
node
->
version
,
node
->
nTrans
)
-
node
->
nTrans
-
(
i
*
tSizes
)
-
tSizes
;
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
return
unpackDelta
(
data
+
at
,
tSizes
,
node
->
end
);
}
// sizes
PackSizes
fstStateSizes
(
FstState
*
s
,
FstSlice
*
slice
)
{
PackSizes
fstStateSizes
(
FstState
*
s
,
FstSlice
*
slice
)
{
assert
(
s
->
state
==
OneTrans
||
s
->
state
==
AnyTrans
);
uint64_t
i
;
if
(
s
->
state
==
OneTrans
)
{
...
...
@@ -431,33 +419,29 @@ PackSizes fstStateSizes(FstState *s, FstSlice *slice) {
i
=
FST_SLICE_LEN
(
slice
)
-
1
-
fstStateNtransLen
(
s
)
-
1
;
}
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
return
(
PackSizes
)(
*
(
data
+
i
));
}
// Output
Output
fstStateOutput
(
FstState
*
s
,
FstNode
*
node
)
{
Output
fstStateOutput
(
FstState
*
s
,
FstNode
*
node
)
{
assert
(
s
->
state
==
OneTrans
);
uint8_t
oSizes
=
FST_GET_OUTPUT_PACK_SIZE
(
node
->
sizes
);
if
(
oSizes
==
0
)
{
return
0
;
}
FstSlice
*
slice
=
&
node
->
data
;
if
(
oSizes
==
0
)
{
return
0
;
}
FstSlice
*
slice
=
&
node
->
data
;
uint8_t
tSizes
=
FST_GET_TRANSITION_PACK_SIZE
(
node
->
sizes
);
uint64_t
i
=
node
->
start
-
fstStateInputLen
(
s
)
-
1
-
tSizes
-
oSizes
;
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
return
unpackUint64
(
data
+
i
,
oSizes
);
}
Output
fstStateOutputForAnyTrans
(
FstState
*
s
,
FstNode
*
node
,
uint64_t
i
)
{
Output
fstStateOutputForAnyTrans
(
FstState
*
s
,
FstNode
*
node
,
uint64_t
i
)
{
assert
(
s
->
state
==
AnyTrans
);
uint8_t
oSizes
=
FST_GET_OUTPUT_PACK_SIZE
(
node
->
sizes
);
if
(
oSizes
==
0
)
{
return
0
;
}
FstSlice
*
slice
=
&
node
->
data
;
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
if
(
oSizes
==
0
)
{
return
0
;
}
FstSlice
*
slice
=
&
node
->
data
;
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
uint64_t
at
=
node
->
start
-
fstStateNtransLen
(
s
)
-
1
// pack size
-
fstStateTotalTransSize
(
s
,
node
->
version
,
node
->
sizes
,
node
->
nTrans
)
-
(
i
*
oSizes
)
-
oSizes
;
...
...
@@ -466,27 +450,23 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) {
// anyTrans specify function
void
fstStateSetFinalState
(
FstState
*
s
,
bool
yes
)
{
void
fstStateSetFinalState
(
FstState
*
s
,
bool
yes
)
{
assert
(
s
->
state
==
AnyTrans
);
if
(
yes
)
{
s
->
val
|=
0
b01000000
;
}
if
(
yes
)
{
s
->
val
|=
0
b01000000
;
}
return
;
}
bool
fstStateIsFinalState
(
FstState
*
s
)
{
bool
fstStateIsFinalState
(
FstState
*
s
)
{
assert
(
s
->
state
==
AnyTrans
);
return
(
s
->
val
&
0
b01000000
)
==
0
b01000000
;
}
void
fstStateSetStateNtrans
(
FstState
*
s
,
uint8_t
n
)
{
void
fstStateSetStateNtrans
(
FstState
*
s
,
uint8_t
n
)
{
assert
(
s
->
state
==
AnyTrans
);
if
(
n
<=
0
b00111111
)
{
s
->
val
=
(
s
->
val
&
0
b11000000
)
|
n
;
}
if
(
n
<=
0
b00111111
)
{
s
->
val
=
(
s
->
val
&
0
b11000000
)
|
n
;
}
return
;
}
// state_ntrans
uint8_t
fstStateStateNtrans
(
FstState
*
s
,
bool
*
null
)
{
uint8_t
fstStateStateNtrans
(
FstState
*
s
,
bool
*
null
)
{
assert
(
s
->
state
==
AnyTrans
);
*
null
=
false
;
uint8_t
n
=
s
->
val
&
0
b00111111
;
...
...
@@ -496,58 +476,52 @@ uint8_t fstStateStateNtrans(FstState *s, bool *null) {
}
return
n
;
}
uint64_t
fstStateTotalTransSize
(
FstState
*
s
,
uint64_t
version
,
PackSizes
sizes
,
uint64_t
nTrans
)
{
uint64_t
fstStateTotalTransSize
(
FstState
*
s
,
uint64_t
version
,
PackSizes
sizes
,
uint64_t
nTrans
)
{
assert
(
s
->
state
==
AnyTrans
);
uint64_t
idxSize
=
fstStateTransIndexSize
(
s
,
version
,
nTrans
);
return
nTrans
+
(
nTrans
*
FST_GET_TRANSITION_PACK_SIZE
(
sizes
))
+
idxSize
;
}
uint64_t
fstStateTransIndexSize
(
FstState
*
s
,
uint64_t
version
,
uint64_t
nTrans
)
{
uint64_t
fstStateTransIndexSize
(
FstState
*
s
,
uint64_t
version
,
uint64_t
nTrans
)
{
assert
(
s
->
state
==
AnyTrans
);
return
(
version
>=
2
&&
nTrans
>
TRANS_INDEX_THRESHOLD
)
?
256
:
0
;
}
uint64_t
fstStateNtransLen
(
FstState
*
s
)
{
uint64_t
fstStateNtransLen
(
FstState
*
s
)
{
assert
(
s
->
state
==
AnyTrans
);
bool
null
=
false
;
fstStateStateNtrans
(
s
,
&
null
);
return
null
==
true
?
1
:
0
;
}
uint64_t
fstStateNtrans
(
FstState
*
s
,
FstSlice
*
slice
)
{
uint64_t
fstStateNtrans
(
FstState
*
s
,
FstSlice
*
slice
)
{
bool
null
=
false
;
uint8_t
n
=
fstStateStateNtrans
(
s
,
&
null
);
if
(
null
!=
true
)
{
return
n
;
}
if
(
null
!=
true
)
{
return
n
;
}
int32_t
len
;
uint8_t
*
data
=
fstSliceData
(
slice
,
&
len
);
uint8_t
*
data
=
fstSliceData
(
slice
,
&
len
);
n
=
data
[
len
-
2
];
// n = data[slice->end - 1]; // data[data.len() - 2]
return
n
==
1
?
256
:
n
;
// // "1" is never a normal legal value here, because if there, // is only 1 transition,
// then it is encoded in the state byte
}
Output
fstStateFinalOutput
(
FstState
*
s
,
uint64_t
version
,
FstSlice
*
slice
,
PackSizes
sizes
,
uint64_t
nTrans
)
{
Output
fstStateFinalOutput
(
FstState
*
s
,
uint64_t
version
,
FstSlice
*
slice
,
PackSizes
sizes
,
uint64_t
nTrans
)
{
uint8_t
oSizes
=
FST_GET_OUTPUT_PACK_SIZE
(
sizes
);
if
(
oSizes
==
0
||
!
fstStateIsFinalState
(
s
))
{
return
0
;
}
if
(
oSizes
==
0
||
!
fstStateIsFinalState
(
s
))
{
return
0
;
}
uint64_t
at
=
FST_SLICE_LEN
(
slice
)
-
1
-
fstStateNtransLen
(
s
)
-
1
// pack size
-
fstStateTotalTransSize
(
s
,
version
,
sizes
,
nTrans
)
-
(
nTrans
*
oSizes
)
-
oSizes
;
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
uint8_t
*
data
=
fstSliceData
(
slice
,
NULL
);
return
unpackUint64
(
data
+
at
,
(
uint8_t
)
oSizes
);
}
uint64_t
fstStateFindInput
(
FstState
*
s
,
FstNode
*
node
,
uint8_t
b
,
bool
*
null
)
{
uint64_t
fstStateFindInput
(
FstState
*
s
,
FstNode
*
node
,
uint8_t
b
,
bool
*
null
)
{
assert
(
s
->
state
==
AnyTrans
);
FstSlice
*
slice
=
&
node
->
data
;
FstSlice
*
slice
=
&
node
->
data
;
if
(
node
->
version
>=
2
&&
node
->
nTrans
>
TRANS_INDEX_THRESHOLD
)
{
uint64_t
at
=
node
->
start
-
fstStateNtransLen
(
s
)
-
1
// pack size
-
fstStateTransIndexSize
(
s
,
node
->
version
,
node
->
nTrans
);
int32_t
dlen
=
0
;
uint8_t
*
data
=
fstSliceData
(
slice
,
&
dlen
);
uint8_t
*
data
=
fstSliceData
(
slice
,
&
dlen
);
uint64_t
i
=
data
[
at
+
b
];
// uint64_t i = slice->data[slice->start + at + b];
if
(
i
>=
node
->
nTrans
)
{
*
null
=
true
;
}
if
(
i
>=
node
->
nTrans
)
{
*
null
=
true
;
}
return
i
;
}
else
{
uint64_t
start
=
node
->
start
-
fstStateNtransLen
(
s
)
-
1
// pack size
...
...
@@ -555,7 +529,7 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) {
uint64_t
end
=
start
+
node
->
nTrans
;
FstSlice
t
=
fstSliceCopy
(
slice
,
start
,
end
-
1
);
int32_t
len
=
0
;
uint8_t
*
data
=
fstSliceData
(
&
t
,
&
len
);
uint8_t
*
data
=
fstSliceData
(
&
t
,
&
len
);
int
i
=
0
;
for
(;
i
<
len
;
i
++
)
{
uint8_t
v
=
data
[
i
];
...
...
@@ -564,20 +538,16 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) {
return
node
->
nTrans
-
i
-
1
;
// bug
}
}
if
(
i
==
len
)
{
*
null
=
true
;
}
if
(
i
==
len
)
{
*
null
=
true
;
}
fstSliceDestroy
(
&
t
);
}
}
// fst node function
FstNode
*
fstNodeCreate
(
int64_t
version
,
CompiledAddr
addr
,
FstSlice
*
slice
)
{
FstNode
*
n
=
(
FstNode
*
)
malloc
(
sizeof
(
FstNode
));
if
(
n
==
NULL
)
{
return
NULL
;
}
FstNode
*
fstNodeCreate
(
int64_t
version
,
CompiledAddr
addr
,
FstSlice
*
slice
)
{
FstNode
*
n
=
(
FstNode
*
)
malloc
(
sizeof
(
FstNode
));
if
(
n
==
NULL
)
{
return
NULL
;
}
FstState
st
=
fstStateCreateFrom
(
slice
,
addr
);
...
...
@@ -625,27 +595,24 @@ FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) {
n
->
isFinal
=
fstStateIsFinalState
(
&
st
);
// s.is_final_state();
n
->
nTrans
=
nTrans
;
n
->
sizes
=
sz
;
n
->
finalOutput
=
fstStateFinalOutput
(
&
st
,
version
,
&
data
,
sz
,
nTrans
);
// s.final_output(version, data, sz, ntrans);
n
->
finalOutput
=
fstStateFinalOutput
(
&
st
,
version
,
&
data
,
sz
,
nTrans
);
// s.final_output(version, data, sz, ntrans);
}
return
n
;
}
// debug state transition
static
const
char
*
fstNodeState
(
FstNode
*
node
)
{
FstState
*
st
=
&
node
->
state
;
static
const
char
*
fstNodeState
(
FstNode
*
node
)
{
FstState
*
st
=
&
node
->
state
;
return
fstStateStr
[
st
->
state
];
}
void
fstNodeDestroy
(
FstNode
*
node
)
{
void
fstNodeDestroy
(
FstNode
*
node
)
{
fstSliceDestroy
(
&
node
->
data
);
free
(
node
);
}
FstTransitions
*
fstNodeTransitions
(
FstNode
*
node
)
{
FstTransitions
*
t
=
malloc
(
sizeof
(
FstTransitions
));
if
(
NULL
==
t
)
{
return
NULL
;
}
FstTransitions
*
fstNodeTransitions
(
FstNode
*
node
)
{
FstTransitions
*
t
=
malloc
(
sizeof
(
FstTransitions
));
if
(
NULL
==
t
)
{
return
NULL
;
}
FstRange
range
=
{.
start
=
0
,
.
end
=
FST_NODE_LEN
(
node
)};
t
->
range
=
range
;
t
->
node
=
node
;
...
...
@@ -653,9 +620,9 @@ FstTransitions *fstNodeTransitions(FstNode *node) {
}
// Returns the transition at index `i`.
bool
fstNodeGetTransitionAt
(
FstNode
*
node
,
uint64_t
i
,
FstTransition
*
trn
)
{
bool
fstNodeGetTransitionAt
(
FstNode
*
node
,
uint64_t
i
,
FstTransition
*
trn
)
{
bool
s
=
true
;
FstState
*
st
=
&
node
->
state
;
FstState
*
st
=
&
node
->
state
;
if
(
st
->
state
==
OneTransNext
)
{
trn
->
inp
=
fstStateInput
(
st
,
node
);
trn
->
out
=
0
;
...
...
@@ -675,9 +642,9 @@ bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *trn) {
}
// Returns the transition address of the `i`th transition
bool
fstNodeGetTransitionAddrAt
(
FstNode
*
node
,
uint64_t
i
,
CompiledAddr
*
res
)
{
bool
fstNodeGetTransitionAddrAt
(
FstNode
*
node
,
uint64_t
i
,
CompiledAddr
*
res
)
{
bool
s
=
true
;
FstState
*
st
=
&
node
->
state
;
FstState
*
st
=
&
node
->
state
;
if
(
st
->
state
==
OneTransNext
)
{
assert
(
i
==
0
);
fstStateTransAddr
(
st
,
node
);
...
...
@@ -696,9 +663,9 @@ bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res) {
// Finds the `i`th transition corresponding to the given input byte.
// If no transition for this byte exists, then `false` is returned.
bool
fstNodeFindInput
(
FstNode
*
node
,
uint8_t
b
,
uint64_t
*
res
)
{
bool
fstNodeFindInput
(
FstNode
*
node
,
uint8_t
b
,
uint64_t
*
res
)
{
bool
s
=
true
;
FstState
*
st
=
&
node
->
state
;
FstState
*
st
=
&
node
->
state
;
if
(
st
->
state
==
OneTransNext
)
{
if
(
fstStateInput
(
st
,
node
)
==
b
)
{
*
res
=
0
;
...
...
@@ -723,7 +690,7 @@ bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res) {
return
s
;
}
bool
fstNodeCompile
(
FstNode
*
node
,
void
*
w
,
CompiledAddr
lastAddr
,
CompiledAddr
addr
,
FstBuilderNode
*
builderNode
)
{
bool
fstNodeCompile
(
FstNode
*
node
,
void
*
w
,
CompiledAddr
lastAddr
,
CompiledAddr
addr
,
FstBuilderNode
*
builderNode
)
{
size_t
sz
=
taosArrayGetSize
(
builderNode
->
trans
);
assert
(
sz
<
256
);
if
(
sz
==
0
&&
builderNode
->
isFinal
&&
builderNode
->
finalOutput
==
0
)
{
...
...
@@ -732,7 +699,7 @@ bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr
fstStateCompileForAnyTrans
(
w
,
addr
,
builderNode
);
// AnyTrans->Compile(w, addr, node);
}
else
{
FstTransition
*
tran
=
taosArrayGet
(
builderNode
->
trans
,
0
);
FstTransition
*
tran
=
taosArrayGet
(
builderNode
->
trans
,
0
);
if
(
tran
->
addr
==
lastAddr
&&
tran
->
out
==
0
)
{
fstStateCompileForOneTransNext
(
w
,
addr
,
tran
->
inp
);
// OneTransNext::compile(w, lastAddr, tran->inp);
...
...
@@ -746,15 +713,13 @@ bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr
return
true
;
}
bool
fstBuilderNodeCompileTo
(
FstBuilderNode
*
b
,
FstCountingWriter
*
wrt
,
CompiledAddr
lastAddr
,
CompiledAddr
startAddr
)
{
bool
fstBuilderNodeCompileTo
(
FstBuilderNode
*
b
,
FstCountingWriter
*
wrt
,
CompiledAddr
lastAddr
,
CompiledAddr
startAddr
)
{
return
fstNodeCompile
(
NULL
,
wrt
,
lastAddr
,
startAddr
,
b
);
}
FstBuilder
*
fstBuilderCreate
(
void
*
w
,
FstType
ty
)
{
FstBuilder
*
b
=
malloc
(
sizeof
(
FstBuilder
));
if
(
NULL
==
b
)
{
return
b
;
}
FstBuilder
*
fstBuilderCreate
(
void
*
w
,
FstType
ty
)
{
FstBuilder
*
b
=
malloc
(
sizeof
(
FstBuilder
));
if
(
NULL
==
b
)
{
return
b
;
}
b
->
wrt
=
fstCountingWriterCreate
(
w
);
b
->
unfinished
=
fstUnFinishedNodesCreate
();
...
...
@@ -764,7 +729,7 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) {
b
->
len
=
0
;
char
buf64
[
8
]
=
{
0
};
void
*
pBuf64
=
buf64
;
void
*
pBuf64
=
buf64
;
taosEncodeFixedU64
(
&
pBuf64
,
VERSION
);
fstCountingWriterWrite
(
b
->
wrt
,
buf64
,
sizeof
(
buf64
));
...
...
@@ -775,10 +740,8 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) {
return
b
;
}
void
fstBuilderDestroy
(
FstBuilder
*
b
)
{
if
(
b
==
NULL
)
{
return
;
}
void
fstBuilderDestroy
(
FstBuilder
*
b
)
{
if
(
b
==
NULL
)
{
return
;
}
fstCountingWriterDestroy
(
b
->
wrt
);
fstUnFinishedNodesDestroy
(
b
->
unfinished
);
...
...
@@ -787,19 +750,19 @@ void fstBuilderDestroy(FstBuilder *b) {
free
(
b
);
}
bool
fstBuilderInsert
(
FstBuilder
*
b
,
FstSlice
bs
,
Output
in
)
{
bool
fstBuilderInsert
(
FstBuilder
*
b
,
FstSlice
bs
,
Output
in
)
{
OrderType
t
=
fstBuilderCheckLastKey
(
b
,
bs
,
true
);
if
(
t
==
Ordered
)
{
// add log info
fstBuilderInsertOutput
(
b
,
bs
,
in
);
return
true
;
}
indexInfo
(
"key must be ordered"
);
indexInfo
(
"
fst write
key must be ordered"
);
return
false
;
}
void
fstBuilderInsertOutput
(
FstBuilder
*
b
,
FstSlice
bs
,
Output
in
)
{
FstSlice
*
s
=
&
bs
;
void
fstBuilderInsertOutput
(
FstBuilder
*
b
,
FstSlice
bs
,
Output
in
)
{
FstSlice
*
s
=
&
bs
;
if
(
fstSliceIsEmpty
(
s
))
{
b
->
len
=
1
;
fstUnFinishedNodesSetRootOutput
(
b
->
unfinished
,
in
);
...
...
@@ -828,8 +791,8 @@ void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) {
return
;
}
OrderType
fstBuilderCheckLastKey
(
FstBuilder
*
b
,
FstSlice
bs
,
bool
ckDup
)
{
FstSlice
*
input
=
&
bs
;
OrderType
fstBuilderCheckLastKey
(
FstBuilder
*
b
,
FstSlice
bs
,
bool
ckDup
)
{
FstSlice
*
input
=
&
bs
;
if
(
fstSliceIsEmpty
(
&
b
->
last
))
{
fstSliceDestroy
(
&
b
->
last
);
// deep copy or not
...
...
@@ -847,10 +810,10 @@ OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) {
}
return
Ordered
;
}
void
fstBuilderCompileFrom
(
FstBuilder
*
b
,
uint64_t
istate
)
{
void
fstBuilderCompileFrom
(
FstBuilder
*
b
,
uint64_t
istate
)
{
CompiledAddr
addr
=
NONE_ADDRESS
;
while
(
istate
+
1
<
FST_UNFINISHED_NODES_LEN
(
b
->
unfinished
))
{
FstBuilderNode
*
bn
=
NULL
;
FstBuilderNode
*
bn
=
NULL
;
if
(
addr
==
NONE_ADDRESS
)
{
bn
=
fstUnFinishedNodesPopEmpty
(
b
->
unfinished
);
}
else
{
...
...
@@ -865,11 +828,11 @@ void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) {
fstUnFinishedNodesTopLastFreeze
(
b
->
unfinished
,
addr
);
return
;
}
CompiledAddr
fstBuilderCompile
(
FstBuilder
*
b
,
FstBuilderNode
*
bn
)
{
CompiledAddr
fstBuilderCompile
(
FstBuilder
*
b
,
FstBuilderNode
*
bn
)
{
if
(
FST_BUILDER_NODE_IS_FINAL
(
bn
)
&&
FST_BUILDER_NODE_TRANS_ISEMPTY
(
bn
)
&&
FST_BUILDER_NODE_FINALOUTPUT_ISZERO
(
bn
))
{
return
EMPTY_ADDRESS
;
}
FstRegistryEntry
*
entry
=
fstRegistryGetEntry
(
b
->
registry
,
bn
);
FstRegistryEntry
*
entry
=
fstRegistryGetEntry
(
b
->
registry
,
bn
);
if
(
entry
->
state
==
FOUND
)
{
CompiledAddr
ret
=
entry
->
addr
;
fstRegistryEntryDestroy
(
entry
);
...
...
@@ -879,23 +842,21 @@ CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn) {
fstBuilderNodeCompileTo
(
bn
,
b
->
wrt
,
b
->
lastAddr
,
startAddr
);
b
->
lastAddr
=
(
CompiledAddr
)(
FST_WRITER_COUNT
(
b
->
wrt
)
-
1
);
if
(
entry
->
state
==
NOTFOUND
)
{
FST_REGISTRY_CELL_INSERT
(
entry
->
cell
,
b
->
lastAddr
);
}
if
(
entry
->
state
==
NOTFOUND
)
{
FST_REGISTRY_CELL_INSERT
(
entry
->
cell
,
b
->
lastAddr
);
}
fstRegistryEntryDestroy
(
entry
);
return
b
->
lastAddr
;
}
void
*
fstBuilderInsertInner
(
FstBuilder
*
b
)
{
void
*
fstBuilderInsertInner
(
FstBuilder
*
b
)
{
fstBuilderCompileFrom
(
b
,
0
);
FstBuilderNode
*
rootNode
=
fstUnFinishedNodesPopRoot
(
b
->
unfinished
);
FstBuilderNode
*
rootNode
=
fstUnFinishedNodesPopRoot
(
b
->
unfinished
);
CompiledAddr
rootAddr
=
fstBuilderCompile
(
b
,
rootNode
);
fstBuilderNodeDestroy
(
rootNode
);
char
buf64
[
8
]
=
{
0
};
void
*
pBuf64
=
buf64
;
void
*
pBuf64
=
buf64
;
taosEncodeFixedU64
(
&
pBuf64
,
b
->
len
);
fstCountingWriterWrite
(
b
->
wrt
,
buf64
,
sizeof
(
buf64
));
...
...
@@ -904,7 +865,7 @@ void *fstBuilderInsertInner(FstBuilder *b) {
fstCountingWriterWrite
(
b
->
wrt
,
buf64
,
sizeof
(
buf64
));
char
buf32
[
4
]
=
{
0
};
void
*
pBuf32
=
buf32
;
void
*
pBuf32
=
buf32
;
uint32_t
sum
=
fstCountingWriterMaskedCheckSum
(
b
->
wrt
);
taosEncodeFixedU32
(
&
pBuf32
,
sum
);
fstCountingWriterWrite
(
b
->
wrt
,
buf32
,
sizeof
(
buf32
));
...
...
@@ -914,31 +875,31 @@ void *fstBuilderInsertInner(FstBuilder *b) {
// b->wrt = NULL;
return
b
->
wrt
;
}
void
fstBuilderFinish
(
FstBuilder
*
b
)
{
fstBuilderInsertInner
(
b
);
}
void
fstBuilderFinish
(
FstBuilder
*
b
)
{
fstBuilderInsertInner
(
b
);
}
FstSlice
fstNodeAsSlice
(
FstNode
*
node
)
{
FstSlice
*
slice
=
&
node
->
data
;
FstSlice
fstNodeAsSlice
(
FstNode
*
node
)
{
FstSlice
*
slice
=
&
node
->
data
;
FstSlice
s
=
fstSliceCopy
(
slice
,
slice
->
end
,
FST_SLICE_LEN
(
slice
)
-
1
);
return
s
;
}
FstLastTransition
*
fstLastTransitionCreate
(
uint8_t
inp
,
Output
out
)
{
FstLastTransition
*
trn
=
malloc
(
sizeof
(
FstLastTransition
));
if
(
trn
==
NULL
)
{
return
NULL
;
}
FstLastTransition
*
fstLastTransitionCreate
(
uint8_t
inp
,
Output
out
)
{
FstLastTransition
*
trn
=
malloc
(
sizeof
(
FstLastTransition
));
if
(
trn
==
NULL
)
{
return
NULL
;
}
trn
->
inp
=
inp
;
trn
->
out
=
out
;
return
trn
;
}
void
fstLastTransitionDestroy
(
FstLastTransition
*
trn
)
{
free
(
trn
);
}
void
fstBuilderNodeUnfinishedLastCompiled
(
FstBuilderNodeUnfinished
*
unNode
,
CompiledAddr
addr
)
{
FstLastTransition
*
trn
=
unNode
->
last
;
if
(
trn
==
NULL
)
{
return
;
}
void
fstLastTransitionDestroy
(
FstLastTransition
*
trn
)
{
free
(
trn
);
}
void
fstBuilderNodeUnfinishedLastCompiled
(
FstBuilderNodeUnfinished
*
unNode
,
CompiledAddr
addr
)
{
FstLastTransition
*
trn
=
unNode
->
last
;
if
(
trn
==
NULL
)
{
return
;
}
FstTransition
t
=
{.
inp
=
trn
->
inp
,
.
out
=
trn
->
out
,
.
addr
=
addr
};
taosArrayPush
(
unNode
->
node
->
trans
,
&
t
);
fstLastTransitionDestroy
(
trn
);
...
...
@@ -946,36 +907,28 @@ void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *unNode, Comp
return
;
}
void
fstBuilderNodeUnfinishedAddOutputPrefix
(
FstBuilderNodeUnfinished
*
unNode
,
Output
out
)
{
if
(
FST_BUILDER_NODE_IS_FINAL
(
unNode
->
node
))
{
unNode
->
node
->
finalOutput
+=
out
;
}
void
fstBuilderNodeUnfinishedAddOutputPrefix
(
FstBuilderNodeUnfinished
*
unNode
,
Output
out
)
{
if
(
FST_BUILDER_NODE_IS_FINAL
(
unNode
->
node
))
{
unNode
->
node
->
finalOutput
+=
out
;
}
size_t
sz
=
taosArrayGetSize
(
unNode
->
node
->
trans
);
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
FstTransition
*
trn
=
taosArrayGet
(
unNode
->
node
->
trans
,
i
);
FstTransition
*
trn
=
taosArrayGet
(
unNode
->
node
->
trans
,
i
);
trn
->
out
+=
out
;
}
if
(
unNode
->
last
)
{
unNode
->
last
->
out
+=
out
;
}
if
(
unNode
->
last
)
{
unNode
->
last
->
out
+=
out
;
}
return
;
}
Fst
*
fstCreate
(
FstSlice
*
slice
)
{
Fst
*
fstCreate
(
FstSlice
*
slice
)
{
int32_t
slen
;
char
*
buf
=
fstSliceData
(
slice
,
&
slen
);
if
(
slen
<
36
)
{
return
NULL
;
}
char
*
buf
=
fstSliceData
(
slice
,
&
slen
);
if
(
slen
<
36
)
{
return
NULL
;
}
uint64_t
len
=
slen
;
uint64_t
skip
=
0
;
uint64_t
version
;
taosDecodeFixedU64
(
buf
,
&
version
);
skip
+=
sizeof
(
version
);
if
(
version
==
0
||
version
>
VERSION
)
{
return
NULL
;
}
if
(
version
==
0
||
version
>
VERSION
)
{
return
NULL
;
}
uint64_t
type
;
taosDecodeFixedU64
(
buf
+
skip
,
&
type
);
...
...
@@ -993,15 +946,11 @@ Fst *fstCreate(FstSlice *slice) {
len
-=
sizeof
(
fstLen
);
taosDecodeFixedU64
(
buf
+
len
,
&
fstLen
);
// TODO(validate root addr)
Fst
*
fst
=
(
Fst
*
)
calloc
(
1
,
sizeof
(
Fst
));
if
(
fst
==
NULL
)
{
return
NULL
;
}
Fst
*
fst
=
(
Fst
*
)
calloc
(
1
,
sizeof
(
Fst
));
if
(
fst
==
NULL
)
{
return
NULL
;
}
fst
->
meta
=
(
FstMeta
*
)
malloc
(
sizeof
(
FstMeta
));
if
(
NULL
==
fst
->
meta
)
{
goto
FST_CREAT_FAILED
;
}
fst
->
meta
=
(
FstMeta
*
)
malloc
(
sizeof
(
FstMeta
));
if
(
NULL
==
fst
->
meta
)
{
goto
FST_CREAT_FAILED
;
}
fst
->
meta
->
version
=
version
;
fst
->
meta
->
rootAddr
=
rootAddr
;
...
...
@@ -1009,7 +958,7 @@ Fst *fstCreate(FstSlice *slice) {
fst
->
meta
->
len
=
fstLen
;
fst
->
meta
->
checkSum
=
checkSum
;
FstSlice
*
s
=
calloc
(
1
,
sizeof
(
FstSlice
));
FstSlice
*
s
=
calloc
(
1
,
sizeof
(
FstSlice
));
*
s
=
fstSliceCopy
(
slice
,
0
,
FST_SLICE_LEN
(
slice
));
fst
->
data
=
s
;
...
...
@@ -1019,7 +968,7 @@ FST_CREAT_FAILED:
free
(
fst
->
meta
);
free
(
fst
);
}
void
fstDestroy
(
Fst
*
fst
)
{
void
fstDestroy
(
Fst
*
fst
)
{
if
(
fst
)
{
free
(
fst
->
meta
);
fstSliceDestroy
(
fst
->
data
);
...
...
@@ -1028,20 +977,18 @@ void fstDestroy(Fst *fst) {
free
(
fst
);
}
bool
fstGet
(
Fst
*
fst
,
FstSlice
*
b
,
Output
*
out
)
{
FstNode
*
root
=
fstGetRoot
(
fst
);
bool
fstGet
(
Fst
*
fst
,
FstSlice
*
b
,
Output
*
out
)
{
FstNode
*
root
=
fstGetRoot
(
fst
);
Output
tOut
=
0
;
int32_t
len
;
uint8_t
*
data
=
fstSliceData
(
b
,
&
len
);
uint8_t
*
data
=
fstSliceData
(
b
,
&
len
);
SArray
*
nodes
=
(
SArray
*
)
taosArrayInit
(
len
,
sizeof
(
FstNode
*
));
SArray
*
nodes
=
(
SArray
*
)
taosArrayInit
(
len
,
sizeof
(
FstNode
*
));
taosArrayPush
(
nodes
,
&
root
);
for
(
uint32_t
i
=
0
;
i
<
len
;
i
++
)
{
uint8_t
inp
=
data
[
i
];
Output
res
=
0
;
if
(
false
==
fstNodeFindInput
(
root
,
inp
,
&
res
))
{
return
false
;
}
if
(
false
==
fstNodeFindInput
(
root
,
inp
,
&
res
))
{
return
false
;
}
FstTransition
trn
;
fstNodeGetTransitionAt
(
root
,
res
,
&
trn
);
...
...
@@ -1056,7 +1003,7 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) {
}
for
(
size_t
i
=
0
;
i
<
taosArrayGetSize
(
nodes
);
i
++
)
{
FstNode
**
node
=
(
FstNode
**
)
taosArrayGet
(
nodes
,
i
);
FstNode
**
node
=
(
FstNode
**
)
taosArrayGet
(
nodes
,
i
);
fstNodeDestroy
(
*
node
);
}
taosArrayDestroy
(
nodes
);
...
...
@@ -1066,30 +1013,36 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) {
return
true
;
}
FstStreamBuilder
*
fstSearch
(
Fst
*
fst
,
AutomationCtx
*
ctx
)
{
return
fstStreamBuilderCreate
(
fst
,
ctx
);
}
StreamWithState
*
streamBuilderIntoStream
(
FstStreamBuilder
*
sb
)
{
if
(
sb
==
NULL
)
{
return
NULL
;
}
FstStreamBuilder
*
fstSearch
(
Fst
*
fst
,
AutomationCtx
*
ctx
)
{
return
fstStreamBuilderCreate
(
fst
,
ctx
);
}
StreamWithState
*
streamBuilderIntoStream
(
FstStreamBuilder
*
sb
)
{
if
(
sb
==
NULL
)
{
return
NULL
;
}
return
streamWithStateCreate
(
sb
->
fst
,
sb
->
aut
,
sb
->
min
,
sb
->
max
);
}
FstStreamWithStateBuilder
*
fstSearchWithState
(
Fst
*
fst
,
AutomationCtx
*
ctx
)
{
return
fstStreamBuilderCreate
(
fst
,
ctx
);
}
FstStreamWithStateBuilder
*
fstSearchWithState
(
Fst
*
fst
,
AutomationCtx
*
ctx
)
{
return
fstStreamBuilderCreate
(
fst
,
ctx
);
}
FstNode
*
fstGetRoot
(
Fst
*
fst
)
{
if
(
fst
->
root
!=
NULL
)
{
return
fst
->
root
;
}
FstNode
*
fstGetRoot
(
Fst
*
fst
)
{
if
(
fst
->
root
!=
NULL
)
{
return
fst
->
root
;
}
CompiledAddr
rAddr
=
fstGetRootAddr
(
fst
);
fst
->
root
=
fstGetNode
(
fst
,
rAddr
);
return
fst
->
root
;
}
FstNode
*
fstGetNode
(
Fst
*
fst
,
CompiledAddr
addr
)
{
return
fstNodeCreate
(
fst
->
meta
->
version
,
addr
,
fst
->
data
);
}
FstType
fstGetType
(
Fst
*
fst
)
{
return
fst
->
meta
->
ty
;
}
CompiledAddr
fstGetRootAddr
(
Fst
*
fst
)
{
return
fst
->
meta
->
rootAddr
;
}
FstNode
*
fstGetNode
(
Fst
*
fst
,
CompiledAddr
addr
)
{
return
fstNodeCreate
(
fst
->
meta
->
version
,
addr
,
fst
->
data
);
}
FstType
fstGetType
(
Fst
*
fst
)
{
return
fst
->
meta
->
ty
;
}
CompiledAddr
fstGetRootAddr
(
Fst
*
fst
)
{
return
fst
->
meta
->
rootAddr
;
}
Output
fstEmptyFinalOutput
(
Fst
*
fst
,
bool
*
null
)
{
Output
fstEmptyFinalOutput
(
Fst
*
fst
,
bool
*
null
)
{
Output
res
=
0
;
FstNode
*
node
=
fstGetRoot
(
fst
);
FstNode
*
node
=
fstGetRoot
(
fst
);
if
(
FST_NODE_IS_FINAL
(
node
))
{
*
null
=
false
;
res
=
FST_NODE_FINAL_OUTPUT
(
node
);
...
...
@@ -1099,23 +1052,19 @@ Output fstEmptyFinalOutput(Fst *fst, bool *null) {
return
res
;
}
bool
fstVerify
(
Fst
*
fst
)
{
bool
fstVerify
(
Fst
*
fst
)
{
uint32_t
checkSum
=
fst
->
meta
->
checkSum
;
int32_t
len
;
uint8_t
*
data
=
fstSliceData
(
fst
->
data
,
&
len
);
uint8_t
*
data
=
fstSliceData
(
fst
->
data
,
&
len
);
TSCKSUM
initSum
=
0
;
if
(
!
taosCheckChecksumWhole
(
data
,
len
))
{
return
false
;
}
if
(
!
taosCheckChecksumWhole
(
data
,
len
))
{
return
false
;
}
return
true
;
}
// data bound function
FstBoundWithData
*
fstBoundStateCreate
(
FstBound
type
,
FstSlice
*
data
)
{
FstBoundWithData
*
b
=
calloc
(
1
,
sizeof
(
FstBoundWithData
));
if
(
b
==
NULL
)
{
return
NULL
;
}
FstBoundWithData
*
fstBoundStateCreate
(
FstBound
type
,
FstSlice
*
data
)
{
FstBoundWithData
*
b
=
calloc
(
1
,
sizeof
(
FstBoundWithData
));
if
(
b
==
NULL
)
{
return
NULL
;
}
if
(
data
!=
NULL
)
{
b
->
data
=
fstSliceCopy
(
data
,
data
->
start
,
data
->
end
);
...
...
@@ -1127,7 +1076,7 @@ FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data) {
return
b
;
}
bool
fstBoundWithDataExceededBy
(
FstBoundWithData
*
bound
,
FstSlice
*
slice
)
{
bool
fstBoundWithDataExceededBy
(
FstBoundWithData
*
bound
,
FstSlice
*
slice
)
{
int
comp
=
fstSliceCompare
(
slice
,
&
bound
->
data
);
if
(
bound
->
type
==
Included
)
{
return
comp
>
0
?
true
:
false
;
...
...
@@ -1137,7 +1086,7 @@ bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) {
return
true
;
}
}
bool
fstBoundWithDataIsEmpty
(
FstBoundWithData
*
bound
)
{
bool
fstBoundWithDataIsEmpty
(
FstBoundWithData
*
bound
)
{
if
(
bound
->
type
==
Unbounded
)
{
return
true
;
}
else
{
...
...
@@ -1145,34 +1094,33 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) {
}
}
bool
fstBoundWithDataIsIncluded
(
FstBoundWithData
*
bound
)
{
return
bound
->
type
==
Excluded
?
false
:
true
;
}
bool
fstBoundWithDataIsIncluded
(
FstBoundWithData
*
bound
)
{
return
bound
->
type
==
Excluded
?
false
:
true
;
}
void
fstBoundDestroy
(
FstBoundWithData
*
bound
)
{
free
(
bound
);
}
void
fstBoundDestroy
(
FstBoundWithData
*
bound
)
{
free
(
bound
);
}
StreamWithState
*
streamWithStateCreate
(
Fst
*
fst
,
AutomationCtx
*
automation
,
FstBoundWithData
*
min
,
FstBoundWithData
*
max
)
{
StreamWithState
*
sws
=
calloc
(
1
,
sizeof
(
StreamWithState
));
if
(
sws
==
NULL
)
{
return
NULL
;
}
StreamWithState
*
streamWithStateCreate
(
Fst
*
fst
,
AutomationCtx
*
automation
,
FstBoundWithData
*
min
,
FstBoundWithData
*
max
)
{
StreamWithState
*
sws
=
calloc
(
1
,
sizeof
(
StreamWithState
));
if
(
sws
==
NULL
)
{
return
NULL
;
}
sws
->
fst
=
fst
;
sws
->
aut
=
automation
;
sws
->
inp
=
(
SArray
*
)
taosArrayInit
(
256
,
sizeof
(
uint8_t
));
sws
->
inp
=
(
SArray
*
)
taosArrayInit
(
256
,
sizeof
(
uint8_t
));
sws
->
emptyOutput
.
null
=
false
;
sws
->
emptyOutput
.
out
=
0
;
sws
->
stack
=
(
SArray
*
)
taosArrayInit
(
256
,
sizeof
(
StreamState
));
sws
->
stack
=
(
SArray
*
)
taosArrayInit
(
256
,
sizeof
(
StreamState
));
sws
->
endAt
=
max
;
streamWithStateSeekMin
(
sws
,
min
);
return
sws
;
}
void
streamWithStateDestroy
(
StreamWithState
*
sws
)
{
if
(
sws
==
NULL
)
{
return
;
}
void
streamWithStateDestroy
(
StreamWithState
*
sws
)
{
if
(
sws
==
NULL
)
{
return
;
}
taosArrayDestroy
(
sws
->
inp
);
taosArrayDestroyEx
(
sws
->
stack
,
streamStateDestroy
);
...
...
@@ -1180,22 +1128,19 @@ void streamWithStateDestroy(StreamWithState *sws) {
free
(
sws
);
}
bool
streamWithStateSeekMin
(
StreamWithState
*
sws
,
FstBoundWithData
*
min
)
{
AutomationCtx
*
aut
=
sws
->
aut
;
bool
streamWithStateSeekMin
(
StreamWithState
*
sws
,
FstBoundWithData
*
min
)
{
AutomationCtx
*
aut
=
sws
->
aut
;
if
(
fstBoundWithDataIsEmpty
(
min
))
{
if
(
fstBoundWithDataIsIncluded
(
min
))
{
sws
->
emptyOutput
.
out
=
fstEmptyFinalOutput
(
sws
->
fst
,
&
(
sws
->
emptyOutput
.
null
));
}
if
(
fstBoundWithDataIsIncluded
(
min
))
{
sws
->
emptyOutput
.
out
=
fstEmptyFinalOutput
(
sws
->
fst
,
&
(
sws
->
emptyOutput
.
null
));
}
StreamState
s
=
{.
node
=
fstGetRoot
(
sws
->
fst
),
.
trans
=
0
,
.
out
=
{.
null
=
false
,
.
out
=
0
},
.
autState
=
automFuncs
[
aut
->
type
].
start
(
aut
)};
// auto.start callback
.
trans
=
0
,
.
out
=
{.
null
=
false
,
.
out
=
0
},
.
autState
=
automFuncs
[
aut
->
type
].
start
(
aut
)};
// auto.start callback
taosArrayPush
(
sws
->
stack
,
&
s
);
return
true
;
}
FstSlice
*
key
=
NULL
;
FstSlice
*
key
=
NULL
;
bool
inclusize
=
false
;
;
if
(
min
->
type
==
Included
)
{
key
=
&
min
->
data
;
...
...
@@ -1206,13 +1151,13 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
return
false
;
}
FstNode
*
node
=
fstGetRoot
(
sws
->
fst
);
FstNode
*
node
=
fstGetRoot
(
sws
->
fst
);
Output
out
=
0
;
// void* autState = sws->aut->start();
void
*
autState
=
automFuncs
[
aut
->
type
].
start
(
aut
);
void
*
autState
=
automFuncs
[
aut
->
type
].
start
(
aut
);
int32_t
len
;
uint8_t
*
data
=
fstSliceData
(
key
,
&
len
);
uint8_t
*
data
=
fstSliceData
(
key
,
&
len
);
for
(
uint32_t
i
=
0
;
i
<
len
;
i
++
)
{
uint8_t
b
=
data
[
i
];
uint64_t
res
=
0
;
...
...
@@ -1220,7 +1165,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
if
(
null
==
false
)
{
FstTransition
trn
;
fstNodeGetTransitionAt
(
node
,
res
,
&
trn
);
void
*
preState
=
autState
;
void
*
preState
=
autState
;
// autState = sws->aut->accept(preState, b);
autState
=
automFuncs
[
aut
->
type
].
accept
(
aut
,
preState
,
b
);
taosArrayPush
(
sws
->
inp
,
&
b
);
...
...
@@ -1235,13 +1180,11 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
// Since this is a minimum bound, we need to find the
// first transition in this node that proceeds the current
// input byte.
FstTransitions
*
trans
=
fstNodeTransitions
(
node
);
FstTransitions
*
trans
=
fstNodeTransitions
(
node
);
uint64_t
i
=
0
;
for
(
i
=
trans
->
range
.
start
;
i
<
trans
->
range
.
end
;
i
++
)
{
FstTransition
trn
;
if
(
fstNodeGetTransitionAt
(
node
,
i
,
&
trn
)
&&
trn
.
inp
>
b
)
{
break
;
}
if
(
fstNodeGetTransitionAt
(
node
,
i
,
&
trn
)
&&
trn
.
inp
>
b
)
{
break
;
}
}
StreamState
s
=
{.
node
=
node
,
.
trans
=
i
,
.
out
=
{.
null
=
false
,
.
out
=
out
},
.
autState
=
autState
};
...
...
@@ -1251,17 +1194,16 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
}
uint32_t
sz
=
taosArrayGetSize
(
sws
->
stack
);
if
(
sz
!=
0
)
{
StreamState
*
s
=
taosArrayGet
(
sws
->
stack
,
sz
-
1
);
StreamState
*
s
=
taosArrayGet
(
sws
->
stack
,
sz
-
1
);
if
(
inclusize
)
{
s
->
trans
-=
1
;
taosArrayPop
(
sws
->
inp
);
}
else
{
FstNode
*
n
=
s
->
node
;
FstNode
*
n
=
s
->
node
;
uint64_t
trans
=
s
->
trans
;
FstTransition
trn
;
fstNodeGetTransitionAt
(
n
,
trans
-
1
,
&
trn
);
StreamState
s
=
{
.
node
=
fstGetNode
(
sws
->
fst
,
trn
.
addr
),
.
trans
=
0
,
.
out
=
{.
null
=
false
,
.
out
=
out
},
.
autState
=
autState
};
StreamState
s
=
{.
node
=
fstGetNode
(
sws
->
fst
,
trn
.
addr
),
.
trans
=
0
,
.
out
=
{.
null
=
false
,
.
out
=
out
},
.
autState
=
autState
};
taosArrayPush
(
sws
->
stack
,
&
s
);
return
true
;
}
...
...
@@ -1269,48 +1211,44 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) {
}
}
StreamWithStateResult
*
streamWithStateNextWith
(
StreamWithState
*
sws
,
StreamCallback
callback
)
{
AutomationCtx
*
aut
=
sws
->
aut
;
StreamWithStateResult
*
streamWithStateNextWith
(
StreamWithState
*
sws
,
StreamCallback
callback
)
{
AutomationCtx
*
aut
=
sws
->
aut
;
FstOutput
output
=
sws
->
emptyOutput
;
if
(
output
.
null
==
false
)
{
FstSlice
emptySlice
=
fstSliceCreate
(
NULL
,
0
);
if
(
fstBoundWithDataExceededBy
(
sws
->
endAt
,
&
emptySlice
))
{
taosArrayDestroyEx
(
sws
->
stack
,
streamStateDestroy
);
sws
->
stack
=
(
SArray
*
)
taosArrayInit
(
256
,
sizeof
(
StreamState
));
sws
->
stack
=
(
SArray
*
)
taosArrayInit
(
256
,
sizeof
(
StreamState
));
return
NULL
;
}
void
*
start
=
automFuncs
[
aut
->
type
].
start
(
aut
);
void
*
start
=
automFuncs
[
aut
->
type
].
start
(
aut
);
if
(
automFuncs
[
aut
->
type
].
isMatch
(
aut
,
start
))
{
FstSlice
s
=
fstSliceCreate
(
NULL
,
0
);
return
swsResultCreate
(
&
s
,
output
,
callback
(
start
));
}
}
SArray
*
nodes
=
taosArrayInit
(
8
,
sizeof
(
FstNode
*
));
SArray
*
nodes
=
taosArrayInit
(
8
,
sizeof
(
FstNode
*
));
while
(
taosArrayGetSize
(
sws
->
stack
)
>
0
)
{
StreamState
*
p
=
(
StreamState
*
)
taosArrayPop
(
sws
->
stack
);
StreamState
*
p
=
(
StreamState
*
)
taosArrayPop
(
sws
->
stack
);
if
(
p
->
trans
>=
FST_NODE_LEN
(
p
->
node
)
||
!
automFuncs
[
aut
->
type
].
canMatch
(
aut
,
p
->
autState
))
{
if
(
FST_NODE_ADDR
(
p
->
node
)
!=
fstGetRootAddr
(
sws
->
fst
))
{
taosArrayPop
(
sws
->
inp
);
}
if
(
FST_NODE_ADDR
(
p
->
node
)
!=
fstGetRootAddr
(
sws
->
fst
))
{
taosArrayPop
(
sws
->
inp
);
}
streamStateDestroy
(
p
);
continue
;
}
FstTransition
trn
;
fstNodeGetTransitionAt
(
p
->
node
,
p
->
trans
,
&
trn
);
Output
out
=
p
->
out
.
out
+
trn
.
out
;
void
*
nextState
=
automFuncs
[
aut
->
type
].
accept
(
aut
,
p
->
autState
,
trn
.
inp
);
void
*
tState
=
callback
(
nextState
);
void
*
nextState
=
automFuncs
[
aut
->
type
].
accept
(
aut
,
p
->
autState
,
trn
.
inp
);
void
*
tState
=
callback
(
nextState
);
bool
isMatch
=
automFuncs
[
aut
->
type
].
isMatch
(
aut
,
nextState
);
FstNode
*
nextNode
=
fstGetNode
(
sws
->
fst
,
trn
.
addr
);
FstNode
*
nextNode
=
fstGetNode
(
sws
->
fst
,
trn
.
addr
);
taosArrayPush
(
nodes
,
&
nextNode
);
taosArrayPush
(
sws
->
inp
,
&
(
trn
.
inp
));
if
(
FST_NODE_IS_FINAL
(
nextNode
))
{
// void *eofState = sws->aut->acceptEof(nextState);
void
*
eofState
=
automFuncs
[
aut
->
type
].
acceptEof
(
aut
,
nextState
);
if
(
eofState
!=
NULL
)
{
isMatch
=
automFuncs
[
aut
->
type
].
isMatch
(
aut
,
eofState
);
}
void
*
eofState
=
automFuncs
[
aut
->
type
].
acceptEof
(
aut
,
nextState
);
if
(
eofState
!=
NULL
)
{
isMatch
=
automFuncs
[
aut
->
type
].
isMatch
(
aut
,
eofState
);
}
}
StreamState
s1
=
{.
node
=
p
->
node
,
.
trans
=
p
->
trans
+
1
,
.
out
=
p
->
out
,
.
autState
=
p
->
autState
};
taosArrayPush
(
sws
->
stack
,
&
s1
);
...
...
@@ -1319,21 +1257,21 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
taosArrayPush
(
sws
->
stack
,
&
s2
);
size_t
isz
=
taosArrayGetSize
(
sws
->
inp
);
uint8_t
*
buf
=
(
uint8_t
*
)
malloc
(
isz
*
sizeof
(
uint8_t
));
uint8_t
*
buf
=
(
uint8_t
*
)
malloc
(
isz
*
sizeof
(
uint8_t
));
for
(
uint32_t
i
=
0
;
i
<
isz
;
i
++
)
{
buf
[
i
]
=
*
(
uint8_t
*
)
taosArrayGet
(
sws
->
inp
,
i
);
buf
[
i
]
=
*
(
uint8_t
*
)
taosArrayGet
(
sws
->
inp
,
i
);
}
FstSlice
slice
=
fstSliceCreate
(
buf
,
taosArrayGetSize
(
sws
->
inp
));
if
(
fstBoundWithDataExceededBy
(
sws
->
endAt
,
&
slice
))
{
taosArrayDestroyEx
(
sws
->
stack
,
streamStateDestroy
);
sws
->
stack
=
(
SArray
*
)
taosArrayInit
(
256
,
sizeof
(
StreamState
));
sws
->
stack
=
(
SArray
*
)
taosArrayInit
(
256
,
sizeof
(
StreamState
));
free
(
buf
);
fstSliceDestroy
(
&
slice
);
return
NULL
;
}
if
(
FST_NODE_IS_FINAL
(
nextNode
)
&&
isMatch
)
{
FstOutput
fOutput
=
{.
null
=
false
,
.
out
=
out
+
FST_NODE_FINAL_OUTPUT
(
nextNode
)};
StreamWithStateResult
*
result
=
swsResultCreate
(
&
slice
,
fOutput
,
tState
);
StreamWithStateResult
*
result
=
swsResultCreate
(
&
slice
,
fOutput
,
tState
);
free
(
buf
);
fstSliceDestroy
(
&
slice
);
return
result
;
...
...
@@ -1342,18 +1280,16 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
fstSliceDestroy
(
&
slice
);
}
for
(
size_t
i
=
0
;
i
<
taosArrayGetSize
(
nodes
);
i
++
)
{
FstNode
**
node
=
(
FstNode
**
)
taosArrayGet
(
nodes
,
i
);
FstNode
**
node
=
(
FstNode
**
)
taosArrayGet
(
nodes
,
i
);
fstNodeDestroy
(
*
node
);
}
taosArrayDestroy
(
nodes
);
return
NULL
;
}
StreamWithStateResult
*
swsResultCreate
(
FstSlice
*
data
,
FstOutput
fOut
,
void
*
state
)
{
StreamWithStateResult
*
result
=
calloc
(
1
,
sizeof
(
StreamWithStateResult
));
if
(
result
==
NULL
)
{
return
NULL
;
}
StreamWithStateResult
*
swsResultCreate
(
FstSlice
*
data
,
FstOutput
fOut
,
void
*
state
)
{
StreamWithStateResult
*
result
=
calloc
(
1
,
sizeof
(
StreamWithStateResult
));
if
(
result
==
NULL
)
{
return
NULL
;
}
result
->
data
=
fstSliceCopy
(
data
,
0
,
FST_SLICE_LEN
(
data
)
-
1
);
result
->
out
=
fOut
;
...
...
@@ -1361,31 +1297,25 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta
return
result
;
}
void
swsResultDestroy
(
StreamWithStateResult
*
result
)
{
if
(
NULL
==
result
)
{
return
;
}
void
swsResultDestroy
(
StreamWithStateResult
*
result
)
{
if
(
NULL
==
result
)
{
return
;
}
fstSliceDestroy
(
&
result
->
data
);
startWithStateValueDestroy
(
result
->
state
);
free
(
result
);
}
void
streamStateDestroy
(
void
*
s
)
{
if
(
NULL
==
s
)
{
return
;
}
StreamState
*
ss
=
(
StreamState
*
)
s
;
void
streamStateDestroy
(
void
*
s
)
{
if
(
NULL
==
s
)
{
return
;
}
StreamState
*
ss
=
(
StreamState
*
)
s
;
fstNodeDestroy
(
ss
->
node
);
// free(s->autoState);
}
FstStreamBuilder
*
fstStreamBuilderCreate
(
Fst
*
fst
,
AutomationCtx
*
aut
)
{
FstStreamBuilder
*
b
=
calloc
(
1
,
sizeof
(
FstStreamBuilder
));
if
(
NULL
==
b
)
{
return
NULL
;
}
FstStreamBuilder
*
fstStreamBuilderCreate
(
Fst
*
fst
,
AutomationCtx
*
aut
)
{
FstStreamBuilder
*
b
=
calloc
(
1
,
sizeof
(
FstStreamBuilder
));
if
(
NULL
==
b
)
{
return
NULL
;
}
b
->
fst
=
fst
;
b
->
aut
=
aut
;
...
...
@@ -1393,17 +1323,15 @@ FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) {
b
->
max
=
fstBoundStateCreate
(
Unbounded
,
NULL
);
return
b
;
}
void
fstStreamBuilderDestroy
(
FstStreamBuilder
*
b
)
{
void
fstStreamBuilderDestroy
(
FstStreamBuilder
*
b
)
{
fstSliceDestroy
(
&
b
->
min
->
data
);
tfree
(
b
->
min
);
fstSliceDestroy
(
&
b
->
max
->
data
);
tfree
(
b
->
max
);
free
(
b
);
}
FstStreamBuilder
*
fstStreamBuilderRange
(
FstStreamBuilder
*
b
,
FstSlice
*
val
,
RangeType
type
)
{
if
(
b
==
NULL
)
{
return
NULL
;
}
FstStreamBuilder
*
fstStreamBuilderRange
(
FstStreamBuilder
*
b
,
FstSlice
*
val
,
RangeType
type
)
{
if
(
b
==
NULL
)
{
return
NULL
;
}
if
(
type
==
GE
)
{
b
->
min
->
type
=
Included
;
...
...
source/libs/index/src/index_fst_automation.c
浏览文件 @
d96963a2
...
...
@@ -15,19 +15,17 @@
#include "index_fst_automation.h"
StartWithStateValue
*
startWithStateValueCreate
(
StartWithStateKind
kind
,
ValueType
ty
,
void
*
val
)
{
StartWithStateValue
*
nsv
=
calloc
(
1
,
sizeof
(
StartWithStateValue
));
if
(
nsv
==
NULL
)
{
return
NULL
;
}
StartWithStateValue
*
startWithStateValueCreate
(
StartWithStateKind
kind
,
ValueType
ty
,
void
*
val
)
{
StartWithStateValue
*
nsv
=
calloc
(
1
,
sizeof
(
StartWithStateValue
));
if
(
nsv
==
NULL
)
{
return
NULL
;
}
nsv
->
kind
=
kind
;
nsv
->
type
=
ty
;
if
(
ty
==
FST_INT
)
{
nsv
->
val
=
*
(
int
*
)
val
;
nsv
->
val
=
*
(
int
*
)
val
;
}
else
if
(
ty
==
FST_CHAR
)
{
size_t
len
=
strlen
((
char
*
)
val
);
nsv
->
ptr
=
(
char
*
)
calloc
(
1
,
len
+
1
);
size_t
len
=
strlen
((
char
*
)
val
);
nsv
->
ptr
=
(
char
*
)
calloc
(
1
,
len
+
1
);
memcpy
(
nsv
->
ptr
,
val
,
len
);
}
else
if
(
ty
==
FST_ARRAY
)
{
// TODO,
...
...
@@ -35,11 +33,9 @@ StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueTyp
}
return
nsv
;
}
void
startWithStateValueDestroy
(
void
*
val
)
{
StartWithStateValue
*
sv
=
(
StartWithStateValue
*
)
val
;
if
(
sv
==
NULL
)
{
return
;
}
void
startWithStateValueDestroy
(
void
*
val
)
{
StartWithStateValue
*
sv
=
(
StartWithStateValue
*
)
val
;
if
(
sv
==
NULL
)
{
return
;
}
if
(
sv
->
type
==
FST_INT
)
{
//
...
...
@@ -50,11 +46,9 @@ void startWithStateValueDestroy(void *val) {
}
free
(
sv
);
}
StartWithStateValue
*
startWithStateValueDump
(
StartWithStateValue
*
sv
)
{
StartWithStateValue
*
nsv
=
calloc
(
1
,
sizeof
(
StartWithStateValue
));
if
(
nsv
==
NULL
)
{
return
NULL
;
}
StartWithStateValue
*
startWithStateValueDump
(
StartWithStateValue
*
sv
)
{
StartWithStateValue
*
nsv
=
calloc
(
1
,
sizeof
(
StartWithStateValue
));
if
(
nsv
==
NULL
)
{
return
NULL
;
}
nsv
->
kind
=
sv
->
kind
;
nsv
->
type
=
sv
->
type
;
...
...
@@ -62,41 +56,40 @@ StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) {
nsv
->
val
=
sv
->
val
;
}
else
if
(
nsv
->
type
==
FST_CHAR
)
{
size_t
len
=
strlen
(
sv
->
ptr
);
nsv
->
ptr
=
(
char
*
)
calloc
(
1
,
len
+
1
);
nsv
->
ptr
=
(
char
*
)
calloc
(
1
,
len
+
1
);
memcpy
(
nsv
->
ptr
,
sv
->
ptr
,
len
);
}
else
if
(
nsv
->
type
==
FST_ARRAY
)
{
//
}
return
nsv
;
}
// prefix query, impl later
static
void
*
prefixStart
(
AutomationCtx
*
ctx
)
{
StartWithStateValue
*
data
=
(
StartWithStateValue
*
)(
ctx
->
stdata
);
static
void
*
prefixStart
(
AutomationCtx
*
ctx
)
{
StartWithStateValue
*
data
=
(
StartWithStateValue
*
)(
ctx
->
stdata
);
return
startWithStateValueDump
(
data
);
};
static
bool
prefixIsMatch
(
AutomationCtx
*
ctx
,
void
*
sv
)
{
StartWithStateValue
*
ssv
=
(
StartWithStateValue
*
)
sv
;
static
bool
prefixIsMatch
(
AutomationCtx
*
ctx
,
void
*
sv
)
{
StartWithStateValue
*
ssv
=
(
StartWithStateValue
*
)
sv
;
return
ssv
->
val
==
strlen
(
ctx
->
data
);
}
static
bool
prefixCanMatch
(
AutomationCtx
*
ctx
,
void
*
sv
)
{
StartWithStateValue
*
ssv
=
(
StartWithStateValue
*
)
sv
;
static
bool
prefixCanMatch
(
AutomationCtx
*
ctx
,
void
*
sv
)
{
StartWithStateValue
*
ssv
=
(
StartWithStateValue
*
)
sv
;
return
ssv
->
val
>=
0
;
}
static
bool
prefixWillAlwaysMatch
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
true
;
}
static
void
*
prefixAccept
(
AutomationCtx
*
ctx
,
void
*
state
,
uint8_t
byte
)
{
StartWithStateValue
*
ssv
=
(
StartWithStateValue
*
)
state
;
if
(
ssv
==
NULL
||
ctx
==
NULL
)
{
return
NULL
;
}
static
bool
prefixWillAlwaysMatch
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
true
;
}
static
void
*
prefixAccept
(
AutomationCtx
*
ctx
,
void
*
state
,
uint8_t
byte
)
{
StartWithStateValue
*
ssv
=
(
StartWithStateValue
*
)
state
;
if
(
ssv
==
NULL
||
ctx
==
NULL
)
{
return
NULL
;
}
char
*
data
=
ctx
->
data
;
if
(
ssv
->
kind
==
Done
)
{
return
startWithStateValueCreate
(
Done
,
FST_INT
,
&
ssv
->
val
);
}
char
*
data
=
ctx
->
data
;
if
(
ssv
->
kind
==
Done
)
{
return
startWithStateValueCreate
(
Done
,
FST_INT
,
&
ssv
->
val
);
}
if
((
strlen
(
data
)
>
ssv
->
val
)
&&
data
[
ssv
->
val
]
==
byte
)
{
int
val
=
ssv
->
val
+
1
;
StartWithStateValue
*
nsv
=
startWithStateValueCreate
(
Running
,
FST_INT
,
&
val
);
StartWithStateValue
*
nsv
=
startWithStateValueCreate
(
Running
,
FST_INT
,
&
val
);
if
(
prefixIsMatch
(
ctx
,
nsv
))
{
nsv
->
kind
=
Done
;
}
else
{
...
...
@@ -106,18 +99,32 @@ static void *prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) {
}
return
NULL
;
}
static
void
*
prefixAcceptEof
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
NULL
;
}
static
void
*
prefixAcceptEof
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
NULL
;
}
// pattern query, impl later
static
void
*
patternStart
(
AutomationCtx
*
ctx
)
{
return
NULL
;
}
static
bool
patternIsMatch
(
AutomationCtx
*
ctx
,
void
*
data
)
{
return
true
;
}
static
bool
patternCanMatch
(
AutomationCtx
*
ctx
,
void
*
data
)
{
return
true
;
}
static
bool
patternWillAlwaysMatch
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
true
;
}
static
void
*
patternStart
(
AutomationCtx
*
ctx
)
{
return
NULL
;
}
static
bool
patternIsMatch
(
AutomationCtx
*
ctx
,
void
*
data
)
{
return
true
;
}
static
bool
patternCanMatch
(
AutomationCtx
*
ctx
,
void
*
data
)
{
return
true
;
}
static
bool
patternWillAlwaysMatch
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
true
;
}
static
void
*
patternAccept
(
AutomationCtx
*
ctx
,
void
*
state
,
uint8_t
byte
)
{
return
NULL
;
}
static
void
*
patternAccept
(
AutomationCtx
*
ctx
,
void
*
state
,
uint8_t
byte
)
{
return
NULL
;
}
static
void
*
patternAcceptEof
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
NULL
;
}
static
void
*
patternAcceptEof
(
AutomationCtx
*
ctx
,
void
*
state
)
{
return
NULL
;
}
AutomationFunc
automFuncs
[]
=
{
{
prefixStart
,
prefixIsMatch
,
prefixCanMatch
,
prefixWillAlwaysMatch
,
prefixAccept
,
prefixAcceptEof
},
...
...
@@ -125,34 +132,32 @@ AutomationFunc automFuncs[] = {
// add more search type
};
AutomationCtx
*
automCtxCreate
(
void
*
data
,
AutomationType
atype
)
{
AutomationCtx
*
ctx
=
calloc
(
1
,
sizeof
(
AutomationCtx
));
if
(
ctx
==
NULL
)
{
return
NULL
;
}
AutomationCtx
*
automCtxCreate
(
void
*
data
,
AutomationType
atype
)
{
AutomationCtx
*
ctx
=
calloc
(
1
,
sizeof
(
AutomationCtx
));
if
(
ctx
==
NULL
)
{
return
NULL
;
}
StartWithStateValue
*
sv
=
NULL
;
StartWithStateValue
*
sv
=
NULL
;
if
(
atype
==
AUTOMATION_PREFIX
)
{
int
val
=
0
;
sv
=
startWithStateValueCreate
(
Running
,
FST_INT
,
&
val
);
ctx
->
stdata
=
(
void
*
)
sv
;
ctx
->
stdata
=
(
void
*
)
sv
;
}
else
if
(
atype
==
AUTMMATION_MATCH
)
{
}
else
{
// add more search type
}
char
*
src
=
(
char
*
)
data
;
char
*
src
=
(
char
*
)
data
;
size_t
len
=
strlen
(
src
);
char
*
dst
=
(
char
*
)
malloc
(
len
*
sizeof
(
char
)
+
1
);
char
*
dst
=
(
char
*
)
malloc
(
len
*
sizeof
(
char
)
+
1
);
memcpy
(
dst
,
src
,
len
);
dst
[
len
]
=
0
;
ctx
->
data
=
dst
;
ctx
->
type
=
atype
;
ctx
->
stdata
=
(
void
*
)
sv
;
ctx
->
stdata
=
(
void
*
)
sv
;
return
ctx
;
}
void
automCtxDestroy
(
AutomationCtx
*
ctx
)
{
void
automCtxDestroy
(
AutomationCtx
*
ctx
)
{
startWithStateValueDestroy
(
ctx
->
stdata
);
free
(
ctx
->
data
);
free
(
ctx
);
...
...
source/libs/index/src/index_fst_common.c
浏览文件 @
d96963a2
...
...
@@ -274,260 +274,20 @@ const uint8_t COMMON_INPUTS[] = {
};
const
char
COMMON_INPUTS_INV
[]
=
{
't'
,
'e'
,
'/'
,
'o'
,
'a'
,
's'
,
'r'
,
'i'
,
'p'
,
'c'
,
'n'
,
'w'
,
'.'
,
'h'
,
'l'
,
'm'
,
'-'
,
'd'
,
'u'
,
'0'
,
'1'
,
'2'
,
'g'
,
'='
,
':'
,
'b'
,
'f'
,
'3'
,
'y'
,
'5'
,
'&'
,
'_'
,
'4'
,
'v'
,
'9'
,
'6'
,
'7'
,
'8'
,
'k'
,
'%'
,
'?'
,
'x'
,
'C'
,
'D'
,
'A'
,
'S'
,
'F'
,
'I'
,
'B'
,
'E'
,
'j'
,
'P'
,
'T'
,
'z'
,
'R'
,
'N'
,
'M'
,
'+'
,
'L'
,
'O'
,
'q'
,
'H'
,
'G'
,
'W'
,
'U'
,
'V'
,
','
,
'Y'
,
'K'
,
'J'
,
'Z'
,
'X'
,
'Q'
,
';'
,
')'
,
'('
,
'~'
,
'['
,
']'
,
'$'
,
'!'
,
'\''
,
'*'
,
'@'
,
'\x00'
,
'\x01'
,
'\x02'
,
'\x03'
,
'\x04'
,
'\x05'
,
'\x06'
,
'\x07'
,
'\x08'
,
'\t'
,
'\n'
,
'\x0b'
,
'\x0c'
,
'\r'
,
'\x0e'
,
'\x0f'
,
'\x10'
,
'\x11'
,
'\x12'
,
'\x13'
,
'\x14'
,
'\x15'
,
'\x16'
,
'\x17'
,
'\x18'
,
'\x19'
,
'\x1a'
,
'\x1b'
,
'\x1c'
,
'\x1d'
,
'\x1e'
,
'\x1f'
,
' '
,
'"'
,
'#'
,
'<'
,
'>'
,
'\\'
,
'^'
,
'`'
,
'{'
,
'|'
,
'}'
,
'\x7f'
,
'\x80'
,
'\x81'
,
'\x82'
,
'\x83'
,
'\x84'
,
'\x85'
,
'\x86'
,
'\x87'
,
'\x88'
,
'\x89'
,
'\x8a'
,
'\x8b'
,
'\x8c'
,
'\x8d'
,
'\x8e'
,
'\x8f'
,
'\x90'
,
'\x91'
,
'\x92'
,
'\x93'
,
'\x94'
,
'\x95'
,
'\x96'
,
'\x97'
,
'\x98'
,
'\x99'
,
'\x9a'
,
'\x9b'
,
'\x9c'
,
'\x9d'
,
'\x9e'
,
'\x9f'
,
'\xa0'
,
'\xa1'
,
'\xa2'
,
'\xa3'
,
'\xa4'
,
'\xa5'
,
'\xa6'
,
'\xa7'
,
'\xa8'
,
'\xa9'
,
'\xaa'
,
'\xab'
,
'\xac'
,
'\xad'
,
'\xae'
,
'\xaf'
,
'\xb0'
,
'\xb1'
,
'\xb2'
,
'\xb3'
,
'\xb4'
,
'\xb5'
,
'\xb6'
,
'\xb7'
,
'\xb8'
,
'\xb9'
,
'\xba'
,
'\xbb'
,
'\xbc'
,
'\xbd'
,
'\xbe'
,
'\xbf'
,
'\xc0'
,
'\xc1'
,
'\xc2'
,
'\xc3'
,
'\xc4'
,
'\xc5'
,
'\xc6'
,
'\xc7'
,
'\xc8'
,
'\xc9'
,
'\xca'
,
'\xcb'
,
'\xcc'
,
'\xcd'
,
'\xce'
,
'\xcf'
,
'\xd0'
,
'\xd1'
,
'\xd2'
,
'\xd3'
,
'\xd4'
,
'\xd5'
,
'\xd6'
,
'\xd7'
,
'\xd8'
,
'\xd9'
,
'\xda'
,
'\xdb'
,
'\xdc'
,
'\xdd'
,
'\xde'
,
'\xdf'
,
'\xe0'
,
'\xe1'
,
'\xe2'
,
'\xe3'
,
'\xe4'
,
'\xe5'
,
'\xe6'
,
'\xe7'
,
'\xe8'
,
'\xe9'
,
'\xea'
,
'\xeb'
,
'\xec'
,
'\xed'
,
'\xee'
,
'\xef'
,
'\xf0'
,
'\xf1'
,
'\xf2'
,
'\xf3'
,
'\xf4'
,
'\xf5'
,
'\xf6'
,
'\xf7'
,
'\xf8'
,
'\xf9'
,
'\xfa'
,
'\xfb'
,
'\xfc'
,
'\xfd'
,
'\xfe'
,
'\xff'
,
't'
,
'e'
,
'/'
,
'o'
,
'a'
,
's'
,
'r'
,
'i'
,
'p'
,
'c'
,
'n'
,
'w'
,
'.'
,
'h'
,
'l'
,
'm'
,
'-'
,
'd'
,
'u'
,
'0'
,
'1'
,
'2'
,
'g'
,
'='
,
':'
,
'b'
,
'f'
,
'3'
,
'y'
,
'5'
,
'&'
,
'_'
,
'4'
,
'v'
,
'9'
,
'6'
,
'7'
,
'8'
,
'k'
,
'%'
,
'?'
,
'x'
,
'C'
,
'D'
,
'A'
,
'S'
,
'F'
,
'I'
,
'B'
,
'E'
,
'j'
,
'P'
,
'T'
,
'z'
,
'R'
,
'N'
,
'M'
,
'+'
,
'L'
,
'O'
,
'q'
,
'H'
,
'G'
,
'W'
,
'U'
,
'V'
,
','
,
'Y'
,
'K'
,
'J'
,
'Z'
,
'X'
,
'Q'
,
';'
,
')'
,
'('
,
'~'
,
'['
,
']'
,
'$'
,
'!'
,
'\''
,
'*'
,
'@'
,
'\x00'
,
'\x01'
,
'\x02'
,
'\x03'
,
'\x04'
,
'\x05'
,
'\x06'
,
'\x07'
,
'\x08'
,
'\t'
,
'\n'
,
'\x0b'
,
'\x0c'
,
'\r'
,
'\x0e'
,
'\x0f'
,
'\x10'
,
'\x11'
,
'\x12'
,
'\x13'
,
'\x14'
,
'\x15'
,
'\x16'
,
'\x17'
,
'\x18'
,
'\x19'
,
'\x1a'
,
'\x1b'
,
'\x1c'
,
'\x1d'
,
'\x1e'
,
'\x1f'
,
' '
,
'"'
,
'#'
,
'<'
,
'>'
,
'\\'
,
'^'
,
'`'
,
'{'
,
'|'
,
'}'
,
'\x7f'
,
'\x80'
,
'\x81'
,
'\x82'
,
'\x83'
,
'\x84'
,
'\x85'
,
'\x86'
,
'\x87'
,
'\x88'
,
'\x89'
,
'\x8a'
,
'\x8b'
,
'\x8c'
,
'\x8d'
,
'\x8e'
,
'\x8f'
,
'\x90'
,
'\x91'
,
'\x92'
,
'\x93'
,
'\x94'
,
'\x95'
,
'\x96'
,
'\x97'
,
'\x98'
,
'\x99'
,
'\x9a'
,
'\x9b'
,
'\x9c'
,
'\x9d'
,
'\x9e'
,
'\x9f'
,
'\xa0'
,
'\xa1'
,
'\xa2'
,
'\xa3'
,
'\xa4'
,
'\xa5'
,
'\xa6'
,
'\xa7'
,
'\xa8'
,
'\xa9'
,
'\xaa'
,
'\xab'
,
'\xac'
,
'\xad'
,
'\xae'
,
'\xaf'
,
'\xb0'
,
'\xb1'
,
'\xb2'
,
'\xb3'
,
'\xb4'
,
'\xb5'
,
'\xb6'
,
'\xb7'
,
'\xb8'
,
'\xb9'
,
'\xba'
,
'\xbb'
,
'\xbc'
,
'\xbd'
,
'\xbe'
,
'\xbf'
,
'\xc0'
,
'\xc1'
,
'\xc2'
,
'\xc3'
,
'\xc4'
,
'\xc5'
,
'\xc6'
,
'\xc7'
,
'\xc8'
,
'\xc9'
,
'\xca'
,
'\xcb'
,
'\xcc'
,
'\xcd'
,
'\xce'
,
'\xcf'
,
'\xd0'
,
'\xd1'
,
'\xd2'
,
'\xd3'
,
'\xd4'
,
'\xd5'
,
'\xd6'
,
'\xd7'
,
'\xd8'
,
'\xd9'
,
'\xda'
,
'\xdb'
,
'\xdc'
,
'\xdd'
,
'\xde'
,
'\xdf'
,
'\xe0'
,
'\xe1'
,
'\xe2'
,
'\xe3'
,
'\xe4'
,
'\xe5'
,
'\xe6'
,
'\xe7'
,
'\xe8'
,
'\xe9'
,
'\xea'
,
'\xeb'
,
'\xec'
,
'\xed'
,
'\xee'
,
'\xef'
,
'\xf0'
,
'\xf1'
,
'\xf2'
,
'\xf3'
,
'\xf4'
,
'\xf5'
,
'\xf6'
,
'\xf7'
,
'\xf8'
,
'\xf9'
,
'\xfa'
,
'\xfb'
,
'\xfc'
,
'\xfd'
,
'\xfe'
,
'\xff'
,
};
source/libs/index/src/index_fst_counting_writer.c
浏览文件 @
d96963a2
...
...
@@ -17,10 +17,8 @@
#include "index_fst_util.h"
#include "tutil.h"
static
int
writeCtxDoWrite
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
)
{
if
(
ctx
->
offset
+
len
>
ctx
->
limit
)
{
return
-
1
;
}
static
int
writeCtxDoWrite
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
)
{
if
(
ctx
->
offset
+
len
>
ctx
->
limit
)
{
return
-
1
;
}
if
(
ctx
->
type
==
TFile
)
{
assert
(
len
==
tfWrite
(
ctx
->
file
.
fd
,
buf
,
len
));
...
...
@@ -30,7 +28,7 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) {
ctx
->
offset
+=
len
;
return
len
;
}
static
int
writeCtxDoRead
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
)
{
static
int
writeCtxDoRead
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
)
{
int
nRead
=
0
;
if
(
ctx
->
type
==
TFile
)
{
nRead
=
tfRead
(
ctx
->
file
.
fd
,
buf
,
len
);
...
...
@@ -41,7 +39,18 @@ static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) {
return
nRead
;
}
static
int
writeCtxDoFlush
(
WriterCtx
*
ctx
)
{
static
int
writeCtxDoReadFrom
(
WriterCtx
*
ctx
,
uint8_t
*
buf
,
int
len
,
int32_t
offset
)
{
int
nRead
=
0
;
if
(
ctx
->
type
==
TFile
)
{
tfLseek
(
ctx
->
file
.
fd
,
offset
,
0
);
nRead
=
tfRead
(
ctx
->
file
.
fd
,
buf
,
len
);
}
else
{
// refactor later
assert
(
0
);
}
return
nRead
;
}
static
int
writeCtxDoFlush
(
WriterCtx
*
ctx
)
{
if
(
ctx
->
type
==
TFile
)
{
// tfFsync(ctx->fd);
// tfFlush(ctx->file.fd);
...
...
@@ -51,11 +60,9 @@ static int writeCtxDoFlush(WriterCtx *ctx) {
return
1
;
}
WriterCtx
*
writerCtxCreate
(
WriterType
type
,
const
char
*
path
,
bool
readOnly
,
int32_t
capacity
)
{
WriterCtx
*
ctx
=
calloc
(
1
,
sizeof
(
WriterCtx
));
if
(
ctx
==
NULL
)
{
return
NULL
;
}
WriterCtx
*
writerCtxCreate
(
WriterType
type
,
const
char
*
path
,
bool
readOnly
,
int32_t
capacity
)
{
WriterCtx
*
ctx
=
calloc
(
1
,
sizeof
(
WriterCtx
));
if
(
ctx
==
NULL
)
{
return
NULL
;
}
ctx
->
type
=
type
;
if
(
ctx
->
type
==
TFile
)
{
...
...
@@ -67,8 +74,8 @@ WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int
ctx
->
file
.
fd
=
tfOpenReadWrite
(
tmpFile
);
}
if
(
ctx
->
file
.
fd
<
0
)
{
goto
END
;
indexError
(
"open file error %d"
,
errno
);
goto
END
;
}
}
else
if
(
ctx
->
type
==
TMemory
)
{
ctx
->
mem
.
buf
=
calloc
(
1
,
sizeof
(
char
)
*
capacity
);
...
...
@@ -77,18 +84,17 @@ WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int
ctx
->
write
=
writeCtxDoWrite
;
ctx
->
read
=
writeCtxDoRead
;
ctx
->
flush
=
writeCtxDoFlush
;
ctx
->
readFrom
=
writeCtxDoReadFrom
;
ctx
->
offset
=
0
;
ctx
->
limit
=
capacity
;
return
ctx
;
END:
if
(
ctx
->
type
==
TMemory
)
{
free
(
ctx
->
mem
.
buf
);
}
if
(
ctx
->
type
==
TMemory
)
{
free
(
ctx
->
mem
.
buf
);
}
free
(
ctx
);
}
void
writerCtxDestroy
(
WriterCtx
*
ctx
)
{
void
writerCtxDestroy
(
WriterCtx
*
ctx
)
{
if
(
ctx
->
type
==
TMemory
)
{
free
(
ctx
->
mem
.
buf
);
}
else
{
...
...
@@ -97,57 +103,53 @@ void writerCtxDestroy(WriterCtx *ctx) {
free
(
ctx
);
}
FstCountingWriter
*
fstCountingWriterCreate
(
void
*
wrt
)
{
FstCountingWriter
*
cw
=
calloc
(
1
,
sizeof
(
FstCountingWriter
));
if
(
cw
==
NULL
)
{
return
NULL
;
}
FstCountingWriter
*
fstCountingWriterCreate
(
void
*
wrt
)
{
FstCountingWriter
*
cw
=
calloc
(
1
,
sizeof
(
FstCountingWriter
));
if
(
cw
==
NULL
)
{
return
NULL
;
}
cw
->
wrt
=
wrt
;
//(void *)(writerCtxCreate(TFile, readOnly));
return
cw
;
}
void
fstCountingWriterDestroy
(
FstCountingWriter
*
cw
)
{
void
fstCountingWriterDestroy
(
FstCountingWriter
*
cw
)
{
// free wrt object: close fd or free mem
fstCountingWriterFlush
(
cw
);
// writerCtxDestroy((WriterCtx *)(cw->wrt));
free
(
cw
);
}
int
fstCountingWriterWrite
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
)
{
if
(
write
==
NULL
)
{
return
0
;
}
int
fstCountingWriterWrite
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
)
{
if
(
write
==
NULL
)
{
return
0
;
}
// update checksum
// write data to file/socket or mem
WriterCtx
*
ctx
=
write
->
wrt
;
WriterCtx
*
ctx
=
write
->
wrt
;
int
nWrite
=
ctx
->
write
(
ctx
,
buf
,
len
);
assert
(
nWrite
==
len
);
write
->
count
+=
len
;
return
len
;
}
int
fstCountingWriterRead
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
)
{
if
(
write
==
NULL
)
{
return
0
;
}
WriterCtx
*
ctx
=
write
->
wrt
;
int
fstCountingWriterRead
(
FstCountingWriter
*
write
,
uint8_t
*
buf
,
uint32_t
len
)
{
if
(
write
==
NULL
)
{
return
0
;
}
WriterCtx
*
ctx
=
write
->
wrt
;
int
nRead
=
ctx
->
read
(
ctx
,
buf
,
len
);
// assert(nRead == len);
return
nRead
;
}
uint32_t
fstCountingWriterMaskedCheckSum
(
FstCountingWriter
*
write
)
{
return
0
;
}
int
fstCountingWriterFlush
(
FstCountingWriter
*
write
)
{
WriterCtx
*
ctx
=
write
->
wrt
;
uint32_t
fstCountingWriterMaskedCheckSum
(
FstCountingWriter
*
write
)
{
return
0
;
}
int
fstCountingWriterFlush
(
FstCountingWriter
*
write
)
{
WriterCtx
*
ctx
=
write
->
wrt
;
ctx
->
flush
(
ctx
);
// write->wtr->flush
return
1
;
}
void
fstCountingWriterPackUintIn
(
FstCountingWriter
*
writer
,
uint64_t
n
,
uint8_t
nBytes
)
{
void
fstCountingWriterPackUintIn
(
FstCountingWriter
*
writer
,
uint64_t
n
,
uint8_t
nBytes
)
{
assert
(
1
<=
nBytes
&&
nBytes
<=
8
);
uint8_t
*
buf
=
calloc
(
8
,
sizeof
(
uint8_t
));
uint8_t
*
buf
=
calloc
(
8
,
sizeof
(
uint8_t
));
for
(
uint8_t
i
=
0
;
i
<
nBytes
;
i
++
)
{
buf
[
i
]
=
(
uint8_t
)
n
;
n
=
n
>>
8
;
...
...
@@ -157,7 +159,7 @@ void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t
return
;
}
uint8_t
fstCountingWriterPackUint
(
FstCountingWriter
*
writer
,
uint64_t
n
)
{
uint8_t
fstCountingWriterPackUint
(
FstCountingWriter
*
writer
,
uint64_t
n
)
{
uint8_t
nBytes
=
packSize
(
n
);
fstCountingWriterPackUintIn
(
writer
,
n
,
nBytes
);
return
nBytes
;
...
...
source/libs/index/src/index_fst_node.c
浏览文件 @
d96963a2
...
...
@@ -14,60 +14,46 @@
*/
#include "index_fst_node.h"
FstBuilderNode
*
fstBuilderNodeDefault
()
{
FstBuilderNode
*
bn
=
malloc
(
sizeof
(
FstBuilderNode
));
FstBuilderNode
*
fstBuilderNodeDefault
()
{
FstBuilderNode
*
bn
=
malloc
(
sizeof
(
FstBuilderNode
));
bn
->
isFinal
=
false
;
bn
->
finalOutput
=
0
;
bn
->
trans
=
taosArrayInit
(
16
,
sizeof
(
FstTransition
));
return
bn
;
}
void
fstBuilderNodeDestroy
(
FstBuilderNode
*
node
)
{
if
(
node
==
NULL
)
{
return
;
}
void
fstBuilderNodeDestroy
(
FstBuilderNode
*
node
)
{
if
(
node
==
NULL
)
{
return
;
}
taosArrayDestroy
(
node
->
trans
);
free
(
node
);
}
bool
fstBuilderNodeEqual
(
FstBuilderNode
*
n1
,
FstBuilderNode
*
n2
)
{
if
(
n1
==
n2
)
{
return
true
;
}
if
(
n1
==
NULL
||
n2
==
NULL
)
{
return
false
;
}
bool
fstBuilderNodeEqual
(
FstBuilderNode
*
n1
,
FstBuilderNode
*
n2
)
{
if
(
n1
==
n2
)
{
return
true
;
}
if
(
n1
==
NULL
||
n2
==
NULL
)
{
return
false
;
}
if
(
n1
->
isFinal
!=
n2
->
isFinal
||
n1
->
finalOutput
!=
n2
->
finalOutput
)
{
return
false
;
}
if
(
n1
->
isFinal
!=
n2
->
isFinal
||
n1
->
finalOutput
!=
n2
->
finalOutput
)
{
return
false
;
}
size_t
s1
=
n1
->
trans
?
taosArrayGetSize
(
n1
->
trans
)
:
0
;
size_t
s2
=
n2
->
trans
?
taosArrayGetSize
(
n2
->
trans
)
:
0
;
if
(
s1
!=
s2
)
{
return
false
;
}
if
(
s1
!=
s2
)
{
return
false
;
}
for
(
size_t
i
=
0
;
i
<
s1
;
i
++
)
{
FstTransition
*
t1
=
taosArrayGet
(
n1
->
trans
,
i
);
FstTransition
*
t2
=
taosArrayGet
(
n2
->
trans
,
i
);
if
(
t1
->
inp
!=
t2
->
inp
||
t1
->
out
!=
t2
->
out
||
t1
->
addr
!=
t2
->
addr
)
{
return
false
;
}
FstTransition
*
t1
=
taosArrayGet
(
n1
->
trans
,
i
);
FstTransition
*
t2
=
taosArrayGet
(
n2
->
trans
,
i
);
if
(
t1
->
inp
!=
t2
->
inp
||
t1
->
out
!=
t2
->
out
||
t1
->
addr
!=
t2
->
addr
)
{
return
false
;
}
}
return
true
;
}
FstBuilderNode
*
fstBuilderNodeClone
(
FstBuilderNode
*
src
)
{
FstBuilderNode
*
node
=
malloc
(
sizeof
(
FstBuilderNode
));
if
(
node
==
NULL
)
{
return
NULL
;
}
FstBuilderNode
*
fstBuilderNodeClone
(
FstBuilderNode
*
src
)
{
FstBuilderNode
*
node
=
malloc
(
sizeof
(
FstBuilderNode
));
if
(
node
==
NULL
)
{
return
NULL
;
}
//
size_t
sz
=
taosArrayGetSize
(
src
->
trans
);
SArray
*
trans
=
taosArrayInit
(
sz
,
sizeof
(
FstTransition
));
SArray
*
trans
=
taosArrayInit
(
sz
,
sizeof
(
FstTransition
));
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
FstTransition
*
tran
=
taosArrayGet
(
src
->
trans
,
i
);
FstTransition
*
tran
=
taosArrayGet
(
src
->
trans
,
i
);
taosArrayPush
(
trans
,
tran
);
}
...
...
@@ -77,10 +63,8 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) {
return
node
;
}
// not destroy src, User's bussiness
void
fstBuilderNodeCloneFrom
(
FstBuilderNode
*
dst
,
FstBuilderNode
*
src
)
{
if
(
dst
==
NULL
||
src
==
NULL
)
{
return
;
}
void
fstBuilderNodeCloneFrom
(
FstBuilderNode
*
dst
,
FstBuilderNode
*
src
)
{
if
(
dst
==
NULL
||
src
==
NULL
)
{
return
;
}
dst
->
isFinal
=
src
->
isFinal
;
dst
->
finalOutput
=
src
->
finalOutput
;
...
...
@@ -90,7 +74,7 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) {
size_t
sz
=
taosArrayGetSize
(
src
->
trans
);
dst
->
trans
=
taosArrayInit
(
sz
,
sizeof
(
FstTransition
));
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
FstTransition
*
trn
=
taosArrayGet
(
src
->
trans
,
i
);
FstTransition
*
trn
=
taosArrayGet
(
src
->
trans
,
i
);
taosArrayPush
(
dst
->
trans
,
trn
);
}
}
...
...
source/libs/index/src/index_fst_registry.c
浏览文件 @
d96963a2
...
...
@@ -15,7 +15,7 @@
#include "index_fst_registry.h"
uint64_t
fstRegistryHash
(
FstRegistry
*
registry
,
FstBuilderNode
*
bNode
)
{
uint64_t
fstRegistryHash
(
FstRegistry
*
registry
,
FstBuilderNode
*
bNode
)
{
// TODO(yihaoDeng): refactor later
const
uint64_t
FNV_PRIME
=
1099511628211
;
uint64_t
h
=
14695981039346656037u
;
...
...
@@ -25,21 +25,19 @@ uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) {
uint32_t
sz
=
(
uint32_t
)
taosArrayGetSize
(
bNode
->
trans
);
for
(
uint32_t
i
=
0
;
i
<
sz
;
i
++
)
{
FstTransition
*
trn
=
taosArrayGet
(
bNode
->
trans
,
i
);
FstTransition
*
trn
=
taosArrayGet
(
bNode
->
trans
,
i
);
h
=
(
h
^
(
uint64_t
)(
trn
->
inp
))
*
FNV_PRIME
;
h
=
(
h
^
(
uint64_t
)(
trn
->
out
))
*
FNV_PRIME
;
h
=
(
h
^
(
uint64_t
)(
trn
->
addr
))
*
FNV_PRIME
;
}
return
h
%
(
registry
->
tableSize
);
}
static
void
fstRegistryCellSwap
(
SArray
*
arr
,
uint32_t
a
,
uint32_t
b
)
{
static
void
fstRegistryCellSwap
(
SArray
*
arr
,
uint32_t
a
,
uint32_t
b
)
{
size_t
sz
=
taosArrayGetSize
(
arr
);
if
(
a
>=
sz
||
b
>=
sz
)
{
return
;
}
if
(
a
>=
sz
||
b
>=
sz
)
{
return
;
}
FstRegistryCell
*
cell1
=
(
FstRegistryCell
*
)
taosArrayGet
(
arr
,
a
);
FstRegistryCell
*
cell2
=
(
FstRegistryCell
*
)
taosArrayGet
(
arr
,
b
);
FstRegistryCell
*
cell1
=
(
FstRegistryCell
*
)
taosArrayGet
(
arr
,
a
);
FstRegistryCell
*
cell2
=
(
FstRegistryCell
*
)
taosArrayGet
(
arr
,
b
);
FstRegistryCell
t
=
{.
addr
=
cell1
->
addr
,
.
node
=
cell1
->
node
};
...
...
@@ -51,11 +49,9 @@ static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) {
return
;
}
static
void
fstRegistryCellPromote
(
SArray
*
arr
,
uint32_t
start
,
uint32_t
end
)
{
static
void
fstRegistryCellPromote
(
SArray
*
arr
,
uint32_t
start
,
uint32_t
end
)
{
size_t
sz
=
taosArrayGetSize
(
arr
);
if
(
start
>=
sz
&&
end
>=
sz
)
{
return
;
}
if
(
start
>=
sz
&&
end
>=
sz
)
{
return
;
}
assert
(
start
>=
end
);
...
...
@@ -67,14 +63,12 @@ static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) {
}
}
FstRegistry
*
fstRegistryCreate
(
uint64_t
tableSize
,
uint64_t
mruSize
)
{
FstRegistry
*
registry
=
malloc
(
sizeof
(
FstRegistry
));
if
(
registry
==
NULL
)
{
return
NULL
;
}
FstRegistry
*
fstRegistryCreate
(
uint64_t
tableSize
,
uint64_t
mruSize
)
{
FstRegistry
*
registry
=
malloc
(
sizeof
(
FstRegistry
));
if
(
registry
==
NULL
)
{
return
NULL
;
}
uint64_t
nCells
=
tableSize
*
mruSize
;
SArray
*
tb
=
(
SArray
*
)
taosArrayInit
(
nCells
,
sizeof
(
FstRegistryCell
));
SArray
*
tb
=
(
SArray
*
)
taosArrayInit
(
nCells
,
sizeof
(
FstRegistryCell
));
if
(
NULL
==
tb
)
{
free
(
registry
);
return
NULL
;
...
...
@@ -91,32 +85,28 @@ FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) {
return
registry
;
}
void
fstRegistryDestroy
(
FstRegistry
*
registry
)
{
if
(
registry
==
NULL
)
{
return
;
}
void
fstRegistryDestroy
(
FstRegistry
*
registry
)
{
if
(
registry
==
NULL
)
{
return
;
}
SArray
*
tb
=
registry
->
table
;
SArray
*
tb
=
registry
->
table
;
size_t
sz
=
taosArrayGetSize
(
tb
);
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
FstRegistryCell
*
cell
=
taosArrayGet
(
tb
,
i
);
FstRegistryCell
*
cell
=
taosArrayGet
(
tb
,
i
);
fstBuilderNodeDestroy
(
cell
->
node
);
}
taosArrayDestroy
(
tb
);
free
(
registry
);
}
FstRegistryEntry
*
fstRegistryGetEntry
(
FstRegistry
*
registry
,
FstBuilderNode
*
bNode
)
{
if
(
taosArrayGetSize
(
registry
->
table
)
<=
0
)
{
return
NULL
;
}
FstRegistryEntry
*
fstRegistryGetEntry
(
FstRegistry
*
registry
,
FstBuilderNode
*
bNode
)
{
if
(
taosArrayGetSize
(
registry
->
table
)
<=
0
)
{
return
NULL
;
}
uint64_t
bucket
=
fstRegistryHash
(
registry
,
bNode
);
uint64_t
start
=
registry
->
mruSize
*
bucket
;
uint64_t
end
=
start
+
registry
->
mruSize
;
FstRegistryEntry
*
entry
=
malloc
(
sizeof
(
FstRegistryEntry
));
FstRegistryEntry
*
entry
=
malloc
(
sizeof
(
FstRegistryEntry
));
if
(
end
-
start
==
1
)
{
FstRegistryCell
*
cell
=
taosArrayGet
(
registry
->
table
,
start
);
FstRegistryCell
*
cell
=
taosArrayGet
(
registry
->
table
,
start
);
// cell->isNode &&
if
(
cell
->
addr
!=
NONE_ADDRESS
&&
fstBuilderNodeEqual
(
cell
->
node
,
bNode
))
{
entry
->
state
=
FOUND
;
...
...
@@ -128,13 +118,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
entry
->
cell
=
cell
;
// copy or not
}
}
else
if
(
end
-
start
==
2
)
{
FstRegistryCell
*
cell1
=
taosArrayGet
(
registry
->
table
,
start
);
FstRegistryCell
*
cell1
=
taosArrayGet
(
registry
->
table
,
start
);
if
(
cell1
->
addr
!=
NONE_ADDRESS
&&
fstBuilderNodeEqual
(
cell1
->
node
,
bNode
))
{
entry
->
state
=
FOUND
;
entry
->
addr
=
cell1
->
addr
;
return
entry
;
}
FstRegistryCell
*
cell2
=
taosArrayGet
(
registry
->
table
,
start
+
1
);
FstRegistryCell
*
cell2
=
taosArrayGet
(
registry
->
table
,
start
+
1
);
if
(
cell2
->
addr
!=
NONE_ADDRESS
&&
fstBuilderNodeEqual
(
cell2
->
node
,
bNode
))
{
entry
->
state
=
FOUND
;
entry
->
addr
=
cell2
->
addr
;
...
...
@@ -146,13 +136,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
fstBuilderNodeCloneFrom
(
cell2
->
node
,
bNode
);
fstRegistryCellSwap
(
registry
->
table
,
start
,
start
+
1
);
FstRegistryCell
*
cCell
=
taosArrayGet
(
registry
->
table
,
start
);
FstRegistryCell
*
cCell
=
taosArrayGet
(
registry
->
table
,
start
);
entry
->
state
=
NOTFOUND
;
entry
->
cell
=
cCell
;
}
else
{
uint32_t
i
=
start
;
for
(;
i
<
end
;
i
++
)
{
FstRegistryCell
*
cell
=
(
FstRegistryCell
*
)
taosArrayGet
(
registry
->
table
,
i
);
FstRegistryCell
*
cell
=
(
FstRegistryCell
*
)
taosArrayGet
(
registry
->
table
,
i
);
if
(
cell
->
addr
!=
NONE_ADDRESS
&&
fstBuilderNodeEqual
(
cell
->
node
,
bNode
))
{
entry
->
state
=
FOUND
;
entry
->
addr
=
cell
->
addr
;
...
...
@@ -162,16 +152,18 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo
}
if
(
i
>=
end
)
{
uint64_t
last
=
end
-
1
;
FstRegistryCell
*
cell
=
(
FstRegistryCell
*
)
taosArrayGet
(
registry
->
table
,
last
);
FstRegistryCell
*
cell
=
(
FstRegistryCell
*
)
taosArrayGet
(
registry
->
table
,
last
);
// clone from bNode, refactor later
fstBuilderNodeCloneFrom
(
cell
->
node
,
bNode
);
fstRegistryCellPromote
(
registry
->
table
,
last
,
start
);
FstRegistryCell
*
cCell
=
taosArrayGet
(
registry
->
table
,
start
);
FstRegistryCell
*
cCell
=
taosArrayGet
(
registry
->
table
,
start
);
entry
->
state
=
NOTFOUND
;
entry
->
cell
=
cCell
;
}
}
return
entry
;
}
void
fstRegistryEntryDestroy
(
FstRegistryEntry
*
entry
)
{
free
(
entry
);
}
void
fstRegistryEntryDestroy
(
FstRegistryEntry
*
entry
)
{
free
(
entry
);
}
source/libs/index/src/index_fst_util.c
浏览文件 @
d96963a2
...
...
@@ -61,9 +61,10 @@ uint8_t packSize(uint64_t n) {
}
}
uint64_t
unpackUint64
(
uint8_t
*
ch
,
uint8_t
sz
)
{
uint64_t
unpackUint64
(
uint8_t
*
ch
,
uint8_t
sz
)
{
uint64_t
n
=
0
;
for
(
uint8_t
i
=
0
;
i
<
sz
;
i
++
)
{
//
n
=
n
|
(
ch
[
i
]
<<
(
8
*
i
));
}
return
n
;
...
...
@@ -75,7 +76,7 @@ uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr) {
return
packSize
(
nodeAddr
-
transAddr
);
}
}
CompiledAddr
unpackDelta
(
char
*
data
,
uint64_t
len
,
uint64_t
nodeAddr
)
{
CompiledAddr
unpackDelta
(
char
*
data
,
uint64_t
len
,
uint64_t
nodeAddr
)
{
uint64_t
delta
=
unpackUint64
(
data
,
len
);
// delta_add = u64_to_usize
if
(
delta
==
EMPTY_ADDRESS
)
{
...
...
@@ -88,8 +89,8 @@ CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) {
// fst slice func
//
FstSlice
fstSliceCreate
(
uint8_t
*
data
,
uint64_t
len
)
{
FstString
*
str
=
(
FstString
*
)
malloc
(
sizeof
(
FstString
));
FstSlice
fstSliceCreate
(
uint8_t
*
data
,
uint64_t
len
)
{
FstString
*
str
=
(
FstString
*
)
malloc
(
sizeof
(
FstString
));
str
->
ref
=
1
;
str
->
len
=
len
;
str
->
data
=
malloc
(
len
*
sizeof
(
uint8_t
));
...
...
@@ -99,8 +100,8 @@ FstSlice fstSliceCreate(uint8_t *data, uint64_t len) {
return
s
;
}
// just shallow copy
FstSlice
fstSliceCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
)
{
FstString
*
str
=
s
->
str
;
FstSlice
fstSliceCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
)
{
FstString
*
str
=
s
->
str
;
str
->
ref
++
;
// uint8_t *buf = fstSliceData(s, &alen);
// start = buf + start - (buf - s->start);
...
...
@@ -109,16 +110,16 @@ FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) {
FstSlice
t
=
{.
str
=
str
,
.
start
=
start
+
s
->
start
,
.
end
=
end
+
s
->
start
};
return
t
;
}
FstSlice
fstSliceDeepCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
)
{
FstSlice
fstSliceDeepCopy
(
FstSlice
*
s
,
int32_t
start
,
int32_t
end
)
{
int32_t
tlen
=
end
-
start
+
1
;
int32_t
slen
;
uint8_t
*
data
=
fstSliceData
(
s
,
&
slen
);
uint8_t
*
data
=
fstSliceData
(
s
,
&
slen
);
assert
(
tlen
<=
slen
);
uint8_t
*
buf
=
malloc
(
sizeof
(
uint8_t
)
*
tlen
);
uint8_t
*
buf
=
malloc
(
sizeof
(
uint8_t
)
*
tlen
);
memcpy
(
buf
,
data
+
start
,
tlen
);
FstString
*
str
=
malloc
(
sizeof
(
FstString
));
FstString
*
str
=
malloc
(
sizeof
(
FstString
));
str
->
data
=
buf
;
str
->
len
=
tlen
;
str
->
ref
=
1
;
...
...
@@ -129,17 +130,17 @@ FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) {
ans
.
end
=
tlen
-
1
;
return
ans
;
}
bool
fstSliceIsEmpty
(
FstSlice
*
s
)
{
return
s
->
str
==
NULL
||
s
->
str
->
len
==
0
||
s
->
start
<
0
||
s
->
end
<
0
;
}
bool
fstSliceIsEmpty
(
FstSlice
*
s
)
{
return
s
->
str
==
NULL
||
s
->
str
->
len
==
0
||
s
->
start
<
0
||
s
->
end
<
0
;
}
uint8_t
*
fstSliceData
(
FstSlice
*
s
,
int32_t
*
size
)
{
FstString
*
str
=
s
->
str
;
if
(
size
!=
NULL
)
{
*
size
=
s
->
end
-
s
->
start
+
1
;
}
uint8_t
*
fstSliceData
(
FstSlice
*
s
,
int32_t
*
size
)
{
FstString
*
str
=
s
->
str
;
if
(
size
!=
NULL
)
{
*
size
=
s
->
end
-
s
->
start
+
1
;
}
return
str
->
data
+
s
->
start
;
}
void
fstSliceDestroy
(
FstSlice
*
s
)
{
FstString
*
str
=
s
->
str
;
void
fstSliceDestroy
(
FstSlice
*
s
)
{
FstString
*
str
=
s
->
str
;
str
->
ref
--
;
if
(
str
->
ref
<=
0
)
{
free
(
str
->
data
);
...
...
@@ -148,10 +149,10 @@ void fstSliceDestroy(FstSlice *s) {
}
}
int
fstSliceCompare
(
FstSlice
*
a
,
FstSlice
*
b
)
{
int
fstSliceCompare
(
FstSlice
*
a
,
FstSlice
*
b
)
{
int32_t
alen
,
blen
;
uint8_t
*
aBuf
=
fstSliceData
(
a
,
&
alen
);
uint8_t
*
bBuf
=
fstSliceData
(
b
,
&
blen
);
uint8_t
*
aBuf
=
fstSliceData
(
a
,
&
alen
);
uint8_t
*
bBuf
=
fstSliceData
(
b
,
&
blen
);
uint32_t
i
,
j
;
for
(
i
=
0
,
j
=
0
;
i
<
alen
&&
j
<
blen
;
i
++
,
j
++
)
{
...
...
source/libs/index/src/index_tfile.c
浏览文件 @
d96963a2
...
...
@@ -21,107 +21,78 @@
#include "index_fst_counting_writer.h"
#include "index_util.h"
#include "taosdef.h"
#include "tcompare.h"
static
FORCE_INLINE
int
tfileReadLoadHeader
(
TFileReader
*
reader
)
{
// TODO simple tfile header later
char
buf
[
TFILE_HADER_PRE_SIZE
];
char
*
p
=
buf
;
TFileReadHeader
*
header
=
&
reader
->
header
;
int64_t
nread
=
reader
->
ctx
->
read
(
reader
->
ctx
,
buf
,
TFILE_HADER_PRE_SIZE
);
assert
(
nread
==
TFILE_HADER_PRE_SIZE
);
#define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t))
memcpy
(
&
header
->
suid
,
p
,
sizeof
(
header
->
suid
));
p
+=
sizeof
(
header
->
suid
);
typedef
struct
TFileValue
{
char
*
colVal
;
// null terminated
SArray
*
tableId
;
int32_t
offset
;
}
TFileValue
;
memcpy
(
&
header
->
version
,
p
,
sizeof
(
header
->
version
)
);
p
+=
sizeof
(
header
->
version
);
static
int
tfileValueCompare
(
const
void
*
a
,
const
void
*
b
,
const
void
*
param
);
static
void
tfileSerialTableIdsToBuf
(
char
*
buf
,
SArray
*
tableIds
);
int32_t
colLen
=
0
;
memcpy
(
&
colLen
,
p
,
sizeof
(
colLen
));
assert
(
colLen
<
sizeof
(
header
->
colName
));
nread
=
reader
->
ctx
->
read
(
reader
->
ctx
,
header
->
colName
,
colLen
);
assert
(
nread
==
colLen
);
static
int
tfileWriteHeader
(
TFileWriter
*
writer
);
static
int
tfileWriteFstOffset
(
TFileWriter
*
tw
,
int32_t
offset
);
static
int
tfileWriteData
(
TFileWriter
*
write
,
TFileValue
*
tval
);
nread
=
reader
->
ctx
->
read
(
reader
->
ctx
,
&
header
->
colType
,
sizeof
(
header
->
colType
));
return
0
;
};
static
int
tfileGetFileList
(
const
char
*
path
,
SArray
*
result
)
{
DIR
*
dir
=
opendir
(
path
);
if
(
NULL
==
dir
)
{
return
-
1
;
}
static
int
tfileReadLoadHeader
(
TFileReader
*
reader
);
static
int
tfileReadLoadFst
(
TFileReader
*
reader
);
static
int
tfileReadLoadTableIds
(
TFileReader
*
reader
,
int32_t
offset
,
SArray
*
result
);
struct
dirent
*
entry
;
while
((
entry
=
readdir
(
dir
))
!=
NULL
)
{
size_t
len
=
strlen
(
entry
->
d_name
);
char
*
buf
=
calloc
(
1
,
len
+
1
);
memcpy
(
buf
,
entry
->
d_name
,
len
);
taosArrayPush
(
result
,
&
buf
);
}
closedir
(
dir
);
return
0
;
}
static
void
tfileDestroyFileName
(
void
*
elem
)
{
char
*
p
=
*
(
char
**
)
elem
;
free
(
p
);
}
static
int
tfileCompare
(
const
void
*
a
,
const
void
*
b
)
{
const
char
*
aName
=
*
(
char
**
)
a
;
const
char
*
bName
=
*
(
char
**
)
b
;
size_t
aLen
=
strlen
(
aName
);
size_t
bLen
=
strlen
(
bName
);
return
strncmp
(
aName
,
bName
,
aLen
>
bLen
?
aLen
:
bLen
);
}
// tfile name suid-colId-version.tindex
static
int
tfileParseFileName
(
const
char
*
filename
,
uint64_t
*
suid
,
int
*
colId
,
int
*
version
)
{
if
(
3
==
sscanf
(
filename
,
"%"
PRIu64
"-%d-%d.tindex"
,
suid
,
colId
,
version
))
{
// read suid & colid & version success
return
0
;
}
return
-
1
;
}
static
void
tfileSerialCacheKey
(
TFileCacheKey
*
key
,
char
*
buf
)
{
SERIALIZE_MEM_TO_BUF
(
buf
,
key
,
suid
);
SERIALIZE_VAR_TO_BUF
(
buf
,
'_'
,
char
);
SERIALIZE_MEM_TO_BUF
(
buf
,
key
,
colType
);
SERIALIZE_VAR_TO_BUF
(
buf
,
'_'
,
char
);
SERIALIZE_MEM_TO_BUF
(
buf
,
key
,
version
);
SERIALIZE_VAR_TO_BUF
(
buf
,
'_'
,
char
);
SERIALIZE_STR_MEM_TO_BUF
(
buf
,
key
,
colName
,
key
->
nColName
);
}
static
int
tfileGetFileList
(
const
char
*
path
,
SArray
*
result
);
static
void
tfileDestroyFileName
(
void
*
elem
);
static
int
tfileCompare
(
const
void
*
a
,
const
void
*
b
);
static
int
tfileParseFileName
(
const
char
*
filename
,
uint64_t
*
suid
,
int
*
colId
,
int
*
version
);
static
void
tfileSerialCacheKey
(
TFileCacheKey
*
key
,
char
*
buf
);
TFileCache
*
tfileCacheCreate
(
const
char
*
path
)
{
TFileCache
*
tcache
=
calloc
(
1
,
sizeof
(
TFileCache
));
if
(
tcache
==
NULL
)
{
return
NULL
;
}
TFileCache
*
tfileCacheCreate
(
const
char
*
path
)
{
TFileCache
*
tcache
=
calloc
(
1
,
sizeof
(
TFileCache
));
if
(
tcache
==
NULL
)
{
return
NULL
;
}
tcache
->
tableCache
=
taosHashInit
(
8
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BINARY
),
true
,
HASH_ENTRY_LOCK
);
tcache
->
capacity
=
64
;
SArray
*
files
=
taosArrayInit
(
4
,
sizeof
(
void
*
));
SArray
*
files
=
taosArrayInit
(
4
,
sizeof
(
void
*
));
tfileGetFileList
(
path
,
files
);
taosArraySort
(
files
,
tfileCompare
);
uint64_t
suid
;
int32_t
colId
,
version
;
for
(
size_t
i
=
0
;
i
<
taosArrayGetSize
(
files
);
i
++
)
{
char
*
file
=
taosArrayGetP
(
files
,
i
);
uint64_t
suid
;
int
colId
,
version
;
if
(
0
!=
tfileParseFileName
(
file
,
&
suid
,
&
colId
,
&
version
))
{
goto
End
;
char
*
file
=
taosArrayGetP
(
files
,
i
);
if
(
0
!=
tfileParseFileName
(
file
,
&
suid
,
(
int
*
)
&
colId
,
(
int
*
)
&
version
))
{
indexInfo
(
"try parse invalid file: %s, skip it"
,
file
);
continue
;
}
WriterCtx
*
wc
=
writerCtxCreate
(
TFile
,
file
,
true
,
1024
*
64
);
WriterCtx
*
wc
=
writerCtxCreate
(
TFile
,
file
,
true
,
1024
*
64
);
if
(
wc
==
NULL
)
{
indexError
(
"failed to open index: %s"
,
file
);
goto
End
;
}
TFileReader
*
reader
=
tfileReaderCreate
(
wc
);
TFileReader
*
reader
=
tfileReaderCreate
(
wc
);
if
(
0
!=
tfileReadLoadHeader
(
reader
))
{
TF
ileReaderDestroy
(
reader
);
indexError
(
"failed to load index header, index
Id
: %s"
,
file
);
tf
ileReaderDestroy
(
reader
);
indexError
(
"failed to load index header, index
file
: %s"
,
file
);
goto
End
;
}
if
(
0
!=
tfileReadLoadFst
(
reader
))
{
tfileReaderDestroy
(
reader
);
indexError
(
"failed to load index fst, index file: %s"
,
file
);
}
// loader fst and validate it
TFileHeader
*
header
=
&
reader
->
header
;
TFileCacheKey
key
=
{.
suid
=
header
->
suid
,
.
version
=
header
->
version
,
.
colName
=
header
->
colName
,
.
nColName
=
strlen
(
header
->
colName
),
.
colType
=
header
->
colType
};
char
buf
[
128
]
=
{
0
};
tfileSerialCacheKey
(
&
key
,
buf
);
taosHashPut
(
tcache
->
tableCache
,
buf
,
strlen
(
buf
),
&
reader
,
sizeof
(
void
*
));
}
taosArrayDestroyEx
(
files
,
tfileDestroyFileName
);
return
tcache
;
...
...
@@ -130,78 +101,329 @@ End:
taosArrayDestroyEx
(
files
,
tfileDestroyFileName
);
return
NULL
;
}
void
tfileCacheDestroy
(
TFileCache
*
tcache
)
{
if
(
tcache
==
NULL
)
{
return
;
}
void
tfileCacheDestroy
(
TFileCache
*
tcache
)
{
if
(
tcache
==
NULL
)
{
return
;
}
// free table cache
TFileReader
**
reader
=
taosHashIterate
(
tcache
->
tableCache
,
NULL
);
TFileReader
**
reader
=
taosHashIterate
(
tcache
->
tableCache
,
NULL
);
while
(
reader
)
{
TFileReader
*
p
=
*
reader
;
indexInfo
(
"drop table cache suid: %"
PRIu64
", colName: %s, colType: %d"
,
p
->
header
.
suid
,
p
->
header
.
colName
,
p
->
header
.
colType
);
TF
ileReaderDestroy
(
p
);
TFileReader
*
p
=
*
reader
;
indexInfo
(
"drop table cache suid: %"
PRIu64
", colName: %s, colType: %d"
,
p
->
header
.
suid
,
p
->
header
.
colName
,
p
->
header
.
colType
);
tf
ileReaderDestroy
(
p
);
reader
=
taosHashIterate
(
tcache
->
tableCache
,
reader
);
}
taosHashCleanup
(
tcache
->
tableCache
);
free
(
tcache
);
}
TFileReader
*
tfileCacheGet
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
)
{
TFileReader
*
tfileCacheGet
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
)
{
char
buf
[
128
]
=
{
0
};
tfileSerialCacheKey
(
key
,
buf
);
TFileReader
*
reader
=
taosHashGet
(
tcache
->
tableCache
,
buf
,
strlen
(
buf
));
TFileReader
*
reader
=
taosHashGet
(
tcache
->
tableCache
,
buf
,
strlen
(
buf
));
return
reader
;
}
void
tfileCachePut
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
,
TFileReader
*
reader
)
{
void
tfileCachePut
(
TFileCache
*
tcache
,
TFileCacheKey
*
key
,
TFileReader
*
reader
)
{
char
buf
[
128
]
=
{
0
};
tfileSerialCacheKey
(
key
,
buf
);
taosHashPut
(
tcache
->
tableCache
,
buf
,
strlen
(
buf
),
&
reader
,
sizeof
(
void
*
));
taosHashPut
(
tcache
->
tableCache
,
buf
,
strlen
(
buf
),
&
reader
,
sizeof
(
void
*
));
return
;
}
TFileReader
*
tfileReaderCreate
(
WriterCtx
*
ctx
)
{
TFileReader
*
reader
=
calloc
(
1
,
sizeof
(
TFileReader
));
if
(
reader
==
NULL
)
{
return
NULL
;
}
reader
->
ctx
=
ctx
;
TFileReader
*
tfileReaderCreate
(
WriterCtx
*
ctx
)
{
TFileReader
*
reader
=
calloc
(
1
,
sizeof
(
TFileReader
));
if
(
reader
==
NULL
)
{
return
NULL
;
}
// T_REF_INC(reader);
reader
->
ctx
=
ctx
;
return
reader
;
}
void
TFileReaderDestroy
(
TFileReader
*
reader
)
{
if
(
reader
==
NULL
)
{
return
;
}
void
tfileReaderDestroy
(
TFileReader
*
reader
)
{
if
(
reader
==
NULL
)
{
return
;
}
// T_REF_INC(reader);
fstDestroy
(
reader
->
fst
);
writerCtxDestroy
(
reader
->
ctx
);
free
(
reader
);
}
TFileWriter
*
tfileWriterCreate
(
const
char
*
suid
,
const
char
*
colName
);
void
tfileWriterDestroy
(
TFileWriter
*
tw
)
;
int
tfileReaderSearch
(
TFileReader
*
reader
,
SIndexTermQuery
*
query
,
SArray
*
result
)
{
SIndexTerm
*
term
=
query
->
term
;
IndexTFile
*
indexTFileCreate
(
const
char
*
path
)
{
IndexTFile
*
tfile
=
calloc
(
1
,
sizeof
(
IndexTFile
));
tfile
->
cache
=
tfileCacheCreate
(
path
);
// refactor to callback later
if
(
query
->
qType
==
QUERY_TERM
)
{
uint64_t
offset
;
FstSlice
key
=
fstSliceCreate
(
term
->
colVal
,
term
->
nColVal
);
if
(
fstGet
(
reader
->
fst
,
&
key
,
&
offset
))
{
return
tfileReadLoadTableIds
(
reader
,
offset
,
result
);
}
else
{
indexInfo
(
"index: %"
PRIu64
", col: %s, colVal: %s, not found in tindex"
,
term
->
suid
,
term
->
colName
,
term
->
colVal
);
}
return
0
;
}
else
if
(
query
->
qType
==
QUERY_PREFIX
)
{
// handle later
//
}
else
{
// handle later
}
return
0
;
}
TFileWriter
*
tfileWriterCreate
(
WriterCtx
*
ctx
,
TFileHeader
*
header
)
{
// char pathBuf[128] = {0};
// sprintf(pathBuf, "%s/% " PRIu64 "-%d-%d.tindex", path, suid, colId, version);
// TFileHeader header = {.suid = suid, .version = version, .colName = {0}, colType = colType};
// memcpy(header.colName, );
// char buf[TFILE_HADER_PRE_SIZE];
// int len = TFILE_HADER_PRE_SIZE;
// if (len != ctx->write(ctx, buf, len)) {
// indexError("index: %" PRIu64 " failed to write header info", header->suid);
// return NULL;
//}
TFileWriter
*
tw
=
calloc
(
1
,
sizeof
(
TFileWriter
));
if
(
tw
==
NULL
)
{
indexError
(
"index: %"
PRIu64
" failed to alloc TFilerWriter"
,
header
->
suid
);
return
NULL
;
}
tw
->
ctx
=
ctx
;
tw
->
header
=
*
header
;
tfileWriteHeader
(
tw
);
return
tw
;
}
int
tfileWriterPut
(
TFileWriter
*
tw
,
void
*
data
)
{
// sort by coltype and write to tindex
__compar_fn_t
fn
=
getComparFunc
(
tw
->
header
.
colType
,
0
);
taosArraySortPWithExt
((
SArray
*
)(
data
),
tfileValueCompare
,
&
fn
);
int32_t
bufLimit
=
4096
,
offset
=
0
;
char
*
buf
=
calloc
(
1
,
sizeof
(
bufLimit
));
char
*
p
=
buf
;
int32_t
sz
=
taosArrayGetSize
((
SArray
*
)
data
);
int32_t
fstOffset
=
tw
->
offset
;
// ugly code, refactor later
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
TFileValue
*
v
=
taosArrayGetP
((
SArray
*
)
data
,
i
);
int32_t
tbsz
=
taosArrayGetSize
(
v
->
tableId
);
fstOffset
+=
TF_TABLE_TATOAL_SIZE
(
tbsz
);
}
// check result or not
tfileWriteFstOffset
(
tw
,
fstOffset
);
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
TFileValue
*
v
=
taosArrayGetP
((
SArray
*
)
data
,
i
);
int32_t
tbsz
=
taosArrayGetSize
(
v
->
tableId
);
// check buf has enough space or not
int32_t
ttsz
=
TF_TABLE_TATOAL_SIZE
(
tbsz
);
if
(
offset
+
ttsz
>
bufLimit
)
{
// batch write
tw
->
ctx
->
write
(
tw
->
ctx
,
buf
,
offset
);
offset
=
0
;
memset
(
buf
,
0
,
bufLimit
);
p
=
buf
;
}
tfileSerialTableIdsToBuf
(
p
,
v
->
tableId
);
offset
+=
ttsz
;
p
=
buf
+
offset
;
// set up value offset
v
->
offset
=
tw
->
offset
;
tw
->
offset
+=
ttsz
;
}
if
(
offset
!=
0
)
{
// write reversed data in buf to tindex
tw
->
ctx
->
write
(
tw
->
ctx
,
buf
,
offset
);
}
// write fst
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
// TODO, fst batch write later
TFileValue
*
v
=
taosArrayGetP
((
SArray
*
)
data
,
i
);
if
(
tfileWriteData
(
tw
,
v
)
==
0
)
{
//
//
}
}
tfree
(
buf
);
return
0
;
}
void
tfileWriterDestroy
(
TFileWriter
*
tw
)
{
if
(
tw
==
NULL
)
{
return
;
}
writerCtxDestroy
(
tw
->
ctx
);
free
(
tw
);
}
IndexTFile
*
indexTFileCreate
(
const
char
*
path
)
{
IndexTFile
*
tfile
=
calloc
(
1
,
sizeof
(
IndexTFile
));
if
(
tfile
==
NULL
)
{
return
NULL
;
}
tfile
->
cache
=
tfileCacheCreate
(
path
);
return
tfile
;
}
void
IndexTFileDestroy
(
IndexTFile
*
tfile
)
{
free
(
tfile
);
}
void
IndexTFileDestroy
(
IndexTFile
*
tfile
)
{
free
(
tfile
);
}
int
indexTFileSearch
(
void
*
tfile
,
SIndexTermQuery
*
query
,
SArray
*
result
)
{
if
(
tfile
==
NULL
)
{
return
-
1
;
}
IndexTFile
*
pTfile
=
(
IndexTFile
*
)
tfile
;
SIndexTerm
*
term
=
query
->
term
;
TFileCacheKey
key
=
{.
suid
=
term
->
suid
,
.
colType
=
term
->
colType
,
.
version
=
0
,
.
colName
=
term
->
colName
,
.
nColName
=
term
->
nColName
};
TFileReader
*
reader
=
tfileCacheGet
(
pTfile
->
cache
,
&
key
);
return
tfileReaderSearch
(
reader
,
query
,
result
);
}
int
indexTFilePut
(
void
*
tfile
,
SIndexTerm
*
term
,
uint64_t
uid
)
{
TFileWriterOpt
wOpt
=
{.
suid
=
term
->
suid
,
.
colType
=
term
->
colType
,
.
colName
=
term
->
colName
,
.
nColName
=
term
->
nColName
,
.
version
=
1
};
return
0
;
}
static
int
tfileValueCompare
(
const
void
*
a
,
const
void
*
b
,
const
void
*
param
)
{
__compar_fn_t
fn
=
*
(
__compar_fn_t
*
)
param
;
TFileValue
*
av
=
(
TFileValue
*
)
a
;
TFileValue
*
bv
=
(
TFileValue
*
)
b
;
return
fn
(
av
->
colVal
,
bv
->
colVal
);
}
static
void
tfileSerialTableIdsToBuf
(
char
*
buf
,
SArray
*
ids
)
{
int
sz
=
taosArrayGetSize
(
ids
);
SERIALIZE_VAR_TO_BUF
(
buf
,
sz
,
int32_t
);
for
(
size_t
i
=
0
;
i
<
sz
;
i
++
)
{
uint64_t
*
v
=
taosArrayGet
(
ids
,
i
);
SERIALIZE_VAR_TO_BUF
(
buf
,
*
v
,
uint64_t
);
}
}
int
indexTFileSearch
(
void
*
tfile
,
SIndexTermQuery
*
query
,
SArray
*
result
)
{
IndexTFile
*
pTfile
=
(
IndexTFile
*
)
tfile
;
static
int
tfileWriteFstOffset
(
TFileWriter
*
tw
,
int32_t
offset
)
{
int32_t
fstOffset
=
offset
+
sizeof
(
tw
->
header
.
fstOffset
);
tw
->
header
.
fstOffset
=
fstOffset
;
if
(
sizeof
(
fstOffset
)
!=
tw
->
ctx
->
write
(
tw
->
ctx
,
(
char
*
)
&
fstOffset
,
sizeof
(
fstOffset
)))
{
return
-
1
;
}
return
0
;
}
static
int
tfileWriteHeader
(
TFileWriter
*
writer
)
{
char
buf
[
TFILE_HEADER_NO_FST
]
=
{
0
};
char
*
p
=
buf
;
SIndexTerm
*
term
=
query
->
term
;
TFileCacheKey
key
=
{
.
suid
=
term
->
suid
,
.
colType
=
term
->
colType
,
.
version
=
0
,
.
colName
=
term
->
colName
,
.
nColName
=
term
->
nColName
};
TFileReader
*
reader
=
tfileCacheGet
(
pTfile
->
cache
,
&
key
);
TFileHeader
*
header
=
&
writer
->
header
;
memcpy
(
buf
,
(
char
*
)
header
,
sizeof
(
buf
));
int
nwrite
=
writer
->
ctx
->
write
(
writer
->
ctx
,
buf
,
sizeof
(
buf
));
if
(
sizeof
(
buf
)
!=
nwrite
)
{
return
-
1
;
}
writer
->
offset
=
nwrite
;
return
0
;
}
static
int
tfileWriteData
(
TFileWriter
*
write
,
TFileValue
*
tval
)
{
TFileHeader
*
header
=
&
write
->
header
;
uint8_t
colType
=
header
->
colType
;
if
(
colType
==
TSDB_DATA_TYPE_BINARY
||
colType
==
TSDB_DATA_TYPE_NCHAR
)
{
FstSlice
key
=
fstSliceCreate
((
uint8_t
*
)(
tval
->
colVal
),
(
size_t
)
strlen
(
tval
->
colVal
));
if
(
fstBuilderInsert
(
write
->
fb
,
key
,
tval
->
offset
))
{
fstSliceDestroy
(
&
key
);
return
0
;
}
fstSliceDestroy
(
&
key
);
return
-
1
;
}
else
{
// handle other type later
}
return
0
;
}
int
indexTFilePut
(
void
*
tfile
,
SIndexTerm
*
term
,
uint64_t
uid
)
{
TFileWriterOpt
wOpt
=
{
.
suid
=
term
->
suid
,
.
colType
=
term
->
colType
,
.
colName
=
term
->
colName
,
.
nColName
=
term
->
nColName
,
.
version
=
1
};
static
int
tfileReadLoadHeader
(
TFileReader
*
reader
)
{
// TODO simple tfile header later
char
buf
[
TFILE_HEADER_SIZE
]
=
{
0
};
char
*
p
=
buf
;
int64_t
nread
=
reader
->
ctx
->
read
(
reader
->
ctx
,
buf
,
sizeof
(
buf
));
assert
(
nread
==
sizeof
(
buf
));
memcpy
(
&
reader
->
header
,
buf
,
sizeof
(
buf
));
return
0
;
}
static
int
tfileReadLoadFst
(
TFileReader
*
reader
)
{
// current load fst into memory, refactor it later
static
int
FST_MAX_SIZE
=
16
*
1024
;
char
*
buf
=
calloc
(
1
,
sizeof
(
char
)
*
FST_MAX_SIZE
);
if
(
buf
==
NULL
)
{
return
-
1
;
}
WriterCtx
*
ctx
=
reader
->
ctx
;
int32_t
nread
=
ctx
->
readFrom
(
ctx
,
buf
,
FST_MAX_SIZE
,
reader
->
header
.
fstOffset
);
// we assuse fst size less than FST_MAX_SIZE
assert
(
nread
>
0
&&
nread
<
FST_MAX_SIZE
);
FstSlice
st
=
fstSliceCreate
((
uint8_t
*
)
buf
,
nread
);
reader
->
fst
=
fstCreate
(
&
st
);
free
(
buf
);
fstSliceDestroy
(
&
st
);
return
reader
->
fst
==
NULL
?
0
:
-
1
;
}
static
int
tfileReadLoadTableIds
(
TFileReader
*
reader
,
int32_t
offset
,
SArray
*
result
)
{
int32_t
nid
;
WriterCtx
*
ctx
=
reader
->
ctx
;
int32_t
nread
=
ctx
->
readFrom
(
ctx
,
(
char
*
)
&
nid
,
sizeof
(
nid
),
offset
);
assert
(
sizeof
(
nid
)
==
nread
);
char
*
buf
=
calloc
(
1
,
sizeof
(
uint64_t
)
*
nid
);
if
(
buf
==
NULL
)
{
return
-
1
;
}
nread
=
ctx
->
read
(
ctx
,
buf
,
sizeof
(
uint64_t
)
*
nid
);
uint64_t
*
ids
=
(
uint64_t
*
)
buf
;
for
(
int32_t
i
=
0
;
i
<
nid
;
i
++
)
{
taosArrayPush
(
result
,
ids
+
i
);
}
free
(
buf
);
return
0
;
}
static
int
tfileGetFileList
(
const
char
*
path
,
SArray
*
result
)
{
DIR
*
dir
=
opendir
(
path
);
if
(
NULL
==
dir
)
{
return
-
1
;
}
struct
dirent
*
entry
;
while
((
entry
=
readdir
(
dir
))
!=
NULL
)
{
size_t
len
=
strlen
(
entry
->
d_name
);
char
*
buf
=
calloc
(
1
,
len
+
1
);
memcpy
(
buf
,
entry
->
d_name
,
len
);
taosArrayPush
(
result
,
&
buf
);
}
closedir
(
dir
);
return
0
;
}
static
void
tfileDestroyFileName
(
void
*
elem
)
{
char
*
p
=
*
(
char
**
)
elem
;
free
(
p
);
}
static
int
tfileCompare
(
const
void
*
a
,
const
void
*
b
)
{
const
char
*
aName
=
*
(
char
**
)
a
;
const
char
*
bName
=
*
(
char
**
)
b
;
size_t
aLen
=
strlen
(
aName
);
size_t
bLen
=
strlen
(
bName
);
return
strncmp
(
aName
,
bName
,
aLen
>
bLen
?
aLen
:
bLen
);
}
// tfile name suid-colId-version.tindex
static
int
tfileParseFileName
(
const
char
*
filename
,
uint64_t
*
suid
,
int
*
colId
,
int
*
version
)
{
if
(
3
==
sscanf
(
filename
,
"%"
PRIu64
"-%d-%d.tindex"
,
suid
,
colId
,
version
))
{
// read suid & colid & version success
return
0
;
}
return
-
1
;
}
static
void
tfileSerialCacheKey
(
TFileCacheKey
*
key
,
char
*
buf
)
{
SERIALIZE_MEM_TO_BUF
(
buf
,
key
,
suid
);
SERIALIZE_VAR_TO_BUF
(
buf
,
'_'
,
char
);
SERIALIZE_MEM_TO_BUF
(
buf
,
key
,
colType
);
SERIALIZE_VAR_TO_BUF
(
buf
,
'_'
,
char
);
SERIALIZE_MEM_TO_BUF
(
buf
,
key
,
version
);
SERIALIZE_VAR_TO_BUF
(
buf
,
'_'
,
char
);
SERIALIZE_STR_MEM_TO_BUF
(
buf
,
key
,
colName
,
key
->
nColName
);
}
source/libs/parser/inc/dataBlockMgt.h
浏览文件 @
d96963a2
...
...
@@ -170,10 +170,11 @@ int32_t schemaIdxCompar(const void *lhs, const void *rhs);
int32_t
boundIdxCompar
(
const
void
*
lhs
,
const
void
*
rhs
);
void
setBoundColumnInfo
(
SParsedDataColInfo
*
pColList
,
SSchema
*
pSchema
,
int32_t
numOfCols
);
void
destroyBoundColumnInfo
(
SParsedDataColInfo
*
pColList
);
void
destroyBlockArrayList
(
SArray
*
pDataBlockList
);
int32_t
initMemRowBuilder
(
SMemRowBuilder
*
pBuilder
,
uint32_t
nRows
,
uint32_t
nCols
,
uint32_t
nBoundCols
,
int32_t
allNullLen
);
int32_t
allocateMemIfNeed
(
STableDataBlocks
*
pDataBlock
,
int32_t
rowSize
,
int32_t
*
numOfRows
);
int32_t
getDataBlockFromList
(
SHashObj
*
pHashList
,
int64_t
id
,
int32_t
size
,
int32_t
startOffset
,
int32_t
rowSize
,
const
STableMeta
*
pTableMeta
,
STableDataBlocks
**
dataBlocks
,
SArray
*
pBlockList
);
int32_t
mergeTableDataBlocks
(
SHashObj
*
pHashObj
,
int8_t
schemaAttached
,
uint8_t
payloadType
,
bool
freeBlockMap
);
int32_t
mergeTableDataBlocks
(
SHashObj
*
pHashObj
,
int8_t
schemaAttached
,
uint8_t
payloadType
,
SArray
**
pVgDataBlocks
);
#endif // TDENGINE_DATABLOCKMGT_H
source/libs/parser/src/dataBlockMgt.c
浏览文件 @
d96963a2
...
...
@@ -269,19 +269,17 @@ void destroyDataBlock(STableDataBlocks* pDataBlock) {
tfree
(
pDataBlock
);
}
void
*
destroyBlockArrayList
(
SArray
*
pDataBlockList
)
{
void
destroyBlockArrayList
(
SArray
*
pDataBlockList
)
{
if
(
pDataBlockList
==
NULL
)
{
return
NULL
;
return
;
}
size_t
size
=
taosArrayGetSize
(
pDataBlockList
);
for
(
int32_t
i
=
0
;
i
<
size
;
i
++
)
{
void
*
d
=
taosArrayGetP
(
pDataBlockList
,
i
);
destroyDataBlock
(
d
);
destroyDataBlock
(
taosArrayGetP
(
pDataBlockList
,
i
));
}
taosArrayDestroy
(
pDataBlockList
);
return
NULL
;
}
// data block is disordered, sort it in ascending order
...
...
@@ -298,6 +296,7 @@ void sortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) {
int32_t
i
=
0
;
int32_t
j
=
1
;
// delete rows with timestamp conflicts
while
(
j
<
pBlocks
->
numOfRows
)
{
TSKEY
ti
=
*
(
TSKEY
*
)(
pBlockData
+
dataBuf
->
rowSize
*
i
);
TSKEY
tj
=
*
(
TSKEY
*
)(
pBlockData
+
dataBuf
->
rowSize
*
j
);
...
...
@@ -430,7 +429,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB
char
*
p
=
pTableDataBlock
->
pData
+
sizeof
(
SSubmitBlk
);
pBlock
->
dataLen
=
0
;
int32_t
numOfRows
=
htons
(
pBlock
->
numOfRows
)
;
int32_t
numOfRows
=
pBlock
->
numOfRows
;
if
(
isRawPayload
)
{
for
(
int32_t
i
=
0
;
i
<
numOfRows
;
++
i
)
{
...
...
@@ -467,18 +466,10 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB
}
}
int32_t
len
=
pBlock
->
dataLen
+
pBlock
->
schemaLen
;
pBlock
->
dataLen
=
htonl
(
pBlock
->
dataLen
);
pBlock
->
schemaLen
=
htonl
(
pBlock
->
schemaLen
);
return
len
;
return
pBlock
->
dataLen
+
pBlock
->
schemaLen
;
}
static
void
extractTableNameList
(
SHashObj
*
pHashObj
,
bool
freeBlockMap
)
{
// todo
}
int32_t
mergeTableDataBlocks
(
SHashObj
*
pHashObj
,
int8_t
schemaAttached
,
uint8_t
payloadType
,
bool
freeBlockMap
)
{
int32_t
mergeTableDataBlocks
(
SHashObj
*
pHashObj
,
int8_t
schemaAttached
,
uint8_t
payloadType
,
SArray
**
pVgDataBlocks
)
{
const
int
INSERT_HEAD_SIZE
=
sizeof
(
SMsgDesc
)
+
sizeof
(
SSubmitMsg
);
int
code
=
0
;
bool
isRawPayload
=
IS_RAW_PAYLOAD
(
payloadType
);
...
...
@@ -537,24 +528,13 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t
(
isRawPayload
?
(
pOneTableBlock
->
rowSize
+
expandSize
)
:
getExtendedRowSize
(
pOneTableBlock
))
+
sizeof
(
STColumn
)
*
getNumOfColumns
(
pOneTableBlock
->
pTableMeta
);
pBlocks
->
tid
=
htonl
(
pBlocks
->
tid
);
pBlocks
->
uid
=
htobe64
(
pBlocks
->
uid
);
pBlocks
->
sversion
=
htonl
(
pBlocks
->
sversion
);
pBlocks
->
numOfRows
=
htons
(
pBlocks
->
numOfRows
);
pBlocks
->
schemaLen
=
0
;
// erase the empty space reserved for binary data
int32_t
finalLen
=
trimDataBlock
(
dataBuf
->
pData
+
dataBuf
->
size
,
pOneTableBlock
,
blkKeyInfo
.
pKeyTuple
,
schemaAttached
,
isRawPayload
);
assert
(
finalLen
<=
len
);
dataBuf
->
size
+=
(
finalLen
+
sizeof
(
SSubmitBlk
));
assert
(
dataBuf
->
size
<=
dataBuf
->
nAllocSize
);
// the length does not include the SSubmitBlk structure
pBlocks
->
dataLen
=
htonl
(
finalLen
);
dataBuf
->
numOfTables
+=
1
;
pBlocks
->
numOfRows
=
0
;
}
p
=
taosHashIterate
(
pHashObj
,
p
);
...
...
@@ -565,12 +545,10 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t
pOneTableBlock
=
*
p
;
}
extractTableNameList
(
pHashObj
,
freeBlockMap
);
// free the table data blocks;
taosHashCleanup
(
pVnodeDataBlockHashList
);
tfree
(
blkKeyInfo
.
pKeyTuple
);
*
pVgDataBlocks
=
pVnodeDataBlockList
;
return
TSDB_CODE_SUCCESS
;
}
...
...
source/libs/parser/src/insertParser.c
浏览文件 @
d96963a2
...
...
@@ -31,33 +31,16 @@
pSql += index; \
} while (0)
#define
CHECK_CODE(expr
) \
#define
NEXT_TOKEN_KEEP_SQL(pSql, sToken, index
) \
do { \
int32_t code = expr; \
if (TSDB_CODE_SUCCESS != code) { \
terrno = code; \
return terrno; \
} \
} while (0)
#define CHECK_CODE_1(expr, destroy) \
do { \
int32_t code = expr; \
if (TSDB_CODE_SUCCESS != code) { \
(void)destroy; \
terrno = code; \
return terrno; \
} \
sToken = tStrGetToken(pSql, &index, false); \
} while (0)
#define CHECK_CODE
_2(expr, destroy1, destroy2
) \
#define CHECK_CODE
(expr
) \
do { \
int32_t code = expr; \
if (TSDB_CODE_SUCCESS != code) { \
(void)destroy1; \
(void)destroy2; \
terrno = code; \
return terrno; \
return code; \
} \
} while (0)
...
...
@@ -70,9 +53,12 @@ typedef struct SInsertParseContext {
SParseContext
*
pComCxt
;
const
char
*
pSql
;
SMsgBuf
msg
;
struct
SCatalog
*
pCatalog
;
STableMeta
*
pTableMeta
;
SHashObj
*
pTableBlockHashObj
;
// data block for each table. need release
SParsedDataColInfo
tags
;
SKVRowBuilder
tagsBuilder
;
SHashObj
*
pTableBlockHashObj
;
SArray
*
pTableDataBlocks
;
SArray
*
pVgDataBlocks
;
int32_t
totalNum
;
SInsertStmtInfo
*
pOutput
;
}
SInsertParseContext
;
...
...
@@ -151,14 +137,6 @@ static int32_t toInt64(const char* z, int16_t type, int32_t n, int64_t* value, b
return
ret
;
}
static
int32_t
createInsertStmtInfo
(
SInsertStmtInfo
**
pInsertInfo
)
{
*
pInsertInfo
=
calloc
(
1
,
sizeof
(
SQueryStmtInfo
));
if
(
NULL
==
*
pInsertInfo
)
{
return
TSDB_CODE_TSC_OUT_OF_MEMORY
;
}
return
TSDB_CODE_SUCCESS
;
}
static
int32_t
skipInsertInto
(
SInsertParseContext
*
pCxt
)
{
SToken
sToken
;
NEXT_TOKEN
(
pCxt
->
pSql
,
sToken
);
...
...
@@ -194,7 +172,7 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) {
char
fullDbName
[
TSDB_FULL_DB_NAME_LEN
]
=
{
0
};
char
tableName
[
TSDB_TABLE_NAME_LEN
]
=
{
0
};
CHECK_CODE
(
buildName
(
pCxt
,
pTname
,
fullDbName
,
tableName
));
CHECK_CODE
(
catalogGetTableMeta
(
pCxt
->
pCatalog
,
pCxt
->
pComCxt
->
pRpc
,
pCxt
->
pComCxt
->
pEpSet
,
fullDbName
,
tableName
,
&
pCxt
->
pTableMeta
));
CHECK_CODE
(
catalogGetTableMeta
(
pCxt
->
pC
omCxt
->
pC
atalog
,
pCxt
->
pComCxt
->
pRpc
,
pCxt
->
pComCxt
->
pEpSet
,
fullDbName
,
tableName
,
&
pCxt
->
pTableMeta
));
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -209,6 +187,51 @@ static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pS
return
-
1
;
}
static
void
fillMsgHeader
(
SVgDataBlocks
*
dst
)
{
SMsgDesc
*
desc
=
(
SMsgDesc
*
)
dst
->
pData
;
desc
->
numOfVnodes
=
htonl
(
1
);
SSubmitMsg
*
submit
=
(
SSubmitMsg
*
)(
desc
+
1
);
submit
->
header
.
vgId
=
htonl
(
dst
->
vgId
);
submit
->
header
.
contLen
=
htonl
(
dst
->
size
-
sizeof
(
SMsgDesc
));
submit
->
length
=
submit
->
header
.
contLen
;
submit
->
numOfBlocks
=
htonl
(
dst
->
numOfTables
);
SSubmitBlk
*
blk
=
(
SSubmitBlk
*
)(
submit
+
1
);
int32_t
numOfBlocks
=
dst
->
numOfTables
;
while
(
numOfBlocks
--
)
{
int32_t
dataLen
=
blk
->
dataLen
;
blk
->
uid
=
htobe64
(
blk
->
uid
);
blk
->
tid
=
htonl
(
blk
->
tid
);
blk
->
padding
=
htonl
(
blk
->
padding
);
blk
->
sversion
=
htonl
(
blk
->
sversion
);
blk
->
dataLen
=
htonl
(
blk
->
dataLen
);
blk
->
schemaLen
=
htonl
(
blk
->
schemaLen
);
blk
->
numOfRows
=
htons
(
blk
->
numOfRows
);
blk
=
(
SSubmitBlk
*
)(
blk
->
data
+
dataLen
);
}
}
static
int32_t
buildOutput
(
SInsertParseContext
*
pCxt
)
{
size_t
numOfVg
=
taosArrayGetSize
(
pCxt
->
pVgDataBlocks
);
pCxt
->
pOutput
->
pDataBlocks
=
taosArrayInit
(
numOfVg
,
POINTER_BYTES
);
if
(
NULL
==
pCxt
->
pOutput
->
pDataBlocks
)
{
return
TSDB_CODE_TSC_OUT_OF_MEMORY
;
}
for
(
size_t
i
=
0
;
i
<
numOfVg
;
++
i
)
{
STableDataBlocks
*
src
=
taosArrayGetP
(
pCxt
->
pVgDataBlocks
,
i
);
SVgDataBlocks
*
dst
=
calloc
(
1
,
sizeof
(
SVgDataBlocks
));
if
(
NULL
==
dst
)
{
return
TSDB_CODE_TSC_OUT_OF_MEMORY
;
}
dst
->
vgId
=
src
->
vgId
;
dst
->
numOfTables
=
src
->
numOfTables
;
dst
->
size
=
src
->
size
;
SWAP
(
dst
->
pData
,
src
->
pData
,
char
*
);
fillMsgHeader
(
dst
);
taosArrayPush
(
pCxt
->
pOutput
->
pDataBlocks
,
&
dst
);
}
return
TSDB_CODE_SUCCESS
;
}
static
int32_t
checkTimestamp
(
STableDataBlocks
*
pDataBlocks
,
const
char
*
start
)
{
// once the data block is disordered, we do NOT keep previous timestamp any more
if
(
!
pDataBlocks
->
ordered
)
{
...
...
@@ -219,16 +242,14 @@ static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start)
if
(
k
==
INT64_MIN
)
{
if
(
pDataBlocks
->
tsSource
==
TSDB_USE_CLI_TS
)
{
return
-
1
;
}
else
if
(
pDataBlocks
->
tsSource
==
-
1
)
{
pDataBlocks
->
tsSource
=
TSDB_USE_SERVER_TS
;
return
TSDB_CODE_FAILED
;
// client time/server time can not be mixed
}
pDataBlocks
->
tsSource
=
TSDB_USE_SERVER_TS
;
}
else
{
if
(
pDataBlocks
->
tsSource
==
TSDB_USE_SERVER_TS
)
{
return
-
1
;
// client time/server time can not be mixed
}
else
if
(
pDataBlocks
->
tsSource
==
-
1
)
{
pDataBlocks
->
tsSource
=
TSDB_USE_CLI_TS
;
return
TSDB_CODE_FAILED
;
// client time/server time can not be mixed
}
pDataBlocks
->
tsSource
=
TSDB_USE_CLI_TS
;
}
if
(
k
<=
pDataBlocks
->
prevTS
&&
(
pDataBlocks
->
tsSource
==
TSDB_USE_CLI_TS
))
{
...
...
@@ -608,26 +629,24 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo*
}
// pSql -> tag1_value, ...)
static
int32_t
parseTagsClause
(
SInsertParseContext
*
pCxt
,
SParsedDataColInfo
*
pSpd
,
SSchema
*
pTagsSchema
,
uint8_t
precision
)
{
SKVRowBuilder
kvRowBuilder
=
{
0
};
if
(
tdInitKVRowBuilder
(
&
kvRowBuilder
)
<
0
)
{
destroyBoundColumnInfo
(
pSpd
);
static
int32_t
parseTagsClause
(
SInsertParseContext
*
pCxt
,
SSchema
*
pTagsSchema
,
uint8_t
precision
)
{
if
(
tdInitKVRowBuilder
(
&
pCxt
->
tagsBuilder
)
<
0
)
{
return
TSDB_CODE_TSC_OUT_OF_MEMORY
;
}
SKvParam
param
=
{.
builder
=
&
kvRow
Builder
};
SKvParam
param
=
{.
builder
=
&
pCxt
->
tags
Builder
};
SToken
sToken
;
char
tmpTokenBuf
[
TSDB_MAX_BYTES_PER_ROW
]
=
{
0
};
// used for deleting Escape character: \\, \', \"
for
(
int
i
=
0
;
i
<
p
Spd
->
numOfBound
;
++
i
)
{
for
(
int
i
=
0
;
i
<
p
Cxt
->
tags
.
numOfBound
;
++
i
)
{
NEXT_TOKEN
(
pCxt
->
pSql
,
sToken
);
SSchema
*
pSchema
=
&
pTagsSchema
[
p
Spd
->
boundedColumns
[
i
]];
SSchema
*
pSchema
=
&
pTagsSchema
[
p
Cxt
->
tags
.
boundedColumns
[
i
]];
param
.
schema
=
pSchema
;
CHECK_CODE
_2
(
parseOneValue
(
pCxt
,
&
sToken
,
pSchema
,
precision
,
tmpTokenBuf
,
KvRowAppend
,
&
param
),
tdDestroyKVRowBuilder
(
&
kvRowBuilder
),
destroyBoundColumnInfo
(
pSpd
));
CHECK_CODE
(
parseOneValue
(
pCxt
,
&
sToken
,
pSchema
,
precision
,
tmpTokenBuf
,
KvRowAppend
,
&
param
));
}
destroyBoundColumnInfo
(
pSpd
);
SKVRow
row
=
tdGetKVRowFromBuilder
(
&
kvRow
Builder
);
tdDestroyKVRowBuilder
(
&
kvRow
Builder
);
destroyBoundColumnInfo
(
&
pCxt
->
tags
);
SKVRow
row
=
tdGetKVRowFromBuilder
(
&
pCxt
->
tags
Builder
);
tdDestroyKVRowBuilder
(
&
pCxt
->
tags
Builder
);
if
(
NULL
==
row
)
{
return
buildInvalidOperationMsg
(
&
pCxt
->
msg
,
"tag value expected"
);
}
...
...
@@ -650,13 +669,12 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken)
}
SSchema
*
pTagsSchema
=
getTableTagSchema
(
pCxt
->
pTableMeta
);
SParsedDataColInfo
spd
=
{
0
};
setBoundColumnInfo
(
&
spd
,
pTagsSchema
,
getNumOfTags
(
pCxt
->
pTableMeta
));
setBoundColumnInfo
(
&
pCxt
->
tags
,
pTagsSchema
,
getNumOfTags
(
pCxt
->
pTableMeta
));
// pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...)
NEXT_TOKEN
(
pCxt
->
pSql
,
sToken
);
if
(
TK_LP
==
sToken
.
type
)
{
CHECK_CODE
_1
(
parseBoundColumns
(
pCxt
,
&
spd
,
pTagsSchema
),
destroyBoundColumnInfo
(
&
spd
));
CHECK_CODE
(
parseBoundColumns
(
pCxt
,
&
pCxt
->
tags
,
pTagsSchema
));
NEXT_TOKEN
(
pCxt
->
pSql
,
sToken
);
}
...
...
@@ -668,7 +686,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken)
if
(
TK_LP
!=
sToken
.
type
)
{
return
buildSyntaxErrMsg
(
&
pCxt
->
msg
,
"( is expected"
,
sToken
.
z
);
}
CHECK_CODE
(
parseTagsClause
(
pCxt
,
&
spd
,
pTagsSchema
,
getTableInfo
(
pCxt
->
pTableMeta
).
precision
));
CHECK_CODE
(
parseTagsClause
(
pCxt
,
pTagsSchema
,
getTableInfo
(
pCxt
->
pTableMeta
).
precision
));
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -732,10 +750,12 @@ static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlo
char
tmpTokenBuf
[
TSDB_MAX_BYTES_PER_ROW
]
=
{
0
};
// used for deleting Escape character: \\, \', \"
SToken
sToken
;
while
(
1
)
{
NEXT_TOKEN
(
pCxt
->
pSql
,
sToken
);
int32_t
index
=
0
;
NEXT_TOKEN_KEEP_SQL
(
pCxt
->
pSql
,
sToken
,
index
);
if
(
TK_LP
!=
sToken
.
type
)
{
break
;
}
pCxt
->
pSql
+=
index
;
if
((
*
numOfRows
)
>=
maxRows
||
pDataBlock
->
size
+
extendedRowSize
>=
pDataBlock
->
nAllocSize
)
{
int32_t
tSize
;
...
...
@@ -815,7 +835,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
if
(
TK_LP
==
sToken
.
type
)
{
// pSql -> field1_name, ...)
CHECK_CODE
_1
(
parseBoundColumns
(
pCxt
,
&
dataBuf
->
boundColumnInfo
,
getTableColumnSchema
(
pCxt
->
pTableMeta
)),
destroyBoundColumnInfo
(
&
dataBuf
->
boundColumnInfo
));
CHECK_CODE
(
parseBoundColumns
(
pCxt
,
&
dataBuf
->
boundColumnInfo
,
getTableColumnSchema
(
pCxt
->
pTableMeta
)
));
NEXT_TOKEN
(
pCxt
->
pSql
,
sToken
);
}
...
...
@@ -842,9 +862,18 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
}
// merge according to vgId
if
(
!
TSDB_QUERY_HAS_TYPE
(
pCxt
->
pOutput
->
insertType
,
TSDB_QUERY_TYPE_STMT_INSERT
)
&&
taosHashGetSize
(
pCxt
->
pTableBlockHashObj
)
>
0
)
{
CHECK_CODE
(
mergeTableDataBlocks
(
pCxt
->
pTableBlockHashObj
,
pCxt
->
pOutput
->
schemaAttache
,
pCxt
->
pOutput
->
payloadType
,
true
));
CHECK_CODE
(
mergeTableDataBlocks
(
pCxt
->
pTableBlockHashObj
,
pCxt
->
pOutput
->
schemaAttache
,
pCxt
->
pOutput
->
payloadType
,
&
pCxt
->
pVgDataBlocks
));
}
return
TSDB_CODE_SUCCESS
;
return
buildOutput
(
pCxt
);
}
static
void
destroyInsertParseContext
(
SInsertParseContext
*
pCxt
)
{
tfree
(
pCxt
->
pTableMeta
);
destroyBoundColumnInfo
(
&
pCxt
->
tags
);
tdDestroyKVRowBuilder
(
&
pCxt
->
tagsBuilder
);
taosHashCleanup
(
pCxt
->
pTableBlockHashObj
);
destroyBlockArrayList
(
pCxt
->
pTableDataBlocks
);
destroyBlockArrayList
(
pCxt
->
pVgDataBlocks
);
}
// INSERT INTO
...
...
@@ -854,26 +883,30 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
// [...];
int32_t
parseInsertSql
(
SParseContext
*
pContext
,
SInsertStmtInfo
**
pInfo
)
{
CHECK_CODE
(
createInsertStmtInfo
(
pInfo
));
SInsertParseContext
context
=
{
.
pComCxt
=
pContext
,
.
pSql
=
pContext
->
pSql
,
.
msg
=
{.
buf
=
pContext
->
pMsg
,
.
len
=
pContext
->
msgLen
},
.
pCatalog
=
NULL
,
.
pTableMeta
=
NULL
,
.
pTableBlockHashObj
=
taosHashInit
(
128
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BIGINT
),
true
,
false
),
.
totalNum
=
0
,
.
pOutput
=
*
pInfo
.
pOutput
=
calloc
(
1
,
sizeof
(
SInsertStmtInfo
))
};
if
(
NULL
==
context
.
pTableBlockHashObj
)
{
return
TSDB_CODE_TSC_OUT_OF_MEMORY
;
if
(
NULL
==
context
.
pTableBlockHashObj
||
NULL
==
context
.
pOutput
)
{
terrno
=
TSDB_CODE_TSC_OUT_OF_MEMORY
;
return
TSDB_CODE_FAILED
;
}
CHECK_CODE
(
catalogGetHandle
(
pContext
->
pClusterId
,
&
context
.
pCatalog
))
;
CHECK_CODE
(
skipInsertInto
(
&
context
))
;
CHECK_CODE
(
parseInsertBody
(
&
context
))
;
*
pInfo
=
context
.
pOutput
;
context
.
pOutput
->
schemaAttache
=
pContext
->
schemaAttached
;
context
.
pOutput
->
payloadType
=
PAYLOAD_TYPE_KV
;
return
TSDB_CODE_SUCCESS
;
int32_t
code
=
skipInsertInto
(
&
context
);
if
(
TSDB_CODE_SUCCESS
==
code
)
{
code
=
parseInsertBody
(
&
context
);
}
destroyInsertParseContext
(
&
context
);
terrno
=
code
;
return
(
TSDB_CODE_SUCCESS
==
code
?
TSDB_CODE_SUCCESS
:
TSDB_CODE_FAILED
);
}
source/libs/parser/test/CMakeLists.txt
浏览文件 @
d96963a2
...
...
@@ -17,5 +17,3 @@ TARGET_LINK_LIBRARIES(
parserTest
PUBLIC os util common parser catalog transport gtest function planner qcom
)
TARGET_LINK_OPTIONS
(
parserTest PRIVATE -Wl,-wrap,malloc
)
source/libs/parser/test/insertTest.cpp
→
source/libs/parser/test/insert
Parser
Test.cpp
浏览文件 @
d96963a2
...
...
@@ -16,7 +16,7 @@
#include <gtest/gtest.h>
#include "insertParser.h"
#include "mockCatalog.h"
//
#include "mockCatalog.h"
using
namespace
std
;
using
namespace
testing
;
...
...
@@ -27,27 +27,6 @@ namespace {
}
}
extern
"C"
{
#include <execinfo.h>
void
*
__real_malloc
(
size_t
);
void
*
__wrap_malloc
(
size_t
c
)
{
// printf("My MALLOC called: %d\n", c);
// void *array[32];
// int size = backtrace(array, 32);
// char **symbols = backtrace_symbols(array, size);
// for (int i = 0; i < size; ++i) {
// cout << symbols[i] << endl;
// }
// free(symbols);
return
__real_malloc
(
c
);
}
}
// syntax:
// INSERT INTO
// tb_name
...
...
@@ -85,6 +64,51 @@ protected:
return
res_
;
}
void
dumpReslut
()
{
size_t
num
=
taosArrayGetSize
(
res_
->
pDataBlocks
);
cout
<<
"schemaAttache:"
<<
(
int32_t
)
res_
->
schemaAttache
<<
", payloadType:"
<<
(
int32_t
)
res_
->
payloadType
<<
", insertType:"
<<
res_
->
insertType
<<
", numOfVgs:"
<<
num
<<
endl
;
for
(
size_t
i
=
0
;
i
<
num
;
++
i
)
{
SVgDataBlocks
*
vg
=
(
SVgDataBlocks
*
)
taosArrayGetP
(
res_
->
pDataBlocks
,
i
);
cout
<<
"vgId:"
<<
vg
->
vgId
<<
", numOfTables:"
<<
vg
->
numOfTables
<<
", dataSize:"
<<
vg
->
size
<<
endl
;
SMsgDesc
*
desc
=
(
SMsgDesc
*
)(
vg
->
pData
);
cout
<<
"numOfVnodes:"
<<
ntohl
(
desc
->
numOfVnodes
)
<<
endl
;
SSubmitMsg
*
submit
=
(
SSubmitMsg
*
)(
desc
+
1
);
cout
<<
"length:"
<<
ntohl
(
submit
->
length
)
<<
", numOfBlocks:"
<<
ntohl
(
submit
->
numOfBlocks
)
<<
endl
;
int32_t
numOfBlocks
=
ntohl
(
submit
->
numOfBlocks
);
SSubmitBlk
*
blk
=
(
SSubmitBlk
*
)(
submit
+
1
);
for
(
int32_t
i
=
0
;
i
<
numOfBlocks
;
++
i
)
{
cout
<<
"Block:"
<<
i
<<
endl
;
cout
<<
"
\t
uid:"
<<
be64toh
(
blk
->
uid
)
<<
", tid:"
<<
ntohl
(
blk
->
tid
)
<<
", padding:"
<<
ntohl
(
blk
->
padding
)
<<
", sversion:"
<<
ntohl
(
blk
->
sversion
)
<<
", dataLen:"
<<
ntohl
(
blk
->
dataLen
)
<<
", schemaLen:"
<<
ntohl
(
blk
->
schemaLen
)
<<
", numOfRows:"
<<
ntohs
(
blk
->
numOfRows
)
<<
endl
;
blk
=
(
SSubmitBlk
*
)(
blk
->
data
+
ntohl
(
blk
->
dataLen
));
}
}
}
void
checkReslut
(
int32_t
numOfTables
,
int16_t
numOfRows1
,
int16_t
numOfRows2
=
-
1
)
{
ASSERT_EQ
(
res_
->
schemaAttache
,
0
);
ASSERT_EQ
(
res_
->
payloadType
,
PAYLOAD_TYPE_KV
);
ASSERT_EQ
(
res_
->
insertType
,
TSDB_QUERY_TYPE_INSERT
);
size_t
num
=
taosArrayGetSize
(
res_
->
pDataBlocks
);
ASSERT_GE
(
num
,
0
);
for
(
size_t
i
=
0
;
i
<
num
;
++
i
)
{
SVgDataBlocks
*
vg
=
(
SVgDataBlocks
*
)
taosArrayGetP
(
res_
->
pDataBlocks
,
i
);
ASSERT_EQ
(
vg
->
numOfTables
,
numOfTables
);
ASSERT_GE
(
vg
->
size
,
0
);
SMsgDesc
*
desc
=
(
SMsgDesc
*
)(
vg
->
pData
);
ASSERT_EQ
(
ntohl
(
desc
->
numOfVnodes
),
1
);
SSubmitMsg
*
submit
=
(
SSubmitMsg
*
)(
desc
+
1
);
ASSERT_GE
(
ntohl
(
submit
->
length
),
0
);
ASSERT_GE
(
ntohl
(
submit
->
numOfBlocks
),
0
);
int32_t
numOfBlocks
=
ntohl
(
submit
->
numOfBlocks
);
SSubmitBlk
*
blk
=
(
SSubmitBlk
*
)(
submit
+
1
);
for
(
int32_t
i
=
0
;
i
<
numOfBlocks
;
++
i
)
{
ASSERT_EQ
(
ntohs
(
blk
->
numOfRows
),
(
0
==
i
?
numOfRows1
:
(
numOfRows2
>
0
?
numOfRows2
:
numOfRows1
)));
blk
=
(
SSubmitBlk
*
)(
blk
->
data
+
ntohl
(
blk
->
dataLen
));
}
}
}
private:
static
const
int
max_err_len
=
1024
;
static
const
int
max_sql_len
=
1024
*
1024
;
...
...
@@ -108,15 +132,44 @@ private:
};
// INSERT INTO tb_name VALUES (field1_value, ...)
TEST_F
(
InsertTest
,
simpleTest
)
{
TEST_F
(
InsertTest
,
singleTableSingleRowTest
)
{
setDatabase
(
"root"
,
"test"
);
bind
(
"insert into t1 values (now, 1,
\"
beijing
\"
)"
);
ASSERT_EQ
(
run
(),
TSDB_CODE_SUCCESS
);
dumpReslut
();
checkReslut
(
1
,
1
);
}
// INSERT INTO tb_name VALUES (field1_value, ...)(field1_value, ...)
TEST_F
(
InsertTest
,
singleTableMultiRowTest
)
{
setDatabase
(
"root"
,
"test"
);
bind
(
"insert into t1 values (now, 1,
\"
beijing
\"
)(now+1s, 2,
\"
shanghai
\"
)(now+2s, 3,
\"
guangzhou
\"
)"
);
ASSERT_EQ
(
run
(),
TSDB_CODE_SUCCESS
);
dumpReslut
();
checkReslut
(
1
,
3
);
}
// INSERT INTO tb1_name VALUES (field1_value, ...) tb2_name VALUES (field1_value, ...)
TEST_F
(
InsertTest
,
multiTableSingleRowTest
)
{
setDatabase
(
"root"
,
"test"
);
bind
(
"insert into st1s1 values (now, 1,
\"
beijing
\"
) st1s2 values (now, 10,
\"
131028
\"
)"
);
ASSERT_EQ
(
run
(),
TSDB_CODE_SUCCESS
);
dumpReslut
();
checkReslut
(
2
,
1
);
}
// INSERT INTO tb1_name VALUES (field1_value, ...) tb2_name VALUES (field1_value, ...)
TEST_F
(
InsertTest
,
multiTableMultiRowTest
)
{
setDatabase
(
"root"
,
"test"
);
bind
(
"insert into t1 values (now, 1,
\"
wxy
\"
)"
);
bind
(
"insert into st1s1 values (now, 1,
\"
beijing
\"
)(now+1s, 2,
\"
shanghai
\"
)(now+2s, 3,
\"
guangzhou
\"
)"
" st1s2 values (now, 10,
\"
131028
\"
)(now+1s, 20,
\"
132028
\"
)"
);
ASSERT_EQ
(
run
(),
TSDB_CODE_SUCCESS
);
SInsertStmtInfo
*
res
=
reslut
();
// todo check
ASSERT_EQ
(
res
->
insertType
,
TSDB_QUERY_TYPE_INSERT
);
// ASSERT_EQ(taosArrayGetSize(res->pDataBlocks), 1);
dumpReslut
();
checkReslut
(
2
,
3
,
2
);
}
TEST_F
(
InsertTest
,
toleranceTest
)
{
...
...
source/libs/parser/test/mockCatalog.cpp
浏览文件 @
d96963a2
...
...
@@ -25,15 +25,15 @@ namespace {
void
generateTestT1
(
MockCatalogService
*
mcs
)
{
ITableBuilder
&
builder
=
mcs
->
createTableBuilder
(
"root.test"
,
"t1"
,
TSDB_NORMAL_TABLE
,
3
)
.
setPrecision
(
TSDB_TIME_PRECISION_MILLI
).
setVgid
(
1
).
addColumn
(
"ts"
,
TSDB_DATA_TYPE_TIMESTAMP
)
.
addColumn
(
"c1"
,
TSDB_DATA_TYPE_INT
).
addColumn
(
"c2"
,
TSDB_DATA_TYPE_BINARY
,
1
0
);
.
addColumn
(
"c1"
,
TSDB_DATA_TYPE_INT
).
addColumn
(
"c2"
,
TSDB_DATA_TYPE_BINARY
,
2
0
);
builder
.
done
();
}
void
generateTestST1
(
MockCatalogService
*
mcs
)
{
ITableBuilder
&
builder
=
mcs
->
createTableBuilder
(
"root.test"
,
"st1"
,
TSDB_SUPER_TABLE
,
3
,
2
)
.
setPrecision
(
TSDB_TIME_PRECISION_MILLI
).
addColumn
(
"ts"
,
TSDB_DATA_TYPE_TIMESTAMP
)
.
addTag
(
"tag1"
,
TSDB_DATA_TYPE_INT
).
addTag
(
"tag2"
,
TSDB_DATA_TYPE_BINARY
,
1
0
)
.
addColumn
(
"c1"
,
TSDB_DATA_TYPE_INT
).
addColumn
(
"c2"
,
TSDB_DATA_TYPE_BINARY
,
1
0
);
.
addTag
(
"tag1"
,
TSDB_DATA_TYPE_INT
).
addTag
(
"tag2"
,
TSDB_DATA_TYPE_BINARY
,
2
0
)
.
addColumn
(
"c1"
,
TSDB_DATA_TYPE_INT
).
addColumn
(
"c2"
,
TSDB_DATA_TYPE_BINARY
,
2
0
);
builder
.
done
();
mcs
->
createSubTable
(
"root.test"
,
"st1"
,
"st1s1"
,
1
);
mcs
->
createSubTable
(
"root.test"
,
"st1"
,
"st1s2"
,
2
);
...
...
source/libs/parser/test/mockCatalog.h
浏览文件 @
d96963a2
...
...
@@ -22,8 +22,4 @@ void initMetaDataEnv();
void
generateMetaData
();
void
destroyMetaDataEnv
();
// mock
// int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle);
// int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta);
#endif // MOCK_CATALOG_H
source/libs/parser/test/mockCatalogService.cpp
浏览文件 @
d96963a2
...
...
@@ -33,6 +33,7 @@ public:
col
->
colId
=
colId_
++
;
col
->
bytes
=
bytes
;
strcpy
(
col
->
name
,
name
.
c_str
());
rowsize_
+=
bytes
;
return
*
this
;
}
...
...
@@ -147,11 +148,11 @@ public:
for
(
const
auto
&
db
:
meta_
)
{
std
::
cout
<<
"Databse:"
<<
db
.
first
<<
std
::
endl
;
std
::
cout
<<
SH
(
"Table"
)
<<
SH
(
"Type"
)
<<
SH
(
"Precision"
)
<<
IH
(
"Vgid"
)
<<
std
::
endl
;
std
::
cout
<<
SH
(
"Table"
)
<<
SH
(
"Type"
)
<<
SH
(
"Precision"
)
<<
IH
(
"Vgid"
)
<<
IH
(
"RowSize"
)
<<
std
::
endl
;
std
::
cout
<<
SL
(
3
,
1
)
<<
std
::
endl
;
for
(
const
auto
&
table
:
db
.
second
)
{
const
auto
&
schema
=
table
.
second
->
schema
;
std
::
cout
<<
SF
(
table
.
first
)
<<
SF
(
ttToString
(
schema
->
tableType
))
<<
SF
(
pToString
(
schema
->
tableInfo
.
precision
))
<<
IF
(
schema
->
vgId
)
<<
std
::
endl
;
std
::
cout
<<
SF
(
table
.
first
)
<<
SF
(
ttToString
(
schema
->
tableType
))
<<
SF
(
pToString
(
schema
->
tableInfo
.
precision
))
<<
IF
(
schema
->
vgId
)
<<
IF
(
schema
->
tableInfo
.
rowSize
)
<<
std
::
endl
;
}
std
::
cout
<<
std
::
endl
;
}
...
...
source/libs/planner/src/physicalPlan.c
浏览文件 @
d96963a2
...
...
@@ -221,7 +221,7 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
node
->
pChildren
=
taosArrayInit
(
TARRAY_MIN_SIZE
,
POINTER_BYTES
);
size_t
size
=
taosArrayGetSize
(
pPlanNode
->
pChildren
);
for
(
int32_t
i
=
0
;
i
<
size
;
++
i
)
{
SPhyNode
*
child
=
createPhyNode
(
pCxt
,
taosArrayGet
(
pPlanNode
->
pChildren
,
i
));
SPhyNode
*
child
=
createPhyNode
(
pCxt
,
taosArrayGet
P
(
pPlanNode
->
pChildren
,
i
));
child
->
pParent
=
node
;
taosArrayPush
(
node
->
pChildren
,
&
child
);
}
...
...
source/libs/qworker/CMakeLists.txt
0 → 100644
浏览文件 @
d96963a2
aux_source_directory
(
src QWORKER_SRC
)
add_library
(
qworker
${
QWORKER_SRC
}
)
target_include_directories
(
qworker
PUBLIC
"
${
CMAKE_SOURCE_DIR
}
/include/libs/qworker"
PRIVATE
"
${
CMAKE_CURRENT_SOURCE_DIR
}
/inc"
)
target_link_libraries
(
qworker
PRIVATE os util transport planner qcom
)
source/libs/qworker/inc/qworkerInt.h
0 → 100644
浏览文件 @
d96963a2
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_QWORKER_INT_H_
#define _TD_QWORKER_INT_H_
#ifdef __cplusplus
extern
"C"
{
#endif
#define QWORKER_DEFAULT_SCHEDULER_NUMBER 10000
#define QWORKER_DEFAULT_RES_CACHE_NUMBER 10000
#define QWORKER_DEFAULT_SCH_TASK_NUMBER 10000
enum
{
QW_READY_NOT_RECEIVED
=
0
,
QW_READY_RECEIVED
,
QW_READY_RESPONSED
,
};
typedef
struct
SQWorkerTaskStatus
{
int8_t
status
;
int8_t
ready
;
}
SQWorkerTaskStatus
;
typedef
struct
SQWorkerResCache
{
void
*
data
;
}
SQWorkerResCache
;
typedef
struct
SQWorkerSchTaskStatus
{
int32_t
lastAccessTs
;
// timestamp in second
SHashObj
*
taskStatus
;
// key:queryId+taskId, value: SQWorkerTaskStatus
}
SQWorkerSchTaskStatus
;
// Qnode/Vnode level task management
typedef
struct
SQWorkerMgmt
{
SQWorkerCfg
cfg
;
SHashObj
*
scheduleHash
;
//key: schedulerId, value: SQWorkerSchTaskStatus
SHashObj
*
resHash
;
//key: queryId+taskId, value: SQWorkerResCache
}
SQWorkerMgmt
;
#define QW_TASK_DONE(status) (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == status == JOB_TASK_STATUS_CANCELLED)
#define QW_SET_QTID(id, qid, tid) do { *(uint64_t *)(id) = (qid); *(uint64_t *)((char *)(id) + sizeof(qid)) = (tid); } while (0)
#define QW_GET_QTID(id, qid, tid) do { (qid) = *(uint64_t *)(id); (tid) = *(uint64_t *)((char *)(id) + sizeof(qid)); } while (0)
#define QW_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define QW_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define QW_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); terrno = _code; return _code; } } while (0)
#define QW_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
#ifdef __cplusplus
}
#endif
#endif
/*_TD_QWORKER_INT_H_*/
source/libs/qworker/src/qworker.c
0 → 100644
浏览文件 @
d96963a2
#include "taosmsg.h"
#include "query.h"
#include "qworker.h"
#include "qworkerInt.h"
#include "planner.h"
int32_t
qwAddTaskStatus
(
SQWorkerMgmt
*
mgmt
,
uint64_t
schedulerId
,
uint64_t
queryId
,
uint64_t
taskId
,
int8_t
taskStatus
)
{
SQWorkerTaskStatus
tStatus
=
{
0
};
tStatus
.
status
=
taskStatus
;
char
id
[
sizeof
(
queryId
)
+
sizeof
(
taskId
)]
=
{
0
};
QW_SET_QTID
(
id
,
queryId
,
taskId
);
SQWorkerSchTaskStatus
*
schStatus
=
taosHashGet
(
mgmt
->
scheduleHash
,
&
schedulerId
,
sizeof
(
schedulerId
));
if
(
NULL
==
schStatus
)
{
SQWorkerSchTaskStatus
newSchStatus
=
{
0
};
newSchStatus
.
taskStatus
=
taosHashInit
(
mgmt
->
cfg
.
maxSchTaskNum
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BINARY
),
false
,
HASH_ENTRY_LOCK
);
if
(
NULL
==
newSchStatus
.
taskStatus
)
{
qError
(
"taosHashInit %d failed"
,
mgmt
->
cfg
.
maxSchTaskNum
);
return
TSDB_CODE_QRY_OUT_OF_MEMORY
;
}
if
(
0
!=
taosHashPut
(
newSchStatus
.
taskStatus
,
id
,
sizeof
(
id
),
&
tStatus
,
sizeof
(
tStatus
)))
{
qError
(
"taosHashPut schedulerId[%"
PRIx64
"]queryId[%"
PRIx64
"] taskId[%"
PRIx64
"] to scheduleHash failed"
,
schedulerId
,
queryId
,
taskId
);
taosHashCleanup
(
newSchStatus
.
taskStatus
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
newSchStatus
.
lastAccessTs
=
taosGetTimestampSec
();
if
(
0
!=
taosHashPut
(
mgmt
->
scheduleHash
,
&
schedulerId
,
sizeof
(
schedulerId
),
&
newSchStatus
,
sizeof
(
newSchStatus
)))
{
qError
(
"taosHashPut schedulerId[%"
PRIx64
"] to scheduleHash failed"
,
schedulerId
);
taosHashCleanup
(
newSchStatus
.
taskStatus
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
return
TSDB_CODE_SUCCESS
;
}
schStatus
->
lastAccessTs
=
taosGetTimestampSec
();
if
(
0
!=
taosHashPut
(
schStatus
->
taskStatus
,
id
,
sizeof
(
id
),
&
tStatus
,
sizeof
(
tStatus
)))
{
qError
(
"taosHashPut schedulerId[%"
PRIx64
"]queryId[%"
PRIx64
"] taskId[%"
PRIx64
"] to scheduleHash failed"
,
schedulerId
,
queryId
,
taskId
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
return
TSDB_CODE_SUCCESS
;
}
int32_t
qwGetTaskStatus
(
SQWorkerMgmt
*
mgmt
,
uint64_t
schedulerId
,
uint64_t
queryId
,
uint64_t
taskId
,
SQWorkerTaskStatus
**
taskStatus
)
{
SQWorkerSchTaskStatus
*
schStatus
=
taosHashGet
(
mgmt
->
scheduleHash
,
&
schedulerId
,
sizeof
(
schedulerId
));
if
(
NULL
==
schStatus
)
{
qError
(
"no scheduler for schedulerId[%"
PRIx64
"]"
,
schedulerId
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
schStatus
->
lastAccessTs
=
taosGetTimestampSec
();
char
id
[
sizeof
(
queryId
)
+
sizeof
(
taskId
)]
=
{
0
};
QW_SET_QTID
(
id
,
queryId
,
taskId
);
SQWorkerTaskStatus
*
tStatus
=
taosHashGet
(
schStatus
->
taskStatus
,
id
,
sizeof
(
id
));
if
(
NULL
==
tStatus
)
{
qError
(
"no task status for schedulerId[%"
PRIx64
"] queryId[%"
PRIx64
"] taskId[%"
PRIx64
"]"
,
schedulerId
,
queryId
,
taskId
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
*
taskStatus
=
tStatus
;
return
TSDB_CODE_SUCCESS
;
}
int32_t
qwAddTaskResult
(
SQWorkerMgmt
*
mgmt
,
uint64_t
queryId
,
uint64_t
taskId
,
void
*
data
)
{
char
id
[
sizeof
(
queryId
)
+
sizeof
(
taskId
)]
=
{
0
};
QW_SET_QTID
(
id
,
queryId
,
taskId
);
SQWorkerResCache
resCache
=
{
0
};
resCache
.
data
=
data
;
if
(
0
!=
taosHashPut
(
mgmt
->
resHash
,
id
,
sizeof
(
id
),
&
resCache
,
sizeof
(
SQWorkerResCache
)))
{
qError
(
"taosHashPut queryId[%"
PRIx64
"] taskId[%"
PRIx64
"] to resHash failed"
,
queryId
,
taskId
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
return
TSDB_CODE_SUCCESS
;
}
int32_t
qwGetTaskResult
(
SQWorkerMgmt
*
mgmt
,
uint64_t
queryId
,
uint64_t
taskId
,
void
**
data
)
{
char
id
[
sizeof
(
queryId
)
+
sizeof
(
taskId
)]
=
{
0
};
QW_SET_QTID
(
id
,
queryId
,
taskId
);
SQWorkerResCache
*
resCache
=
taosHashGet
(
mgmt
->
resHash
,
id
,
sizeof
(
id
));
if
(
NULL
==
resCache
)
{
qError
(
"no task res for queryId[%"
PRIx64
"] taskId[%"
PRIx64
"]"
,
queryId
,
taskId
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
*
data
=
resCache
->
data
;
return
TSDB_CODE_SUCCESS
;
}
int32_t
qwUpdateSchLastAccess
(
SQWorkerMgmt
*
mgmt
,
uint64_t
schedulerId
)
{
SQWorkerSchTaskStatus
*
schStatus
=
taosHashGet
(
mgmt
->
scheduleHash
,
&
schedulerId
,
sizeof
(
schedulerId
));
if
(
NULL
==
schStatus
)
{
qError
(
"no scheduler for schedulerId[%"
PRIx64
"]"
,
schedulerId
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
schStatus
->
lastAccessTs
=
taosGetTimestampSec
();
return
TSDB_CODE_SUCCESS
;
}
int32_t
qwGetSchTasksStatus
(
SQWorkerMgmt
*
mgmt
,
uint64_t
schedulerId
,
SSchedulerStatusRsp
**
rsp
)
{
SQWorkerSchTaskStatus
*
schStatus
=
taosHashGet
(
mgmt
->
scheduleHash
,
&
schedulerId
,
sizeof
(
schedulerId
));
if
(
NULL
==
schStatus
)
{
qError
(
"no scheduler for schedulerId[%"
PRIx64
"]"
,
schedulerId
);
return
TSDB_CODE_QRY_APP_ERROR
;
}
schStatus
->
lastAccessTs
=
taosGetTimestampSec
();
int32_t
i
=
0
;
int32_t
taskNum
=
taosHashGetSize
(
schStatus
->
taskStatus
);
int32_t
size
=
sizeof
(
SSchedulerStatusRsp
)
+
sizeof
((
*
rsp
)
->
status
[
0
])
*
taskNum
;
*
rsp
=
calloc
(
1
,
size
);
if
(
NULL
==
*
rsp
)
{
qError
(
"calloc %d failed"
,
size
);
return
TSDB_CODE_QRY_OUT_OF_MEMORY
;
}
void
*
key
=
NULL
;
size_t
keyLen
=
0
;
void
*
pIter
=
taosHashIterate
(
schStatus
->
taskStatus
,
NULL
);
while
(
pIter
)
{
SQWorkerTaskStatus
*
taskStatus
=
(
SQWorkerTaskStatus
*
)
pIter
;
taosHashGetKey
(
pIter
,
&
key
,
&
keyLen
);
QW_GET_QTID
(
key
,
(
*
rsp
)
->
status
[
i
].
queryId
,
(
*
rsp
)
->
status
[
i
].
taskId
);
(
*
rsp
)
->
status
[
i
].
status
=
taskStatus
->
status
;
pIter
=
taosHashIterate
(
schStatus
->
taskStatus
,
pIter
);
}
(
*
rsp
)
->
num
=
taskNum
;
return
TSDB_CODE_SUCCESS
;
}
int32_t
qwBuildRspMsg
(
void
*
data
,
int32_t
msgType
);
int32_t
qWorkerInit
(
SQWorkerCfg
*
cfg
,
void
**
qWorkerMgmt
)
{
SQWorkerMgmt
*
mgmt
=
calloc
(
1
,
sizeof
(
SQWorkerMgmt
));
if
(
NULL
==
mgmt
)
{
qError
(
"calloc %d failed"
,
(
int32_t
)
sizeof
(
SQWorkerMgmt
));
return
TSDB_CODE_QRY_OUT_OF_MEMORY
;
}
if
(
cfg
)
{
mgmt
->
cfg
=
*
cfg
;
}
else
{
mgmt
->
cfg
.
maxSchedulerNum
=
QWORKER_DEFAULT_SCHEDULER_NUMBER
;
mgmt
->
cfg
.
maxResCacheNum
=
QWORKER_DEFAULT_RES_CACHE_NUMBER
;
mgmt
->
cfg
.
maxSchTaskNum
=
QWORKER_DEFAULT_SCH_TASK_NUMBER
;
}
mgmt
->
scheduleHash
=
taosHashInit
(
mgmt
->
cfg
.
maxSchedulerNum
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_UBIGINT
),
false
,
HASH_ENTRY_LOCK
);
if
(
NULL
==
mgmt
->
scheduleHash
)
{
tfree
(
mgmt
);
QW_ERR_LRET
(
TSDB_CODE_QRY_OUT_OF_MEMORY
,
"init %d schduler hash failed"
,
mgmt
->
cfg
.
maxSchedulerNum
);
}
mgmt
->
resHash
=
taosHashInit
(
mgmt
->
cfg
.
maxResCacheNum
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BINARY
),
false
,
HASH_ENTRY_LOCK
);
if
(
NULL
==
mgmt
->
resHash
)
{
taosHashCleanup
(
mgmt
->
scheduleHash
);
mgmt
->
scheduleHash
=
NULL
;
tfree
(
mgmt
);
QW_ERR_LRET
(
TSDB_CODE_QRY_OUT_OF_MEMORY
,
"init %d res cache hash failed"
,
mgmt
->
cfg
.
maxResCacheNum
);
}
*
qWorkerMgmt
=
mgmt
;
return
TSDB_CODE_SUCCESS
;
}
int32_t
qWorkerProcessQueryMsg
(
void
*
qWorkerMgmt
,
SSchedulerQueryMsg
*
msg
,
SRpcMsg
*
rsp
)
{
if
(
NULL
==
qWorkerMgmt
||
NULL
==
msg
||
NULL
==
rsp
)
{
return
TSDB_CODE_QRY_INVALID_INPUT
;
}
SSubplan
*
plan
=
NULL
;
SQWorkerTaskStatus
*
tStatus
=
NULL
;
int32_t
code
=
qStringToSubplan
(
msg
->
msg
,
&
plan
);
if
(
TSDB_CODE_SUCCESS
!=
code
)
{
qError
(
"schId:%"
PRIx64
",qId:%"
PRIx64
",taskId:%"
PRIx64
" string to subplan failed, code:%d"
,
msg
->
schedulerId
,
msg
->
queryId
,
msg
->
taskId
,
code
);
return
code
;
}
//TODO call executer to init subquery
QW_ERR_JRET
(
qwAddTaskStatus
(
qWorkerMgmt
,
msg
->
schedulerId
,
msg
->
queryId
,
msg
->
taskId
,
JOB_TASK_STATUS_EXECUTING
));
QW_ERR_JRET
(
qwBuildRspMsg
(
NULL
,
TSDB_MSG_TYPE_QUERY_RSP
));
//TODO call executer to execute subquery
code
=
0
;
void
*
data
=
NULL
;
//TODO call executer to execute subquery
QW_ERR_JRET
(
qwGetTaskStatus
(
qWorkerMgmt
,
msg
->
schedulerId
,
msg
->
queryId
,
msg
->
taskId
,
&
tStatus
));
tStatus
->
status
=
(
code
)
?
JOB_TASK_STATUS_FAILED
:
JOB_TASK_STATUS_SUCCEED
;
QW_ERR_JRET
(
qwAddTaskResult
(
qWorkerMgmt
,
msg
->
queryId
,
msg
->
taskId
,
data
));
_return:
if
(
tStatus
&&
QW_TASK_DONE
(
tStatus
->
status
)
&&
QW_READY_RECEIVED
==
tStatus
->
ready
)
{
QW_ERR_RET
(
qwBuildRspMsg
(
NULL
,
TSDB_MSG_TYPE_RES_READY_RSP
));
}
qDestroySubplan
(
plan
);
return
code
;
}
int32_t
qWorkerProcessReadyMsg
(
void
*
qWorkerMgmt
,
SSchedulerReadyMsg
*
msg
,
SRpcMsg
*
rsp
){
if
(
NULL
==
qWorkerMgmt
||
NULL
==
msg
||
NULL
==
rsp
)
{
return
TSDB_CODE_QRY_INVALID_INPUT
;
}
SQWorkerTaskStatus
*
tStatus
=
NULL
;
QW_ERR_RET
(
qwGetTaskStatus
(
qWorkerMgmt
,
msg
->
schedulerId
,
msg
->
queryId
,
msg
->
taskId
,
&
tStatus
));
if
(
QW_TASK_DONE
(
tStatus
->
status
))
{
QW_ERR_RET
(
qwBuildRspMsg
(
tStatus
,
TSDB_MSG_TYPE_RES_READY_RSP
));
}
else
{
tStatus
->
ready
=
QW_READY_RECEIVED
;
return
TSDB_CODE_SUCCESS
;
}
tStatus
->
ready
=
QW_READY_RESPONSED
;
return
TSDB_CODE_SUCCESS
;
}
int32_t
qWorkerProcessStatusMsg
(
void
*
qWorkerMgmt
,
SSchedulerStatusMsg
*
msg
,
SRpcMsg
*
rsp
)
{
if
(
NULL
==
qWorkerMgmt
||
NULL
==
msg
||
NULL
==
rsp
)
{
return
TSDB_CODE_QRY_INVALID_INPUT
;
}
SSchedulerStatusRsp
*
sStatus
=
NULL
;
QW_ERR_RET
(
qwGetSchTasksStatus
(
qWorkerMgmt
,
msg
->
schedulerId
,
&
sStatus
));
return
TSDB_CODE_SUCCESS
;
}
int32_t
qWorkerProcessFetchMsg
(
void
*
qWorkerMgmt
,
SSchedulerFetchMsg
*
msg
,
SRpcMsg
*
rsp
)
{
if
(
NULL
==
qWorkerMgmt
||
NULL
==
msg
||
NULL
==
rsp
)
{
return
TSDB_CODE_QRY_INVALID_INPUT
;
}
QW_ERR_RET
(
qwUpdateSchLastAccess
(
qWorkerMgmt
,
msg
->
schedulerId
));
void
*
data
=
NULL
;
QW_ERR_RET
(
qwGetTaskResult
(
qWorkerMgmt
,
msg
->
queryId
,
msg
->
taskId
,
&
data
));
QW_ERR_RET
(
qwBuildRspMsg
(
data
,
TSDB_MSG_TYPE_FETCH_RSP
));
return
TSDB_CODE_SUCCESS
;
}
int32_t
qWorkerProcessCancelMsg
(
void
*
qWorkerMgmt
,
SSchedulerCancelMsg
*
msg
,
SRpcMsg
*
rsp
);
void
qWorkerDestroy
(
void
**
qWorkerMgmt
)
{
if
(
NULL
==
qWorkerMgmt
||
NULL
==
*
qWorkerMgmt
)
{
return
;
}
SQWorkerMgmt
*
mgmt
=
*
qWorkerMgmt
;
//TODO STOP ALL QUERY
//TODO FREE ALL
tfree
(
*
qWorkerMgmt
);
}
source/libs/scheduler/inc/schedulerInt.h
浏览文件 @
d96963a2
...
...
@@ -31,17 +31,9 @@ extern "C" {
#define SCH_MAX_CONDIDATE_EP_NUM TSDB_MAX_REPLICA
enum
{
SCH_STATUS_NOT_START
=
1
,
SCH_STATUS_EXECUTING
,
SCH_STATUS_SUCCEED
,
SCH_STATUS_FAILED
,
SCH_STATUS_CANCELLING
,
SCH_STATUS_CANCELLED
};
typedef
struct
SSchedulerMgmt
{
uint64_t
taskId
;
uint64_t
taskId
;
uint64_t
schedulerId
;
SSchedulerCfg
cfg
;
SHashObj
*
Jobs
;
// key: queryId, value: SQueryJob*
}
SSchedulerMgmt
;
...
...
source/libs/scheduler/src/scheduler.c
浏览文件 @
d96963a2
...
...
@@ -161,7 +161,7 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) {
SArray
*
levelPlans
=
NULL
;
int32_t
levelPlanNum
=
0
;
level
.
status
=
SCH
_STATUS_NOT_START
;
level
.
status
=
JOB_TASK
_STATUS_NOT_START
;
for
(
int32_t
i
=
0
;
i
<
levelNum
;
++
i
)
{
level
.
level
=
i
;
...
...
@@ -191,7 +191,7 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) {
task
.
taskId
=
atomic_add_fetch_64
(
&
schMgmt
.
taskId
,
1
);
task
.
plan
=
plan
;
task
.
status
=
SCH
_STATUS_NOT_START
;
task
.
status
=
JOB_TASK
_STATUS_NOT_START
;
void
*
p
=
taosArrayPush
(
level
.
subTasks
,
&
task
);
if
(
NULL
==
p
)
{
...
...
@@ -304,13 +304,15 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) {
}
SSchedulerQueryMsg
*
pMsg
=
msg
;
pMsg
->
queryId
=
job
->
queryId
;
pMsg
->
taskId
=
task
->
taskId
;
pMsg
->
contentLen
=
len
;
pMsg
->
schedulerId
=
htobe64
(
schMgmt
.
schedulerId
);
pMsg
->
queryId
=
htobe64
(
job
->
queryId
);
pMsg
->
taskId
=
htobe64
(
task
->
taskId
);
pMsg
->
contentLen
=
htonl
(
len
);
memcpy
(
pMsg
->
msg
,
task
->
msg
,
len
);
break
;
}
case
TSDB_MSG_TYPE_R
SP
_READY
:
{
case
TSDB_MSG_TYPE_R
ES
_READY
:
{
msgSize
=
sizeof
(
SSchedulerReadyMsg
);
msg
=
calloc
(
1
,
msgSize
);
if
(
NULL
==
msg
)
{
...
...
@@ -319,8 +321,8 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) {
}
SSchedulerReadyMsg
*
pMsg
=
msg
;
pMsg
->
queryId
=
job
->
queryId
;
pMsg
->
taskId
=
task
->
taskId
;
pMsg
->
queryId
=
htobe64
(
job
->
queryId
)
;
pMsg
->
taskId
=
htobe64
(
task
->
taskId
)
;
break
;
}
case
TSDB_MSG_TYPE_FETCH
:
{
...
...
@@ -332,8 +334,8 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) {
}
SSchedulerFetchMsg
*
pMsg
=
msg
;
pMsg
->
queryId
=
job
->
queryId
;
pMsg
->
taskId
=
task
->
taskId
;
pMsg
->
queryId
=
htobe64
(
job
->
queryId
)
;
pMsg
->
taskId
=
htobe64
(
task
->
taskId
)
;
break
;
}
default:
...
...
@@ -376,7 +378,7 @@ _return:
int32_t
schProcessOnJobSuccess
(
SQueryJob
*
job
)
{
job
->
status
=
SCH
_STATUS_SUCCEED
;
job
->
status
=
JOB_TASK
_STATUS_SUCCEED
;
if
(
job
->
userFetch
)
{
SCH_ERR_RET
(
schFetchFromRemote
(
job
));
...
...
@@ -386,7 +388,7 @@ int32_t schProcessOnJobSuccess(SQueryJob *job) {
}
int32_t
schProcessOnJobFailure
(
SQueryJob
*
job
)
{
job
->
status
=
SCH
_STATUS_FAILED
;
job
->
status
=
JOB_TASK
_STATUS_FAILED
;
atomic_val_compare_exchange_32
(
&
job
->
remoteFetch
,
1
,
0
);
...
...
@@ -413,7 +415,7 @@ int32_t schProcessOnTaskSuccess(SQueryJob *job, SQueryTask *task) {
return
TSDB_CODE_SUCCESS
;
}
task
->
status
=
SCH
_STATUS_SUCCEED
;
task
->
status
=
JOB_TASK
_STATUS_SUCCEED
;
int32_t
parentNum
=
(
int32_t
)
taosArrayGetSize
(
task
->
parents
);
if
(
parentNum
==
0
)
{
...
...
@@ -459,7 +461,7 @@ int32_t schProcessOnTaskFailure(SQueryJob *job, SQueryTask *task, int32_t errCod
if
(
!
needRetry
)
{
SCH_TASK_ERR_LOG
(
"task failed[%x], no more retry"
,
errCode
);
job
->
status
=
SCH
_STATUS_FAILED
;
job
->
status
=
JOB_TASK
_STATUS_FAILED
;
SCH_ERR_RET
(
schProcessOnJobFailure
(
job
));
return
TSDB_CODE_SUCCESS
;
...
...
@@ -478,13 +480,13 @@ int32_t schHandleRspMsg(SQueryJob *job, SQueryTask *task, int32_t msgType, int32
if
(
rspCode
!=
TSDB_CODE_SUCCESS
)
{
SCH_ERR_JRET
(
schProcessOnTaskFailure
(
job
,
task
,
rspCode
));
}
else
{
code
=
schAsyncSendMsg
(
job
,
task
,
TSDB_MSG_TYPE_R
SP
_READY
);
code
=
schAsyncSendMsg
(
job
,
task
,
TSDB_MSG_TYPE_R
ES
_READY
);
if
(
code
)
{
goto
_task_error
;
}
}
break
;
case
TSDB_MSG_TYPE_R
SP
_READY
:
case
TSDB_MSG_TYPE_R
ES
_READY
:
if
(
rspCode
!=
TSDB_CODE_SUCCESS
)
{
SCH_ERR_JRET
(
schProcessOnTaskFailure
(
job
,
task
,
rspCode
));
}
else
{
...
...
@@ -534,7 +536,7 @@ int32_t schLaunchTask(SQueryJob *job, SQueryTask *task) {
SCH_ERR_RET
(
schPushTaskToExecList
(
job
,
task
));
task
->
status
=
SCH
_STATUS_EXECUTING
;
task
->
status
=
JOB_TASK
_STATUS_EXECUTING
;
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -546,22 +548,26 @@ int32_t schLaunchJob(SQueryJob *job) {
SCH_ERR_RET
(
schLaunchTask
(
job
,
task
));
}
job
->
status
=
SCH
_STATUS_EXECUTING
;
job
->
status
=
JOB_TASK
_STATUS_EXECUTING
;
return
TSDB_CODE_SUCCESS
;
}
int32_t
schedulerInit
(
SSchedulerCfg
*
cfg
)
{
schMgmt
.
Jobs
=
taosHashInit
(
SCHEDULE_DEFAULT_JOB_NUMBER
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_UBIGINT
),
false
,
HASH_ENTRY_LOCK
);
if
(
NULL
==
schMgmt
.
Jobs
)
{
SCH_ERR_LRET
(
TSDB_CODE_QRY_OUT_OF_MEMORY
,
"init %d schduler jobs failed"
,
SCHEDULE_DEFAULT_JOB_NUMBER
);
}
if
(
cfg
)
{
schMgmt
.
cfg
=
*
cfg
;
}
else
{
schMgmt
.
cfg
.
maxJobNum
=
SCHEDULE_DEFAULT_JOB_NUMBER
;
}
schMgmt
.
Jobs
=
taosHashInit
(
schMgmt
.
cfg
.
maxJobNum
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_UBIGINT
),
false
,
HASH_ENTRY_LOCK
);
if
(
NULL
==
schMgmt
.
Jobs
)
{
SCH_ERR_LRET
(
TSDB_CODE_QRY_OUT_OF_MEMORY
,
"init %d schduler jobs failed"
,
schMgmt
.
cfg
.
maxJobNum
);
}
schMgmt
.
schedulerId
=
1
;
//TODO GENERATE A UUID
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -605,7 +611,7 @@ int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, voi
SCH_ERR_JRET
(
TSDB_CODE_SCH_INTERNAL_ERROR
);
}
job
->
status
=
SCH
_STATUS_NOT_START
;
job
->
status
=
JOB_TASK
_STATUS_NOT_START
;
SCH_ERR_JRET
(
schLaunchJob
(
job
));
...
...
@@ -634,7 +640,7 @@ int32_t scheduleFetchRows(void *pJob, void **data) {
return
TSDB_CODE_QRY_APP_ERROR
;
}
if
(
job
->
status
==
SCH
_STATUS_SUCCEED
)
{
if
(
job
->
status
==
JOB_TASK
_STATUS_SUCCEED
)
{
SCH_ERR_JRET
(
schFetchFromRemote
(
job
));
}
...
...
@@ -668,7 +674,7 @@ void scheduleFreeJob(void *pJob) {
return
;
}
if
(
job
->
status
==
SCH
_STATUS_EXECUTING
)
{
if
(
job
->
status
==
JOB_TASK
_STATUS_EXECUTING
)
{
scheduleCancelJob
(
pJob
);
}
}
...
...
source/util/src/thash.c
浏览文件 @
d96963a2
...
...
@@ -776,9 +776,16 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) {
return
(
pHashObj
->
capacity
*
(
sizeof
(
SHashEntry
)
+
POINTER_BYTES
))
+
sizeof
(
SHashNode
)
*
taosHashGetSize
(
pHashObj
)
+
sizeof
(
SHashObj
);
}
FORCE_INLINE
void
*
taosHashGetDataKey
(
SHashObj
*
pHashObj
,
void
*
data
)
{
FORCE_INLINE
int32_t
taosHashGetKey
(
void
*
data
,
void
**
key
,
size_t
*
keyLen
)
{
if
(
NULL
==
data
||
NULL
==
key
)
{
return
-
1
;
}
SHashNode
*
node
=
GET_HASH_PNODE
(
data
);
return
GET_HASH_NODE_KEY
(
node
);
*
key
=
GET_HASH_NODE_KEY
(
node
);
*
keyLen
=
node
->
keyLen
;
return
0
;
}
FORCE_INLINE
uint32_t
taosHashGetDataKeyLen
(
SHashObj
*
pHashObj
,
void
*
data
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录