Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
10a7547b
TDengine
项目概览
taosdata
/
TDengine
12 个月 前同步成功
通知
1180
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
10a7547b
编写于
12月 17, 2021
作者:
dengyihao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
update index cache
上级
7daca7ff
变更
4
显示空白变更内容
内联
并排
Showing
4 changed file
with
88 addition
and
39 deletion
+88
-39
source/libs/index/inc/indexInt.h
source/libs/index/inc/indexInt.h
+6
-3
source/libs/index/inc/index_cache.h
source/libs/index/inc/index_cache.h
+4
-4
source/libs/index/src/index.c
source/libs/index/src/index.c
+33
-7
source/libs/index/src/index_cache.c
source/libs/index/src/index_cache.c
+45
-25
未找到文件。
source/libs/index/inc/indexInt.h
浏览文件 @
10a7547b
...
...
@@ -37,9 +37,11 @@ struct SIndex {
#endif
void
*
cache
;
void
*
tindex
;
SHashObj
*
fieldObj
;
// <field name, field id>
uint64_t
suid
;
int
fieldId
;
SHashObj
*
fieldObj
;
// < field name, field id>
int64_t
suid
;
// current super table id, -1 is normal table
int
fieldId
;
// field id allocated to cache
int32_t
cVersion
;
// current version allocated to cache
pthread_mutex_t
mtx
;
};
...
...
@@ -58,6 +60,7 @@ struct SIndexMultiTermQuery {
// field and key;
typedef
struct
SIndexTerm
{
uint8_t
type
;
// term data type, str/interger/json
char
*
key
;
int32_t
nKey
;
char
*
val
;
...
...
source/libs/index/inc/index_cache.h
浏览文件 @
10a7547b
...
...
@@ -20,8 +20,8 @@
// ----------------- row structure in skiplist ---------------------
/* A data row, the format is like below:
*
|<--totalLen-->|<-- fieldId-->|<-- value len--->|<-- value
-->|<--version--->|<-- itermType -->|
*
*
content: |<--totalLen-->|<-- fieldid-->|<--field type -->|<-- value len--->|<-- value -->|<-- uid
-->|<--version--->|<-- itermType -->|
*
len : |<--int32_t -->|<-- int16_t-->|<-- int16_t --->|<--- int32_t --->|<--valuelen->|<--uint64_t->|<-- int32_t-->|<-- int8_t --->|
*/
#ifdef __cplusplus
...
...
@@ -30,7 +30,6 @@ extern "C" {
typedef
struct
IndexCache
{
T_REF_DECLARE
()
int
cVersion
;
//
}
IndexCache
;
...
...
@@ -39,7 +38,8 @@ IndexCache *indexCacheCreate();
void
indexCacheDestroy
(
IndexCache
*
cache
);
int
indexCachePut
(
IndexCache
*
cache
,
int32_t
fieldId
,
const
char
*
fieldVale
,
int32_t
fvlen
,
uint64_t
uid
,
int8_t
operaType
);
int
indexCachePut
(
IndexCache
*
cache
,
int16_t
fieldId
,
int16_t
fieldType
,
const
char
*
fieldValue
,
int32_t
fvLen
,
uint32_t
version
,
uint64_t
uid
,
int8_t
operType
);
int
indexCacheGet
(
IndexCache
*
cache
,
uint64_t
*
rst
);
int
indexCacheSearch
(
IndexCache
*
cache
,
SIndexMultiTermQuery
*
query
,
SArray
*
result
);
...
...
source/libs/index/src/index.c
浏览文件 @
10a7547b
...
...
@@ -23,7 +23,8 @@
typedef
struct
SIdxFieldInfo
{
int
id
;
// generated by index internal
int
fieldId
;
// generated by index internal
int
cVersion
;
int
type
;
// field type
}
SIdxFieldInfo
;
...
...
@@ -39,7 +40,7 @@ static int indexMergeCacheIntoTindex(struct SIndex *sIdx) {
}
SIndex
*
indexOpen
(
SIndexOpts
*
opts
,
const
char
*
path
)
{
pthread_once
(
&
isInit
,
indexInit
);
SIndex
*
sIdx
=
malloc
(
sizeof
(
SIndex
));
SIndex
*
sIdx
=
calloc
(
1
,
sizeof
(
SIndex
));
#ifdef USE_LUCENE
index_t
*
index
=
index_open
(
path
);
...
...
@@ -49,6 +50,8 @@ SIndex *indexOpen(SIndexOpts *opts, const char *path) {
sIdx
->
cache
=
(
void
*
)
indexCacheCreate
();
sIdx
->
tindex
=
NULL
;
sIdx
->
fieldObj
=
taosHashInit
(
8
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BINARY
),
true
,
HASH_ENTRY_LOCK
);
sIdx
->
fieldId
=
1
;
sIdx
->
cVersion
=
1
;
pthread_mutex_init
(
&
sIdx
->
mtx
,
NULL
);
return
sIdx
;
}
...
...
@@ -65,7 +68,7 @@ void indexClose(SIndex *sIdx) {
return
;
}
int
indexPut
(
SIndex
*
index
,
SArray
*
f
ield_v
als
,
int
uid
)
{
int
indexPut
(
SIndex
*
index
,
SArray
*
f
V
als
,
int
uid
)
{
#ifdef USE_LUCENE
index_document_t
*
doc
=
index_document_create
();
...
...
@@ -73,8 +76,8 @@ int indexPut(SIndex *index, SArray* field_vals, int uid) {
char
buf
[
16
]
=
{
0
};
sprintf
(
buf
,
"%d"
,
uid
);
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
f
ield_v
als
);
i
++
)
{
SIndexTerm
*
p
=
taosArrayGetP
(
f
ield_v
als
,
i
);
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
f
V
als
);
i
++
)
{
SIndexTerm
*
p
=
taosArrayGetP
(
f
V
als
,
i
);
index_document_add
(
doc
,
(
const
char
*
)(
p
->
key
),
p
->
nKey
,
(
const
char
*
)(
p
->
val
),
p
->
nVal
,
1
);
}
index_document_add
(
doc
,
NULL
,
0
,
buf
,
strlen
(
buf
),
0
);
...
...
@@ -82,10 +85,33 @@ int indexPut(SIndex *index, SArray* field_vals, int uid) {
index_put
(
index
->
index
,
doc
);
index_document_destroy
(
doc
);
#endif
//TODO(yihao): reduce the lock range
pthread_mutex_lock
(
&
index
->
mtx
);
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
fVals
);
i
++
)
{
SIndexTerm
*
p
=
taosArrayGetP
(
fVals
,
i
);
SIdxFieldInfo
*
fi
=
taosHashGet
(
index
->
fieldObj
,
p
->
key
,
p
->
nKey
);
if
(
fi
==
NULL
)
{
SIdxFieldInfo
tfi
=
{.
fieldId
=
index
->
fieldId
,
.
type
=
p
->
type
};
index
->
cVersion
++
;
index
->
fieldId
++
;
taosHashPut
(
index
->
fieldObj
,
p
->
key
,
p
->
nKey
,
&
tfi
,
sizeof
(
tfi
));
}
else
{
//TODO, del
}
}
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
fVals
);
i
++
)
{
SIndexTerm
*
p
=
taosArrayGetP
(
fVals
,
i
);
SIdxFieldInfo
*
fi
=
taosHashGet
(
index
->
fieldObj
,
p
->
key
,
p
->
nKey
);
assert
(
fi
!=
NULL
);
int32_t
fieldId
=
fi
->
fieldId
;
int32_t
colType
=
fi
->
type
;
int32_t
version
=
index
->
cVersion
;
}
pthread_mutex_unlock
(
&
index
->
mtx
);
return
1
;
}
int
indexSearch
(
SIndex
*
index
,
SIndexMultiTermQuery
*
multiQuerys
,
SArray
*
result
)
{
#ifdef USE_LUCENE
...
...
@@ -152,7 +178,7 @@ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) {
SIndexMultiTermQuery
*
p
=
(
SIndexMultiTermQuery
*
)
malloc
(
sizeof
(
SIndexMultiTermQuery
));
if
(
p
==
NULL
)
{
return
NULL
;
}
p
->
opera
=
opera
;
p
->
query
=
taosArrayInit
(
1
,
sizeof
(
SIndexTermQuery
));
p
->
query
=
taosArrayInit
(
4
,
sizeof
(
SIndexTermQuery
));
return
p
;
}
void
indexMultiTermQueryDestroy
(
SIndexMultiTermQuery
*
pQuery
)
{
...
...
source/libs/index/src/index_cache.c
浏览文件 @
10a7547b
...
...
@@ -19,7 +19,7 @@ static int32_t compareKey(const void *l, const void *r) {
char
*
lp
=
(
char
*
)
l
;
char
*
rp
=
(
char
*
)
r
;
// skip total len
// skip total len
, not compare
int32_t
ll
,
rl
;
// len
memcpy
(
&
ll
,
lp
,
sizeof
(
int32_t
));
memcpy
(
&
rl
,
rp
,
sizeof
(
int32_t
));
...
...
@@ -27,7 +27,7 @@ static int32_t compareKey(const void *l, const void *r) {
rp
+=
sizeof
(
int32_t
);
// compare field id
int
32
_t
lf
,
rf
;
// field id
int
16
_t
lf
,
rf
;
// field id
memcpy
(
&
lf
,
lp
,
sizeof
(
lf
));
memcpy
(
&
rf
,
rp
,
sizeof
(
rf
));
if
(
lf
!=
rf
)
{
...
...
@@ -36,14 +36,22 @@ static int32_t compareKey(const void *l, const void *r) {
lp
+=
sizeof
(
lf
);
rp
+=
sizeof
(
rf
);
// compare field value
// compare field type
int16_t
lft
,
rft
;
memcpy
(
&
lft
,
lp
,
sizeof
(
lft
));
memcpy
(
&
rft
,
rp
,
sizeof
(
rft
));
lp
+=
sizeof
(
lft
);
rp
+=
sizeof
(
rft
);
assert
(
rft
==
rft
);
// skip value len
int32_t
lfl
,
rfl
;
memcpy
(
&
lfl
,
lp
,
sizeof
(
lfl
));
memcpy
(
&
rfl
,
rp
,
sizeof
(
rfl
));
lp
+=
sizeof
(
lfl
);
rp
+=
sizeof
(
rfl
);
//
refator later
//
compare value
int32_t
i
,
j
;
for
(
i
=
0
,
j
=
0
;
i
<
lfl
&&
j
<
rfl
;
i
++
,
j
++
)
{
if
(
lp
[
i
]
==
rp
[
j
])
{
continue
;
}
...
...
@@ -54,7 +62,14 @@ static int32_t compareKey(const void *l, const void *r) {
lp
+=
lfl
;
rp
+=
rfl
;
// compare version
// skip uid
uint64_t
lu
,
ru
;
memcpy
(
&
lu
,
lp
,
sizeof
(
lu
));
memcpy
(
&
ru
,
rp
,
sizeof
(
ru
));
lp
+=
sizeof
(
lu
);
rp
+=
sizeof
(
ru
);
// compare version, desc order
int32_t
lv
,
rv
;
memcpy
(
&
lv
,
lp
,
sizeof
(
lv
));
memcpy
(
&
rv
,
rp
,
sizeof
(
rv
));
...
...
@@ -63,7 +78,7 @@ static int32_t compareKey(const void *l, const void *r) {
}
lp
+=
sizeof
(
lv
);
rp
+=
sizeof
(
rv
);
// not care item type
return
0
;
...
...
@@ -77,35 +92,40 @@ void indexCacheDestroy(IndexCache *cache) {
free
(
cache
);
}
int
indexCachePut
(
IndexCache
*
cache
,
int32_t
fieldId
,
const
char
*
fieldValue
,
int32_t
fvlen
,
uint64_t
uid
,
int8_t
operType
)
{
int
indexCachePut
(
IndexCache
*
cache
,
int16_t
fieldId
,
int16_t
fieldType
,
const
char
*
fieldValue
,
int32_t
fvLen
,
uint32_t
version
,
uint64_t
uid
,
int8_t
operType
)
{
if
(
cache
==
NULL
)
{
return
-
1
;}
int32_t
version
=
T_REF_INC
(
cache
);
int32_t
total
=
sizeof
(
int32_t
)
+
sizeof
(
fieldId
)
+
4
+
fvl
en
+
sizeof
(
version
)
+
sizeof
(
uid
)
+
sizeof
(
operType
);
int32_t
total
=
sizeof
(
int32_t
)
+
sizeof
(
fieldId
)
+
sizeof
(
fieldType
)
+
sizeof
(
fvLen
)
+
fvL
en
+
sizeof
(
version
)
+
sizeof
(
uid
)
+
sizeof
(
operType
);
char
*
buf
=
calloc
(
1
,
total
);
char
*
p
=
buf
;
memcpy
(
buf
,
&
total
,
sizeof
(
total
));
total
+=
total
;
memcpy
(
p
,
&
total
,
sizeof
(
total
));
p
+=
sizeof
(
total
)
;
memcpy
(
buf
,
&
fieldId
,
sizeof
(
fieldId
));
buf
+=
sizeof
(
fieldId
);
memcpy
(
p
,
&
fieldId
,
sizeof
(
fieldId
));
p
+=
sizeof
(
fieldId
);
memcpy
(
buf
,
&
fvlen
,
sizeof
(
fvlen
));
buf
+=
sizeof
(
fvlen
);
memcpy
(
buf
,
fieldValue
,
fvlen
);
buf
+=
fvlen
;
memcpy
(
p
,
&
fieldType
,
sizeof
(
fieldType
));
p
+=
sizeof
(
fieldType
);
memcpy
(
buf
,
&
version
,
sizeof
(
version
));
buf
+=
sizeof
(
version
);
memcpy
(
p
,
&
fvLen
,
sizeof
(
fvLen
));
p
+=
sizeof
(
fvLen
);
memcpy
(
p
,
fieldValue
,
fvLen
);
p
+=
fvLen
;
memcpy
(
buf
,
&
uid
,
sizeof
(
uid
));
buf
+=
sizeof
(
uid
);
memcpy
(
p
,
&
version
,
sizeof
(
version
));
p
+=
sizeof
(
version
);
memcpy
(
buf
,
&
operType
,
sizeof
(
operType
));
buf
+=
sizeof
(
operType
);
memcpy
(
p
,
&
uid
,
sizeof
(
uid
));
p
+=
sizeof
(
uid
);
memcpy
(
p
,
&
operType
,
sizeof
(
operType
));
p
+=
sizeof
(
operType
);
}
int
indexCacheDel
(
IndexCache
*
cache
,
int32_t
fieldId
,
const
char
*
fieldValue
,
int32_t
fvlen
,
uint64_t
uid
,
int8_t
operType
)
{
}
int
indexCacheSearch
(
IndexCache
*
cache
,
SIndexMultiTermQuery
*
query
,
SArray
*
result
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录