Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
int
Ip2region
提交
288859f7
I
Ip2region
项目概览
int
/
Ip2region
上一次同步 大约 1 年
通知
21
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
I
Ip2region
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
288859f7
编写于
7月 01, 2022
作者:
L
Lion
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
wrap the header/vector/content buffer
上级
420bc3fb
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
173 addition
and
97 deletion
+173
-97
binding/c/main.c
binding/c/main.c
+4
-4
binding/c/util_test.c
binding/c/util_test.c
+16
-14
binding/c/xdb_searcher.c
binding/c/xdb_searcher.c
+96
-49
binding/c/xdb_searcher.h
binding/c/xdb_searcher.h
+57
-30
未找到文件。
binding/c/main.c
浏览文件 @
288859f7
...
...
@@ -11,8 +11,8 @@
struct
searcher_test_entry
{
xdb_searcher_t
searcher
;
char
*
v_index
;
char
*
c_buffer
;
xdb_vector_index_t
*
v_index
;
xdb_content_t
*
c_buffer
;
};
typedef
struct
searcher_test_entry
searcher_test_t
;
...
...
@@ -64,13 +64,13 @@ void destroy_searcher_test(searcher_test_t *test) {
// check and free the vector index
if
(
test
->
v_index
!=
NULL
)
{
xdb_
free
(
test
->
v_index
);
xdb_
close_vector_index
(
test
->
v_index
);
test
->
v_index
=
NULL
;
}
// check and free the content buffer
if
(
test
->
c_buffer
!=
NULL
)
{
xdb_
free
(
test
->
c_buffer
);
xdb_
close_content
(
test
->
c_buffer
);
test
->
c_buffer
=
NULL
;
}
}
...
...
binding/c/util_test.c
浏览文件 @
288859f7
...
...
@@ -37,10 +37,9 @@ void test_check_ip() {
}
void
test_load_header
()
{
xdb_header_t
header
;
int
err
=
xdb_load_header_from_file
(
"../../data/ip2region.xdb"
,
&
header
);
if
(
err
!=
0
)
{
printf
(
"failed to load header with errcode=%d
\n
"
,
err
);
xdb_header_t
*
header
=
xdb_load_header_from_file
(
"../../data/ip2region.xdb"
);
if
(
header
==
NULL
)
{
printf
(
"failed to load header"
);
}
else
{
printf
(
"header loaded: {
\n
"
" version: %d,
\n
"
...
...
@@ -48,33 +47,36 @@ void test_load_header() {
" created_at: %u,
\n
"
" start_index_ptr: %d,
\n
"
" end_index_ptr: %d
\n
"
" length: %d
\n
"
"}
\n
"
,
header
.
version
,
header
.
index_policy
,
header
.
created_at
,
header
.
start_index_ptr
,
header
.
end_index_ptr
header
->
version
,
header
->
index_policy
,
header
->
created_at
,
header
->
start_index_ptr
,
header
->
end_index_ptr
,
header
->
length
);
}
xdb_close_header
(
header
);
}
void
test_load_vector_index
()
{
char
*
ptr
=
xdb_load_vector_index_from_file
(
"../../data/ip2region.xdb"
);
if
(
ptr
==
NULL
)
{
xdb_vector_index_t
*
v_index
=
xdb_load_vector_index_from_file
(
"../../data/ip2region.xdb"
);
if
(
v_index
==
NULL
)
{
printf
(
"failed to load vector index from file
\n
"
);
}
else
{
printf
(
"vector index loaded from file
\n
"
);
printf
(
"vector index loaded from file
, length=%d
\n
"
,
v_index
->
length
);
}
xdb_
free
(
ptr
);
xdb_
close_vector_index
(
v_index
);
}
void
test_load_content
()
{
char
*
ptr
=
xdb_load_content_from_file
(
"../../data/ip2region.xdb"
);
if
(
ptr
==
NULL
)
{
xdb_content_t
*
content
=
xdb_load_content_from_file
(
"../../data/ip2region.xdb"
);
if
(
content
==
NULL
)
{
printf
(
"failed to load content from file
\n
"
);
}
else
{
printf
(
"content loaded from file
\n
"
);
printf
(
"content loaded from file
, length=%d
\n
"
,
content
->
length
);
}
xdb_
free
(
ptr
);
xdb_
close_content
(
content
);
}
// valgrind --tool=memcheck --leak-check=full ./a.out
...
...
binding/c/xdb_searcher.c
浏览文件 @
288859f7
...
...
@@ -12,13 +12,13 @@
// internal function prototype define
XDB_PRIVATE
(
int
)
read
(
xdb_searcher_t
*
,
long
offset
,
char
*
,
size_t
length
);
XDB_PRIVATE
(
int
)
xdb_new_base
(
xdb_searcher_t
*
xdb
,
const
char
*
db_path
,
const
char
*
vIndex
,
const
char
*
cBuff
)
{
XDB_PRIVATE
(
int
)
xdb_new_base
(
xdb_searcher_t
*
xdb
,
const
char
*
db_path
,
const
xdb_vector_index_t
*
v_index
,
const
xdb_content_t
*
c_buffer
)
{
memset
(
xdb
,
0x00
,
sizeof
(
xdb_searcher_t
));
// check the content buffer first
if
(
c
Buff
!=
NULL
)
{
xdb
->
v
ector
_index
=
NULL
;
xdb
->
content
_buff
=
cBuff
;
if
(
c
_buffer
!=
NULL
)
{
xdb
->
v_index
=
NULL
;
xdb
->
content
=
c_buffer
;
return
0
;
}
...
...
@@ -29,7 +29,7 @@ XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const ch
}
xdb
->
handle
=
handle
;
xdb
->
v
ector_index
=
vI
ndex
;
xdb
->
v
_index
=
v_i
ndex
;
return
0
;
}
...
...
@@ -39,11 +39,11 @@ XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *xdb, const char *db_path)
return
xdb_new_base
(
xdb
,
db_path
,
NULL
,
NULL
);
}
XDB_PUBLIC
(
int
)
xdb_new_with_vector_index
(
xdb_searcher_t
*
xdb
,
const
char
*
db_path
,
const
char
*
vI
ndex
)
{
return
xdb_new_base
(
xdb
,
db_path
,
v
I
ndex
,
NULL
);
XDB_PUBLIC
(
int
)
xdb_new_with_vector_index
(
xdb_searcher_t
*
xdb
,
const
char
*
db_path
,
const
xdb_vector_index_t
*
v_i
ndex
)
{
return
xdb_new_base
(
xdb
,
db_path
,
v
_i
ndex
,
NULL
);
}
XDB_PUBLIC
(
int
)
xdb_new_with_buffer
(
xdb_searcher_t
*
xdb
,
const
char
*
c_buffer
)
{
XDB_PUBLIC
(
int
)
xdb_new_with_buffer
(
xdb_searcher_t
*
xdb
,
const
xdb_content_t
*
c_buffer
)
{
return
xdb_new_base
(
xdb
,
NULL
,
NULL
,
c_buffer
);
}
...
...
@@ -78,12 +78,12 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_bu
il0
=
((
int
)
(
ip
>>
24
))
&
0xFF
;
il1
=
((
int
)
(
ip
>>
16
))
&
0xFF
;
idx
=
il0
*
xdb_vector_index_cols
*
xdb_vector_index_size
+
il1
*
xdb_vector_index_size
;
if
(
xdb
->
v
ector
_index
!=
NULL
)
{
s_ptr
=
xdb_get_uint
(
xdb
->
v
ector_index
,
idx
);
e_ptr
=
xdb_get_uint
(
xdb
->
v
ector_index
,
idx
+
4
);
}
else
if
(
xdb
->
content
_buff
!=
NULL
)
{
s_ptr
=
xdb_get_uint
(
xdb
->
content
_buff
,
xdb_header_info_length
+
idx
);
e_ptr
=
xdb_get_uint
(
xdb
->
content
_buff
,
xdb_header_info_length
+
idx
+
4
);
if
(
xdb
->
v_index
!=
NULL
)
{
s_ptr
=
xdb_get_uint
(
xdb
->
v
_index
->
buffer
,
idx
);
e_ptr
=
xdb_get_uint
(
xdb
->
v
_index
->
buffer
,
idx
+
4
);
}
else
if
(
xdb
->
content
!=
NULL
)
{
s_ptr
=
xdb_get_uint
(
xdb
->
content
->
buffer
,
xdb_header_info_length
+
idx
);
e_ptr
=
xdb_get_uint
(
xdb
->
content
->
buffer
,
xdb_header_info_length
+
idx
+
4
);
}
else
{
err
=
read
(
xdb
,
xdb_header_info_length
+
idx
,
vector_buffer
,
sizeof
(
vector_buffer
));
if
(
err
!=
0
)
{
...
...
@@ -147,8 +147,8 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_bu
XDB_PRIVATE
(
int
)
read
(
xdb_searcher_t
*
xdb
,
long
offset
,
char
*
buffer
,
size_t
length
)
{
// check the xdb content cache first
if
(
xdb
->
content
_buff
!=
NULL
)
{
memcpy
(
buffer
,
xdb
->
content
_buff
+
offset
,
length
);
if
(
xdb
->
content
!=
NULL
)
{
memcpy
(
buffer
,
xdb
->
content
->
buffer
+
offset
,
length
);
return
0
;
}
...
...
@@ -172,39 +172,58 @@ XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *xdb) {
// --- buffer load util functions
XDB_PUBLIC
(
int
)
xdb_load_header
(
FILE
*
handle
,
xdb_header_t
*
header
)
{
char
buffer
[
256
];
XDB_PUBLIC
(
xdb_header_t
*
)
xdb_load_header
(
FILE
*
handle
)
{
xdb_header_t
*
header
;
unsigned
int
size
=
xdb_header_info_length
;
// entry alloc
header
=
(
xdb_header_t
*
)
xdb_malloc
(
sizeof
(
xdb_header_t
));
if
(
header
==
NULL
)
{
return
NULL
;
}
if
(
fseek
(
handle
,
0
,
SEEK_SET
)
==
-
1
)
{
return
1
;
xdb_free
(
header
);
return
NULL
;
}
if
(
fread
(
buffer
,
1
,
256
,
handle
)
!=
256
)
{
return
2
;
if
(
fread
(
header
->
buffer
,
1
,
size
,
handle
)
!=
size
)
{
xdb_free
(
header
);
return
NULL
;
}
// fill the fields
header
->
version
=
(
unsigned
short
)
xdb_get_ushort
(
buffer
,
0
);
header
->
index_policy
=
(
unsigned
short
)
xdb_get_ushort
(
buffer
,
2
);
header
->
created_at
=
xdb_get_uint
(
buffer
,
4
);
header
->
start_index_ptr
=
xdb_get_uint
(
buffer
,
8
);
header
->
end_index_ptr
=
xdb_get_uint
(
buffer
,
12
);
return
0
;
header
->
length
=
size
;
header
->
version
=
(
unsigned
short
)
xdb_get_ushort
(
header
->
buffer
,
0
);
header
->
index_policy
=
(
unsigned
short
)
xdb_get_ushort
(
header
->
buffer
,
2
);
header
->
created_at
=
xdb_get_uint
(
header
->
buffer
,
4
);
header
->
start_index_ptr
=
xdb_get_uint
(
header
->
buffer
,
8
);
header
->
end_index_ptr
=
xdb_get_uint
(
header
->
buffer
,
12
);
return
header
;
}
XDB_PUBLIC
(
int
)
xdb_load_header_from_file
(
const
char
*
db_path
,
xdb_header_t
*
header
)
{
XDB_PUBLIC
(
xdb_header_t
*
)
xdb_load_header_from_file
(
const
char
*
db_path
)
{
FILE
*
handle
=
fopen
(
db_path
,
"r"
);
if
(
handle
==
NULL
)
{
return
10
;
return
NULL
;
}
return
xdb_load_header
(
handle
,
header
);
return
xdb_load_header
(
handle
);
}
XDB_PUBLIC
(
void
)
xdb_close_header
(
xdb_header_t
*
header
)
{
if
(
header
->
length
>
0
)
{
header
->
length
=
0
;
xdb_free
(
header
);
}
}
XDB_PUBLIC
(
char
*
)
xdb_load_vector_index
(
FILE
*
handle
)
{
char
*
ptr
=
NULL
;
int
size
=
xdb_vector_index_length
;
// --- vector index
XDB_PUBLIC
(
xdb_vector_index_t
*
)
xdb_load_vector_index
(
FILE
*
handle
)
{
xdb_vector_index_t
*
v_index
;
unsigned
int
size
=
xdb_vector_index_length
;
// seek to the vector index offset
if
(
fseek
(
handle
,
xdb_header_info_length
,
SEEK_SET
)
==
-
1
)
{
...
...
@@ -212,20 +231,21 @@ XDB_PUBLIC(char *) xdb_load_vector_index(FILE *handle) {
}
// do the buffer read
ptr
=
(
char
*
)
xdb_malloc
(
size
);
if
(
ptr
==
NULL
)
{
v_index
=
(
xdb_vector_index_t
*
)
xdb_malloc
(
sizeof
(
xdb_vector_index_t
)
);
if
(
v_index
==
NULL
)
{
return
NULL
;
}
if
(
fread
(
ptr
,
1
,
size
,
handle
)
!=
size
)
{
xdb_free
(
ptr
);
v_index
->
length
=
size
;
if
(
fread
(
v_index
->
buffer
,
1
,
size
,
handle
)
!=
size
)
{
xdb_free
(
v_index
);
return
NULL
;
}
return
ptr
;
return
v_index
;
}
XDB_PUBLIC
(
char
*
)
xdb_load_vector_index_from_file
(
const
char
*
db_path
)
{
XDB_PUBLIC
(
xdb_vector_index_t
*
)
xdb_load_vector_index_from_file
(
const
char
*
db_path
)
{
FILE
*
handle
=
fopen
(
db_path
,
"r"
);
if
(
handle
==
NULL
)
{
return
NULL
;
...
...
@@ -234,36 +254,54 @@ XDB_PUBLIC(char *) xdb_load_vector_index_from_file(const char *db_path) {
return
xdb_load_vector_index
(
handle
);
}
XDB_PUBLIC
(
char
*
)
xdb_load_content
(
FILE
*
handle
)
{
long
filesize
;
char
*
ptr
=
NULL
;
XDB_PUBLIC
(
void
)
xdb_close_vector_index
(
xdb_vector_index_t
*
v_index
)
{
if
(
v_index
->
length
>
0
)
{
v_index
->
length
=
0
;
xdb_free
(
v_index
);
}
}
// --- content buffer
XDB_PUBLIC
(
xdb_content_t
*
)
xdb_load_content
(
FILE
*
handle
)
{
unsigned
int
size
;
xdb_content_t
*
content
;
char
*
ptr
;
// determine the file size
if
(
fseek
(
handle
,
0
,
SEEK_END
)
==
-
1
)
{
return
NULL
;
}
filesize
=
ftell
(
handle
);
size
=
(
unsigned
int
)
ftell
(
handle
);
if
(
fseek
(
handle
,
0
,
SEEK_SET
)
==
-
1
)
{
return
NULL
;
}
// do the file read
ptr
=
(
char
*
)
xdb_malloc
(
filesize
);
if
(
ptr
==
NULL
)
{
content
=
(
xdb_content_t
*
)
xdb_malloc
(
sizeof
(
xdb_content_t
));
if
(
content
==
NULL
)
{
return
NULL
;
}
// do the buffer alloc
content
->
buffer
=
(
char
*
)
xdb_malloc
(
size
);
if
(
content
->
buffer
==
NULL
)
{
xdb_free
(
content
);
return
NULL
;
}
// read the content into the buffer
if
(
fread
(
ptr
,
1
,
filesize
,
handle
)
!=
filesize
)
{
content
->
length
=
size
;
if
(
fread
(
content
->
buffer
,
1
,
size
,
handle
)
!=
size
)
{
xdb_free
(
ptr
);
return
NULL
;
}
return
ptr
;
return
content
;
}
XDB_PUBLIC
(
char
*
)
xdb_load_content_from_file
(
const
char
*
db_path
)
{
XDB_PUBLIC
(
xdb_content_t
*
)
xdb_load_content_from_file
(
const
char
*
db_path
)
{
FILE
*
handle
=
fopen
(
db_path
,
"r"
);
if
(
handle
==
NULL
)
{
return
NULL
;
...
...
@@ -272,6 +310,15 @@ XDB_PUBLIC(char *) xdb_load_content_from_file(const char *db_path) {
return
xdb_load_content
(
handle
);
}
XDB_PUBLIC
(
void
)
xdb_close_content
(
xdb_content_t
*
content
)
{
if
(
content
->
length
>
0
)
{
content
->
length
=
0
;
xdb_free
(
content
->
buffer
);
content
->
buffer
=
NULL
;
xdb_free
(
content
);
}
}
// --- End
// get unsigned long (4bytes) from a specified buffer start from the specified offset
...
...
binding/c/xdb_searcher.h
浏览文件 @
288859f7
...
...
@@ -38,6 +38,59 @@
#define xdb_vector_index_length 524288
// --- buffer load util functions
// use the following buffer struct to wrap the binary buffer data
// since the buffer data could not be operated with the string API.
struct
xdb_header
{
unsigned
short
version
;
unsigned
short
index_policy
;
unsigned
int
created_at
;
unsigned
int
start_index_ptr
;
unsigned
int
end_index_ptr
;
// the original buffer
unsigned
int
length
;
char
buffer
[
xdb_header_info_length
];
};
typedef
struct
xdb_header
xdb_header_t
;
XDB_PUBLIC
(
xdb_header_t
*
)
xdb_load_header
(
FILE
*
);
XDB_PUBLIC
(
xdb_header_t
*
)
xdb_load_header_from_file
(
const
char
*
);
XDB_PUBLIC
(
void
)
xdb_close_header
(
xdb_header_t
*
);
// --- vector index buffer
struct
xdb_vector_index
{
unsigned
int
length
;
char
buffer
[
xdb_vector_index_length
];
};
typedef
struct
xdb_vector_index
xdb_vector_index_t
;
XDB_PUBLIC
(
xdb_vector_index_t
*
)
xdb_load_vector_index
(
FILE
*
);
XDB_PUBLIC
(
xdb_vector_index_t
*
)
xdb_load_vector_index_from_file
(
const
char
*
);
XDB_PUBLIC
(
void
)
xdb_close_vector_index
(
xdb_vector_index_t
*
);
// --- content buffer
struct
xdb_content
{
unsigned
int
length
;
char
*
buffer
;
};
typedef
struct
xdb_content
xdb_content_t
;
XDB_PUBLIC
(
xdb_content_t
*
)
xdb_load_content
(
FILE
*
);
XDB_PUBLIC
(
xdb_content_t
*
)
xdb_load_content_from_file
(
const
char
*
);
XDB_PUBLIC
(
void
)
xdb_close_content
(
xdb_content_t
*
);
// --- End buffer load
// xdb searcher structure
struct
xdb_searcher_entry
{
FILE
*
handle
;
...
...
@@ -49,20 +102,20 @@ struct xdb_searcher_entry {
// vector index buffer cache.
// preload the vector index will reduce the number of IO operations
// thus speedup the search process.
const
char
*
vector
_index
;
const
xdb_vector_index_t
*
v
_index
;
// content buffer.
// cache the whole xdb content.
const
char
*
content_buff
;
const
xdb_content_t
*
content
;
};
typedef
struct
xdb_searcher_entry
xdb_searcher_t
;
// xdb searcher new api define
XDB_PUBLIC
(
int
)
xdb_new_with_file_only
(
xdb_searcher_t
*
,
const
char
*
);
XDB_PUBLIC
(
int
)
xdb_new_with_vector_index
(
xdb_searcher_t
*
,
const
char
*
,
const
char
*
);
XDB_PUBLIC
(
int
)
xdb_new_with_vector_index
(
xdb_searcher_t
*
,
const
char
*
,
const
xdb_vector_index_t
*
);
XDB_PUBLIC
(
int
)
xdb_new_with_buffer
(
xdb_searcher_t
*
,
const
char
*
);
XDB_PUBLIC
(
int
)
xdb_new_with_buffer
(
xdb_searcher_t
*
,
const
xdb_content_t
*
);
XDB_PUBLIC
(
void
)
xdb_close
(
xdb_searcher_t
*
);
...
...
@@ -74,32 +127,6 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *, unsigned int, char *, size_t);
XDB_PUBLIC
(
int
)
xdb_get_io_count
(
xdb_searcher_t
*
);
// --- buffer load util functions
struct
xdb_header
{
unsigned
short
version
;
unsigned
short
index_policy
;
unsigned
int
created_at
;
unsigned
int
start_index_ptr
;
unsigned
int
end_index_ptr
;
};
typedef
struct
xdb_header
xdb_header_t
;
XDB_PUBLIC
(
int
)
xdb_load_header
(
FILE
*
,
xdb_header_t
*
);
XDB_PUBLIC
(
int
)
xdb_load_header_from_file
(
const
char
*
,
xdb_header_t
*
);
XDB_PUBLIC
(
char
*
)
xdb_load_vector_index
(
FILE
*
);
XDB_PUBLIC
(
char
*
)
xdb_load_vector_index_from_file
(
const
char
*
);
XDB_PUBLIC
(
char
*
)
xdb_load_content
(
FILE
*
);
XDB_PUBLIC
(
char
*
)
xdb_load_content_from_file
(
const
char
*
);
// --- End buffer load
// get unsigned long (4bytes) from a specified buffer start from the specified offset with little-endian
XDB_PUBLIC
(
unsigned
int
)
xdb_get_uint
(
const
char
*
,
int
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录