Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
Serving
提交
39fede5b
S
Serving
项目概览
PaddlePaddle
/
Serving
12 个月 前同步成功
通知
183
Star
833
Fork
253
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
105
列表
看板
标记
里程碑
合并请求
10
Wiki
2
Wiki
分析
仓库
DevOps
项目成员
Pages
S
Serving
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
105
Issue
105
列表
看板
标记
里程碑
合并请求
10
合并请求
10
Pages
分析
分析
仓库分析
DevOps
Wiki
2
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
39fede5b
编写于
7月 04, 2019
作者:
J
Jiawei Wang
提交者:
GitHub
7月 04, 2019
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #18 from bjjwwang/my_cool_stuff
Modifications about code-format and
上级
115fa191
c3478214
变更
31
隐藏空白更改
内联
并排
Showing
31 changed file
with
555 addition
and
636 deletion
+555
-636
.pre-commit-config.yaml
.pre-commit-config.yaml
+0
-6
cmake/external/opencv.cmake
cmake/external/opencv.cmake
+1
-0
cmake/external/redis++.cmake
cmake/external/redis++.cmake
+0
-45
demo-client/src/echo_kvdb.cpp
demo-client/src/echo_kvdb.cpp
+7
-7
demo-serving/op/kvdb_echo_op.cpp
demo-serving/op/kvdb_echo_op.cpp
+25
-27
demo-serving/op/kvdb_echo_op.h
demo-serving/op/kvdb_echo_op.h
+9
-8
demo-serving/op/reader_op.cpp
demo-serving/op/reader_op.cpp
+3
-2
demo-serving/proto/echo_kvdb_service.proto
demo-serving/proto/echo_kvdb_service.proto
+5
-5
kvdb/include/kvdb/kvdb_impl.h
kvdb/include/kvdb/kvdb_impl.h
+103
-92
kvdb/include/kvdb/paddle_rocksdb.h
kvdb/include/kvdb/paddle_rocksdb.h
+12
-13
kvdb/include/kvdb/rocksdb_impl.h
kvdb/include/kvdb/rocksdb_impl.h
+13
-17
kvdb/src/gtest_db_func.cpp
kvdb/src/gtest_db_func.cpp
+34
-43
kvdb/src/gtest_db_thread.cpp
kvdb/src/gtest_db_thread.cpp
+38
-42
kvdb/src/gtest_kvdb.cpp
kvdb/src/gtest_kvdb.cpp
+114
-116
kvdb/src/mock_param_dict_impl.cpp
kvdb/src/mock_param_dict_impl.cpp
+94
-101
kvdb/src/paddle_rocksdb.cpp
kvdb/src/paddle_rocksdb.cpp
+25
-24
kvdb/src/param_dict_mgr_impl.cpp
kvdb/src/param_dict_mgr_impl.cpp
+5
-5
kvdb/src/rockskvdb_impl.cpp
kvdb/src/rockskvdb_impl.cpp
+10
-17
kvdb/src/test_rocksdb.cpp
kvdb/src/test_rocksdb.cpp
+20
-23
pdcodegen/src/pdcodegen.cpp
pdcodegen/src/pdcodegen.cpp
+6
-3
predictor/CMakeLists.txt
predictor/CMakeLists.txt
+0
-1
predictor/framework/memory.cpp
predictor/framework/memory.cpp
+10
-9
predictor/framework/memory.h
predictor/framework/memory.h
+1
-1
predictor/framework/predictor_metric.h
predictor/framework/predictor_metric.h
+1
-1
predictor/framework/resource.cpp
predictor/framework/resource.cpp
+3
-7
predictor/framework/resource.h
predictor/framework/resource.h
+2
-3
predictor/framework/server.cpp
predictor/framework/server.cpp
+1
-1
predictor/proto/builtin_format.proto
predictor/proto/builtin_format.proto
+4
-9
predictor/src/pdserving.cpp
predictor/src/pdserving.cpp
+2
-1
sdk-cpp/include/common.h
sdk-cpp/include/common.h
+2
-2
sdk-cpp/proto/echo_kvdb_service.proto
sdk-cpp/proto/echo_kvdb_service.proto
+5
-5
未找到文件。
.pre-commit-config.yaml
浏览文件 @
39fede5b
...
...
@@ -42,12 +42,6 @@ repos:
entry
:
bash ./tools/codestyle/pylint_pre_commit.hook
language
:
system
files
:
\.(py)$
-
repo
:
https://github.com/PaddlePaddle/pre-commit-golang
sha
:
8337620115c25ff8333f1b1a493bd031049bd7c0
hooks
:
-
id
:
go-fmt
types
:
-
go
-
repo
:
local
hooks
:
-
id
:
copyright_checker
...
...
cmake/external/opencv.cmake
浏览文件 @
39fede5b
...
...
@@ -38,6 +38,7 @@ ExternalProject_Add(
-DCMAKE_INSTALL_PREFIX=
${
OPENCV_INSTALL_DIR
}
-DCMAKE_INSTALL_LIBDIR=
${
OPENCV_INSTALL_DIR
}
/lib
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DWITH_GTK=OFF
-DBUILD_TESTS=OFF
-DBUILD_PERF_TESTS=OFF
-DCMAKE_BUILD_TYPE=
${
THIRD_PARTY_BUILD_TYPE
}
...
...
cmake/external/redis++.cmake
已删除
100644 → 0
浏览文件 @
115fa191
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
INCLUDE
(
ExternalProject
)
SET
(
REDISCLIENT_SOURCES_DIR
${
THIRD_PARTY_PATH
}
/redis++
)
SET
(
REDISCLIENT_INSTALL_DIR
${
THIRD_PARTY_PATH
}
/install/redis++
)
SET
(
REDISCLIENT_INCLUDE_DIR
"
${
REDISCLIENT_INSTALL_DIR
}
/include"
CACHE PATH
"redis++ include directory."
FORCE
)
SET
(
REDISCLIENT_LIBRARIES
"
${
REDISCLIENT_INSTALL_DIR
}
/lib/libredis++.a"
CACHE FILEPATH
"redis++ library."
FORCE
)
INCLUDE_DIRECTORIES
(
${
REDISCLIENT_INCLUDE_DIR
}
)
ExternalProject_Add
(
extern_redis++
${
EXTERNAL_PROJECT_LOG_ARGS
}
PREFIX
${
REDISCLIENT_SOURCES_DIR
}
GIT_REPOSITORY
"https://github.com/sewenew/redis-plus-plus"
GIT_TAG master
UPDATE_COMMAND
""
CONFIGURE_COMMAND
""
BUILD_COMMAND cmake . && CXXFLAGS=-fPIC make -j
${
NUM_OF_PROCESSOR
}
static
INSTALL_COMMAND mkdir -p
${
REDISCLIENT_INSTALL_DIR
}
/lib/
&& cp
${
REDISCLIENT_SOURCES_DIR
}
/src/extern_redis++/lib/libredis++.a
${
REDISCLIENT_LIBRARIES
}
&& cp -r
${
REDISCLIENT_SOURCES_DIR
}
/src/extern_redis++/src/sw/redis++/
${
REDISCLIENT_INSTALL_DIR
}
/include/redis++/
BUILD_IN_SOURCE 1
)
ADD_DEPENDENCIES
(
extern_redis++ snappy
)
ADD_LIBRARY
(
redis++ STATIC IMPORTED GLOBAL
)
SET_PROPERTY
(
TARGET redis++ PROPERTY IMPORTED_LOCATION
${
ROCKSDB_LIBRARIES
}
)
ADD_DEPENDENCIES
(
redis++ extern_redis++
)
LIST
(
APPEND external_project_dependencies redis++
)
demo-client/src/echo_kvdb.cpp
浏览文件 @
39fede5b
...
...
@@ -16,12 +16,12 @@
#include <sys/types.h>
#include <unistd.h>
#include <algorithm>
#include <fstream>
#include "sdk-cpp/builtin_format.pb.h"
#include "sdk-cpp/echo_kvdb_service.pb.h"
#include "sdk-cpp/include/common.h"
#include "sdk-cpp/include/predictor_sdk.h"
#include <algorithm>
using
baidu
::
paddle_serving
::
sdk_cpp
::
Predictor
;
using
baidu
::
paddle_serving
::
sdk_cpp
::
PredictorApi
;
using
baidu
::
paddle_serving
::
predictor
::
echo_kvdb_service
::
Request
;
...
...
@@ -48,11 +48,11 @@ void print_res(const Request& req,
uint64_t
elapse_ms
)
{
LOG
(
INFO
)
<<
"Receive Response size: "
<<
res
.
ress_size
();
for
(
size_t
i
=
0
;
i
<
res
.
ress_size
();
i
++
)
{
KVDBRes
val
=
res
.
ress
(
i
);
LOG
(
INFO
)
<<
"Receive value from demo-server: "
<<
val
.
value
();
}
LOG
(
INFO
)
<<
"Succ call predictor[echo_kvdb_service], the tag is: "
<<
route_tag
<<
", elapse_ms: "
<<
elapse_ms
;
KVDBRes
val
=
res
.
ress
(
i
);
LOG
(
INFO
)
<<
"Receive value from demo-server: "
<<
val
.
value
();
}
LOG
(
INFO
)
<<
"Succ call predictor[echo_kvdb_service], the tag is: "
<<
route_tag
<<
", elapse_ms: "
<<
elapse_ms
;
}
int
main
(
int
argc
,
char
**
argv
)
{
...
...
@@ -100,7 +100,7 @@ int main(int argc, char** argv) {
while
(
true
)
{
if
(
global_key
>
10000
)
{
break
;
break
;
}
timeval
start
;
gettimeofday
(
&
start
,
NULL
);
...
...
demo-serving/op/kvdb_echo_op.cpp
浏览文件 @
39fede5b
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
...
...
@@ -23,36 +23,34 @@ using baidu::paddle_serving::predictor::format::KVDBRes;
using
baidu
::
paddle_serving
::
predictor
::
echo_kvdb_service
::
Request
;
using
baidu
::
paddle_serving
::
predictor
::
echo_kvdb_service
::
Response
;
int
KVDBEchoOp
::
inference
()
{
debug
();
}
int
KVDBEchoOp
::
inference
()
{
debug
();
}
int
KVDBEchoOp
::
debug
()
{
//TODO: implement DEBUG mode
baidu
::
paddle_serving
::
predictor
::
Resource
&
resource
=
baidu
::
paddle_serving
::
predictor
::
Resource
::
instance
();
std
::
shared_ptr
<
RocksDBWrapper
>
db
=
resource
.
getDB
();
const
Request
*
req
=
dynamic_cast
<
const
Request
*>
(
get_request_message
());
Response
*
res
=
mutable_data
<
Response
>
();
LOG
(
INFO
)
<<
"Receive request in KVDB echo service: "
<<
req
->
ShortDebugString
();
for
(
size_t
i
=
0
;
i
<
req
->
reqs_size
();
i
++
)
{
auto
kvdbreq
=
req
->
reqs
(
i
);
std
::
string
op
=
kvdbreq
.
op
();
std
::
string
key
=
kvdbreq
.
key
();
std
::
string
val
=
kvdbreq
.
value
();
if
(
op
==
"SET"
)
{
db
->
Put
(
key
,
val
);
KVDBRes
*
kvdb_value_res
=
res
->
mutable_ress
()
->
Add
();
kvdb_value_res
->
set_value
(
"OK"
);
}
else
if
(
op
==
"GET"
)
{
std
::
string
getvalue
=
db
->
Get
(
key
);
KVDBRes
*
kvdb_value_res
=
res
->
mutable_ress
()
->
Add
();
kvdb_value_res
->
set_value
(
getvalue
);
}
// TODO: implement DEBUG mode
baidu
::
paddle_serving
::
predictor
::
Resource
&
resource
=
baidu
::
paddle_serving
::
predictor
::
Resource
::
instance
();
std
::
shared_ptr
<
RocksDBWrapper
>
db
=
resource
.
getDB
();
const
Request
*
req
=
dynamic_cast
<
const
Request
*>
(
get_request_message
());
Response
*
res
=
mutable_data
<
Response
>
();
LOG
(
INFO
)
<<
"Receive request in KVDB echo service: "
<<
req
->
ShortDebugString
();
for
(
size_t
i
=
0
;
i
<
req
->
reqs_size
();
i
++
)
{
auto
kvdbreq
=
req
->
reqs
(
i
);
std
::
string
op
=
kvdbreq
.
op
();
std
::
string
key
=
kvdbreq
.
key
();
std
::
string
val
=
kvdbreq
.
value
();
if
(
op
==
"SET"
)
{
db
->
Put
(
key
,
val
);
KVDBRes
*
kvdb_value_res
=
res
->
mutable_ress
()
->
Add
();
kvdb_value_res
->
set_value
(
"OK"
);
}
else
if
(
op
==
"GET"
)
{
std
::
string
getvalue
=
db
->
Get
(
key
);
KVDBRes
*
kvdb_value_res
=
res
->
mutable_ress
()
->
Add
();
kvdb_value_res
->
set_value
(
getvalue
);
}
return
0
;
}
return
0
;
}
DEFINE_OP
(
KVDBEchoOp
);
}
}
...
...
demo-serving/op/kvdb_echo_op.h
浏览文件 @
39fede5b
...
...
@@ -14,23 +14,24 @@
#pragma once
#include "demo-serving/echo_kvdb_service.pb.h"
#include "
predictor/framework/resource
.h"
#include "
kvdb/paddle_rocksdb
.h"
#include "predictor/common/inner_common.h"
#include "predictor/framework/channel.h"
#include "predictor/framework/op_repository.h"
#include "predictor/framework/resource.h"
#include "predictor/op/op.h"
#include "kvdb/paddle_rocksdb.h"
namespace
baidu
{
namespace
paddle_serving
{
namespace
predictor
{
class
KVDBEchoOp
:
public
OpWithChannel
<
baidu
::
paddle_serving
::
predictor
::
echo_kvdb_service
::
Response
>
{
public:
DECLARE_OP
(
KVDBEchoOp
);
int
inference
();
int
debug
();
class
KVDBEchoOp
:
public
OpWithChannel
<
baidu
::
paddle_serving
::
predictor
::
echo_kvdb_service
::
Response
>
{
public:
DECLARE_OP
(
KVDBEchoOp
);
int
inference
();
int
debug
();
};
}
// namespace predictor
}
// namespace paddle_serving
...
...
demo-serving/op/reader_op.cpp
浏览文件 @
39fede5b
...
...
@@ -64,9 +64,10 @@ int ReaderOp::inference() {
size_t
dense_capacity
=
3
*
resize
.
width
*
resize
.
height
;
size_t
len
=
dense_capacity
*
sizeof
(
float
)
*
sample_size
;
// Allocate buffer in PaddleTensor, so that buffer will be managed by the Tensor
// Allocate buffer in PaddleTensor, so that buffer will be managed by the
// Tensor
in_tensor
.
data
.
Resize
(
len
);
float
*
data
=
reinterpret_cast
<
float
*>
(
in_tensor
.
data
.
data
());
float
*
data
=
reinterpret_cast
<
float
*>
(
in_tensor
.
data
.
data
());
if
(
in_tensor
.
data
.
data
()
==
NULL
)
{
LOG
(
ERROR
)
<<
"Failed create temp float array, "
<<
"size="
<<
dense_capacity
*
sample_size
*
sizeof
(
float
);
...
...
demo-serving/proto/echo_kvdb_service.proto
浏览文件 @
39fede5b
...
...
@@ -20,15 +20,15 @@ package baidu.paddle_serving.predictor.echo_kvdb_service;
option
cc_generic_services
=
true
;
message
Request
{
repeated
baidu.paddle_serving.predictor.format.KVDBReq
reqs
=
1
;
repeated
baidu.paddle_serving.predictor.format.KVDBReq
reqs
=
1
;
};
message
Response
{
repeated
baidu.paddle_serving.predictor.format.KVDBRes
ress
=
1
;
repeated
baidu.paddle_serving.predictor.format.KVDBRes
ress
=
1
;
};
service
EchoKVDBService
{
rpc
inference
(
Request
)
returns
(
Response
);
rpc
debug
(
Request
)
returns
(
Response
);
option
(
pds.options
)
.
generate_impl
=
true
;
rpc
inference
(
Request
)
returns
(
Response
);
rpc
debug
(
Request
)
returns
(
Response
);
option
(
pds.options
)
.
generate_impl
=
true
;
};
kvdb/include/kvdb/kvdb_impl.h
浏览文件 @
39fede5b
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
//
...
...
@@ -14,10 +27,10 @@
// limitations under the License.
#pragma once
#include <vector>
#include <unordered_map>
#include <memory>
#include <chrono>
#include <memory>
#include <unordered_map>
#include <vector>
class
AbstractKVDB
;
class
FileReader
;
class
ParamDict
;
...
...
@@ -27,104 +40,102 @@ typedef std::shared_ptr<FileReader> FileReaderPtr;
typedef
std
::
shared_ptr
<
ParamDict
>
ParamDictPtr
;
class
AbstractKVDB
{
public:
virtual
void
CreateDB
()
=
0
;
virtual
void
SetDBName
(
std
::
string
)
=
0
;
virtual
void
Set
(
std
::
string
key
,
std
::
string
value
)
=
0
;
virtual
std
::
string
Get
(
std
::
string
key
)
=
0
;
virtual
~
AbstractKVDB
()
=
0
;
public:
virtual
void
CreateDB
()
=
0
;
virtual
void
SetDBName
(
std
::
string
)
=
0
;
virtual
void
Set
(
std
::
string
key
,
std
::
string
value
)
=
0
;
virtual
std
::
string
Get
(
std
::
string
key
)
=
0
;
virtual
~
AbstractKVDB
()
=
0
;
};
// TODO: Implement RedisKVDB
//class RedisKVDB;
class
FileReader
{
public:
inline
virtual
std
::
string
GetFileName
()
{
return
this
->
filename_
;
}
inline
virtual
void
SetFileName
(
std
::
string
filename
)
{
this
->
filename_
=
filename
;
this
->
last_md5_val_
=
this
->
GetMD5
();
this
->
time_stamp_
=
std
::
chrono
::
system_clock
::
now
();
}
inline
virtual
std
::
string
GetMD5
()
{
auto
getCmdOut
=
[]
(
std
::
string
cmd
)
{
std
::
string
data
;
FILE
*
stream
=
nullptr
;
const
int
max_buffer
=
256
;
char
buffer
[
max_buffer
];
cmd
.
append
(
" 2>&1"
);
stream
=
popen
(
cmd
.
c_str
(),
"r"
);
if
(
stream
)
{
if
(
fgets
(
buffer
,
max_buffer
,
stream
)
!=
NULL
)
{
data
.
append
(
buffer
);
}
}
return
data
;
};
std
::
string
cmd
=
"md5sum "
+
this
->
filename_
;
//TODO: throw exception if error occurs during execution of shell command
std
::
string
md5val
=
getCmdOut
(
cmd
);
this
->
time_stamp_
=
md5val
==
this
->
last_md5_val_
?
this
->
time_stamp_
:
std
::
chrono
::
system_clock
::
now
();
this
->
last_md5_val_
=
md5val
;
return
md5val
;
}
inline
virtual
bool
CheckDiff
()
{
return
this
->
GetMD5
()
==
this
->
last_md5_val_
;
}
inline
virtual
std
::
chrono
::
system_clock
::
time_point
GetTimeStamp
()
{
return
this
->
time_stamp_
;
}
inline
virtual
~
FileReader
()
{};
protected:
std
::
string
filename_
;
std
::
string
last_md5_val_
;
std
::
chrono
::
system_clock
::
time_point
time_stamp_
;
public:
inline
virtual
std
::
string
GetFileName
()
{
return
this
->
filename_
;
}
inline
virtual
void
SetFileName
(
std
::
string
filename
)
{
this
->
filename_
=
filename
;
this
->
last_md5_val_
=
this
->
GetMD5
();
this
->
time_stamp_
=
std
::
chrono
::
system_clock
::
now
();
}
inline
virtual
std
::
string
GetMD5
()
{
auto
getCmdOut
=
[](
std
::
string
cmd
)
{
std
::
string
data
;
FILE
*
stream
=
nullptr
;
const
int
max_buffer
=
256
;
char
buffer
[
max_buffer
];
cmd
.
append
(
" 2>&1"
);
stream
=
popen
(
cmd
.
c_str
(),
"r"
);
if
(
stream
)
{
if
(
fgets
(
buffer
,
max_buffer
,
stream
)
!=
NULL
)
{
data
.
append
(
buffer
);
}
}
return
data
;
};
std
::
string
cmd
=
"md5sum "
+
this
->
filename_
;
// TODO: throw exception if error occurs during execution of shell command
std
::
string
md5val
=
getCmdOut
(
cmd
);
this
->
time_stamp_
=
md5val
==
this
->
last_md5_val_
?
this
->
time_stamp_
:
std
::
chrono
::
system_clock
::
now
();
this
->
last_md5_val_
=
md5val
;
return
md5val
;
}
inline
virtual
bool
CheckDiff
()
{
return
this
->
GetMD5
()
==
this
->
last_md5_val_
;
}
inline
virtual
std
::
chrono
::
system_clock
::
time_point
GetTimeStamp
()
{
return
this
->
time_stamp_
;
}
inline
virtual
~
FileReader
(){};
private:
std
::
string
filename_
;
std
::
string
last_md5_val_
;
std
::
chrono
::
system_clock
::
time_point
time_stamp_
;
};
class
ParamDict
{
typedef
std
::
string
Key
;
typedef
std
::
vector
<
std
::
string
>
Value
;
public:
virtual
std
::
vector
<
FileReaderPtr
>
GetDictReaderLst
();
virtual
void
SetFileReaderLst
(
std
::
vector
<
std
::
string
>
lst
);
virtual
std
::
vector
<
float
>
GetSparseValue
(
int64_t
,
int64_t
);
virtual
std
::
vector
<
float
>
GetSparseValue
(
std
::
string
,
std
::
string
);
virtual
bool
InsertSparseValue
(
int64_t
,
int64_t
,
const
std
::
vector
<
float
>&
);
virtual
bool
InsertSparseValue
(
std
::
string
,
std
::
string
,
const
std
::
vector
<
float
>&
);
virtual
void
SetReader
(
std
::
function
<
std
::
pair
<
Key
,
Value
>
(
std
::
string
)
>
);
virtual
void
UpdateBaseModel
();
virtual
void
UpdateDeltaModel
();
virtual
std
::
pair
<
AbsKVDBPtr
,
AbsKVDBPtr
>
GetKVDB
();
virtual
void
SetKVDB
(
std
::
pair
<
AbsKVDBPtr
,
AbsKVDBPtr
>
);
virtual
void
CreateKVDB
();
virtual
~
ParamDict
();
protected:
std
::
function
<
std
::
pair
<
Key
,
Value
>
(
std
::
string
)
>
read_func_
;
std
::
vector
<
FileReaderPtr
>
file_reader_lst_
;
AbsKVDBPtr
front_db
,
back_db
;
};
typedef
std
::
string
Key
;
typedef
std
::
vector
<
std
::
string
>
Value
;
public:
virtual
std
::
vector
<
FileReaderPtr
>
GetDictReaderLst
();
virtual
void
SetFileReaderLst
(
std
::
vector
<
std
::
string
>
lst
);
virtual
std
::
vector
<
float
>
GetSparseValue
(
int64_t
,
int64_t
);
virtual
std
::
vector
<
float
>
GetSparseValue
(
std
::
string
,
std
::
string
);
class
ParamDictMgr
{
public:
void
UpdateAll
();
void
InsertParamDict
(
std
::
string
,
ParamDictPtr
);
virtual
bool
InsertSparseValue
(
int64_t
,
int64_t
,
const
std
::
vector
<
float
>
&
);
virtual
bool
InsertSparseValue
(
std
::
string
,
std
::
string
,
const
std
::
vector
<
float
>
&
);
virtual
void
SetReader
(
std
::
function
<
std
::
pair
<
Key
,
Value
>
(
std
::
string
)
>
);
virtual
void
UpdateBaseModel
();
virtual
void
UpdateDeltaModel
();
virtual
std
::
pair
<
AbsKVDBPtr
,
AbsKVDBPtr
>
GetKVDB
();
virtual
void
SetKVDB
(
std
::
pair
<
AbsKVDBPtr
,
AbsKVDBPtr
>
);
virtual
void
CreateKVDB
();
virtual
~
ParamDict
();
protected:
std
::
unordered_map
<
std
::
string
,
ParamDictPtr
>
ParamDictMap
;
private:
std
::
function
<
std
::
pair
<
Key
,
Value
>
(
std
::
string
)
>
read_func_
;
std
::
vector
<
FileReaderPtr
>
file_reader_lst_
;
AbsKVDBPtr
front_db
,
back_db
;
};
class
ParamDictMgr
{
public:
void
UpdateAll
();
void
InsertParamDict
(
std
::
string
,
ParamDictPtr
);
private:
std
::
unordered_map
<
std
::
string
,
ParamDictPtr
>
ParamDictMap
;
};
kvdb/include/kvdb/paddle_rocksdb.h
浏览文件 @
39fede5b
...
...
@@ -15,26 +15,25 @@
#pragma once
#include <memory>
#include <string>
#include "rocksdb/compaction_filter.h"
#include "rocksdb/db.h"
#include "rocksdb/filter_policy.h"
#include "rocksdb/options.h"
#include "rocksdb/slice.h"
#include "rocksdb/sst_file_writer.h"
#include "rocksdb/table.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/filter_policy.h"
class
RocksDBWrapper
{
public:
RocksDBWrapper
(
std
::
string
db_name
);
std
::
string
Get
(
std
::
string
key
);
public:
RocksDBWrapper
(
std
::
string
db_name
);
std
::
string
Get
(
std
::
string
key
);
bool
Put
(
std
::
string
key
,
std
::
string
value
);
void
SetDBName
(
std
::
string
db_name
);
static
std
::
shared_ptr
<
RocksDBWrapper
>
RocksDBWrapperFactory
(
std
::
string
db_name
=
"SparseMatrix"
);
bool
Put
(
std
::
string
key
,
std
::
string
value
);
void
SetDBName
(
std
::
string
db_name
);
static
std
::
shared_ptr
<
RocksDBWrapper
>
RocksDBWrapperFactory
(
std
::
string
db_name
=
"SparseMatrix"
);
protected
:
rocksdb
::
DB
*
db_
;
std
::
string
db_name_
;
private
:
rocksdb
::
DB
*
db_
;
std
::
string
db_name_
;
};
kvdb/include/kvdb/rocksdb_impl.h
浏览文件 @
39fede5b
...
...
@@ -13,23 +13,19 @@
// limitations under the License.
#pragma once
#include "kvdb_impl.h"
#include "paddle_rocksdb.h"
class
RocksKVDB
:
public
AbstractKVDB
{
public:
void
CreateDB
();
void
SetDBName
(
std
::
string
);
void
Set
(
std
::
string
key
,
std
::
string
value
);
std
::
string
Get
(
std
::
string
key
);
~
RocksKVDB
();
#include "kvdb
/kvdb
_impl.h"
#include "
kvdb/
paddle_rocksdb.h"
class
RocksKVDB
:
public
AbstractKVDB
{
public:
void
CreateDB
();
void
SetDBName
(
std
::
string
);
void
Set
(
std
::
string
key
,
std
::
string
value
);
std
::
string
Get
(
std
::
string
key
);
~
RocksKVDB
();
protected
:
std
::
shared_ptr
<
RocksDBWrapper
>
db_
;
private
:
std
::
shared_ptr
<
RocksDBWrapper
>
db_
;
public:
static
int
db_count
;
public:
static
int
db_count
;
};
kvdb/src/gtest_db_func.cpp
浏览文件 @
39fede5b
...
...
@@ -12,65 +12,56 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "kvdb/rocksdb_impl.h"
#include "kvdb/kvdb_impl.h"
#include "kvdb/paddle_rocksdb.h"
#include <gtest/gtest.h>
#include <
string
>
#include <
chrono
>
#include <fstream>
#include <sstream>
#include <
chrono
>
#include <
string
>
#include <thread>
#include "kvdb/kvdb_impl.h"
#include "kvdb/paddle_rocksdb.h"
#include "kvdb/rocksdb_impl.h"
class
KVDBTest
:
public
::
testing
::
Test
{
protected:
void
SetUp
()
override
{
}
static
void
SetUpTestCase
()
{
}
protected:
void
SetUp
()
override
{}
static
void
SetUpTestCase
()
{}
};
int
my_argc
;
char
**
my_argv
;
std
::
vector
<
std
::
string
>
StringSplit
(
std
::
string
str
,
char
split
)
{
std
::
vector
<
std
::
string
>
strs
;
std
::
istringstream
f
(
str
);
std
::
string
s
;
while
(
getline
(
f
,
s
,
split
))
{
strs
.
push_back
(
s
);
}
return
strs
;
std
::
vector
<
std
::
string
>
strs
;
std
::
istringstream
f
(
str
);
std
::
string
s
;
while
(
getline
(
f
,
s
,
split
))
{
strs
.
push_back
(
s
);
}
return
strs
;
}
TEST_F
(
KVDBTest
,
AbstractKVDB_Func_Test
)
{
AbsKVDBPtr
kvdb
=
std
::
make_shared
<
RocksKVDB
>
();
kvdb
->
CreateDB
();
std
::
string
set_list
=
"setlist.txt"
;
std
::
string
get_list
=
"getlist.txt"
;
std
::
ifstream
set_file
(
set_list
);
std
::
ifstream
get_file
(
get_list
);
for
(
std
::
string
line
;
getline
(
set_file
,
line
);
)
{
std
::
vector
<
std
::
string
>
strs
=
StringSplit
(
line
,
' '
);
kvdb
->
Set
(
strs
[
0
],
strs
[
1
]);
}
AbsKVDBPtr
kvdb
=
std
::
make_shared
<
RocksKVDB
>
();
kvdb
->
CreateDB
();
std
::
string
set_list
=
"setlist.txt"
;
std
::
string
get_list
=
"getlist.txt"
;
std
::
ifstream
set_file
(
set_list
);
std
::
ifstream
get_file
(
get_list
);
for
(
std
::
string
line
;
getline
(
set_file
,
line
);)
{
std
::
vector
<
std
::
string
>
strs
=
StringSplit
(
line
,
' '
);
kvdb
->
Set
(
strs
[
0
],
strs
[
1
]);
}
for
(
std
::
string
line
;
getline
(
get_file
,
line
);
)
{
std
::
vector
<
std
::
string
>
strs
=
StringSplit
(
line
,
' '
);
std
::
string
val
=
kvdb
->
Get
(
strs
[
0
]);
ASSERT_EQ
(
val
,
strs
[
1
]);
}
for
(
std
::
string
line
;
getline
(
get_file
,
line
);
)
{
std
::
vector
<
std
::
string
>
strs
=
StringSplit
(
line
,
' '
);
std
::
string
val
=
kvdb
->
Get
(
strs
[
0
]);
ASSERT_EQ
(
val
,
strs
[
1
]);
}
}
int
main
(
int
argc
,
char
**
argv
)
{
my_argc
=
argc
;
my_argv
=
argv
;
::
testing
::
InitGoogleTest
(
&
argc
,
argv
);
return
RUN_ALL_TESTS
();
my_argc
=
argc
;
my_argv
=
argv
;
::
testing
::
InitGoogleTest
(
&
argc
,
argv
);
return
RUN_ALL_TESTS
();
}
kvdb/src/gtest_db_thread.cpp
浏览文件 @
39fede5b
...
...
@@ -12,63 +12,59 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "kvdb/rocksdb_impl.h"
#include "kvdb/kvdb_impl.h"
#include "kvdb/paddle_rocksdb.h"
#include <gtest/gtest.h>
#include <string>
#include <fstream>
#include <chrono>
#include <fstream>
#include <string>
#include <thread>
#include "kvdb/kvdb_impl.h"
#include "kvdb/paddle_rocksdb.h"
#include "kvdb/rocksdb_impl.h"
class
KVDBTest
:
public
::
testing
::
Test
{
protected:
void
SetUp
()
override
{
}
static
void
SetUpTestCase
()
{
protected:
void
SetUp
()
override
{}
}
static
void
SetUpTestCase
()
{
}
};
int
my_argc
;
char
**
my_argv
;
void
db_thread_test
(
AbsKVDBPtr
kvdb
,
int
size
)
{
for
(
int
i
=
0
;
i
<
size
;
i
++
)
{
kvdb
->
Set
(
std
::
to_string
(
i
),
std
::
to_string
(
i
));
kvdb
->
Get
(
std
::
to_string
(
i
));
}
for
(
int
i
=
0
;
i
<
size
;
i
++
)
{
kvdb
->
Set
(
std
::
to_string
(
i
),
std
::
to_string
(
i
));
kvdb
->
Get
(
std
::
to_string
(
i
));
}
}
TEST_F
(
KVDBTest
,
AbstractKVDB_Thread_Test
)
{
if
(
my_argc
!=
3
)
{
std
::
cerr
<<
"illegal input! should be db_thread ${num_of_thread} ${num_of_ops_each_thread}"
<<
std
::
endl
;
return
;
}
int
num_of_thread
=
atoi
(
my_argv
[
1
]);
int
nums_of_ops_each_thread
=
atoi
(
my_argv
[
2
]);
std
::
vector
<
AbsKVDBPtr
>
kvdbptrs
;
for
(
int
i
=
0
;
i
<
num_of_thread
;
i
++
)
{
kvdbptrs
.
push_back
(
std
::
make_shared
<
RocksKVDB
>
());
kvdbptrs
[
i
]
->
CreateDB
();
}
std
::
vector
<
std
::
thread
>
tarr
;
for
(
int
i
=
0
;
i
<
num_of_thread
;
i
++
)
{
tarr
.
push_back
(
std
::
thread
(
db_thread_test
,
kvdbptrs
[
i
],
nums_of_ops_each_thread
));
}
for
(
int
i
=
0
;
i
<
num_of_thread
;
i
++
)
{
tarr
[
i
].
join
();
}
if
(
my_argc
!=
3
)
{
std
::
cerr
<<
"illegal input! should be db_thread ${num_of_thread} "
"${num_of_ops_each_thread}"
<<
std
::
endl
;
return
;
}
int
num_of_thread
=
atoi
(
my_argv
[
1
]);
int
nums_of_ops_each_thread
=
atoi
(
my_argv
[
2
]);
std
::
vector
<
AbsKVDBPtr
>
kvdbptrs
;
for
(
int
i
=
0
;
i
<
num_of_thread
;
i
++
)
{
kvdbptrs
.
push_back
(
std
::
make_shared
<
RocksKVDB
>
());
kvdbptrs
[
i
]
->
CreateDB
();
}
std
::
vector
<
std
::
thread
>
tarr
;
for
(
int
i
=
0
;
i
<
num_of_thread
;
i
++
)
{
tarr
.
push_back
(
std
::
thread
(
db_thread_test
,
kvdbptrs
[
i
],
nums_of_ops_each_thread
));
}
for
(
int
i
=
0
;
i
<
num_of_thread
;
i
++
)
{
tarr
[
i
].
join
();
}
return
;
}
int
main
(
int
argc
,
char
**
argv
)
{
my_argc
=
argc
;
my_argv
=
argv
;
::
testing
::
InitGoogleTest
(
&
argc
,
argv
);
return
RUN_ALL_TESTS
();
my_argc
=
argc
;
my_argv
=
argv
;
::
testing
::
InitGoogleTest
(
&
argc
,
argv
);
return
RUN_ALL_TESTS
();
}
kvdb/src/gtest_kvdb.cpp
浏览文件 @
39fede5b
...
...
@@ -12,32 +12,29 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "kvdb/rocksdb_impl.h"
#include "kvdb/kvdb_impl.h"
#include "kvdb/paddle_rocksdb.h"
#include <gtest/gtest.h>
#include <chrono>
#include <fstream>
#include <functional>
#include <string>
#include <fstream>
#include <chrono>
#include <thread>
#include "kvdb/kvdb_impl.h"
#include "kvdb/paddle_rocksdb.h"
#include "kvdb/rocksdb_impl.h"
class
KVDBTest
:
public
::
testing
::
Test
{
protected:
void
SetUp
()
override
{
}
static
void
SetUpTestCase
()
{
kvdb
=
std
::
make_shared
<
RocksKVDB
>
();
dict_reader
=
std
::
make_shared
<
FileReader
>
();
param_dict
=
std
::
make_shared
<
ParamDict
>
();
}
static
AbsKVDBPtr
kvdb
;
static
FileReaderPtr
dict_reader
;
static
ParamDictPtr
param_dict
;
static
ParamDictMgr
dict_mgr
;
protected:
void
SetUp
()
override
{}
static
void
SetUpTestCase
()
{
kvdb
=
std
::
make_shared
<
RocksKVDB
>
();
dict_reader
=
std
::
make_shared
<
FileReader
>
();
param_dict
=
std
::
make_shared
<
ParamDict
>
();
}
static
AbsKVDBPtr
kvdb
;
static
FileReaderPtr
dict_reader
;
static
ParamDictPtr
param_dict
;
static
ParamDictMgr
dict_mgr
;
};
AbsKVDBPtr
KVDBTest
::
kvdb
;
FileReaderPtr
KVDBTest
::
dict_reader
;
...
...
@@ -48,116 +45,117 @@ void GenerateTestIn(std::string);
void
UpdateTestIn
(
std
::
string
);
TEST_F
(
KVDBTest
,
AbstractKVDB_Unit_Test
)
{
kvdb
->
CreateDB
();
kvdb
->
SetDBName
(
"test_kvdb"
);
for
(
int
i
=
0
;
i
<
100
;
i
++
)
{
kvdb
->
Set
(
std
::
to_string
(
i
),
std
::
to_string
(
i
*
2
));
}
for
(
int
i
=
0
;
i
<
100
;
i
++
)
{
std
::
string
val
=
kvdb
->
Get
(
std
::
to_string
(
i
));
ASSERT_EQ
(
val
,
std
::
to_string
(
i
*
2
));
}
kvdb
->
CreateDB
();
kvdb
->
SetDBName
(
"test_kvdb"
);
for
(
int
i
=
0
;
i
<
100
;
i
++
)
{
kvdb
->
Set
(
std
::
to_string
(
i
),
std
::
to_string
(
i
*
2
));
}
for
(
int
i
=
0
;
i
<
100
;
i
++
)
{
std
::
string
val
=
kvdb
->
Get
(
std
::
to_string
(
i
));
ASSERT_EQ
(
val
,
std
::
to_string
(
i
*
2
));
}
}
TEST_F
(
KVDBTest
,
FileReader_Unit_Test
)
{
std
::
string
test_in_filename
=
"abs_dict_reader_test_in.txt"
;
GenerateTestIn
(
test_in_filename
);
dict_reader
->
SetFileName
(
test_in_filename
);
std
::
string
md5_1
=
dict_reader
->
GetMD5
();
std
::
chrono
::
system_clock
::
time_point
timestamp_1
=
dict_reader
->
GetTimeStamp
();
std
::
string
md5_2
=
dict_reader
->
GetMD5
();
std
::
chrono
::
system_clock
::
time_point
timestamp_2
=
dict_reader
->
GetTimeStamp
();
ASSERT_EQ
(
md5_1
,
md5_2
);
ASSERT_EQ
(
timestamp_1
,
timestamp_2
);
UpdateTestIn
(
test_in_filename
);
std
::
string
md5_3
=
dict_reader
->
GetMD5
();
std
::
chrono
::
system_clock
::
time_point
timestamp_3
=
dict_reader
->
GetTimeStamp
();
ASSERT_NE
(
md5_2
,
md5_3
);
ASSERT_NE
(
timestamp_2
,
timestamp_3
);
std
::
string
test_in_filename
=
"abs_dict_reader_test_in.txt"
;
GenerateTestIn
(
test_in_filename
);
dict_reader
->
SetFileName
(
test_in_filename
);
std
::
string
md5_1
=
dict_reader
->
GetMD5
();
std
::
chrono
::
system_clock
::
time_point
timestamp_1
=
dict_reader
->
GetTimeStamp
();
std
::
string
md5_2
=
dict_reader
->
GetMD5
();
std
::
chrono
::
system_clock
::
time_point
timestamp_2
=
dict_reader
->
GetTimeStamp
();
ASSERT_EQ
(
md5_1
,
md5_2
);
ASSERT_EQ
(
timestamp_1
,
timestamp_2
);
UpdateTestIn
(
test_in_filename
);
std
::
string
md5_3
=
dict_reader
->
GetMD5
();
std
::
chrono
::
system_clock
::
time_point
timestamp_3
=
dict_reader
->
GetTimeStamp
();
ASSERT_NE
(
md5_2
,
md5_3
);
ASSERT_NE
(
timestamp_2
,
timestamp_3
);
}
#include <cmath>
TEST_F
(
KVDBTest
,
ParamDict_Unit_Test
)
{
std
::
string
test_in_filename
=
"abs_dict_reader_test_in.txt"
;
param_dict
->
SetFileReaderLst
({
test_in_filename
});
param_dict
->
SetReader
(
[]
(
std
::
string
text
)
{
auto
split
=
[](
const
std
::
string
&
s
,
std
::
vector
<
std
::
string
>&
sv
,
const
char
*
delim
=
" "
)
{
sv
.
clear
();
char
*
buffer
=
new
char
[
s
.
size
()
+
1
];
std
::
copy
(
s
.
begin
(),
s
.
end
(),
buffer
);
char
*
p
=
strtok
(
buffer
,
delim
);
do
{
sv
.
push_back
(
p
);
}
while
((
p
=
strtok
(
NULL
,
delim
)));
return
;
};
std
::
vector
<
std
::
string
>
text_split
;
split
(
text
,
text_split
,
" "
);
std
::
string
key
=
text_split
[
0
];
text_split
.
erase
(
text_split
.
begin
());
return
make_pair
(
key
,
text_split
);
});
param_dict
->
CreateKVDB
();
GenerateTestIn
(
test_in_filename
);
param_dict
->
UpdateBaseModel
();
std
::
this_thread
::
sleep_for
(
std
::
chrono
::
seconds
(
2
));
std
::
vector
<
float
>
test_vec
=
param_dict
->
GetSparseValue
(
"1"
,
""
);
ASSERT_LT
(
fabs
(
test_vec
[
0
]
-
1.0
),
1e-2
);
UpdateTestIn
(
test_in_filename
);
param_dict
->
UpdateDeltaModel
();
std
::
string
test_in_filename
=
"abs_dict_reader_test_in.txt"
;
param_dict
->
SetFileReaderLst
({
test_in_filename
});
param_dict
->
SetReader
([](
std
::
string
text
)
{
auto
split
=
[](
const
std
::
string
&
s
,
std
::
vector
<
std
::
string
>&
sv
,
const
char
*
delim
=
" "
)
{
sv
.
clear
();
char
*
buffer
=
new
char
[
s
.
size
()
+
1
];
std
::
copy
(
s
.
begin
(),
s
.
end
(),
buffer
);
char
*
p
=
strtok
(
buffer
,
delim
);
do
{
sv
.
push_back
(
p
);
}
while
((
p
=
strtok
(
NULL
,
delim
)));
return
;
};
std
::
vector
<
std
::
string
>
text_split
;
split
(
text
,
text_split
,
" "
);
std
::
string
key
=
text_split
[
0
];
text_split
.
erase
(
text_split
.
begin
());
return
make_pair
(
key
,
text_split
);
});
param_dict
->
CreateKVDB
();
GenerateTestIn
(
test_in_filename
);
param_dict
->
UpdateBaseModel
();
std
::
this_thread
::
sleep_for
(
std
::
chrono
::
seconds
(
2
));
std
::
vector
<
float
>
test_vec
=
param_dict
->
GetSparseValue
(
"1"
,
""
);
ASSERT_LT
(
fabs
(
test_vec
[
0
]
-
1.0
),
1e-2
);
UpdateTestIn
(
test_in_filename
);
param_dict
->
UpdateDeltaModel
();
}
void
GenerateTestIn
(
std
::
string
filename
)
{
std
::
ifstream
in_file
(
filename
);
if
(
in_file
.
good
())
{
in_file
.
close
();
std
::
string
cmd
=
"rm -rf "
+
filename
;
system
(
cmd
.
c_str
());
std
::
ifstream
in_file
(
filename
);
if
(
in_file
.
good
())
{
in_file
.
close
();
std
::
string
cmd
=
"rm -rf "
+
filename
;
system
(
cmd
.
c_str
());
}
std
::
ofstream
out_file
(
filename
);
for
(
size_t
i
=
0
;
i
<
100000
;
i
++
)
{
out_file
<<
i
<<
" "
<<
i
<<
" "
;
for
(
size_t
j
=
0
;
j
<
3
;
j
++
)
{
out_file
<<
i
<<
" "
;
}
std
::
ofstream
out_file
(
filename
);
for
(
size_t
i
=
0
;
i
<
100000
;
i
++
)
{
out_file
<<
i
<<
" "
<<
i
<<
" "
;
for
(
size_t
j
=
0
;
j
<
3
;
j
++
)
{
out_file
<<
i
<<
" "
;
}
out_file
<<
std
::
endl
;
}
out_file
.
close
();
out_file
<<
std
::
endl
;
}
out_file
.
close
();
}
void
UpdateTestIn
(
std
::
string
filename
)
{
std
::
ifstream
in_file
(
filename
);
if
(
in_file
.
good
())
{
in_file
.
close
();
std
::
string
cmd
=
"rm -rf "
+
filename
;
system
(
cmd
.
c_str
());
}
std
::
ofstream
out_file
(
filename
);
for
(
size_t
i
=
0
;
i
<
10000
;
i
++
)
{
out_file
<<
i
<<
" "
<<
i
<<
" "
;
for
(
size_t
j
=
0
;
j
<
3
;
j
++
)
{
out_file
<<
i
+
1
<<
" "
;
}
out_file
<<
std
::
endl
;
std
::
ifstream
in_file
(
filename
);
if
(
in_file
.
good
())
{
in_file
.
close
();
std
::
string
cmd
=
"rm -rf "
+
filename
;
system
(
cmd
.
c_str
());
}
std
::
ofstream
out_file
(
filename
);
for
(
size_t
i
=
0
;
i
<
10000
;
i
++
)
{
out_file
<<
i
<<
" "
<<
i
<<
" "
;
for
(
size_t
j
=
0
;
j
<
3
;
j
++
)
{
out_file
<<
i
+
1
<<
" "
;
}
out_file
.
close
();
out_file
<<
std
::
endl
;
}
out_file
.
close
();
}
int
main
(
int
argc
,
char
**
argv
)
{
::
testing
::
InitGoogleTest
(
&
argc
,
argv
);
return
RUN_ALL_TESTS
();
::
testing
::
InitGoogleTest
(
&
argc
,
argv
);
return
RUN_ALL_TESTS
();
}
kvdb/src/mock_param_dict_impl.cpp
浏览文件 @
39fede5b
...
...
@@ -12,139 +12,132 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "kvdb/rocksdb_impl.h"
#include <thread>
#include <iterator>
#include <fstream>
#include <algorithm>
#include <fstream>
#include <iterator>
#include <sstream>
#include <thread>
#include "kvdb/rocksdb_impl.h"
std
::
vector
<
FileReaderPtr
>
ParamDict
::
GetDictReaderLst
()
{
return
this
->
file_reader_lst_
;
return
this
->
file_reader_lst_
;
}
void
ParamDict
::
SetFileReaderLst
(
std
::
vector
<
std
::
string
>
lst
)
{
for
(
size_t
i
=
0
;
i
<
lst
.
size
();
i
++
)
{
FileReaderPtr
fr
=
std
::
make_shared
<
FileReader
>
();
fr
->
SetFileName
(
lst
[
i
]);
this
->
file_reader_lst_
.
push_back
(
fr
);
}
for
(
size_t
i
=
0
;
i
<
lst
.
size
();
i
++
)
{
FileReaderPtr
fr
=
std
::
make_shared
<
FileReader
>
();
fr
->
SetFileName
(
lst
[
i
]);
this
->
file_reader_lst_
.
push_back
(
fr
);
}
}
std
::
vector
<
float
>
ParamDict
::
GetSparseValue
(
std
::
string
feasign
,
std
::
string
slot
)
{
auto
BytesToFloat
=
[](
uint8_t
*
byteArray
){
return
*
((
float
*
)
byteArray
);
};
//TODO: the concatation of feasign and slot is TBD.
std
::
string
result
=
front_db
->
Get
(
feasign
+
slot
);
std
::
vector
<
float
>
value
;
if
(
result
==
"NOT_FOUND"
)
return
value
;
uint8_t
*
raw_values_ptr
=
reinterpret_cast
<
uint8_t
*>
(
&
result
[
0
]);
for
(
size_t
i
=
0
;
i
<
result
.
size
();
i
+=
4
)
{
float
temp
=
BytesToFloat
(
raw_values_ptr
+
i
);
value
.
push_back
(
temp
);
}
return
value
;
std
::
vector
<
float
>
ParamDict
::
GetSparseValue
(
std
::
string
feasign
,
std
::
string
slot
)
{
auto
BytesToFloat
=
[](
uint8_t
*
byte_array
)
{
return
*
((
float
*
)
byte_array
);
};
// TODO: the concatation of feasign and slot is TBD.
std
::
string
result
=
front_db
->
Get
(
feasign
+
slot
);
std
::
vector
<
float
>
value
;
if
(
result
==
"NOT_FOUND"
)
return
value
;
uint8_t
*
raw_values_ptr
=
reinterpret_cast
<
uint8_t
*>
(
&
result
[
0
]);
for
(
size_t
i
=
0
;
i
<
result
.
size
();
i
+=
sizeof
(
float
))
{
float
temp
=
BytesToFloat
(
raw_values_ptr
+
i
);
value
.
push_back
(
temp
);
}
return
value
;
}
void
ParamDict
::
SetReader
(
std
::
function
<
std
::
pair
<
Key
,
Value
>
(
std
::
string
)
>
func
)
{
read_func_
=
func
;
void
ParamDict
::
SetReader
(
std
::
function
<
std
::
pair
<
Key
,
Value
>
(
std
::
string
)
>
func
)
{
read_func_
=
func
;
}
std
::
vector
<
float
>
ParamDict
::
GetSparseValue
(
int64_t
feasign
,
int64_t
slot
)
{
return
this
->
GetSparseValue
(
std
::
to_string
(
feasign
),
std
::
to_string
(
slot
));
return
this
->
GetSparseValue
(
std
::
to_string
(
feasign
),
std
::
to_string
(
slot
));
}
bool
ParamDict
::
InsertSparseValue
(
int64_t
feasign
,
int64_t
slot
,
const
std
::
vector
<
float
>&
values
)
{
return
this
->
InsertSparseValue
(
std
::
to_string
(
feasign
),
std
::
to_string
(
slot
),
values
);
bool
ParamDict
::
InsertSparseValue
(
int64_t
feasign
,
int64_t
slot
,
const
std
::
vector
<
float
>&
values
)
{
return
this
->
InsertSparseValue
(
std
::
to_string
(
feasign
),
std
::
to_string
(
slot
),
values
);
}
bool
ParamDict
::
InsertSparseValue
(
std
::
string
feasign
,
std
::
string
slot
,
const
std
::
vector
<
float
>&
values
)
{
auto
FloatToBytes
=
[](
float
fvalue
,
uint8_t
*
arr
){
unsigned
char
*
pf
=
nullptr
;
unsigned
char
*
px
=
nullptr
;
unsigned
char
i
=
0
;
pf
=
(
unsigned
char
*
)
&
fvalue
;
px
=
arr
;
for
(
i
=
0
;
i
<
4
;
i
++
)
{
*
(
px
+
i
)
=*
(
pf
+
i
);
}
};
std
::
string
key
=
feasign
+
slot
;
uint8_t
*
values_ptr
=
new
uint8_t
[
values
.
size
()
*
4
];
std
::
string
value
;
for
(
size_t
i
=
0
;
i
<
values
.
size
();
i
++
)
{
FloatToBytes
(
values
[
i
],
values_ptr
+
4
*
i
);
bool
ParamDict
::
InsertSparseValue
(
std
::
string
feasign
,
std
::
string
slot
,
const
std
::
vector
<
float
>&
values
)
{
auto
FloatToBytes
=
[](
float
fvalue
,
uint8_t
*
arr
)
{
unsigned
char
*
pf
=
nullptr
;
unsigned
char
*
px
=
nullptr
;
unsigned
char
i
=
0
;
pf
=
(
unsigned
char
*
)
&
fvalue
;
px
=
arr
;
for
(
i
=
0
;
i
<
sizeof
(
float
);
i
++
)
{
*
(
px
+
i
)
=
*
(
pf
+
i
);
}
char
*
raw_values_ptr
=
reinterpret_cast
<
char
*>
(
values_ptr
);
for
(
size_t
i
=
0
;
i
<
values
.
size
()
*
4
;
i
++
)
{
value
.
push_back
(
raw_values_ptr
[
i
]);
}
back_db
->
Set
(
key
,
value
);
//TODO: change stateless to stateful
return
true
;
};
std
::
string
key
=
feasign
+
slot
;
uint8_t
*
values_ptr
=
new
uint8_t
[
values
.
size
()
*
sizeof
(
float
)];
std
::
string
value
;
for
(
size_t
i
=
0
;
i
<
values
.
size
();
i
++
)
{
FloatToBytes
(
values
[
i
],
values_ptr
+
sizeof
(
float
)
*
i
);
}
char
*
raw_values_ptr
=
reinterpret_cast
<
char
*>
(
values_ptr
);
for
(
size_t
i
=
0
;
i
<
values
.
size
()
*
sizeof
(
float
);
i
++
)
{
value
.
push_back
(
raw_values_ptr
[
i
]);
}
back_db
->
Set
(
key
,
value
);
// TODO: change stateless to stateful
return
true
;
}
void
ParamDict
::
UpdateBaseModel
()
{
auto
is_number
=
[]
(
const
std
::
string
&
s
)
{
return
!
s
.
empty
()
&&
std
::
find_if
(
s
.
begin
(),
s
.
end
(),
[](
char
c
)
{
return
!
std
::
isdigit
(
c
);
})
==
s
.
end
();
};
std
::
thread
t
([
&
]
()
{
for
(
FileReaderPtr
file_reader
:
this
->
file_reader_lst_
)
{
std
::
string
line
;
std
::
ifstream
infile
(
file_reader
->
GetFileName
());
if
(
infile
.
is_open
())
{
while
(
getline
(
infile
,
line
))
{
std
::
pair
<
Key
,
Value
>
kvpair
=
read_func_
(
line
);
std
::
vector
<
float
>
nums
;
for
(
size_t
i
=
0
;
i
<
kvpair
.
second
.
size
();
i
++
)
{
if
(
is_number
(
kvpair
.
second
[
i
]))
{
nums
.
push_back
(
std
::
stof
(
kvpair
.
second
[
i
]));
}
}
this
->
InsertSparseValue
(
kvpair
.
first
,
""
,
nums
);
}
auto
is_number
=
[](
const
std
::
string
&
s
)
{
return
!
s
.
empty
()
&&
std
::
find_if
(
s
.
begin
(),
s
.
end
(),
[](
char
c
)
{
return
!
std
::
isdigit
(
c
);
})
==
s
.
end
();
};
std
::
thread
t
([
&
]()
{
for
(
FileReaderPtr
file_reader
:
this
->
file_reader_lst_
)
{
std
::
string
line
;
std
::
ifstream
infile
(
file_reader
->
GetFileName
());
if
(
infile
.
is_open
())
{
while
(
getline
(
infile
,
line
))
{
std
::
pair
<
Key
,
Value
>
kvpair
=
read_func_
(
line
);
std
::
vector
<
float
>
nums
;
for
(
size_t
i
=
0
;
i
<
kvpair
.
second
.
size
();
i
++
)
{
if
(
is_number
(
kvpair
.
second
[
i
]))
{
nums
.
push_back
(
std
::
stof
(
kvpair
.
second
[
i
]));
}
infile
.
close
();
}
this
->
InsertSparseValue
(
kvpair
.
first
,
""
,
nums
);
}
AbsKVDBPtr
temp
=
front_db
;
front_db
=
back_db
;
back_db
=
temp
;
});
t
.
detach
();
}
infile
.
close
();
}
AbsKVDBPtr
temp
=
front_db
;
front_db
=
back_db
;
back_db
=
temp
;
});
t
.
detach
();
}
void
ParamDict
::
UpdateDeltaModel
()
{
UpdateBaseModel
();
}
void
ParamDict
::
UpdateDeltaModel
()
{
UpdateBaseModel
();
}
std
::
pair
<
AbsKVDBPtr
,
AbsKVDBPtr
>
ParamDict
::
GetKVDB
()
{
return
{
front_db
,
back_db
};
std
::
pair
<
AbsKVDBPtr
,
AbsKVDBPtr
>
ParamDict
::
GetKVDB
()
{
return
{
front_db
,
back_db
};
}
void
ParamDict
::
SetKVDB
(
std
::
pair
<
AbsKVDBPtr
,
AbsKVDBPtr
>
kvdbs
)
{
this
->
front_db
=
kvdbs
.
first
;
this
->
back_db
=
kvdbs
.
second
;
this
->
front_db
=
kvdbs
.
first
;
this
->
back_db
=
kvdbs
.
second
;
}
void
ParamDict
::
CreateKVDB
()
{
this
->
front_db
=
std
::
make_shared
<
RocksKVDB
>
();
this
->
back_db
=
std
::
make_shared
<
RocksKVDB
>
();
this
->
front_db
->
CreateDB
();
this
->
back_db
->
CreateDB
();
this
->
front_db
=
std
::
make_shared
<
RocksKVDB
>
();
this
->
back_db
=
std
::
make_shared
<
RocksKVDB
>
();
this
->
front_db
->
CreateDB
();
this
->
back_db
->
CreateDB
();
}
ParamDict
::~
ParamDict
()
{
}
ParamDict
::~
ParamDict
()
{}
kvdb/src/paddle_rocksdb.cpp
浏览文件 @
39fede5b
...
...
@@ -15,39 +15,40 @@
#include "kvdb/paddle_rocksdb.h"
RocksDBWrapper
::
RocksDBWrapper
(
std
::
string
db_name
)
{
rocksdb
::
Options
options
;
options
.
create_if_missing
=
true
;
db_name_
=
db_name
;
db_
=
nullptr
;
rocksdb
::
Status
s
=
rocksdb
::
DB
::
Open
(
options
,
db_name
,
&
db_
);
return
;
rocksdb
::
Options
options
;
options
.
create_if_missing
=
true
;
db_name_
=
db_name
;
db_
=
nullptr
;
rocksdb
::
Status
s
=
rocksdb
::
DB
::
Open
(
options
,
db_name
,
&
db_
);
return
;
}
std
::
string
RocksDBWrapper
::
Get
(
std
::
string
key
)
{
rocksdb
::
ReadOptions
options
;
options
.
verify_checksums
=
true
;
std
::
string
result
;
rocksdb
::
Status
s
=
db_
->
Get
(
options
,
key
,
&
result
);
if
(
s
.
IsNotFound
())
{
result
=
"NOT_FOUND"
;
}
return
result
;
rocksdb
::
ReadOptions
options
;
options
.
verify_checksums
=
true
;
std
::
string
result
;
rocksdb
::
Status
s
=
db_
->
Get
(
options
,
key
,
&
result
);
if
(
s
.
IsNotFound
())
{
result
=
"NOT_FOUND"
;
}
return
result
;
}
bool
RocksDBWrapper
::
Put
(
std
::
string
key
,
std
::
string
value
)
{
rocksdb
::
WriteOptions
options
;
rocksdb
::
Status
s
=
db_
->
Put
(
options
,
key
,
value
);
if
(
s
.
ok
())
{
return
true
;
}
else
{
return
false
;
}
rocksdb
::
WriteOptions
options
;
rocksdb
::
Status
s
=
db_
->
Put
(
options
,
key
,
value
);
if
(
s
.
ok
())
{
return
true
;
}
else
{
return
false
;
}
}
void
RocksDBWrapper
::
SetDBName
(
std
::
string
db_name
)
{
this
->
db_name_
=
db_name
;
this
->
db_name_
=
db_name
;
}
std
::
shared_ptr
<
RocksDBWrapper
>
RocksDBWrapper
::
RocksDBWrapperFactory
(
std
::
string
db_name
)
{
return
std
::
make_shared
<
RocksDBWrapper
>
(
db_name
);
std
::
shared_ptr
<
RocksDBWrapper
>
RocksDBWrapper
::
RocksDBWrapperFactory
(
std
::
string
db_name
)
{
return
std
::
make_shared
<
RocksDBWrapper
>
(
db_name
);
}
kvdb/src/param_dict_mgr_impl.cpp
浏览文件 @
39fede5b
...
...
@@ -15,14 +15,14 @@
#include "kvdb/kvdb_impl.h"
void
ParamDictMgr
::
UpdateAll
()
{
for
(
auto
it
=
this
->
ParamDictMap
.
begin
();
it
!=
this
->
ParamDictMap
.
end
();
++
it
)
{
it
->
second
->
UpdateBaseModel
();
}
for
(
auto
it
=
this
->
ParamDictMap
.
begin
();
it
!=
this
->
ParamDictMap
.
end
();
++
it
)
{
it
->
second
->
UpdateBaseModel
();
}
}
void
ParamDictMgr
::
InsertParamDict
(
std
::
string
key
,
ParamDictPtr
value
)
{
this
->
ParamDictMap
.
insert
(
std
::
make_pair
(
key
,
value
));
this
->
ParamDictMap
.
insert
(
std
::
make_pair
(
key
,
value
));
}
AbstractKVDB
::~
AbstractKVDB
()
{}
kvdb/src/rockskvdb_impl.cpp
浏览文件 @
39fede5b
...
...
@@ -16,29 +16,22 @@
int
RocksKVDB
::
db_count
;
void
RocksKVDB
::
CreateDB
()
{
this
->
db_
=
RocksDBWrapper
::
RocksDBWrapperFactory
(
"RocksDB_"
+
std
::
to_string
(
RocksKVDB
::
db_count
));
RocksKVDB
::
db_count
++
;
return
;
this
->
db_
=
RocksDBWrapper
::
RocksDBWrapperFactory
(
"RocksDB_"
+
std
::
to_string
(
RocksKVDB
::
db_count
));
RocksKVDB
::
db_count
++
;
return
;
}
void
RocksKVDB
::
SetDBName
(
std
::
string
db_name
)
{
this
->
db_
->
SetDBName
(
db_name
);
return
;
this
->
db_
->
SetDBName
(
db_name
);
return
;
}
void
RocksKVDB
::
Set
(
std
::
string
key
,
std
::
string
value
)
{
this
->
db_
->
Put
(
key
,
value
);
return
;
this
->
db_
->
Put
(
key
,
value
);
return
;
}
std
::
string
RocksKVDB
::
Get
(
std
::
string
key
)
{
return
this
->
db_
->
Get
(
key
);
}
RocksKVDB
::~
RocksKVDB
()
{
}
std
::
string
RocksKVDB
::
Get
(
std
::
string
key
)
{
return
this
->
db_
->
Get
(
key
);
}
RocksKVDB
::~
RocksKVDB
()
{}
kvdb/src/test_rocksdb.cpp
浏览文件 @
39fede5b
...
...
@@ -12,37 +12,34 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "kvdb/rocksdb_impl.h"
#include "kvdb/paddle_rocksdb.h"
#include <iostream>
#include "kvdb/paddle_rocksdb.h"
#include "kvdb/rocksdb_impl.h"
void
test_rockskvdb
()
{
RocksKVDB
db
;
db
.
CreateDB
();
db
.
SetDBName
(
"Sparse Matrix"
);
db
.
Set
(
"1"
,
"One"
);
std
::
cout
<<
db
.
Get
(
"1"
)
<<
std
::
endl
;
return
;
RocksKVDB
db
;
db
.
CreateDB
();
db
.
SetDBName
(
"Sparse Matrix"
);
db
.
Set
(
"1"
,
"One"
);
std
::
cout
<<
db
.
Get
(
"1"
)
<<
std
::
endl
;
return
;
}
void
test_rocksdbwrapper
()
{
std
::
shared_ptr
<
RocksDBWrapper
>
db
=
RocksDBWrapper
::
RocksDBWrapperFactory
(
"TEST"
);
for
(
size_t
i
=
0
;
i
<
1000
;
i
++
)
{
db
->
Put
(
std
::
to_string
(
i
),
std
::
to_string
(
i
*
2
));
}
for
(
size_t
i
=
0
;
i
<
1000
;
i
++
)
{
std
::
string
res
=
db
->
Get
(
std
::
to_string
(
i
));
std
::
cout
<<
res
<<
" "
;
}
std
::
cout
<<
std
::
endl
;
std
::
shared_ptr
<
RocksDBWrapper
>
db
=
RocksDBWrapper
::
RocksDBWrapperFactory
(
"TEST"
);
for
(
size_t
i
=
0
;
i
<
1000
;
i
++
)
{
db
->
Put
(
std
::
to_string
(
i
),
std
::
to_string
(
i
*
2
));
}
for
(
size_t
i
=
0
;
i
<
1000
;
i
++
)
{
std
::
string
res
=
db
->
Get
(
std
::
to_string
(
i
));
std
::
cout
<<
res
<<
" "
;
}
std
::
cout
<<
std
::
endl
;
}
#ifdef RAW_TEST
int
main
()
{
test_rockskvdb
();
test_rocksdbwrapper
();
test_rockskvdb
();
test_rocksdbwrapper
();
}
#endif
pdcodegen/src/pdcodegen.cpp
浏览文件 @
39fede5b
...
...
@@ -484,7 +484,8 @@ class PdsCodeGenerator : public CodeGenerator {
"response);
\n
"
"}
\n
"
"tt.stop();
\n
"
"if (ret.flags != baidu::rpc::SKIP_SUB_CHANNEL && ret.method != NULL) {
\n
"
"if (ret.flags != baidu::rpc::SKIP_SUB_CHANNEL && ret.method != "
"NULL) {
\n
"
" _stub_handler->update_latency(tt.u_elapsed(),
\"
pack_map
\"
);
\n
"
"}
\n
"
"return ret;
\n
"
);
...
...
@@ -498,7 +499,8 @@ class PdsCodeGenerator : public CodeGenerator {
////////////////////////////////////////////////////////////////
printer
->
Print
(
"class $name$_StubResponseMerger : public baidu::rpc::ResponseMerger {
\n
"
"class $name$_StubResponseMerger : public baidu::rpc::ResponseMerger "
"{
\n
"
"private:
\n
"
" uint32_t _package_size;
\n
"
" baidu::paddle_serving::sdk_cpp::Stub* _stub_handler;
\n
"
...
...
@@ -600,7 +602,8 @@ class PdsCodeGenerator : public CodeGenerator {
"}
\n
"
);
"LOG(INFO)
\n
"
" <<
\"
[default] Succ map, channel_index:
\"
<< channel_index;
\n
"
;
printer
->
Print
(
"return baidu::rpc::SubCall(method, request, cur_res, 0);
\n
"
);
printer
->
Print
(
"return baidu::rpc::SubCall(method, request, cur_res, 0);
\n
"
);
return
true
;
}
bool
generate_paddle_serving_stub_default_merger
(
...
...
predictor/CMakeLists.txt
浏览文件 @
39fede5b
...
...
@@ -37,4 +37,3 @@ install(FILES ${CMAKE_CURRENT_LIST_DIR}/mempool/mempool.h
DESTINATION
${
PADDLE_SERVING_INSTALL_DIR
}
/include/predictor/mempool
)
install
(
FILES
${
CMAKE_CURRENT_LIST_DIR
}
/op/op.h
DESTINATION
${
PADDLE_SERVING_INSTALL_DIR
}
/include/predictor/op
)
predictor/framework/memory.cpp
浏览文件 @
39fede5b
...
...
@@ -20,11 +20,10 @@ namespace paddle_serving {
namespace
predictor
{
struct
MempoolRegion
{
MempoolRegion
(
im
::
fugue
::
memory
::
Region
*
region
,
im
::
Mempool
*
mempool
)
:
_region
(
region
),
_mempool
(
mempool
){}
im
::
fugue
::
memory
::
Region
*
region
()
{
return
_region
;}
im
::
Mempool
*
mempool
()
{
return
_mempool
;}
MempoolRegion
(
im
::
fugue
::
memory
::
Region
*
region
,
im
::
Mempool
*
mempool
)
:
_region
(
region
),
_mempool
(
mempool
)
{}
im
::
fugue
::
memory
::
Region
*
region
()
{
return
_region
;
}
im
::
Mempool
*
mempool
()
{
return
_mempool
;
}
im
::
fugue
::
memory
::
Region
*
_region
;
im
::
Mempool
*
_mempool
;
...
...
@@ -54,10 +53,10 @@ int MempoolWrapper::initialize() {
}
int
MempoolWrapper
::
thread_initialize
()
{
im
::
fugue
::
memory
::
Region
*
region
=
new
im
::
fugue
::
memory
::
Region
();
im
::
fugue
::
memory
::
Region
*
region
=
new
im
::
fugue
::
memory
::
Region
();
region
->
init
();
im
::
Mempool
*
mempool
=
new
(
std
::
nothrow
)
im
::
Mempool
(
region
);
MempoolRegion
*
mempool_region
=
new
MempoolRegion
(
region
,
mempool
);
MempoolRegion
*
mempool_region
=
new
MempoolRegion
(
region
,
mempool
);
if
(
mempool
==
NULL
)
{
LOG
(
ERROR
)
<<
"Failed create thread mempool"
;
return
-
1
;
...
...
@@ -76,7 +75,8 @@ int MempoolWrapper::thread_initialize() {
}
int
MempoolWrapper
::
thread_clear
()
{
MempoolRegion
*
mempool_region
=
(
MempoolRegion
*
)
THREAD_GETSPECIFIC
(
_bspec_key
);
MempoolRegion
*
mempool_region
=
(
MempoolRegion
*
)
THREAD_GETSPECIFIC
(
_bspec_key
);
if
(
mempool_region
==
NULL
)
{
LOG
(
WARNING
)
<<
"THREAD_GETSPECIFIC() returned NULL"
;
return
-
1
;
...
...
@@ -91,7 +91,8 @@ int MempoolWrapper::thread_clear() {
}
void
*
MempoolWrapper
::
malloc
(
size_t
size
)
{
MempoolRegion
*
mempool_region
=
(
MempoolRegion
*
)
THREAD_GETSPECIFIC
(
_bspec_key
);
MempoolRegion
*
mempool_region
=
(
MempoolRegion
*
)
THREAD_GETSPECIFIC
(
_bspec_key
);
if
(
mempool_region
==
NULL
)
{
LOG
(
WARNING
)
<<
"THREAD_GETSPECIFIC() returned NULL"
;
return
NULL
;
...
...
predictor/framework/memory.h
浏览文件 @
39fede5b
...
...
@@ -39,7 +39,7 @@ class MempoolWrapper {
void
*
malloc
(
size_t
size
);
private:
//im::fugue::memory::Region _region;
//
im::fugue::memory::Region _region;
THREAD_KEY_T
_bspec_key
;
};
...
...
predictor/framework/predictor_metric.h
浏览文件 @
39fede5b
...
...
@@ -24,7 +24,7 @@
#include <butil/scoped_lock.h> // BAIDU_SCOPED_LOCK
#endif
#include <bvar/bvar.h>
// bvar
#include <bvar/bvar.h> // bvar
#include <string>
#ifdef BCLOUD
...
...
predictor/framework/resource.cpp
浏览文件 @
39fede5b
...
...
@@ -36,13 +36,9 @@ DynamicResource::DynamicResource() {}
DynamicResource
::~
DynamicResource
()
{}
int
DynamicResource
::
initialize
()
{
return
0
;
}
int
DynamicResource
::
initialize
()
{
return
0
;
}
std
::
shared_ptr
<
RocksDBWrapper
>
Resource
::
getDB
()
{
return
db
;
}
std
::
shared_ptr
<
RocksDBWrapper
>
Resource
::
getDB
()
{
return
db
;
}
int
DynamicResource
::
clear
()
{
return
0
;
}
...
...
@@ -86,7 +82,7 @@ int Resource::initialize(const std::string& path, const std::string& file) {
LOG
(
ERROR
)
<<
"unable to create tls_bthread_key of thrd_data"
;
return
-
1
;
}
//
init rocksDB instance
//
init rocksDB instance
if
(
db
.
get
()
==
nullptr
)
{
db
=
RocksDBWrapper
::
RocksDBWrapperFactory
(
"kvdb"
);
}
...
...
predictor/framework/resource.h
浏览文件 @
39fede5b
...
...
@@ -14,9 +14,9 @@
#pragma once
#include <string>
#include "kvdb/paddle_rocksdb.h"
#include "predictor/common/inner_common.h"
#include "predictor/framework/memory.h"
#include "kvdb/paddle_rocksdb.h"
namespace
baidu
{
namespace
paddle_serving
{
...
...
@@ -31,7 +31,6 @@ struct DynamicResource {
int
initialize
();
int
clear
();
};
class
Resource
{
...
...
@@ -65,7 +64,7 @@ class Resource {
private:
int
thread_finalize
()
{
return
0
;
}
std
::
shared_ptr
<
RocksDBWrapper
>
db
;
THREAD_KEY_T
_tls_bspec_key
;
};
...
...
predictor/framework/server.cpp
浏览文件 @
39fede5b
...
...
@@ -16,7 +16,7 @@
#ifdef BCLOUD
#include <baidu/rpc/policy/nova_pbrpc_protocol.h> // NovaServiceAdaptor
#include <baidu/rpc/policy/nshead_mcpack_protocol.h> // NsheadMcpackAdaptor
#include <baidu/rpc/policy/public_pbrpc_protocol.h>
// PublicPbrpcServiceAdaptor
#include <baidu/rpc/policy/public_pbrpc_protocol.h> // PublicPbrpcServiceAdaptor
#else
#include <brpc/policy/nova_pbrpc_protocol.h> // NovaServiceAdaptor
#include <brpc/policy/nshead_mcpack_protocol.h> // NsheadMcpackAdaptor
...
...
predictor/proto/builtin_format.proto
浏览文件 @
39fede5b
...
...
@@ -17,16 +17,11 @@ package baidu.paddle_serving.predictor.format;
// echo kvdb formant
message
KVDBReq
{
required
string
op
=
1
;
required
string
key
=
2
;
optional
string
value
=
3
;
required
string
op
=
1
;
required
string
key
=
2
;
optional
string
value
=
3
;
};
message
KVDBRes
{
required
string
value
=
2
;
};
message
KVDBRes
{
required
string
value
=
2
;
};
// dense format
message
DenseInstance
{
repeated
float
features
=
1
;
};
...
...
predictor/src/pdserving.cpp
浏览文件 @
39fede5b
...
...
@@ -143,7 +143,8 @@ int main(int argc, char** argv) {
std
::
string
filename
(
argv
[
0
]);
filename
=
filename
.
substr
(
filename
.
find_last_of
(
'/'
)
+
1
);
settings
.
log_file
=
strdup
((
std
::
string
(
"./log/"
)
+
filename
+
".log"
).
c_str
());
settings
.
log_file
=
strdup
((
std
::
string
(
"./log/"
)
+
filename
+
".log"
).
c_str
());
settings
.
delete_old
=
logging
::
DELETE_OLD_LOG_FILE
;
logging
::
InitLogging
(
settings
);
...
...
sdk-cpp/include/common.h
浏览文件 @
39fede5b
...
...
@@ -31,11 +31,11 @@
#include "baidu/rpc/channel.h"
#include "baidu/rpc/parallel_channel.h"
#include "baidu/rpc/traceprintf.h"
#include "bthread.h"
#include "base/logging.h"
#include "base/comlog_sink.h"
#include "base/logging.h"
#include "base/object_pool.h"
#include "base/time.h"
#include "bthread.h"
#else
#include "brpc/channel.h"
#include "brpc/parallel_channel.h"
...
...
sdk-cpp/proto/echo_kvdb_service.proto
浏览文件 @
39fede5b
...
...
@@ -20,15 +20,15 @@ package baidu.paddle_serving.predictor.echo_kvdb_service;
option
cc_generic_services
=
true
;
message
Request
{
repeated
baidu.paddle_serving.predictor.format.KVDBReq
reqs
=
1
;
repeated
baidu.paddle_serving.predictor.format.KVDBReq
reqs
=
1
;
};
message
Response
{
repeated
baidu.paddle_serving.predictor.format.KVDBRes
ress
=
1
;
repeated
baidu.paddle_serving.predictor.format.KVDBRes
ress
=
1
;
};
service
EchoKVDBService
{
rpc
inference
(
Request
)
returns
(
Response
);
rpc
debug
(
Request
)
returns
(
Response
);
option
(
pds.options
)
.
generate_stub
=
true
;
rpc
inference
(
Request
)
returns
(
Response
);
rpc
debug
(
Request
)
returns
(
Response
);
option
(
pds.options
)
.
generate_stub
=
true
;
};
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录