Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
Serving
提交
982d3e34
S
Serving
项目概览
PaddlePaddle
/
Serving
大约 1 年 前同步成功
通知
186
Star
833
Fork
253
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
105
列表
看板
标记
里程碑
合并请求
10
Wiki
2
Wiki
分析
仓库
DevOps
项目成员
Pages
S
Serving
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
105
Issue
105
列表
看板
标记
里程碑
合并请求
10
合并请求
10
Pages
分析
分析
仓库分析
DevOps
Wiki
2
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
982d3e34
编写于
8月 02, 2019
作者:
D
dangyifei
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
delete some comment
上级
dd65660f
变更
8
显示空白变更内容
内联
并排
Showing
8 changed file
with
3 addition
and
60 deletion
+3
-60
cube/cube-builder/include/cube-builder/crovl_builder_increment.h
...be-builder/include/cube-builder/crovl_builder_increment.h
+0
-10
cube/cube-builder/include/cube-builder/raw_reader.h
cube/cube-builder/include/cube-builder/raw_reader.h
+1
-1
cube/cube-builder/include/cube-builder/seqfile_reader.h
cube/cube-builder/include/cube-builder/seqfile_reader.h
+0
-6
cube/cube-builder/include/cube-builder/vtext.h
cube/cube-builder/include/cube-builder/vtext.h
+0
-27
cube/cube-builder/src/builder_job.cpp
cube/cube-builder/src/builder_job.cpp
+0
-5
cube/cube-builder/src/crovl_builder_increment.cpp
cube/cube-builder/src/crovl_builder_increment.cpp
+2
-6
cube/cube-builder/src/curl_simple.cpp
cube/cube-builder/src/curl_simple.cpp
+0
-3
cube/cube-builder/src/main.cpp
cube/cube-builder/src/main.cpp
+0
-2
未找到文件。
cube/cube-builder/include/cube-builder/crovl_builder_increment.h
浏览文件 @
982d3e34
...
...
@@ -27,7 +27,6 @@ class CROVLBuilderIncremental {
static
const
uint32_t
INDEX_BUFFER
=
4194304
;
int
_index_type
;
uint32_t
_data_file_len
;
// hdfs _index_file;
uint64_t
_data_file
;
uint32_t
_data_buf_len
;
uint32_t
_index_buf_len
;
...
...
@@ -39,12 +38,6 @@ class CROVLBuilderIncremental {
uint32_t
_fix_len
;
uint32_t
_rec_size
;
// hdfsFS _fs;
// hdfsFile _file;
// AfsFileSystem* _afs_conn;
// Writer* _index_writer;
uint64_t
*
_index
;
char
*
_data
;
std
::
vector
<
uint32_t
>
_data_file_list
;
...
...
@@ -74,15 +67,12 @@ class CROVLBuilderIncremental {
std
::
string
_id
;
std
::
string
_key
;
std
::
string
_extra
;
// std::string _afs_regin;
public:
CROVLBuilderIncremental
();
~
CROVLBuilderIncremental
();
// bool set_sort_arg(uint64_t inner_sort_size, uint64_t memory_quota);
bool
Init
(
int
index_type
,
uint32_t
data_file_len
,
const
char
*
mode
,
...
...
cube/cube-builder/include/cube-builder/raw_reader.h
浏览文件 @
982d3e34
...
...
@@ -82,7 +82,7 @@ class FileRawReader : public RawReader {
return
-
1
;
}
virtual
int64_t
read
(
char
*
v
)
{
_reader
.
read
(
reinterpret_cast
<
char
*>
(
*
v
),
sizeof
(
char
));
_reader
.
read
(
reinterpret_cast
<
char
*>
(
v
),
sizeof
(
char
));
if
(
_reader
.
good
())
return
sizeof
(
char
);
else
if
(
_reader
.
eof
())
...
...
cube/cube-builder/include/cube-builder/seqfile_reader.h
浏览文件 @
982d3e34
...
...
@@ -22,11 +22,6 @@
class
SequenceFileRecordReader
:
public
RecordReader
{
public:
SequenceFileRecordReader
()
{}
// SequenceFileRecordReader(const std::string& path, AfsFileSystemPtr fs) {
// _path = path;
// _fs = fs;
// _raw_reader = new AFsRawReader(_path, _fs);
// }
explicit
SequenceFileRecordReader
(
const
std
::
string
&
path
)
{
_path
=
path
;
_raw_reader
=
new
FileRawReader
(
_path
);
...
...
@@ -44,7 +39,6 @@ class SequenceFileRecordReader : public RecordReader {
private:
std
::
string
_path
{
""
};
// AfsFileSystemPtr _fs {nullptr};
RawReader
*
_raw_reader
{
nullptr
};
Header
_header
;
};
...
...
cube/cube-builder/include/cube-builder/vtext.h
浏览文件 @
982d3e34
...
...
@@ -32,33 +32,6 @@ class VInt {
static
bool
is_negative_vint
(
const
char
&
value
)
{
return
value
<
-
120
||
(
value
>=
-
112
&&
value
<
0
);
}
// todo
/*static int32_t writeVInt(OutputStream& os, int i) {
int32_t rc = 0;
if (i >= -112 && i <= 127) {
rc = os.write((char*)&i, (int64_t)sizeof(char));
return rc;
}
int len = -112;
if (i < 0) {
i ^= -1L; // take one's complement'
len = -120;
}
int32_t tmp = i;
while (tmp != 0) {
tmp = tmp >> 8;
len--;
}
os.write((char*)&len, sizeof(char));
len = (len < -120) ? -(len + 120) : -(len + 112);
for (int idx = len - 1; idx >= 0; idx--) {
int32_t shiftbits = idx * 8;
int mask = 0xFFL << shiftbits;
int val = (i & mask) >> shiftbits;
os.write((char*)&val, sizeof(char));
}
return rc;
}*/
static
bool
read_vint
(
RawReader
*
reader
,
int32_t
*
vint
)
{
char
first_byte
;
if
(
reader
->
read
(
&
first_byte
)
<=
0
)
{
...
...
cube/cube-builder/src/builder_job.cpp
浏览文件 @
982d3e34
...
...
@@ -61,11 +61,6 @@ void mapFileLocal(Job job,
uint64_t
key
=
*
reinterpret_cast
<
uint64_t
*>
(
const_cast
<
char
*>
(
record
.
key
.
data
()));
// printf("key:%lu yu:%lu value:%s \n", key, key%job.get_shard_num(),
// string_to_hex(record.value).c_str());
// printf("key_length:%llu value_length:%llu \n", record.key.length(),
// record.value.length());
// fprintf(stdout, "key:%lu value:%s\n", key, record.value.c_str());
total_count
++
;
int
part
=
key
%
job
.
get_shard_num
();
int64_t
value_length
=
record
.
record_len
-
record
.
key_len
;
...
...
cube/cube-builder/src/crovl_builder_increment.cpp
浏览文件 @
982d3e34
...
...
@@ -156,10 +156,6 @@ bool CROVLBuilderIncremental::Init(int index_type,
if
(
_master_address
==
""
)
{
if
(
!
read_last_meta_from_local
())
{
LOG
(
ERROR
)
<<
"read last meta from db error! "
;
// if (!read_last_meta_from_hdfs()) {
// fprintf(stderr, "read last meta from hdfs error\n");
// return false;
//}
return
false
;
}
}
else
{
...
...
@@ -182,7 +178,7 @@ bool CROVLBuilderIncremental::flush_data() {
}
char
file
[
MAX_DATA_DIR_LEN
*
2
];
snprintf
(
file
,
sizeof
(
file
),
"%s/data.%l
l
u"
,
_data_dir
,
_data_file
);
snprintf
(
file
,
sizeof
(
file
),
"%s/data.%lu"
,
_data_dir
,
_data_file
);
FILE
*
fp
;
if
((
fp
=
fopen
(
file
,
"wb"
))
==
NULL
)
{
LOG
(
ERROR
)
<<
"open file failed! "
<<
file
;
...
...
@@ -390,7 +386,7 @@ bool CROVLBuilderIncremental::done() {
// fix empty data bug
// if the version no data, filter this version, index_file_num no add
// only can patch is null, not bae
// only can patch is null, not ba
s
e
if
(
_cur_count
>
0
)
{
*
reinterpret_cast
<
uint32_t
*>
(
buffer
)
=
++
_index_file_num
;
}
else
{
...
...
cube/cube-builder/src/curl_simple.cpp
浏览文件 @
982d3e34
...
...
@@ -143,9 +143,6 @@ std::string CurlSimple::curl_post(
if
(
res
!=
CURLE_OK
)
{
std
::
cerr
<<
"curl_easy_perform() failed:"
<<
curl_easy_strerror
(
res
)
<<
std
::
endl
;
// return "";
// fprintf(stderr, "curl_easy_perform() failed: %s\n",
// curl_easy_strerror(res));
}
else
{
/*
* Now, our chunk.memory points to a memory block that is chunk.size
...
...
cube/cube-builder/src/main.cpp
浏览文件 @
982d3e34
...
...
@@ -44,8 +44,6 @@ int main(int argc, char *argv[]) {
LOG
(
INFO
)
<<
"start"
;
// string f = "/";
// string output_path = argv[7] + f + depend_version + "_" + cur_version;
string
last_version
=
std
::
to_string
(
FLAGS_last_version
);
string
cur_version
=
std
::
to_string
(
FLAGS_cur_version
);
string
depend_version
=
std
::
to_string
(
FLAGS_depend_version
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录