Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
机器未来
Paddle
提交
1fe54416
P
Paddle
项目概览
机器未来
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
1fe54416
编写于
2月 22, 2019
作者:
D
dongdaxiang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
move fs.cc and shell.cc into paddle/fluid/framework/io
test=develop
上级
53fbab5d
变更
13
隐藏空白更改
内联
并排
Showing
13 changed file
with
121 addition
and
97 deletion
+121
-97
paddle/fluid/framework/CMakeLists.txt
paddle/fluid/framework/CMakeLists.txt
+1
-3
paddle/fluid/framework/async_executor.cc
paddle/fluid/framework/async_executor.cc
+0
-2
paddle/fluid/framework/data_feed.cc
paddle/fluid/framework/data_feed.cc
+46
-39
paddle/fluid/framework/data_feed.h
paddle/fluid/framework/data_feed.h
+3
-2
paddle/fluid/framework/data_feed.proto
paddle/fluid/framework/data_feed.proto
+1
-0
paddle/fluid/framework/executor_thread_worker.cc
paddle/fluid/framework/executor_thread_worker.cc
+1
-6
paddle/fluid/framework/io/CMakeLists.txt
paddle/fluid/framework/io/CMakeLists.txt
+0
-0
paddle/fluid/framework/io/fs.cc
paddle/fluid/framework/io/fs.cc
+42
-36
paddle/fluid/framework/io/fs.h
paddle/fluid/framework/io/fs.h
+2
-2
paddle/fluid/framework/io/shell.cc
paddle/fluid/framework/io/shell.cc
+4
-4
paddle/fluid/framework/io/shell.h
paddle/fluid/framework/io/shell.h
+1
-1
paddle/fluid/string/string_helper.h
paddle/fluid/string/string_helper.h
+2
-2
python/paddle/fluid/data_feed_desc.py
python/paddle/fluid/data_feed_desc.py
+18
-0
未找到文件。
paddle/fluid/framework/CMakeLists.txt
浏览文件 @
1fe54416
...
@@ -23,11 +23,9 @@ endfunction()
...
@@ -23,11 +23,9 @@ endfunction()
add_subdirectory
(
ir
)
add_subdirectory
(
ir
)
add_subdirectory
(
details
)
add_subdirectory
(
details
)
<<<<<<< HEAD
add_subdirectory
(
fleet
)
add_subdirectory
(
fleet
)
=======
add_subdirectory
(
common
)
add_subdirectory
(
common
)
>>>>>>> add fs_local_open example
add_subdirectory
(
io
)
#ddim lib
#ddim lib
proto_library
(
framework_proto SRCS framework.proto
)
proto_library
(
framework_proto SRCS framework.proto
)
proto_library
(
async_executor_proto SRCS data_feed.proto
)
proto_library
(
async_executor_proto SRCS data_feed.proto
)
...
...
paddle/fluid/framework/async_executor.cc
浏览文件 @
1fe54416
...
@@ -18,8 +18,6 @@ limitations under the License. */
...
@@ -18,8 +18,6 @@ limitations under the License. */
#include "google/protobuf/text_format.h"
#include "google/protobuf/text_format.h"
#include "gflags/gflags.h"
#include "gflags/gflags.h"
#include "paddle/fluid/framework/common/fs.h"
#include "paddle/fluid/framework/common/shell.h"
#include "paddle/fluid/framework/data_feed_factory.h"
#include "paddle/fluid/framework/data_feed_factory.h"
#include "paddle/fluid/framework/executor_thread_worker.h"
#include "paddle/fluid/framework/executor_thread_worker.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
...
...
paddle/fluid/framework/data_feed.cc
浏览文件 @
1fe54416
...
@@ -12,15 +12,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
...
@@ -12,15 +12,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
See the License for the specific language governing permissions and
limitations under the License. */
limitations under the License. */
#include "paddle/fluid/framework/data_feed.h"
#include <stdio_ext.h>
#include <stdio_ext.h>
#include "gflags/gflags.h"
#include "google/protobuf/io/zero_copy_stream_impl.h"
#include "google/protobuf/io/zero_copy_stream_impl.h"
#include "google/protobuf/message.h"
#include "google/protobuf/message.h"
#include "google/protobuf/text_format.h"
#include "google/protobuf/text_format.h"
#include "io/fs.h"
#include "common/fs.h"
#include "io/shell.h"
#include "common/shell.h"
#include "gflags/gflags.h"
#include "paddle/fluid/framework/data_feed.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/feed_fetch_type.h"
#include "paddle/fluid/framework/feed_fetch_type.h"
...
@@ -94,24 +93,8 @@ void PrivateQueueDataFeed<T>::SetQueueSize(int queue_size) {
...
@@ -94,24 +93,8 @@ void PrivateQueueDataFeed<T>::SetQueueSize(int queue_size) {
template
<
typename
T
>
template
<
typename
T
>
bool
PrivateQueueDataFeed
<
T
>::
Start
()
{
bool
PrivateQueueDataFeed
<
T
>::
Start
()
{
CheckSetFileList
();
CheckSetFileList
();
std
::
string
filename
;
read_thread_
=
std
::
thread
(
&
PrivateQueueDataFeed
::
ReadThread
,
this
);
if
(
PickOneFile
(
&
filename
))
{
read_thread_
.
detach
();
int
err_no
=
0
;
std
::
string
pipeline_cmd
=
"cat"
;
std
::
string
path
=
"/home/users/dongdaxiang/pslib_ctr/local/data_mod/part-00012"
;
fp_
=
fs_open_read
(
path
,
&
err_no
,
pipeline_cmd
);
__fsetlocking
(
&*
fp_
,
FSETLOCKING_BYCALLER
);
thread_local
LineFileReader
reader
;
while
(
reader
.
getline
(
&*
(
fp_
.
get
())))
{
LOG
(
ERROR
)
<<
"read a line"
;
}
read_thread_
=
std
::
thread
(
&
PrivateQueueDataFeed
::
ReadThread
,
this
);
read_thread_
.
detach
();
}
queue_
->
Close
();
finish_start_
=
true
;
finish_start_
=
true
;
return
true
;
return
true
;
...
@@ -119,10 +102,18 @@ bool PrivateQueueDataFeed<T>::Start() {
...
@@ -119,10 +102,18 @@ bool PrivateQueueDataFeed<T>::Start() {
template
<
typename
T
>
template
<
typename
T
>
void
PrivateQueueDataFeed
<
T
>::
ReadThread
()
{
void
PrivateQueueDataFeed
<
T
>::
ReadThread
()
{
T
instance
;
std
::
string
filename
;
while
(
ParseOneInstanceFromPipe
(
&
instance
))
{
while
(
PickOneFile
(
&
filename
))
{
queue_
->
Send
(
instance
);
int
err_no
=
0
;
fp_
=
fs_open_read
(
filename
,
&
err_no
,
pipe_command_
);
__fsetlocking
(
&*
fp_
,
FSETLOCKING_BYCALLER
);
thread_local
string
::
LineFileReader
reader
;
T
instance
;
while
(
ParseOneInstanceFromPipe
(
&
instance
))
{
queue_
->
Send
(
instance
);
}
}
}
queue_
->
Close
();
}
}
template
<
typename
T
>
template
<
typename
T
>
...
@@ -177,15 +168,26 @@ void MultiSlotDataFeed::Init(
...
@@ -177,15 +168,26 @@ void MultiSlotDataFeed::Init(
}
}
}
}
feed_vec_
.
resize
(
use_slots_
.
size
());
feed_vec_
.
resize
(
use_slots_
.
size
());
pipe_command_
=
data_feed_desc
.
pipe_command
();
finish_init_
=
true
;
finish_init_
=
true
;
}
}
void
MultiSlotDataFeed
::
ReadThread
()
{
void
MultiSlotDataFeed
::
ReadThread
()
{
LOG
(
ERROR
)
<<
"Haha"
;
std
::
string
filename
;
std
::
vector
<
MultiSlotType
>
instance
;
while
(
PickOneFile
(
&
filename
))
{
while
(
ParseOneInstanceFromPipe
(
&
instance
))
{
int
err_no
=
0
;
queue_
->
Send
(
instance
);
fp_
=
fs_open_read
(
filename
,
&
err_no
,
pipe_command_
);
__fsetlocking
(
&*
fp_
,
FSETLOCKING_BYCALLER
);
thread_local
string
::
LineFileReader
reader
;
std
::
vector
<
MultiSlotType
>
instance
;
int
ins_num
=
0
;
while
(
ParseOneInstanceFromPipe
(
&
instance
))
{
ins_num
++
;
queue_
->
Send
(
instance
);
}
LOG
(
ERROR
)
<<
"filename: "
<<
filename
<<
" inst num: "
<<
ins_num
;
}
}
queue_
->
Close
();
}
}
bool
MultiSlotDataFeed
::
CheckFile
(
const
char
*
filename
)
{
bool
MultiSlotDataFeed
::
CheckFile
(
const
char
*
filename
)
{
...
@@ -301,26 +303,32 @@ bool MultiSlotDataFeed::CheckFile(const char* filename) {
...
@@ -301,26 +303,32 @@ bool MultiSlotDataFeed::CheckFile(const char* filename) {
bool
MultiSlotDataFeed
::
ParseOneInstanceFromPipe
(
bool
MultiSlotDataFeed
::
ParseOneInstanceFromPipe
(
std
::
vector
<
MultiSlotType
>*
instance
)
{
std
::
vector
<
MultiSlotType
>*
instance
)
{
LOG
(
ERROR
)
<<
"hehe"
;
thread_local
string
::
LineFileReader
reader
;
thread_local
LineFileReader
reader
;
/*
while (reader.getline(&*(fp_.get()))) {
while (reader.getline(&*(fp_.get()))) {
/*
*/
/*
const char* str = reader.get();
const char* str = reader.get();
std::string line = std::string(str);
std::string line = std::string(str);
LOG(ERROR) << line;
LOG(ERROR) << line;
*/
*/
/*
LOG(ERROR) << "read a line";
LOG(ERROR) << "read a line";
}
}
return
true
;
*/
/*
if (!reader.getline(
fp_.get(
))) {
if
(
!
reader
.
getline
(
&*
(
fp_
.
get
()
)))
{
return
false
;
return
false
;
}
else
{
}
else
{
// std::string& line = reader_.get();
// std::string& line = reader_.get();
// const char* str = line.c_str();
// const char* str = line.c_str();
int
use_slots_num
=
use_slots_
.
size
();
instance
->
resize
(
use_slots_num
);
const
char
*
str
=
reader
.
get
();
const
char
*
str
=
reader
.
get
();
std
::
string
line
=
std
::
string
(
str
);
std
::
string
line
=
std
::
string
(
str
);
LOG(ERROR) << line;
//
LOG(ERROR) << line;
char
*
endptr
=
const_cast
<
char
*>
(
str
);
char
*
endptr
=
const_cast
<
char
*>
(
str
);
int
pos
=
0
;
int
pos
=
0
;
for
(
size_t
i
=
0
;
i
<
use_slots_index_
.
size
();
++
i
)
{
for
(
size_t
i
=
0
;
i
<
use_slots_index_
.
size
();
++
i
)
{
...
@@ -355,7 +363,6 @@ bool MultiSlotDataFeed::ParseOneInstanceFromPipe(
...
@@ -355,7 +363,6 @@ bool MultiSlotDataFeed::ParseOneInstanceFromPipe(
}
}
return
true
;
return
true
;
}
}
*/
}
}
bool
MultiSlotDataFeed
::
ParseOneInstance
(
std
::
vector
<
MultiSlotType
>*
instance
)
{
bool
MultiSlotDataFeed
::
ParseOneInstance
(
std
::
vector
<
MultiSlotType
>*
instance
)
{
...
...
paddle/fluid/framework/data_feed.h
浏览文件 @
1fe54416
...
@@ -21,12 +21,12 @@ limitations under the License. */
...
@@ -21,12 +21,12 @@ limitations under the License. */
#include <thread> // NOLINT
#include <thread> // NOLINT
#include <vector>
#include <vector>
#include "paddle/fluid/framework/common/ps_string.h"
#include "paddle/fluid/framework/data_feed.pb.h"
#include "paddle/fluid/framework/data_feed.pb.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/reader.h"
#include "paddle/fluid/framework/reader.h"
#include "paddle/fluid/framework/variable.h"
#include "paddle/fluid/framework/variable.h"
#include "paddle/fluid/operators/reader/blocking_queue.h"
#include "paddle/fluid/operators/reader/blocking_queue.h"
#include "paddle/fluid/string/string_helper.h"
namespace
paddle
{
namespace
paddle
{
namespace
framework
{
namespace
framework
{
...
@@ -115,6 +115,7 @@ class DataFeed {
...
@@ -115,6 +115,7 @@ class DataFeed {
bool
finish_init_
;
bool
finish_init_
;
static
bool
finish_set_filelist_
;
static
bool
finish_set_filelist_
;
bool
finish_start_
;
bool
finish_start_
;
std
::
string
pipe_command_
;
};
};
// PrivateQueueDataFeed is the base virtual class for ohther DataFeeds.
// PrivateQueueDataFeed is the base virtual class for ohther DataFeeds.
...
@@ -154,7 +155,7 @@ class PrivateQueueDataFeed : public DataFeed {
...
@@ -154,7 +155,7 @@ class PrivateQueueDataFeed : public DataFeed {
std
::
ifstream
file_
;
std
::
ifstream
file_
;
std
::
shared_ptr
<
FILE
>
fp_
;
std
::
shared_ptr
<
FILE
>
fp_
;
size_t
queue_size_
;
size_t
queue_size_
;
LineFileReader
reader_
;
string
::
LineFileReader
reader_
;
// The queue for store parsed data
// The queue for store parsed data
std
::
unique_ptr
<
paddle
::
operators
::
reader
::
BlockingQueue
<
T
>>
queue_
;
std
::
unique_ptr
<
paddle
::
operators
::
reader
::
BlockingQueue
<
T
>>
queue_
;
};
};
...
...
paddle/fluid/framework/data_feed.proto
浏览文件 @
1fe54416
...
@@ -27,4 +27,5 @@ message DataFeedDesc {
...
@@ -27,4 +27,5 @@ message DataFeedDesc {
optional
string
name
=
1
;
optional
string
name
=
1
;
optional
int32
batch_size
=
2
[
default
=
32
];
optional
int32
batch_size
=
2
[
default
=
32
];
optional
MultiSlotDesc
multi_slot_desc
=
3
;
optional
MultiSlotDesc
multi_slot_desc
=
3
;
optional
string
pipe_command
=
4
;
}
}
paddle/fluid/framework/executor_thread_worker.cc
浏览文件 @
1fe54416
...
@@ -13,15 +13,12 @@ See the License for the specific language governing permissions and
...
@@ -13,15 +13,12 @@ See the License for the specific language governing permissions and
limitations under the License. */
limitations under the License. */
#include "paddle/fluid/framework/executor_thread_worker.h"
#include "paddle/fluid/framework/executor_thread_worker.h"
#include <stdio_ext.h>
#include <algorithm>
#include <algorithm>
#include "google/protobuf/io/zero_copy_stream_impl.h"
#include "google/protobuf/io/zero_copy_stream_impl.h"
#include "google/protobuf/message.h"
#include "google/protobuf/message.h"
#include "google/protobuf/text_format.h"
#include "google/protobuf/text_format.h"
#include "gflags/gflags.h"
#include "gflags/gflags.h"
#include "paddle/fluid/framework/common/fs.h"
#include "paddle/fluid/framework/common/shell.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/feed_fetch_type.h"
#include "paddle/fluid/framework/feed_fetch_type.h"
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_rank_table.h"
...
@@ -247,8 +244,7 @@ void ExecutorThreadWorker::TrainFilesWithTimer() {
...
@@ -247,8 +244,7 @@ void ExecutorThreadWorker::TrainFilesWithTimer() {
platform
::
SetNumThreads
(
1
);
platform
::
SetNumThreads
(
1
);
SetDevice
();
SetDevice
();
thread_reader_
->
Start
();
thread_reader_
->
Start
();
exit
(
0
);
/*
std
::
vector
<
double
>
op_total_time
;
std
::
vector
<
double
>
op_total_time
;
std
::
vector
<
std
::
string
>
op_name
;
std
::
vector
<
std
::
string
>
op_name
;
for
(
auto
&
op
:
ops_
)
{
for
(
auto
&
op
:
ops_
)
{
...
@@ -292,7 +288,6 @@ void ExecutorThreadWorker::TrainFilesWithTimer() {
...
@@ -292,7 +288,6 @@ void ExecutorThreadWorker::TrainFilesWithTimer() {
}
}
timeline
.
Start
();
timeline
.
Start
();
}
}
*/
}
}
void
ExecutorThreadWorker
::
TrainFiles
()
{
void
ExecutorThreadWorker
::
TrainFiles
()
{
...
...
paddle/fluid/framework/
common
/CMakeLists.txt
→
paddle/fluid/framework/
io
/CMakeLists.txt
浏览文件 @
1fe54416
文件已移动
paddle/fluid/framework/
common
/fs.cc
→
paddle/fluid/framework/
io
/fs.cc
浏览文件 @
1fe54416
...
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
...
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
See the License for the specific language governing permissions and
limitations under the License. */
limitations under the License. */
#include "paddle/fluid/framework/
common
/fs.h"
#include "paddle/fluid/framework/
io
/fs.h"
namespace
paddle
{
namespace
paddle
{
namespace
framework
{
namespace
framework
{
...
@@ -25,10 +25,11 @@ static void fs_add_read_converter_internal(std::string& path, // NOLINT
...
@@ -25,10 +25,11 @@ static void fs_add_read_converter_internal(std::string& path, // NOLINT
}
}
if
(
!
is_pipe
)
{
if
(
!
is_pipe
)
{
path
=
format_string
(
"( %s ) <
\"
%s
\"
"
,
converter
.
c_str
(),
path
.
c_str
());
path
=
string
::
format_string
(
"( %s ) <
\"
%s
\"
"
,
converter
.
c_str
(),
path
.
c_str
());
is_pipe
=
true
;
is_pipe
=
true
;
}
else
{
}
else
{
path
=
format_string
(
"%s | %s"
,
path
.
c_str
(),
converter
.
c_str
());
path
=
string
::
format_string
(
"%s | %s"
,
path
.
c_str
(),
converter
.
c_str
());
}
}
}
}
...
@@ -40,10 +41,11 @@ static void fs_add_write_converter_internal(std::string& path, // NOLINT
...
@@ -40,10 +41,11 @@ static void fs_add_write_converter_internal(std::string& path, // NOLINT
}
}
if
(
!
is_pipe
)
{
if
(
!
is_pipe
)
{
path
=
format_string
(
"( %s ) >
\"
%s
\"
"
,
converter
.
c_str
(),
path
.
c_str
());
path
=
string
::
format_string
(
"( %s ) >
\"
%s
\"
"
,
converter
.
c_str
(),
path
.
c_str
());
is_pipe
=
true
;
is_pipe
=
true
;
}
else
{
}
else
{
path
=
format_string
(
"%s | %s"
,
converter
.
c_str
(),
path
.
c_str
());
path
=
string
::
format_string
(
"%s | %s"
,
converter
.
c_str
(),
path
.
c_str
());
}
}
}
}
...
@@ -108,7 +110,8 @@ std::shared_ptr<FILE> localfs_open_read(std::string path,
...
@@ -108,7 +110,8 @@ std::shared_ptr<FILE> localfs_open_read(std::string path,
std
::
shared_ptr
<
FILE
>
localfs_open_write
(
std
::
string
path
,
std
::
shared_ptr
<
FILE
>
localfs_open_write
(
std
::
string
path
,
const
std
::
string
&
converter
)
{
const
std
::
string
&
converter
)
{
shell_execute
(
format_string
(
"mkdir -p $(dirname
\"
%s
\"
)"
,
path
.
c_str
()));
shell_execute
(
string
::
format_string
(
"mkdir -p $(dirname
\"
%s
\"
)"
,
path
.
c_str
()));
bool
is_pipe
=
false
;
bool
is_pipe
=
false
;
...
@@ -134,7 +137,7 @@ void localfs_remove(const std::string& path) {
...
@@ -134,7 +137,7 @@ void localfs_remove(const std::string& path) {
return
;
return
;
}
}
shell_execute
(
format_string
(
"rm -rf %s"
,
path
.
c_str
()));
shell_execute
(
string
::
format_string
(
"rm -rf %s"
,
path
.
c_str
()));
}
}
std
::
vector
<
std
::
string
>
localfs_list
(
const
std
::
string
&
path
)
{
std
::
vector
<
std
::
string
>
localfs_list
(
const
std
::
string
&
path
)
{
...
@@ -144,9 +147,10 @@ std::vector<std::string> localfs_list(const std::string& path) {
...
@@ -144,9 +147,10 @@ std::vector<std::string> localfs_list(const std::string& path) {
std
::
shared_ptr
<
FILE
>
pipe
;
std
::
shared_ptr
<
FILE
>
pipe
;
int
err_no
=
0
;
int
err_no
=
0
;
pipe
=
shell_popen
(
format_string
(
"find %s -type f -maxdepth 1"
,
path
.
c_str
()),
pipe
=
shell_popen
(
"r"
,
&
err_no
);
string
::
format_string
(
"find %s -type f -maxdepth 1"
,
path
.
c_str
()),
"r"
,
LineFileReader
reader
;
&
err_no
);
string
::
LineFileReader
reader
;
std
::
vector
<
std
::
string
>
list
;
std
::
vector
<
std
::
string
>
list
;
while
(
reader
.
getline
(
&*
pipe
))
{
while
(
reader
.
getline
(
&*
pipe
))
{
...
@@ -161,21 +165,22 @@ std::string localfs_tail(const std::string& path) {
...
@@ -161,21 +165,22 @@ std::string localfs_tail(const std::string& path) {
return
""
;
return
""
;
}
}
return
shell_get_command_output
(
format_string
(
"tail -1 %s "
,
path
.
c_str
()));
return
shell_get_command_output
(
string
::
format_string
(
"tail -1 %s "
,
path
.
c_str
()));
}
}
bool
localfs_exists
(
const
std
::
string
&
path
)
{
bool
localfs_exists
(
const
std
::
string
&
path
)
{
std
::
string
test_f
=
shell_get_command_output
(
std
::
string
test_f
=
shell_get_command_output
(
format_string
(
"[ -f %s ] ; echo $?"
,
path
.
c_str
()));
string
::
format_string
(
"[ -f %s ] ; echo $?"
,
path
.
c_str
()));
if
(
trim_spaces
(
test_f
)
==
"0"
)
{
if
(
string
::
trim_spaces
(
test_f
)
==
"0"
)
{
return
true
;
return
true
;
}
}
std
::
string
test_d
=
shell_get_command_output
(
std
::
string
test_d
=
shell_get_command_output
(
format_string
(
"[ -d %s ] ; echo $?"
,
path
.
c_str
()));
string
::
format_string
(
"[ -d %s ] ; echo $?"
,
path
.
c_str
()));
if
(
trim_spaces
(
test_d
)
==
"0"
)
{
if
(
string
::
trim_spaces
(
test_d
)
==
"0"
)
{
return
true
;
return
true
;
}
}
...
@@ -187,7 +192,7 @@ void localfs_mkdir(const std::string& path) {
...
@@ -187,7 +192,7 @@ void localfs_mkdir(const std::string& path) {
return
;
return
;
}
}
shell_execute
(
format_string
(
"mkdir -p %s"
,
path
.
c_str
()));
shell_execute
(
string
::
format_string
(
"mkdir -p %s"
,
path
.
c_str
()));
}
}
static
size_t
&
hdfs_buffer_size_internal
()
{
static
size_t
&
hdfs_buffer_size_internal
()
{
...
@@ -211,11 +216,11 @@ void hdfs_set_command(const std::string& x) { hdfs_command_internal() = x; }
...
@@ -211,11 +216,11 @@ void hdfs_set_command(const std::string& x) { hdfs_command_internal() = x; }
std
::
shared_ptr
<
FILE
>
hdfs_open_read
(
std
::
string
path
,
int
*
err_no
,
std
::
shared_ptr
<
FILE
>
hdfs_open_read
(
std
::
string
path
,
int
*
err_no
,
const
std
::
string
&
converter
)
{
const
std
::
string
&
converter
)
{
if
(
fs_end_with_internal
(
path
,
".gz"
))
{
if
(
fs_end_with_internal
(
path
,
".gz"
))
{
path
=
path
=
string
::
format_string
(
"%s -text
\"
%s
\"
"
,
hdfs_command
().
c_str
(),
format_string
(
"%s -text
\"
%s
\"
"
,
hdfs_command
().
c_str
(),
path
.
c_str
());
path
.
c_str
());
}
else
{
}
else
{
path
=
path
=
string
::
format_string
(
"%s -cat
\"
%s
\"
"
,
hdfs_command
().
c_str
(),
format_string
(
"%s -cat
\"
%s
\"
"
,
hdfs_command
().
c_str
(),
path
.
c_str
());
path
.
c_str
());
}
}
bool
is_pipe
=
true
;
bool
is_pipe
=
true
;
...
@@ -225,8 +230,8 @@ std::shared_ptr<FILE> hdfs_open_read(std::string path, int* err_no,
...
@@ -225,8 +230,8 @@ std::shared_ptr<FILE> hdfs_open_read(std::string path, int* err_no,
std
::
shared_ptr
<
FILE
>
hdfs_open_write
(
std
::
string
path
,
int
*
err_no
,
std
::
shared_ptr
<
FILE
>
hdfs_open_write
(
std
::
string
path
,
int
*
err_no
,
const
std
::
string
&
converter
)
{
const
std
::
string
&
converter
)
{
path
=
path
=
string
::
format_string
(
"%s -put -
\"
%s
\"
"
,
hdfs_command
().
c_str
(),
format_string
(
"%s -put -
\"
%s
\"
"
,
hdfs_command
().
c_str
(),
path
.
c_str
());
path
.
c_str
());
bool
is_pipe
=
true
;
bool
is_pipe
=
true
;
if
(
fs_end_with_internal
(
path
,
".gz
\"
"
))
{
if
(
fs_end_with_internal
(
path
,
".gz
\"
"
))
{
...
@@ -242,8 +247,8 @@ void hdfs_remove(const std::string& path) {
...
@@ -242,8 +247,8 @@ void hdfs_remove(const std::string& path) {
return
;
return
;
}
}
shell_execute
(
format_string
(
"%s -rmr %s &>/dev/null; true"
,
shell_execute
(
string
::
format_string
(
"%s -rmr %s &>/dev/null; true"
,
hdfs_command
().
c_str
(),
path
.
c_str
()));
hdfs_command
().
c_str
(),
path
.
c_str
()));
}
}
std
::
vector
<
std
::
string
>
hdfs_list
(
const
std
::
string
&
path
)
{
std
::
vector
<
std
::
string
>
hdfs_list
(
const
std
::
string
&
path
)
{
...
@@ -261,14 +266,15 @@ std::vector<std::string> hdfs_list(const std::string& path) {
...
@@ -261,14 +266,15 @@ std::vector<std::string> hdfs_list(const std::string& path) {
do
{
do
{
err_no
=
0
;
err_no
=
0
;
std
::
shared_ptr
<
FILE
>
pipe
;
std
::
shared_ptr
<
FILE
>
pipe
;
pipe
=
shell_popen
(
format_string
(
"%s -ls %s | ( grep ^- ; [ $? != 2 ] )"
,
pipe
=
shell_popen
(
hdfs_command
().
c_str
(),
path
.
c_str
()),
string
::
format_string
(
"%s -ls %s | ( grep ^- ; [ $? != 2 ] )"
,
"r"
,
&
err_no
);
hdfs_command
().
c_str
(),
path
.
c_str
()),
LineFileReader
reader
;
"r"
,
&
err_no
);
string
::
LineFileReader
reader
;
list
.
clear
();
list
.
clear
();
while
(
reader
.
getline
(
&*
pipe
))
{
while
(
reader
.
getline
(
&*
pipe
))
{
std
::
vector
<
std
::
string
>
line
=
split_string
(
reader
.
get
());
std
::
vector
<
std
::
string
>
line
=
s
tring
::
s
plit_string
(
reader
.
get
());
if
(
line
.
size
()
!=
8
)
{
if
(
line
.
size
()
!=
8
)
{
continue
;
continue
;
}
}
...
@@ -283,15 +289,15 @@ std::string hdfs_tail(const std::string& path) {
...
@@ -283,15 +289,15 @@ std::string hdfs_tail(const std::string& path) {
return
""
;
return
""
;
}
}
return
shell_get_command_output
(
format_string
(
return
shell_get_command_output
(
string
::
format_string
(
"%s -text %s | tail -1 "
,
hdfs_command
().
c_str
(),
path
.
c_str
()));
"%s -text %s | tail -1 "
,
hdfs_command
().
c_str
(),
path
.
c_str
()));
}
}
bool
hdfs_exists
(
const
std
::
string
&
path
)
{
bool
hdfs_exists
(
const
std
::
string
&
path
)
{
std
::
string
test
=
shell_get_command_output
(
format_string
(
std
::
string
test
=
shell_get_command_output
(
string
::
format_string
(
"%s -test -e %s ; echo $?"
,
hdfs_command
().
c_str
(),
path
.
c_str
()));
"%s -test -e %s ; echo $?"
,
hdfs_command
().
c_str
(),
path
.
c_str
()));
if
(
trim_spaces
(
test
)
==
"0"
)
{
if
(
string
::
trim_spaces
(
test
)
==
"0"
)
{
return
true
;
return
true
;
}
}
...
@@ -303,8 +309,8 @@ void hdfs_mkdir(const std::string& path) {
...
@@ -303,8 +309,8 @@ void hdfs_mkdir(const std::string& path) {
return
;
return
;
}
}
shell_execute
(
format_string
(
"%s -mkdir %s; true"
,
hdfs_command
().
c_str
()
,
shell_execute
(
string
::
format_string
(
"%s -mkdir %s; true"
,
path
.
c_str
()));
hdfs_command
().
c_str
(),
path
.
c_str
()));
}
}
int
fs_select_internal
(
const
std
::
string
&
path
)
{
int
fs_select_internal
(
const
std
::
string
&
path
)
{
...
@@ -445,5 +451,5 @@ void fs_mkdir(const std::string& path) {
...
@@ -445,5 +451,5 @@ void fs_mkdir(const std::string& path) {
LOG
(
FATAL
)
<<
"Not supported"
;
LOG
(
FATAL
)
<<
"Not supported"
;
}
}
}
}
}
// namespace framework
}
//
end
namespace framework
}
// namespace paddle
}
//
end
namespace paddle
paddle/fluid/framework/
common
/fs.h
→
paddle/fluid/framework/
io
/fs.h
浏览文件 @
1fe54416
...
@@ -18,8 +18,8 @@
...
@@ -18,8 +18,8 @@
#include <string>
#include <string>
#include <vector>
#include <vector>
#include "glog/logging.h"
#include "glog/logging.h"
#include "paddle/fluid/framework/
common/ps_string
.h"
#include "paddle/fluid/framework/
io/shell
.h"
#include "paddle/fluid/
framework/common/shell
.h"
#include "paddle/fluid/
string/string_helper
.h"
namespace
paddle
{
namespace
paddle
{
namespace
framework
{
namespace
framework
{
...
...
paddle/fluid/framework/
common
/shell.cc
→
paddle/fluid/framework/
io
/shell.cc
浏览文件 @
1fe54416
...
@@ -12,7 +12,7 @@
...
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// See the License for the specific language governing permissions and
// limitations under the License.
// limitations under the License.
#include "paddle/fluid/framework/
common
/shell.h"
#include "paddle/fluid/framework/
io
/shell.h"
namespace
paddle
{
namespace
paddle
{
namespace
framework
{
namespace
framework
{
...
@@ -282,7 +282,7 @@ std::string shell_get_command_output(const std::string& cmd) {
...
@@ -282,7 +282,7 @@ std::string shell_get_command_output(const std::string& cmd) {
do
{
do
{
err_no
=
0
;
err_no
=
0
;
std
::
shared_ptr
<
FILE
>
pipe
=
shell_popen
(
cmd
,
"r"
,
&
err_no
);
std
::
shared_ptr
<
FILE
>
pipe
=
shell_popen
(
cmd
,
"r"
,
&
err_no
);
LineFileReader
reader
;
string
::
LineFileReader
reader
;
if
(
reader
.
getdelim
(
&*
pipe
,
0
))
{
if
(
reader
.
getdelim
(
&*
pipe
,
0
))
{
pipe
=
nullptr
;
pipe
=
nullptr
;
...
@@ -294,5 +294,5 @@ std::string shell_get_command_output(const std::string& cmd) {
...
@@ -294,5 +294,5 @@ std::string shell_get_command_output(const std::string& cmd) {
return
""
;
return
""
;
}
}
}
// namespace framework
}
//
end
namespace framework
}
// namespace paddle
}
//
end
namespace paddle
paddle/fluid/framework/
common
/shell.h
→
paddle/fluid/framework/
io
/shell.h
浏览文件 @
1fe54416
...
@@ -23,7 +23,7 @@
...
@@ -23,7 +23,7 @@
#include <string>
#include <string>
#include <utility>
#include <utility>
#include "glog/logging.h"
#include "glog/logging.h"
#include "paddle/fluid/
framework/common/ps_string
.h"
#include "paddle/fluid/
string/string_helper
.h"
namespace
paddle
{
namespace
paddle
{
namespace
framework
{
namespace
framework
{
...
...
paddle/fluid/
framework/common/ps_string
.h
→
paddle/fluid/
string/string_helper
.h
浏览文件 @
1fe54416
...
@@ -23,7 +23,7 @@
...
@@ -23,7 +23,7 @@
#include "glog/logging.h"
#include "glog/logging.h"
namespace
paddle
{
namespace
paddle
{
namespace
framework
{
namespace
string
{
inline
size_t
count_spaces
(
const
char
*
s
)
{
inline
size_t
count_spaces
(
const
char
*
s
)
{
size_t
count
=
0
;
size_t
count
=
0
;
...
@@ -234,5 +234,5 @@ class LineFileReader {
...
@@ -234,5 +234,5 @@ class LineFileReader {
size_t
_buf_size
=
0
;
size_t
_buf_size
=
0
;
size_t
_length
=
0
;
size_t
_length
=
0
;
};
};
}
// end namespace
framework
}
// end namespace
string
}
// end namespace paddle
}
// end namespace paddle
python/paddle/fluid/data_feed_desc.py
浏览文件 @
1fe54416
...
@@ -114,6 +114,24 @@ class DataFeedDesc(object):
...
@@ -114,6 +114,24 @@ class DataFeedDesc(object):
self
.
proto_desc
.
multi_slot_desc
.
slots
[
self
.
__name_to_index
[
self
.
proto_desc
.
multi_slot_desc
.
slots
[
self
.
__name_to_index
[
name
]].
is_dense
=
True
name
]].
is_dense
=
True
def
set_pipe_command
(
self
,
pipe_command
):
"""
Pipeline command will be set with this function. In IO runtime,
pipeline command will be executed given user provided input raw
files.
Example:
>>> data_feed = fluid.DataFeedDesc('data.proto')
>>> data_feed.set_pipe_command('awk -F '
\t
' '{print $2}'')
Args:
pipe_command: a command string of shell command
Note:
Default is cat, i.e., cat user's input file list to data feed
"""
self
.
proto_desc
.
pipe_command
=
pipe_command
def
set_use_slots
(
self
,
use_slots_name
):
def
set_use_slots
(
self
,
use_slots_name
):
"""
"""
Set if a specific slot will be used for training. A dataset shall
Set if a specific slot will be used for training. A dataset shall
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录