Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
PaddleDetection
提交
1377b332
P
PaddleDetection
项目概览
PaddlePaddle
/
PaddleDetection
大约 1 年 前同步成功
通知
694
Star
11112
Fork
2696
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
184
列表
看板
标记
里程碑
合并请求
40
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
PaddleDetection
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
184
Issue
184
列表
看板
标记
里程碑
合并请求
40
合并请求
40
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
1377b332
编写于
7月 06, 2018
作者:
M
minqiyang
浏览文件
操作
浏览文件
下载
差异文件
Merge branch 'develop' of
https://github.com/PaddlePaddle/Paddle
into fix_grpc_destroy_bug
上级
a947445a
f86198e6
变更
17
隐藏空白更改
内联
并排
Showing
17 changed file
with
111 addition
and
99 deletion
+111
-99
Dockerfile
Dockerfile
+1
-1
doc/fluid/api/transpiler.rst
doc/fluid/api/transpiler.rst
+9
-0
paddle/fluid/framework/details/build_strategy.h
paddle/fluid/framework/details/build_strategy.h
+1
-1
paddle/fluid/framework/details/data_balance_op_handle.cc
paddle/fluid/framework/details/data_balance_op_handle.cc
+3
-3
paddle/fluid/framework/details/multi_devices_graph_builder.cc
...le/fluid/framework/details/multi_devices_graph_builder.cc
+5
-0
paddle/fluid/framework/op_registry.h
paddle/fluid/framework/op_registry.h
+9
-15
paddle/fluid/framework/op_registry_test.cc
paddle/fluid/framework/op_registry_test.cc
+2
-7
paddle/fluid/framework/operator.h
paddle/fluid/framework/operator.h
+0
-35
paddle/fluid/framework/operator_test.cc
paddle/fluid/framework/operator_test.cc
+0
-23
paddle/fluid/framework/var_type_inference_test.cc
paddle/fluid/framework/var_type_inference_test.cc
+11
-0
paddle/fluid/operators/read_op.cc
paddle/fluid/operators/read_op.cc
+7
-3
paddle/legacy/gserver/gradientmachines/MultiGradientMachine.cpp
.../legacy/gserver/gradientmachines/MultiGradientMachine.cpp
+4
-0
paddle/legacy/gserver/gradientmachines/ParallelNeuralNetwork.cpp
...legacy/gserver/gradientmachines/ParallelNeuralNetwork.cpp
+1
-0
python/paddle/fluid/tests/unittests/test_data_balance.py
python/paddle/fluid/tests/unittests/test_data_balance.py
+10
-3
python/paddle/fluid/transpiler/inference_transpiler.py
python/paddle/fluid/transpiler/inference_transpiler.py
+1
-1
python/paddle/libs/__init__.py
python/paddle/libs/__init__.py
+15
-0
python/setup.py.in
python/setup.py.in
+32
-7
未找到文件。
Dockerfile
浏览文件 @
1377b332
...
...
@@ -23,7 +23,7 @@ ENV HOME /root
COPY
./paddle/scripts/docker/root/ /root/
RUN
apt-get update
&&
\
apt-get
install
-y
--allow-downgrades
\
apt-get
install
-y
--allow-downgrades
patchelf
\
git python-pip python-dev python-opencv openssh-server bison
\
libnccl2
=
2.1.2-1+cuda8.0 libnccl-dev
=
2.1.2-1+cuda8.0
\
wget unzip unrar
tar
xz-utils bzip2
gzip
coreutils ntp
\
...
...
doc/fluid/api/transpiler.rst
浏览文件 @
1377b332
...
...
@@ -14,6 +14,15 @@ DistributeTranspiler
:members:
:noindex:
.. _api_fluid_transpiler_InferenceTranspiler:
InferenceTranspiler
-------------------
.. autoclass:: paddle.fluid.transpiler.InferenceTranspiler
:members:
:noindex:
.. _api_fluid_transpiler_memory_optimize:
memory_optimize
...
...
paddle/fluid/framework/details/build_strategy.h
浏览文件 @
1377b332
...
...
@@ -34,7 +34,7 @@ struct BuildStrategy {
std
::
string
debug_graphviz_path_
{
""
};
bool
enable_data_balance_
{
tru
e
};
bool
enable_data_balance_
{
fals
e
};
};
}
// namespace details
...
...
paddle/fluid/framework/details/data_balance_op_handle.cc
浏览文件 @
1377b332
...
...
@@ -86,9 +86,9 @@ std::vector<std::array<int, 3>> DataBalanceOpHandle::GetBalancePlan(
}
void
DataBalanceOpHandle
::
RunImpl
()
{
if
(
places_
.
size
()
==
1
)
{
return
;
}
PADDLE_ENFORCE_GT
(
places_
.
size
(),
1
,
"Data balance can only be enabled when the number of "
"places to run larger than 1."
);
auto
in_var_handles
=
DynamicCast
<
VarHandle
>
(
inputs_
);
auto
out_var_handles
=
DynamicCast
<
VarHandle
>
(
outputs_
);
PADDLE_ENFORCE
(
in_var_handles
.
size
()
%
places_
.
size
()
==
0
);
...
...
paddle/fluid/framework/details/multi_devices_graph_builder.cc
浏览文件 @
1377b332
...
...
@@ -59,6 +59,11 @@ MultiDevSSAGraphBuilder::MultiDevSSAGraphBuilder(
grad_names_
.
insert
(
GradVarName
(
p
));
}
balance_vars_
.
resize
(
places_
.
size
(),
0
);
if
(
strategy_
.
enable_data_balance_
&&
places_
.
size
()
==
1
)
{
LOG
(
WARNING
)
<<
"It is no need to enable data balance when there is only "
"one place. enable_data_balance is set to False."
;
strategy_
.
enable_data_balance_
=
false
;
}
}
void
MultiDevSSAGraphBuilder
::
CreateOpHandleIOs
(
SSAGraph
*
result
,
...
...
paddle/fluid/framework/op_registry.h
浏览文件 @
1377b332
...
...
@@ -182,21 +182,15 @@ struct OpKernelRegistrarFunctorEx<PlaceType, false, I,
VarTypeInference
InferShapeBase
*/
#define REGISTER_OPERATOR(op_type, op_class, ...) \
STATIC_ASSERT_GLOBAL_NAMESPACE( \
__reg_op__##op_type, \
"REGISTER_OPERATOR must be called in global namespace"); \
class _OpClass_##op_type##_ : public op_class { \
public: \
DEFINE_OP_CLONE_METHOD(_OpClass_##op_type##_); \
DEFINE_OP_CONSTRUCTOR(_OpClass_##op_type##_, op_class); \
}; \
static ::paddle::framework::OperatorRegistrar<_OpClass_##op_type##_, \
##__VA_ARGS__> \
__op_registrar_##op_type##__(#op_type); \
int TouchOpRegistrar_##op_type() { \
__op_registrar_##op_type##__.Touch(); \
return 0; \
#define REGISTER_OPERATOR(op_type, op_class, ...) \
STATIC_ASSERT_GLOBAL_NAMESPACE( \
__reg_op__##op_type, \
"REGISTER_OPERATOR must be called in global namespace"); \
static ::paddle::framework::OperatorRegistrar<op_class, ##__VA_ARGS__> \
__op_registrar_##op_type##__(#op_type); \
int TouchOpRegistrar_##op_type() { \
__op_registrar_##op_type##__.Touch(); \
return 0; \
}
#define REGISTER_OP_WITHOUT_GRADIENT(op_type, op_class, op_maker_class) \
...
...
paddle/fluid/framework/op_registry_test.cc
浏览文件 @
1377b332
...
...
@@ -193,15 +193,10 @@ TEST(OpRegistry, CustomChecker) {
ASSERT_EQ
(
test_attr
,
4
);
}
class
CosineOpComplete
:
public
paddle
::
framework
::
CosineOp
{
public:
DEFINE_OP_CONSTRUCTOR
(
CosineOpComplete
,
paddle
::
framework
::
CosineOp
);
DEFINE_OP_CLONE_METHOD
(
CosineOpComplete
);
};
TEST
(
OperatorRegistrar
,
Test
)
{
paddle
::
framework
::
OperatorRegistrar
<
CosineOpComplete
,
paddle
::
framework
::
CosineOpProtoAndCheckerMaker
>
paddle
::
framework
::
CosineOp
,
paddle
::
framework
::
CosineOpProtoAndCheckerMaker
>
reg
(
"cos"
);
}
...
...
paddle/fluid/framework/operator.h
浏览文件 @
1377b332
...
...
@@ -121,10 +121,6 @@ class OperatorBase {
//! Get all outputs variable names
virtual
std
::
vector
<
std
::
string
>
OutputVars
(
bool
has_intermediate
)
const
;
// Return a new operator instance, which is as same as this.
// Use unique_ptr to prevent caller forget to delete this pointer.
virtual
std
::
unique_ptr
<
OperatorBase
>
Clone
()
const
=
0
;
protected:
std
::
string
type_
;
// NOTE: in case of OpGrad, inputs_ contains:
...
...
@@ -145,37 +141,6 @@ class OperatorBase {
const
platform
::
Place
&
place
)
const
=
0
;
};
// Macro for define a clone method.
// If you are writing an kernel operator, `Clone` will be defined when you
// register it. i.e. `Clone` method is not needed to define by yourself.
#define DEFINE_OP_CLONE_METHOD(cls) \
std::unique_ptr<::paddle::framework::OperatorBase> Clone() const final { \
return std::unique_ptr<::paddle::framework::OperatorBase>(new cls(*this)); \
}
// Macro for define a default constructor for Operator.
// You can also use
// using PARENT_CLASS::PARENT_CLASS;
// to use parent's constructor.
#define DEFINE_OP_CONSTRUCTOR(cls, parent_cls) \
cls(const std::string& type, \
const ::paddle::framework::VariableNameMap& inputs, \
const ::paddle::framework::VariableNameMap& outputs, \
const paddle::framework::AttributeMap& attrs) \
: parent_cls(type, inputs, outputs, attrs) {}
class
NOP
:
public
OperatorBase
{
public:
using
OperatorBase
::
OperatorBase
;
std
::
unique_ptr
<
OperatorBase
>
Clone
()
const
override
{
return
std
::
unique_ptr
<
OperatorBase
>
(
new
NOP
(
*
this
));
}
private:
void
RunImpl
(
const
Scope
&
scope
,
const
platform
::
Place
&
place
)
const
override
{}
};
class
ExecutionContext
{
public:
ExecutionContext
(
const
OperatorBase
&
op
,
const
Scope
&
scope
,
...
...
paddle/fluid/framework/operator_test.cc
浏览文件 @
1377b332
...
...
@@ -247,26 +247,3 @@ TEST(OpKernel, multi_inputs) {
auto
op
=
paddle
::
framework
::
OpRegistry
::
CreateOp
(
op_desc
);
op
->
Run
(
scope
,
cpu_place
);
}
class
OperatorClone
:
public
paddle
::
framework
::
OperatorBase
{
public:
DEFINE_OP_CLONE_METHOD
(
OperatorClone
);
OperatorClone
(
const
std
::
string
&
type
,
const
paddle
::
framework
::
VariableNameMap
&
inputs
,
const
paddle
::
framework
::
VariableNameMap
&
outputs
,
const
paddle
::
framework
::
AttributeMap
&
attrs
)
:
OperatorBase
(
type
,
inputs
,
outputs
,
attrs
)
{}
private:
void
RunImpl
(
const
paddle
::
framework
::
Scope
&
scope
,
const
paddle
::
platform
::
Place
&
place
)
const
override
{}
};
TEST
(
Operator
,
Clone
)
{
paddle
::
framework
::
InitDevices
(
true
);
OperatorClone
a
(
"ABC"
,
paddle
::
framework
::
VariableNameMap
{},
paddle
::
framework
::
VariableNameMap
{},
paddle
::
framework
::
AttributeMap
{});
auto
b
=
a
.
Clone
();
ASSERT_EQ
(
a
.
Type
(),
b
->
Type
());
}
paddle/fluid/framework/var_type_inference_test.cc
浏览文件 @
1377b332
...
...
@@ -22,6 +22,17 @@ limitations under the License. */
namespace
paddle
{
namespace
framework
{
class
NOP
:
public
OperatorBase
{
public:
NOP
(
const
std
::
string
&
type
,
const
VariableNameMap
&
inputs
,
const
VariableNameMap
&
outputs
,
const
AttributeMap
&
attrs
)
:
OperatorBase
(
type
,
inputs
,
outputs
,
attrs
)
{}
private:
void
RunImpl
(
const
Scope
&
scope
,
const
platform
::
Place
&
place
)
const
override
{}
};
class
SumOpMaker
:
public
OpProtoAndCheckerMaker
{
public:
void
Make
()
{
...
...
paddle/fluid/operators/read_op.cc
浏览文件 @
1377b332
...
...
@@ -92,9 +92,13 @@ class ReadOpMaker : public framework::OpProtoAndCheckerMaker {
void
Make
()
override
{
AddInput
(
"Reader"
,
"(ReaderHolder) The executed reader."
);
AddOutput
(
"Out"
,
"(LoDTensor) The output data."
).
AsDuplicable
();
AddAttr
<
bool
>
(
"throw_eof_exp"
,
"If set true, an exception will be thrown when the Reader "
"yields empty (which means there is no next data)."
)
AddAttr
<
bool
>
(
"throw_eof_exp"
,
"If set true, an exception will be thrown when the Reader "
"yields empty (which means there is no next data).
\n
"
"NOTES: This flag must be true always. It will be set to false"
" only when the data-balance is enabled in ParallelExecutor"
" and it is set by ParallelExecutor instance, not users."
)
.
SetDefault
(
true
);
AddComment
(
R"DOC(
Read Operator
...
...
paddle/legacy/gserver/gradientmachines/MultiGradientMachine.cpp
浏览文件 @
1377b332
...
...
@@ -532,6 +532,7 @@ void TrainerThread::computeThread() {
break
;
}
}
hl_fini
();
}
void
TrainerThread
::
prefetch
()
{
...
...
@@ -651,6 +652,7 @@ void TrainerThread::copyGradToBufferThread() {
}
partnerThread
->
notifyGradientCollect
(
pid
);
}
hl_fini
();
}
void
TrainerThread
::
gradCollectThread
()
{
...
...
@@ -693,6 +695,7 @@ void TrainerThread::gradCollectThread() {
notifyCopyGradToBuffer
(
pid
);
}
}
hl_fini
();
}
void
TrainerThread
::
doCallback
(
int
pid
)
{
...
...
@@ -741,6 +744,7 @@ void TrainerThread::valueDispatchThread() {
thread
->
notifyValueReady
(
pid
);
}
hl_fini
();
}
void
TrainerThread
::
notifyValueReady
(
int
paramId
)
{
...
...
paddle/legacy/gserver/gradientmachines/ParallelNeuralNetwork.cpp
浏览文件 @
1377b332
...
...
@@ -197,6 +197,7 @@ void ParallelThread::computeThread() {
job_work
.
layer_
->
markAllInputGrad
();
}
}
hl_fini
();
}
void
ParallelThread
::
start
()
{
...
...
python/paddle/fluid/tests/unittests/test_data_balance.py
浏览文件 @
1377b332
...
...
@@ -103,8 +103,12 @@ class TestDataBalance(unittest.TestCase):
exe
=
fluid
.
Executor
(
place
)
exe
.
run
(
startup_prog
)
build_strategy
=
fluid
.
BuildStrategy
()
build_strategy
.
enable_data_balance
=
True
parallel_exe
=
fluid
.
ParallelExecutor
(
use_cuda
=
self
.
use_cuda
,
main_program
=
main_prog
)
use_cuda
=
self
.
use_cuda
,
main_program
=
main_prog
,
build_strategy
=
build_strategy
)
if
(
parallel_exe
.
device_count
>
self
.
batch_size
):
print
(
"WARNING: Unittest TestDataBalance skipped.
\
...
...
@@ -145,9 +149,12 @@ class TestDataBalance(unittest.TestCase):
place
=
fluid
.
CUDAPlace
(
0
)
if
self
.
use_cuda
else
fluid
.
CPUPlace
()
exe
=
fluid
.
Executor
(
place
)
exe
.
run
(
startup_prog
)
build_strategy
=
fluid
.
BuildStrategy
()
build_strategy
.
enable_data_balance
=
True
parallel_exe
=
fluid
.
ParallelExecutor
(
use_cuda
=
self
.
use_cuda
,
main_program
=
main_prog
)
use_cuda
=
self
.
use_cuda
,
main_program
=
main_prog
,
build_strategy
=
build_strategy
)
if
(
parallel_exe
.
device_count
>
self
.
batch_size
):
print
(
"WARNING: Unittest TestDataBalance skipped.
\
...
...
python/paddle/fluid/transpiler/inference_transpiler.py
浏览文件 @
1377b332
...
...
@@ -19,7 +19,7 @@ from ..framework import Program
from
..executor
import
global_scope
class
InferenceTranspiler
:
class
InferenceTranspiler
(
object
)
:
'''
Convert the fluid program to optimized inference program.
...
...
python/paddle/libs/__init__.py
0 → 100644
浏览文件 @
1377b332
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# used for setup.py.in to store the thirdparty shared libraries
python/setup.py.in
浏览文件 @
1377b332
from setuptools import setup, Distribution, Extension
import subprocess
import shutil
import os
class BinaryDistribution(Distribution):
def has_ext_modules(foo):
return True
...
...
@@ -62,6 +64,7 @@ write_version_py(filename='@PADDLE_BINARY_DIR@/python/paddle/version.py')
packages=['paddle',
'paddle.libs',
'paddle.utils',
'paddle.dataset',
'paddle.reader',
...
...
@@ -113,12 +116,35 @@ package_dir={
}
if '${WITH_FLUID_ONLY}'== 'OFF':
package_dir['py_paddle']='${PADDLE_BINARY_DIR}/python/py_paddle'
paddle_rt_lib_dir = 'lib'
paddle_rt_libs = ['${WARPCTC_LIBRARIES}']
if '${MKL_SHARED_LIBS}'!= '':
paddle_rt_libs += '${MKL_SHARED_LIBS}'.split(';')
# put all thirdparty libraries in paddle.libs
package_data['paddle.libs']=['libwarpctc.so']
libs_path='${PADDLE_BINARY_DIR}/python/paddle/libs'
shutil.copy('${WARPCTC_LIBRARIES}', libs_path)
if '${WITH_MKL}' == 'ON':
shutil.copy('${MKLML_LIB}', libs_path)
shutil.copy('${MKLML_IOMP_LIB}', libs_path)
package_data['paddle.libs']+=['libmklml_intel.so','libiomp5.so']
if '${WITH_MKLDNN}' == 'ON':
# change rpath of libmkldnn.so.0, add $ORIGIN/ to it.
# The reason is that all thirdparty libraries in the same directory,
# thus, libmkldnn.so.0 will find libmklml_intel.so and libiomp5.so.
command = "patchelf --set-rpath '$ORIGIN/' ${MKLDNN_SHARED_LIB}"
if os.system(command) != 0:
raise Exception("patchelf --set-rpath for libmkldnn.so.0 fails")
package_data['paddle.libs']+=['libmkldnn.so.0']
shutil.copy('${MKLDNN_SHARED_LIB}', libs_path)
# remove unused paddle/libs/__init__.py
os.remove(libs_path+'/__init__.py')
package_dir['paddle.libs']=libs_path
# change rpath of core.so, add $ORIGIN/../libs/ to it.
# The reason is that libwarpctc.so, libiomp5.so etc are in paddle.libs, and
# core.so is in paddle.fluid, thus paddle/fluid/../libs will pointer to above libraries.
# This operation will fix https://github.com/PaddlePaddle/Paddle/issues/3213
command = "patchelf --set-rpath '$ORIGIN/../libs/' ${PADDLE_BINARY_DIR}/python/paddle/fluid/core.so"
if os.system(command) != 0:
raise Exception("patchelf --set-rpath for core.so fails")
setup(name='${PACKAGE_NAME}',
version='${PADDLE_VERSION}',
...
...
@@ -128,6 +154,5 @@ setup(name='${PACKAGE_NAME}',
ext_modules=[Extension('_foo', ['stub.cc'])],
package_data=package_data,
package_dir=package_dir,
scripts=paddle_bins,
data_files=[(paddle_rt_lib_dir, paddle_rt_libs)]
scripts=paddle_bins
)
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录