Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
兽拳
Paddle
提交
78fb3a62
P
Paddle
项目概览
兽拳
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
78fb3a62
编写于
3月 26, 2019
作者:
S
sneaxiy
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix env variable settting bug
test=develop
上级
a7d0ac50
变更
10
隐藏空白更改
内联
并排
Showing
10 changed file
with
49 addition
and
28 deletion
+49
-28
paddle/fluid/framework/CMakeLists.txt
paddle/fluid/framework/CMakeLists.txt
+1
-1
paddle/fluid/framework/details/eager_deletion_pass.cc
paddle/fluid/framework/details/eager_deletion_pass.cc
+5
-10
paddle/fluid/framework/executor_gc_helper.cc
paddle/fluid/framework/executor_gc_helper.cc
+0
-1
paddle/fluid/framework/garbage_collector.cc
paddle/fluid/framework/garbage_collector.cc
+18
-2
paddle/fluid/framework/garbage_collector.h
paddle/fluid/framework/garbage_collector.h
+4
-1
paddle/fluid/operators/batch_norm_op.cc
paddle/fluid/operators/batch_norm_op.cc
+1
-0
paddle/fluid/operators/concat_op.cc
paddle/fluid/operators/concat_op.cc
+17
-2
paddle/fluid/pybind/pybind.cc
paddle/fluid/pybind/pybind.cc
+1
-11
python/paddle/fluid/tests/unittests/test_eager_deletion_mnist.py
...paddle/fluid/tests/unittests/test_eager_deletion_mnist.py
+1
-0
python/paddle/fluid/tests/unittests/test_eager_deletion_transformer.py
.../fluid/tests/unittests/test_eager_deletion_transformer.py
+1
-0
未找到文件。
paddle/fluid/framework/CMakeLists.txt
浏览文件 @
78fb3a62
...
...
@@ -63,7 +63,7 @@ cc_library(lod_tensor SRCS lod_tensor.cc DEPS ddim place tensor framework_proto
cc_test
(
lod_tensor_test SRCS lod_tensor_test.cc DEPS lod_tensor memory
)
nv_test
(
lod_tensor_gpu_test SRCS lod_tensor_test.cu DEPS lod_tensor
)
cc_library
(
garbage_collector SRCS garbage_collector.cc DEPS device_context memory gflags
)
cc_library
(
garbage_collector SRCS garbage_collector.cc DEPS device_context memory gflags
glog
)
cc_library
(
reader SRCS reader.cc DEPS lod_tensor ddim
)
cc_test
(
reader_test SRCS reader_test.cc DEPS reader
)
...
...
paddle/fluid/framework/details/eager_deletion_pass.cc
浏览文件 @
78fb3a62
...
...
@@ -22,14 +22,9 @@
#include "paddle/fluid/framework/details/computation_op_handle.h"
#include "paddle/fluid/framework/details/eager_deletion_op_handle.h"
#include "paddle/fluid/framework/details/multi_devices_helper.h"
#include "paddle/fluid/framework/garbage_collector.h"
#include "paddle/fluid/framework/ir/graph_helper.h"
DEFINE_double
(
memory_fraction_of_eager_deletion
,
1.0
,
"Fraction of eager deletion. If less than 1.0, all variables in "
"the program would be sorted according to its memory size, and "
"only the FLAGS_memory_fraction_of_eager_deletion of the largest "
"variables would be deleted."
);
namespace
paddle
{
namespace
framework
{
namespace
details
{
...
...
@@ -206,8 +201,9 @@ std::unique_ptr<ir::Graph> EagerDeletionPass::ApplyImpl(
}
}
op_vars_map
=
ShrinkGCVars
(
op_vars_map
,
vars
,
places
,
FLAGS_memory_fraction_of_eager_deletion
);
double
memory_fraction
=
framework
::
GetEagerDeletionMemoryFraction
();
op_vars_map
=
ShrinkGCVars
(
op_vars_map
,
vars
,
places
,
memory_fraction
);
for
(
auto
&
pair
:
op_vars_map
)
{
auto
*
op
=
pair
.
first
;
...
...
@@ -239,8 +235,7 @@ std::unique_ptr<ir::Graph> EagerDeletionPass::ApplyImpl(
eager_deletion_op
->
AddOutput
(
dummy_leaf
);
}
VLOG
(
10
)
<<
"FLAGS_memory_fraction_of_eager_deletion = "
<<
FLAGS_memory_fraction_of_eager_deletion
;
VLOG
(
10
)
<<
"FLAGS_memory_fraction_of_eager_deletion = "
<<
memory_fraction
;
VLOG
(
10
)
<<
"Create "
<<
op_vars_map
.
size
()
<<
" EagerDeletionOpHandle(s)"
;
auto
while_op_eager_deletion_pass
=
...
...
paddle/fluid/framework/executor_gc_helper.cc
浏览文件 @
78fb3a62
...
...
@@ -93,7 +93,6 @@ std::unordered_map<OperatorBase *, std::vector<std::string>> GetUnusedVars(
const
BlockDesc
&
block
,
const
std
::
vector
<
std
::
unique_ptr
<
OperatorBase
>>
&
ops
,
const
std
::
vector
<
std
::
string
>
&
skip_var_list
)
{
UseGarbageCollectorGFlags
();
std
::
unordered_set
<
std
::
string
>
skip_vars
(
skip_var_list
.
begin
(),
skip_var_list
.
end
());
...
...
paddle/fluid/framework/garbage_collector.cc
浏览文件 @
78fb3a62
...
...
@@ -22,6 +22,7 @@
#include "paddle/fluid/platform/cuda_device_guard.h"
#endif
#include "gflags/gflags.h"
#include "glog/logging.h"
#include "paddle/fluid/framework/garbage_collector.h"
namespace
paddle
{
...
...
@@ -36,6 +37,12 @@ DEFINE_bool(fast_eager_deletion_mode, true,
"Fast eager deletion mode. If enabled, memory would release "
"immediately without waiting GPU kernel ends."
);
DEFINE_double
(
memory_fraction_of_eager_deletion
,
1.0
,
"Fraction of eager deletion. If less than 1.0, all variables in "
"the program would be sorted according to its memory size, and "
"only the FLAGS_memory_fraction_of_eager_deletion of the largest "
"variables would be deleted."
);
GarbageCollector
::
GarbageCollector
(
const
platform
::
Place
&
place
,
size_t
max_memory_size
)
:
max_memory_size_
((
std
::
max
)(
max_memory_size
,
static_cast
<
size_t
>
(
1
)))
{
...
...
@@ -101,8 +108,6 @@ void StreamGarbageCollector::ClearCallback(
}
#endif
void
UseGarbageCollectorGFlags
()
{}
int64_t
GetEagerDeletionThreshold
()
{
return
FLAGS_eager_delete_tensor_gb
<
0
?
-
1
...
...
@@ -111,5 +116,16 @@ int64_t GetEagerDeletionThreshold() {
}
bool
IsFastEagerDeletionModeEnabled
()
{
return
FLAGS_fast_eager_deletion_mode
;
}
void
SetEagerDeletionMode
(
double
threshold
,
double
fraction
,
bool
fast_mode
)
{
FLAGS_eager_delete_tensor_gb
=
threshold
;
FLAGS_memory_fraction_of_eager_deletion
=
fraction
;
FLAGS_fast_eager_deletion_mode
=
fast_mode
;
}
double
GetEagerDeletionMemoryFraction
()
{
return
FLAGS_memory_fraction_of_eager_deletion
;
}
}
// namespace framework
}
// namespace paddle
paddle/fluid/framework/garbage_collector.h
浏览文件 @
78fb3a62
...
...
@@ -19,6 +19,7 @@
#include <memory>
#include <mutex> // NOLINT
#include <utility>
#include "gflags/gflags.h"
#include "paddle/fluid/platform/device_context.h"
namespace
paddle
{
...
...
@@ -130,7 +131,9 @@ void GarbageCollector::Add(Container &&objs, Callback &&callback) {
int64_t
GetEagerDeletionThreshold
();
bool
IsFastEagerDeletionModeEnabled
();
extern
void
UseGarbageCollectorGFlags
();
void
SetEagerDeletionMode
(
double
threshold
,
double
fraction
,
bool
fast_mode
);
double
GetEagerDeletionMemoryFraction
();
}
// namespace framework
}
// namespace paddle
paddle/fluid/operators/batch_norm_op.cc
浏览文件 @
78fb3a62
...
...
@@ -567,6 +567,7 @@ std::unique_ptr<framework::OpDesc> BatchNormGradMaker::Apply() const {
op
->
SetInput
(
framework
::
GradVarName
(
"Y"
),
OutputGrad
(
"Y"
));
op
->
SetInput
(
"Scale"
,
Input
(
"Scale"
));
op
->
SetInput
(
"Bias"
,
Input
(
"Bias"
));
op
->
SetInput
(
"SavedMean"
,
Output
(
"SavedMean"
));
op
->
SetInput
(
"SavedVariance"
,
Output
(
"SavedVariance"
));
...
...
paddle/fluid/operators/concat_op.cc
浏览文件 @
78fb3a62
...
...
@@ -152,13 +152,28 @@ class ConcatOpGrad : public framework::OperatorWithKernel {
DECLARE_NO_NEED_BUFFER_VARS_INFERENCE
(
ConcatOpGradNoNeedBufferVarInference
,
"X"
);
class
ConcatGradOpDescMaker
:
public
framework
::
SingleGradOpDescMaker
{
public:
using
framework
::
SingleGradOpDescMaker
::
SingleGradOpDescMaker
;
protected:
std
::
unique_ptr
<
framework
::
OpDesc
>
Apply
()
const
override
{
std
::
unique_ptr
<
framework
::
OpDesc
>
op
(
new
framework
::
OpDesc
());
op
->
SetType
(
"concat_grad"
);
op
->
SetInput
(
"X"
,
Input
(
"X"
));
op
->
SetInput
(
framework
::
GradVarName
(
"Out"
),
OutputGrad
(
"Out"
));
op
->
SetOutput
(
framework
::
GradVarName
(
"X"
),
InputGrad
(
"X"
,
false
));
op
->
SetAttrMap
(
Attrs
());
return
op
;
}
};
}
// namespace operators
}
// namespace paddle
namespace
ops
=
paddle
::
operators
;
REGISTER_OPERATOR
(
concat
,
ops
::
ConcatOp
,
ops
::
ConcatOpMaker
,
paddle
::
framework
::
DefaultGradOpDescMaker
<
false
>
/* set false to disable empty grad */
);
ops
::
ConcatGradOpDescMaker
);
REGISTER_OPERATOR
(
concat_grad
,
ops
::
ConcatOpGrad
,
ops
::
ConcatOpGradNoNeedBufferVarInference
);
REGISTER_OP_CPU_KERNEL
(
...
...
paddle/fluid/pybind/pybind.cc
浏览文件 @
78fb3a62
...
...
@@ -76,10 +76,6 @@ DEFINE_bool(reader_queue_speed_test_mode, false,
"If set true, the queue.pop will only get data from queue but not "
"remove the data from queue for speed testing"
);
DECLARE_double
(
eager_delete_tensor_gb
);
DECLARE_bool
(
fast_eager_deletion_mode
);
DECLARE_double
(
memory_fraction_of_eager_deletion
);
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE
(
paddle
::
framework
::
LoDTensorArray
);
...
...
@@ -144,7 +140,6 @@ PYBIND11_MODULE(core, m) {
paddle
::
platform
::
CpuTotalPhysicalMemory
();
paddle
::
memory
::
allocation
::
UseAllocatorStrategyGFlag
();
paddle
::
framework
::
UseGarbageCollectorGFlags
();
m
.
doc
()
=
"C++ core of PaddlePaddle"
;
...
...
@@ -163,12 +158,7 @@ PYBIND11_MODULE(core, m) {
// NOTE(zjl): ctest would load environment variables at the beginning even
// though we have not `import paddle.fluid as fluid`. So we add this API
// to enable eager deletion mode in unittest.
m
.
def
(
"_set_eager_deletion_mode"
,
[](
double
threshold
,
double
fraction
,
bool
fast_mode
)
{
FLAGS_eager_delete_tensor_gb
=
threshold
;
FLAGS_memory_fraction_of_eager_deletion
=
fraction
;
FLAGS_fast_eager_deletion_mode
=
fast_mode
;
});
m
.
def
(
"_set_eager_deletion_mode"
,
&
paddle
::
framework
::
SetEagerDeletionMode
);
m
.
add_object
(
"_cleanup"
,
py
::
capsule
([]()
{
ScopePool
::
Instance
().
Clear
();
}));
...
...
python/paddle/fluid/tests/unittests/test_eager_deletion_mnist.py
浏览文件 @
78fb3a62
...
...
@@ -14,6 +14,7 @@
import
os
import
unittest
import
paddle.fluid
as
fluid
fluid
.
core
.
_set_eager_deletion_mode
(
0.0
,
1.0
,
True
)
...
...
python/paddle/fluid/tests/unittests/test_eager_deletion_transformer.py
浏览文件 @
78fb3a62
...
...
@@ -14,6 +14,7 @@
import
os
import
unittest
import
paddle.fluid
as
fluid
fluid
.
core
.
_set_eager_deletion_mode
(
0.0
,
1.0
,
True
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录