Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
wzpei9
Paddle
提交
d655417f
P
Paddle
项目概览
wzpei9
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
d655417f
编写于
4月 18, 2018
作者:
W
Wu Yi
提交者:
GitHub
4月 18, 2018
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #9956 from typhoonzero/split_byref_op
Split byref op
上级
f7fbef17
ff0d9341
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
193 addition
and
21 deletion
+193
-21
paddle/fluid/operators/detail/sendrecvop_utils.cc
paddle/fluid/operators/detail/sendrecvop_utils.cc
+5
-4
paddle/fluid/operators/split_byref_op.cc
paddle/fluid/operators/split_byref_op.cc
+101
-0
paddle/fluid/operators/split_byref_op.cu.cc
paddle/fluid/operators/split_byref_op.cu.cc
+19
-0
paddle/fluid/operators/split_byref_op.h
paddle/fluid/operators/split_byref_op.h
+43
-0
paddle/fluid/operators/split_op.cc
paddle/fluid/operators/split_op.cc
+0
-15
paddle/fluid/operators/split_op.h
paddle/fluid/operators/split_op.h
+15
-0
python/paddle/fluid/distribute_transpiler.py
python/paddle/fluid/distribute_transpiler.py
+1
-1
python/paddle/fluid/tests/unittests/test_split_op.py
python/paddle/fluid/tests/unittests/test_split_op.py
+9
-1
未找到文件。
paddle/fluid/operators/detail/sendrecvop_utils.cc
浏览文件 @
d655417f
...
...
@@ -82,7 +82,7 @@ void SerializeToByteBuffer(const std::string& name, framework::Variable* var,
platform
::
CPUPlace
cpu
;
auto
&
gpu_dev_ctx
=
static_cast
<
const
platform
::
CUDADeviceContext
&>
(
ctx
);
auto
copy_size
=
tensor
.
memory_size
(
);
auto
copy_size
=
tensor
.
numel
()
*
framework
::
SizeOfType
(
tensor
.
type
()
);
payload
=
memory
::
Alloc
(
cpu
,
copy_size
);
memory
::
Copy
(
cpu
,
payload
,
...
...
@@ -99,7 +99,7 @@ void SerializeToByteBuffer(const std::string& name, framework::Variable* var,
}
else
{
payload
=
tensor
.
data
<
void
>
();
}
payload_size
=
tensor
.
memory_size
(
);
payload_size
=
tensor
.
numel
()
*
framework
::
SizeOfType
(
tensor
.
type
()
);
e
.
WriteVarlengthBeginning
(
VarMsg
::
kSerializedFieldNumber
,
payload_size
);
}
break
;
case
framework
::
proto
::
VarType_Type_SELECTED_ROWS
:
{
...
...
@@ -118,7 +118,8 @@ void SerializeToByteBuffer(const std::string& name, framework::Variable* var,
platform
::
CPUPlace
cpu
;
auto
&
gpu_dev_ctx
=
static_cast
<
const
platform
::
CUDADeviceContext
&>
(
ctx
);
auto
copy_size
=
tensor
->
memory_size
();
auto
copy_size
=
tensor
->
numel
()
*
framework
::
SizeOfType
(
tensor
->
type
());
payload
=
memory
::
Alloc
(
cpu
,
copy_size
);
memory
::
Copy
(
cpu
,
payload
,
boost
::
get
<
platform
::
CUDAPlace
>
(
tensor
->
place
()),
...
...
@@ -133,7 +134,7 @@ void SerializeToByteBuffer(const std::string& name, framework::Variable* var,
}
else
{
payload
=
slr
->
mutable_value
()
->
data
<
void
>
();
}
payload_size
=
tensor
->
memory_size
(
);
payload_size
=
tensor
->
numel
()
*
framework
::
SizeOfType
(
tensor
->
type
()
);
e
.
WriteVarlengthBeginning
(
VarMsg
::
kSerializedFieldNumber
,
payload_size
);
}
break
;
default:
...
...
paddle/fluid/operators/split_byref_op.cc
0 → 100644
浏览文件 @
d655417f
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/fluid/operators/split_byref_op.h"
#include "paddle/fluid/operators/split_op.h"
namespace
paddle
{
namespace
operators
{
using
framework
::
Tensor
;
class
SplitByrefOp
:
public
framework
::
OperatorWithKernel
{
public:
using
framework
::
OperatorWithKernel
::
OperatorWithKernel
;
void
InferShape
(
framework
::
InferShapeContext
*
ctx
)
const
override
{
PADDLE_ENFORCE
(
ctx
->
HasInput
(
"X"
),
"Input(X) of SplitOp should not be null."
);
PADDLE_ENFORCE_GE
(
ctx
->
Outputs
(
"Out"
).
size
(),
1UL
,
"Outputs(Out) of SplitOp should not be empty."
);
auto
in_dims
=
ctx
->
GetInputDim
(
"X"
);
auto
outs_names
=
ctx
->
Outputs
(
"Out"
);
size_t
num
=
static_cast
<
size_t
>
(
ctx
->
Attrs
().
Get
<
int
>
(
"num"
));
std
::
vector
<
int
>
sections
=
static_cast
<
std
::
vector
<
int
>>
(
ctx
->
Attrs
().
Get
<
std
::
vector
<
int
>>
(
"sections"
));
const
size_t
outs_number
=
outs_names
.
size
();
std
::
vector
<
framework
::
DDim
>
outs_dims
;
outs_dims
.
reserve
(
outs_number
);
if
(
num
>
0
)
{
int64_t
in_axis_dim
=
in_dims
[
0
];
PADDLE_ENFORCE_EQ
(
in_axis_dim
%
num
,
0
,
"tensor split does not result"
" in an equal division"
);
size_t
out_axis_dim
=
in_axis_dim
/
num
;
for
(
size_t
i
=
0
;
i
<
outs_number
;
++
i
)
{
auto
dim
=
in_dims
;
dim
[
0
]
=
out_axis_dim
;
outs_dims
.
push_back
(
dim
);
}
}
else
if
(
sections
.
size
()
>
0
)
{
PADDLE_ENFORCE_EQ
(
sections
.
size
(),
outs_number
,
"tensor split sections size"
"should be equal to output size."
);
for
(
size_t
i
=
0
;
i
<
outs_number
;
++
i
)
{
auto
dim
=
in_dims
;
dim
[
0
]
=
sections
[
i
];
outs_dims
.
push_back
(
dim
);
}
}
ctx
->
SetOutputsDim
(
"Out"
,
outs_dims
);
}
};
class
SplitByrefOpMaker
:
public
framework
::
OpProtoAndCheckerMaker
{
public:
SplitByrefOpMaker
(
OpProto
*
proto
,
OpAttrChecker
*
op_checker
)
:
OpProtoAndCheckerMaker
(
proto
,
op_checker
)
{
AddInput
(
"X"
,
"(Tensor) Input tensor of the split operator."
);
AddOutput
(
"Out"
,
"(Tensor) Output tensors of the split operator."
)
.
AsDuplicable
();
AddComment
(
R"DOC(
SplitByref operator
Split source tensor to sevaral tensors by axis 0. No copy in this operator
is performed, output tensor shares the same blocks of memory.
)DOC"
);
AddAttr
<
std
::
vector
<
int
>>
(
"sections"
,
"(vector<int>) "
"the length of each output along the "
"specified axis."
)
.
SetDefault
(
std
::
vector
<
int
>
{});
AddAttr
<
int
>
(
"num"
,
"(int, default 0)"
"Number of sub-tensors. This must evenly divide "
"Input.dims()[axis]"
)
.
SetDefault
(
0
);
}
};
}
// namespace operators
}
// namespace paddle
namespace
ops
=
paddle
::
operators
;
// NOTE: concat op default axis must be 0!
USE_CPU_ONLY_OP
(
concat
);
REGISTER_OPERATOR
(
split_byref
,
ops
::
SplitByrefOp
,
ops
::
SplitByrefOpMaker
,
ops
::
SplitGradMaker
);
REGISTER_OP_CPU_KERNEL
(
split_byref
,
ops
::
SplitByrefOpKernel
<
paddle
::
platform
::
CPUPlace
,
float
>
);
paddle/fluid/operators/split_byref_op.cu.cc
0 → 100644
浏览文件 @
d655417f
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/fluid/operators/split_byref_op.h"
namespace
ops
=
paddle
::
operators
;
REGISTER_OP_CUDA_KERNEL
(
split_byref
,
ops
::
SplitByrefOpKernel
<
paddle
::
platform
::
CUDADeviceContext
,
float
>
);
paddle/fluid/operators/split_byref_op.h
0 → 100644
浏览文件 @
d655417f
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <vector>
#include "paddle/fluid/framework/op_registry.h"
namespace
paddle
{
namespace
operators
{
template
<
typename
DeviceContext
,
typename
T
>
class
SplitByrefOpKernel
:
public
framework
::
OpKernel
<
T
>
{
public:
void
Compute
(
const
framework
::
ExecutionContext
&
ctx
)
const
override
{
auto
*
in
=
ctx
.
Input
<
framework
::
Tensor
>
(
"X"
);
auto
outs
=
ctx
.
MultiOutput
<
framework
::
Tensor
>
(
"Out"
);
auto
place
=
ctx
.
GetPlace
();
size_t
row_offset
=
0
;
for
(
size_t
i
=
0
;
i
<
outs
.
size
();
++
i
)
{
// NOTE: no need to call mutable_data here to allocate memory.
auto
*
out
=
outs
[
i
];
VLOG
(
3
)
<<
"spliting by ref: "
<<
row_offset
<<
" "
<<
out
->
dims
()[
0
];
*
out
=
std
::
move
(
in
->
Slice
(
row_offset
,
row_offset
+
out
->
dims
()[
0
]));
row_offset
+=
out
->
dims
()[
0
];
}
}
};
}
// namespace operators
}
// namespace paddle
paddle/fluid/operators/split_op.cc
浏览文件 @
d655417f
...
...
@@ -108,21 +108,6 @@ Example:
}
};
class
SplitGradMaker
:
public
framework
::
SingleGradOpDescMaker
{
public:
using
framework
::
SingleGradOpDescMaker
::
SingleGradOpDescMaker
;
protected:
std
::
unique_ptr
<
framework
::
OpDesc
>
Apply
()
const
override
{
auto
op
=
new
framework
::
OpDesc
();
op
->
SetType
(
"concat"
);
op
->
SetInput
(
"X"
,
OutputGrad
(
"Out"
));
op
->
SetOutput
(
"Out"
,
InputGrad
(
"X"
));
op
->
SetAttrMap
(
Attrs
());
return
std
::
unique_ptr
<
framework
::
OpDesc
>
(
op
);
}
};
}
// namespace operators
}
// namespace paddle
...
...
paddle/fluid/operators/split_op.h
浏览文件 @
d655417f
...
...
@@ -44,5 +44,20 @@ class SplitOpKernel : public framework::OpKernel<T> {
}
};
class
SplitGradMaker
:
public
framework
::
SingleGradOpDescMaker
{
public:
using
framework
::
SingleGradOpDescMaker
::
SingleGradOpDescMaker
;
protected:
std
::
unique_ptr
<
framework
::
OpDesc
>
Apply
()
const
override
{
auto
op
=
new
framework
::
OpDesc
();
op
->
SetType
(
"concat"
);
op
->
SetInput
(
"X"
,
OutputGrad
(
"Out"
));
op
->
SetOutput
(
"Out"
,
InputGrad
(
"X"
));
op
->
SetAttrMap
(
Attrs
());
return
std
::
unique_ptr
<
framework
::
OpDesc
>
(
op
);
}
};
}
// namespace operators
}
// namespace paddle
python/paddle/fluid/distribute_transpiler.py
浏览文件 @
d655417f
...
...
@@ -825,7 +825,7 @@ class DistributeTranspiler:
for
v
in
splited_vars
:
sections
.
append
(
v
.
shape
[
0
])
program
.
global_block
().
append_op
(
type
=
"split"
,
type
=
"split
_byref
"
,
inputs
=
{
"X"
:
orig_var
},
outputs
=
{
"Out"
:
splited_vars
},
attrs
=
{
"sections"
:
sections
}
# assume split evenly
...
...
python/paddle/fluid/tests/unittests/test_split_op.py
浏览文件 @
d655417f
...
...
@@ -19,7 +19,7 @@ from op_test import OpTest
class
TestSplitOp
(
OpTest
):
def
setUp
(
self
):
self
.
op_type
=
"split"
self
.
_set_op_type
()
axis
=
1
x
=
np
.
random
.
random
((
4
,
5
,
6
)).
astype
(
'float32'
)
out
=
np
.
split
(
x
,
[
2
,
3
],
axis
)
...
...
@@ -28,6 +28,9 @@ class TestSplitOp(OpTest):
self
.
outputs
=
{
'Out'
:
[(
'out%d'
%
i
,
out
[
i
])
\
for
i
in
xrange
(
len
(
out
))]}
def
_set_op_type
(
self
):
self
.
op_type
=
"split"
def
test_check_output
(
self
):
self
.
check_output
()
...
...
@@ -35,5 +38,10 @@ class TestSplitOp(OpTest):
self
.
check_grad
([
'X'
],
[
'out0'
,
'out1'
,
'out2'
])
class
TestSplitByrefOp
(
OpTest
):
def
_set_op_type
(
self
):
self
.
op_type
=
"split_byref"
if
__name__
==
'__main__'
:
unittest
.
main
()
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录