Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
Paddle-Lite
提交
91b8d2be
P
Paddle-Lite
项目概览
PaddlePaddle
/
Paddle-Lite
通知
338
Star
4
Fork
1
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
271
列表
看板
标记
里程碑
合并请求
78
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle-Lite
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
271
Issue
271
列表
看板
标记
里程碑
合并请求
78
合并请求
78
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
91b8d2be
编写于
1月 05, 2019
作者:
H
hjchen2
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Optimize int8 5x5 depthwise conv, add aarch64 macros to make compilation no problem
上级
b901235e
变更
8
显示空白变更内容
内联
并排
Showing
8 changed file
with
1134 addition
and
122 deletion
+1134
-122
src/operators/kernel/arm/conv_kernel.cpp
src/operators/kernel/arm/conv_kernel.cpp
+18
-5
src/operators/kernel/central-arm-func/conv_arm_func.h
src/operators/kernel/central-arm-func/conv_arm_func.h
+25
-4
src/operators/math/depthwise_conv5x5.cpp
src/operators/math/depthwise_conv5x5.cpp
+0
-1
src/operators/math/depthwise_conv5x5_int8.cpp
src/operators/math/depthwise_conv5x5_int8.cpp
+1041
-0
src/operators/math/gemm.cpp
src/operators/math/gemm.cpp
+2
-0
src/operators/math/pooling2x2.cpp
src/operators/math/pooling2x2.cpp
+2
-1
src/operators/op_param.h
src/operators/op_param.h
+2
-2
test/operators/test_conv_op.cpp
test/operators/test_conv_op.cpp
+44
-109
未找到文件。
src/operators/kernel/arm/conv_kernel.cpp
浏览文件 @
91b8d2be
...
...
@@ -31,12 +31,19 @@ bool ConvKernel<CPU, float>::Init(ConvParam<CPU> *param) {
bool
depth5x5
=
conv5x5
&&
param
->
Groups
()
==
param
->
Input
()
->
dims
()[
1
]
&&
param
->
Input
()
->
dims
()[
1
]
==
param
->
Output
()
->
dims
()[
1
];
if
(
param
->
Filter
()
->
type
()
==
typeid
(
int8_t
))
{
#ifndef __aarch64__
if
(
depth3x3
&&
param
->
Strides
()[
0
]
<
3
&&
param
->
Strides
()[
0
]
==
param
->
Strides
()[
1
])
{
param
->
ExecMode
()
=
ConvParam
<
CPU
>::
EXEC_DEPTHWISE3x3_INT8
;
}
else
if
(
depth5x5
&&
param
->
Strides
()[
0
]
<
2
&&
param
->
Strides
()[
0
]
==
param
->
Strides
()[
1
])
{
param
->
ExecMode
()
=
ConvParam
<
CPU
>::
EXEC_DEPTHWISE5x5_INT8
;
}
else
{
#endif // __aarch64__
param
->
ExecMode
()
=
ConvParam
<
CPU
>::
EXEC_GEMM_INT8
;
#ifndef __aarch64__
}
#endif // __aarch64__
}
else
{
if
(
depth3x3
&&
param
->
Strides
()[
0
]
==
param
->
Strides
()[
1
]
&&
param
->
Strides
()[
0
]
==
1
&&
param
->
Paddings
()[
0
]
==
1
&&
...
...
@@ -50,10 +57,10 @@ bool ConvKernel<CPU, float>::Init(ConvParam<CPU> *param) {
param
->
Strides
()[
0
]
==
2
&&
param
->
Paddings
()[
0
]
==
1
&&
param
->
Paddings
()[
0
]
==
param
->
Paddings
()[
1
])
{
param
->
ExecMode
()
=
ConvParam
<
CPU
>::
EXEC_DEPTHWISE3x3S2P1_FLOAT
;
#ifndef __aarch64__
}
else
if
(
depth5x5
&&
param
->
Strides
()[
0
]
==
param
->
Strides
()[
1
]
&&
param
->
Strides
()[
0
]
==
1
)
{
param
->
ExecMode
()
=
ConvParam
<
CPU
>::
EXEC_DEPTHWISE5x5S1_FLOAT
;
#ifndef __aarch64__
param
->
ExecMode
()
=
ConvParam
<
CPU
>::
EXEC_DEPTHWISE5x5_FLOAT
;
}
else
if
(
conv3x3
&&
param
->
Strides
()[
0
]
==
param
->
Strides
()[
1
]
&&
param
->
Dilations
()[
0
]
==
param
->
Dilations
()[
1
]
&&
param
->
Strides
()[
0
]
==
1
&&
param
->
Dilations
()[
0
]
==
1
&&
...
...
@@ -79,9 +86,14 @@ void ConvKernel<CPU, float>::Compute(const ConvParam<CPU> ¶m) {
case
ConvParam
<
CPU
>::
EXEC_GEMM_INT8
:
GemmConv
<
int8_t
,
int32_t
>
(
param
);
break
;
#ifndef __aarch64__
case
ConvParam
<
CPU
>::
EXEC_DEPTHWISE3x3_INT8
:
DepthwiseConv3x3
<
int8_t
,
int32_t
>
(
param
);
break
;
case
ConvParam
<
CPU
>::
EXEC_DEPTHWISE5x5_INT8
:
DepthwiseConv5x5
<
int8_t
,
int32_t
>
(
param
);
break
;
#endif // __aarch64__
case
ConvParam
<
CPU
>::
EXEC_DEPTHWISE3x3S1P1_FLOAT
:
math
::
DepthwiseConv3x3s1p1
(
param
.
Input
(),
param
.
Filter
(),
param
.
Output
(),
nullptr
,
false
);
...
...
@@ -94,13 +106,14 @@ void ConvKernel<CPU, float>::Compute(const ConvParam<CPU> ¶m) {
math
::
DepthwiseConv3x3s2p0
(
param
.
Input
(),
param
.
Filter
(),
param
.
Output
(),
nullptr
,
false
);
break
;
case
ConvParam
<
CPU
>::
EXEC_DEPTHWISE5x5S1_FLOAT
:
math
::
DepthwiseConv5x5S1
<
float
,
float
>
(
*
param
.
Input
(),
*
param
.
Filter
(),
param
.
Paddings
(),
param
.
Output
()
);
#ifndef __aarch64__
case
ConvParam
<
CPU
>::
EXEC_DEPTHWISE5x5_FLOAT
:
DepthwiseConv5x5
<
float
,
float
>
(
param
);
break
;
case
ConvParam
<
CPU
>::
EXEC_WINOGRAD3X3_FLOAT
:
WinogradConv3x3
<
8
,
3
>
(
param
);
break
;
#endif // __aarch64__
case
ConvParam
<
CPU
>::
EXEC_GEMM_FLOAT
:
GemmConv
<
float
,
float
>
(
param
);
break
;
...
...
src/operators/kernel/central-arm-func/conv_arm_func.h
浏览文件 @
91b8d2be
...
...
@@ -161,6 +161,7 @@ inline void WinogradConv3x3(const ConvParam<CPU> ¶m) {
}
}
#ifndef __aarch64__
template
<
typename
Itype
,
typename
Otype
>
inline
void
DepthwiseConv3x3
(
const
ConvParam
<
CPU
>
&
param
)
{
const
Tensor
*
input
=
param
.
Input
();
...
...
@@ -181,13 +182,33 @@ inline void DepthwiseConv3x3(const ConvParam<CPU> ¶m) {
math
::
DepthwiseConv3x3S2
<
Itype
,
Otype
>
(
in_batch
,
*
filter
,
paddings
,
&
out_batch
);
}
else
{
// math::DepthwiseConv3x3<Itype, Otype>(input_pad, *filter,
// &out_batch);
PADDLE_MOBILE_THROW_EXCEPTION
(
"Depthwise conv with generic strides has not been implemented."
);
GemmConv
<
Itype
,
Otype
>
(
param
);
}
}
}
#endif // __aarch64__
template
<
typename
Itype
,
typename
Otype
>
inline
void
DepthwiseConv5x5
(
const
ConvParam
<
CPU
>
&
param
)
{
const
Tensor
*
input
=
param
.
Input
();
const
Tensor
*
filter
=
param
.
Filter
();
const
std
::
vector
<
int
>
&
paddings
=
param
.
Paddings
();
const
std
::
vector
<
int
>
&
strides
=
param
.
Strides
();
const
int
batch_size
=
input
->
dims
()[
0
];
Tensor
*
output
=
param
.
Output
();
output
->
mutable_data
<
Otype
>
();
if
(
strides
[
0
]
==
1
)
{
for
(
int
i
=
0
;
i
<
batch_size
;
i
++
)
{
Tensor
in_batch
=
input
->
Slice
(
i
,
i
+
1
);
Tensor
out_batch
=
output
->
Slice
(
i
,
i
+
1
);
math
::
DepthwiseConv5x5S1
<
Itype
,
Otype
>
(
in_batch
,
*
filter
,
paddings
,
&
out_batch
);
}
}
else
{
GemmConv
<
Itype
,
Otype
>
(
param
);
}
}
}
// namespace operators
}
// namespace paddle_mobile
...
...
src/operators/math/depthwise_conv5x5.cpp
浏览文件 @
91b8d2be
...
...
@@ -16,7 +16,6 @@ limitations under the License. */
#include "operators/math/depthwise_conv5x5.h"
#include <arm_neon.h>
#include <iostream>
namespace
paddle_mobile
{
namespace
operators
{
...
...
src/operators/math/depthwise_conv5x5_int8.cpp
0 → 100644
浏览文件 @
91b8d2be
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#if defined(__ARM_NEON__) && !defined(__aarch64__)
#include <arm_neon.h>
#include "operators/math/depthwise_conv5x5.h"
namespace
paddle_mobile
{
namespace
operators
{
namespace
math
{
#ifndef __aarch64__
inline
int32x4_t
vpaddq_s32
(
int32x4_t
r0
,
int32x4_t
r1
)
{
int32x2_t
sum0
=
vpadd_s32
(
vget_low_s32
(
r0
),
vget_high_s32
(
r0
));
int32x2_t
sum1
=
vpadd_s32
(
vget_low_s32
(
r1
),
vget_high_s32
(
r1
));
return
vcombine_s32
(
sum0
,
sum1
);
}
#endif
template
<
int
Stride
=
1
>
inline
void
Depth5x5NormalRowLoadInput
(
const
int8_t
*
input
,
int16x4_t
*
y
)
{
int16x8_t
x
=
vmovl_s8
(
vld1_s8
(
input
));
y
[
0
]
=
vget_low_s16
(
x
);
y
[
4
]
=
vget_high_s16
(
x
);
y
[
1
]
=
vext_s16
(
y
[
0
],
y
[
4
],
1
);
y
[
2
]
=
vext_s16
(
y
[
0
],
y
[
4
],
2
);
y
[
3
]
=
vext_s16
(
y
[
0
],
y
[
4
],
3
);
}
template
<
>
inline
void
Depth5x5NormalRowLoadInput
<
2
>
(
const
int8_t
*
input
,
int16x4_t
*
y
)
{
int8x8x2_t
x
=
vld2_s8
(
input
);
y
[
0
]
=
vget_low_s16
(
vmovl_s8
(
x
.
val
[
0
]));
y
[
1
]
=
vget_low_s16
(
vmovl_s8
(
x
.
val
[
1
]));
y
[
2
]
=
vext_s16
(
y
[
0
],
y
[
0
],
1
);
y
[
3
]
=
vext_s16
(
y
[
1
],
y
[
1
],
1
);
y
[
4
]
=
vext_s16
(
y
[
0
],
y
[
0
],
2
);
}
#define DEPTHWISE_CONV_NORMAL_BORDER(start, end) \
for (int w = start; w < end; ++w) { \
const int w_in_start = -padding_w + w * Stride_w; \
const int w_in_end = w_in_start + 5; \
const int w_start = w_in_start > 0 ? w_in_start : 0; \
const int w_end = w_in_end < input_w ? w_in_end : input_w; \
int32_t value = 0; \
for (int h_in = h_start; h_in < h_end; ++h_in) { \
for (int w_in = w_start; w_in < w_end; ++w_in) { \
value += filter[(h_in - h_in_start) * 5 + (w_in - w_in_start)] * \
input[h_in * input_w + w_in]; \
} \
} \
output_ptr[w] = value; \
}
template
<
int
Stride_h
,
int
Stride_w
>
inline
void
DepthwiseConv5x5NormalRow
(
const
int8_t
*
input
,
const
int8_t
*
filter
,
const
int
h_output
,
const
int
input_h
,
const
int
input_w
,
const
int
padding_h
,
const
int
padding_w
,
const
int
output_w
,
int32_t
*
output
,
int16x4_t
*
ker
,
int16_t
*
ker1
)
{
const
int
h_in_start
=
-
padding_h
+
h_output
*
Stride_h
;
const
int
h_in_end
=
h_in_start
+
5
;
const
int
h_start
=
h_in_start
>
0
?
h_in_start
:
0
;
const
int
h_end
=
h_in_end
<
input_h
?
h_in_end
:
input_h
;
int
valid_w_start
=
(
padding_w
+
Stride_w
-
1
)
/
Stride_w
;
int
valid_w_end
=
output_w
-
valid_w_start
;
int32_t
*
output_ptr
=
output
+
h_output
*
output_w
;
// border left
DEPTHWISE_CONV_NORMAL_BORDER
(
0
,
valid_w_start
)
// middle
int
output_tiles
=
(
valid_w_end
-
valid_w_start
)
>>
2
;
int16x4_t
_x
[
5
];
int32x4_t
_sum
;
// valid w
for
(
int
w
=
0
;
w
<
output_tiles
*
4
;
w
+=
4
)
{
_sum
=
vdupq_n_s32
(
0
);
int
output_offset
=
valid_w_start
+
w
;
int
input_w_offset
=
output_offset
*
Stride_w
-
padding_w
;
for
(
int
h_in
=
h_start
;
h_in
<
h_end
;
++
h_in
)
{
int
index
=
h_in
-
h_in_start
;
Depth5x5NormalRowLoadInput
<
Stride_w
>
(
input
+
h_in
*
input_w
+
input_w_offset
,
_x
);
_sum
=
vmlal_n_s16
(
_sum
,
_x
[
0
],
ker1
[
index
]);
_sum
=
vmlal_lane_s16
(
_sum
,
_x
[
1
],
ker
[
index
],
0
);
_sum
=
vmlal_lane_s16
(
_sum
,
_x
[
2
],
ker
[
index
],
1
);
_sum
=
vmlal_lane_s16
(
_sum
,
_x
[
3
],
ker
[
index
],
2
);
_sum
=
vmlal_lane_s16
(
_sum
,
_x
[
4
],
ker
[
index
],
3
);
}
vst1q_s32
(
output_ptr
+
output_offset
,
_sum
);
}
// remain valid w
int
remain
=
(
valid_w_end
-
valid_w_start
)
&
0x3
;
if
(
remain
>
0
)
{
_sum
=
vdupq_n_s32
(
0
);
int
remain_start
=
valid_w_start
+
(
output_tiles
<<
2
);
int
input_w_offset
=
remain_start
*
Stride_w
-
padding_w
;
int32_t
*
output_ptr0
=
output_ptr
+
remain_start
;
for
(
int
h_in
=
h_start
;
h_in
<
h_end
;
++
h_in
)
{
int
index
=
h_in
-
h_in_start
;
Depth5x5NormalRowLoadInput
<
Stride_w
>
(
input
+
h_in
*
input_w
+
input_w_offset
,
_x
);
_sum
=
vmlal_n_s16
(
_sum
,
_x
[
0
],
ker1
[
index
]);
_sum
=
vmlal_lane_s16
(
_sum
,
_x
[
1
],
ker
[
index
],
0
);
_sum
=
vmlal_lane_s16
(
_sum
,
_x
[
2
],
ker
[
index
],
1
);
_sum
=
vmlal_lane_s16
(
_sum
,
_x
[
3
],
ker
[
index
],
2
);
_sum
=
vmlal_lane_s16
(
_sum
,
_x
[
4
],
ker
[
index
],
3
);
}
switch
(
remain
)
{
case
1
:
vst1_lane_s32
(
output_ptr0
,
vget_low_s32
(
_sum
),
0
);
break
;
case
2
:
vst1_s32
(
output_ptr0
,
vget_low_s32
(
_sum
));
break
;
case
3
:
vst1_s32
(
output_ptr0
,
vget_low_s32
(
_sum
));
vst1_lane_s32
(
output_ptr0
+
2
,
vget_high_s32
(
_sum
),
0
);
break
;
}
}
// border right
DEPTHWISE_CONV_NORMAL_BORDER
(
valid_w_end
,
output_w
)
}
template
<
>
void
DepthwiseConv5x5S1
<
int8_t
,
int32_t
>
(
const
framework
::
Tensor
&
input
,
const
framework
::
Tensor
&
filter
,
const
std
::
vector
<
int
>
&
paddings
,
framework
::
Tensor
*
output
)
{
const
int8_t
*
input_data
=
input
.
data
<
int8_t
>
();
const
int8_t
*
filter_data
=
filter
.
data
<
int8_t
>
();
int32_t
*
out_data
=
output
->
mutable_data
<
int32_t
>
();
int
input_h
=
input
.
dims
()[
2
];
int
input_w
=
input
.
dims
()[
3
];
int
output_h
=
output
->
dims
()[
2
];
int
output_w
=
output
->
dims
()[
3
];
int
padding_h
=
paddings
[
0
];
int
padding_w
=
paddings
[
1
];
int
image_size
=
input_h
*
input_w
;
int
out_image_size
=
output_h
*
output_w
;
int
valid_h_start
=
padding_h
;
int
valid_h_end
=
output_h
-
valid_h_start
;
int
valid_h
=
valid_h_end
-
valid_h_start
;
int
valid_w_start
=
padding_w
;
int
valid_w_end
=
output_w
-
valid_w_start
;
int
valid_w
=
valid_w_end
-
valid_w_start
;
#pragma omp parallel for
for
(
int
g
=
0
;
g
<
input
.
dims
()[
1
];
++
g
)
{
const
int8_t
*
input_ptr
=
input_data
+
g
*
image_size
;
const
int8_t
*
filter_ptr
=
filter_data
+
g
*
25
;
int32_t
*
output_ptr
=
out_data
+
g
*
out_image_size
;
const
int8_t
*
filter_ptr0
=
filter_ptr
;
const
int8_t
*
filter_ptr1
=
filter_ptr0
+
5
;
const
int8_t
*
filter_ptr2
=
filter_ptr1
+
5
;
const
int8_t
*
filter_ptr3
=
filter_ptr2
+
5
;
const
int8_t
*
filter_ptr4
=
filter_ptr3
+
5
;
int16_t
kernel
[
5
]
=
{
*
filter_ptr0
,
*
filter_ptr1
,
*
filter_ptr2
,
*
filter_ptr3
,
*
filter_ptr4
};
int16x4_t
_k0
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
filter_ptr0
+
1
)));
int16x4_t
_k1
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
filter_ptr1
+
1
)));
int16x4_t
_k2
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
filter_ptr2
+
1
)));
int16x4_t
_k3
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
filter_ptr3
+
1
)));
int16x4_t
_k4
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
filter_ptr4
+
1
)));
int16x4_t
_k5
=
vld1_s16
(
kernel
);
int16x4_t
_k6
=
vld1_s16
(
kernel
+
4
);
int16x8_t
_ker0
=
vcombine_s16
(
_k0
,
_k1
);
int16x8_t
_ker1
=
vcombine_s16
(
_k2
,
_k3
);
int16x8_t
_ker2
=
vcombine_s16
(
_k4
,
_k5
);
int16x8_t
_ker3
=
vcombine_s16
(
_k6
,
_k6
);
int16x4_t
_ker
[
7
]
=
{
_k0
,
_k1
,
_k2
,
_k3
,
_k4
,
_k5
,
_k6
};
// pad top
for
(
int
h
=
0
;
h
<
valid_h_start
;
++
h
)
{
DepthwiseConv5x5NormalRow
<
1
,
1
>
(
input_ptr
,
filter_ptr
,
h
,
input_h
,
input_w
,
padding_h
,
padding_w
,
output_w
,
output_ptr
,
_ker
,
kernel
);
}
// output 4x4
int
output_w_tiles
=
valid_w
/
8
;
int
output_w_remain
=
valid_w
-
output_w_tiles
*
8
;
for
(
int
h
=
valid_h_start
;
h
<
valid_h_end
-
1
;
h
+=
2
)
{
const
int8_t
*
input_ptr0
=
input_ptr
+
(
h
-
padding_h
)
*
input_w
;
const
int8_t
*
input_ptr1
=
input_ptr0
+
input_w
;
const
int8_t
*
input_ptr2
=
input_ptr1
+
input_w
;
const
int8_t
*
input_ptr3
=
input_ptr2
+
input_w
;
const
int8_t
*
input_ptr4
=
input_ptr3
+
input_w
;
const
int8_t
*
input_ptr5
=
input_ptr4
+
input_w
;
int32_t
*
output_ptr0
=
output_ptr
+
h
*
output_w
;
int32_t
*
output_ptr1
=
output_ptr0
+
output_w
;
// pad left
if
(
padding_w
)
{
int16x4_t
row0
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr0
)));
int16x4_t
row1
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr1
)));
int16x4_t
row2
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr2
)));
int16x4_t
row3
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr3
)));
int16x4_t
row4
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr4
)));
int16x4_t
row5
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr5
)));
int16x4_t
zero
=
vdup_n_s16
(
0
);
int32x4_t
acc0
,
acc1
;
for
(
int
w
=
valid_w_start
-
1
;
w
>=
0
;
--
w
)
{
int
padding
=
padding_w
-
w
;
if
(
padding
>=
5
)
{
output_ptr0
[
w
]
=
0
;
output_ptr1
[
w
]
=
0
;
}
else
{
acc0
=
vmull_s16
(
row0
,
_ker
[
0
]);
acc0
=
vmlal_s16
(
acc0
,
row1
,
_ker
[
1
]);
acc0
=
vmlal_s16
(
acc0
,
row2
,
_ker
[
2
]);
acc0
=
vmlal_s16
(
acc0
,
row3
,
_ker
[
3
]);
acc0
=
vmlal_s16
(
acc0
,
row4
,
_ker
[
4
]);
acc1
=
vmull_s16
(
row1
,
_ker
[
0
]);
acc1
=
vmlal_s16
(
acc1
,
row2
,
_ker
[
1
]);
acc1
=
vmlal_s16
(
acc1
,
row3
,
_ker
[
2
]);
acc1
=
vmlal_s16
(
acc1
,
row4
,
_ker
[
3
]);
acc1
=
vmlal_s16
(
acc1
,
row5
,
_ker
[
4
]);
acc0
=
vpaddq_s32
(
acc0
,
acc1
);
int32x2_t
sum
=
vpadd_s32
(
vget_low_s32
(
acc0
),
vget_high_s32
(
acc0
));
vst1_lane_s32
(
output_ptr0
+
w
,
sum
,
0
);
vst1_lane_s32
(
output_ptr1
+
w
,
sum
,
1
);
row0
=
vext_s16
(
zero
,
row0
,
3
);
row1
=
vext_s16
(
zero
,
row1
,
3
);
row2
=
vext_s16
(
zero
,
row2
,
3
);
row3
=
vext_s16
(
zero
,
row3
,
3
);
row4
=
vext_s16
(
zero
,
row4
,
3
);
row5
=
vext_s16
(
zero
,
row5
,
3
);
}
}
output_ptr0
+=
valid_w_start
;
output_ptr1
+=
valid_w_start
;
}
// valid
int
loop
=
output_w_tiles
;
int
w_remain
=
output_w_remain
;
asm
volatile
(
"cmp %[loop], #0
\n
"
"ble start_remain4_%=
\n
"
"mov r0, #8
\n
"
"loop_2h8w_%=:
\n
"
"vld1.s8 {d10-d11}, [%[input_ptr0]], r0
\n
"
"vld1.s8 {d12-d13}, [%[input_ptr1]], r0
\n
"
"vld1.s8 {d14-d15}, [%[input_ptr2]], r0
\n
"
"vmovl.s8 q8, d10
\n
"
"vmovl.s8 q9, d11
\n
"
"vmull.s16 q12, d16, %f[ker2][0]
\n
"
"vmull.s16 q13, d17, %f[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker0][0]
\n
"
"vmlal.s16 q13, d21, %e[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker0][1]
\n
"
"vmlal.s16 q13, d21, %e[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker0][2]
\n
"
"vmlal.s16 q13, d21, %e[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker0][3]
\n
"
"vmlal.s16 q13, d21, %e[ker0][3]
\n
"
"vmovl.s8 q8, d12
\n
"
"vmovl.s8 q9, d13
\n
"
"vmlal.s16 q12, d16, %f[ker2][1]
\n
"
"vmlal.s16 q13, d17, %f[ker2][1]
\n
"
"vmull.s16 q14, d16, %f[ker2][0]
\n
"
"vmull.s16 q15, d17, %f[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %f[ker0][0]
\n
"
"vmlal.s16 q13, d21, %f[ker0][0]
\n
"
"vmlal.s16 q14, d20, %e[ker0][0]
\n
"
"vmlal.s16 q15, d21, %e[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %f[ker0][1]
\n
"
"vmlal.s16 q13, d21, %f[ker0][1]
\n
"
"vmlal.s16 q14, d20, %e[ker0][1]
\n
"
"vmlal.s16 q15, d21, %e[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %f[ker0][2]
\n
"
"vmlal.s16 q13, d21, %f[ker0][2]
\n
"
"vmlal.s16 q14, d20, %e[ker0][2]
\n
"
"vmlal.s16 q15, d21, %e[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %f[ker0][3]
\n
"
"vmlal.s16 q13, d21, %f[ker0][3]
\n
"
"vmlal.s16 q14, d20, %e[ker0][3]
\n
"
"vmlal.s16 q15, d21, %e[ker0][3]
\n
"
"vmovl.s8 q8, d14
\n
"
"vmovl.s8 q9, d15
\n
"
"vmlal.s16 q12, d16, %f[ker2][2]
\n
"
"vmlal.s16 q13, d17, %f[ker2][2]
\n
"
"vmlal.s16 q14, d16, %f[ker2][1]
\n
"
"vmlal.s16 q15, d17, %f[ker2][1]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker1][0]
\n
"
"vmlal.s16 q13, d21, %e[ker1][0]
\n
"
"vmlal.s16 q14, d20, %f[ker0][0]
\n
"
"vmlal.s16 q15, d21, %f[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker1][1]
\n
"
"vmlal.s16 q13, d21, %e[ker1][1]
\n
"
"vmlal.s16 q14, d20, %f[ker0][1]
\n
"
"vmlal.s16 q15, d21, %f[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker1][2]
\n
"
"vmlal.s16 q13, d21, %e[ker1][2]
\n
"
"vmlal.s16 q14, d20, %f[ker0][2]
\n
"
"vmlal.s16 q15, d21, %f[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker1][3]
\n
"
"vmlal.s16 q13, d21, %e[ker1][3]
\n
"
"vmlal.s16 q14, d20, %f[ker0][3]
\n
"
"vmlal.s16 q15, d21, %f[ker0][3]
\n
"
"vld1.s8 {d10-d11}, [%[input_ptr3]], r0
\n
"
"vld1.s8 {d12-d13}, [%[input_ptr4]], r0
\n
"
"vld1.s8 {d14-d15}, [%[input_ptr5]], r0
\n
"
"vmovl.s8 q8, d10
\n
"
"vmovl.s8 q9, d11
\n
"
"vmlal.s16 q12, d16, %f[ker2][3]
\n
"
"vmlal.s16 q13, d17, %f[ker2][3]
\n
"
"vmlal.s16 q14, d16, %f[ker2][2]
\n
"
"vmlal.s16 q15, d17, %f[ker2][2]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %f[ker1][0]
\n
"
"vmlal.s16 q13, d21, %f[ker1][0]
\n
"
"vmlal.s16 q14, d20, %e[ker1][0]
\n
"
"vmlal.s16 q15, d21, %e[ker1][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %f[ker1][1]
\n
"
"vmlal.s16 q13, d21, %f[ker1][1]
\n
"
"vmlal.s16 q14, d20, %e[ker1][1]
\n
"
"vmlal.s16 q15, d21, %e[ker1][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %f[ker1][2]
\n
"
"vmlal.s16 q13, d21, %f[ker1][2]
\n
"
"vmlal.s16 q14, d20, %e[ker1][2]
\n
"
"vmlal.s16 q15, d21, %e[ker1][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %f[ker1][3]
\n
"
"vmlal.s16 q13, d21, %f[ker1][3]
\n
"
"vmlal.s16 q14, d20, %e[ker1][3]
\n
"
"vmlal.s16 q15, d21, %e[ker1][3]
\n
"
"vmovl.s8 q8, d12
\n
"
"vmovl.s8 q9, d13
\n
"
"vmlal.s16 q12, d16, %e[ker3][0]
\n
"
"vmlal.s16 q13, d17, %e[ker3][0]
\n
"
"vmlal.s16 q14, d16, %f[ker2][3]
\n
"
"vmlal.s16 q15, d17, %f[ker2][3]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker2][0]
\n
"
"vmlal.s16 q13, d21, %e[ker2][0]
\n
"
"vmlal.s16 q14, d20, %f[ker1][0]
\n
"
"vmlal.s16 q15, d21, %f[ker1][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker2][1]
\n
"
"vmlal.s16 q13, d21, %e[ker2][1]
\n
"
"vmlal.s16 q14, d20, %f[ker1][1]
\n
"
"vmlal.s16 q15, d21, %f[ker1][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker2][2]
\n
"
"vmlal.s16 q13, d21, %e[ker2][2]
\n
"
"vmlal.s16 q14, d20, %f[ker1][2]
\n
"
"vmlal.s16 q15, d21, %f[ker1][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker2][3]
\n
"
"vmlal.s16 q13, d21, %e[ker2][3]
\n
"
"vmlal.s16 q14, d20, %f[ker1][3]
\n
"
"vmlal.s16 q15, d21, %f[ker1][3]
\n
"
"vmovl.s8 q8, d14
\n
"
"vmovl.s8 q9, d15
\n
"
"vmlal.s16 q14, d16, %e[ker3][0]
\n
"
"vmlal.s16 q15, d17, %e[ker3][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q14, d20, %e[ker2][0]
\n
"
"vmlal.s16 q15, d21, %e[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q14, d20, %e[ker2][1]
\n
"
"vmlal.s16 q15, d21, %e[ker2][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q14, d20, %e[ker2][2]
\n
"
"vmlal.s16 q15, d21, %e[ker2][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q14, d20, %e[ker2][3]
\n
"
"vmlal.s16 q15, d21, %e[ker2][3]
\n
"
// restore output
"vst1.32 {q12-q13}, [%[output_ptr0]]!
\n
"
"vst1.32 {q14-q15}, [%[output_ptr1]]!
\n
"
"subs %[loop], #1
\n
"
"bne loop_2h8w_%=
\n
"
"start_remain4_%=:
\n
"
"cmp %[remain], #4
\n
"
"blt start_remain_%=
\n
"
"mov r0, #4
\n
"
"vld1.s8 {d10}, [%[input_ptr0]], r0
\n
"
"vld1.s8 {d12}, [%[input_ptr1]], r0
\n
"
"vld1.s8 {d14}, [%[input_ptr2]], r0
\n
"
"vmovl.s8 q8, d10
\n
"
"vmull.s16 q12, d16, %f[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker0][3]
\n
"
"vmovl.s8 q8, d12
\n
"
"vmlal.s16 q12, d16, %f[ker2][1]
\n
"
"vmull.s16 q14, d16, %f[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %f[ker0][0]
\n
"
"vmlal.s16 q14, d20, %e[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %f[ker0][1]
\n
"
"vmlal.s16 q14, d20, %e[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %f[ker0][2]
\n
"
"vmlal.s16 q14, d20, %e[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %f[ker0][3]
\n
"
"vmlal.s16 q14, d20, %e[ker0][3]
\n
"
"vmovl.s8 q8, d14
\n
"
"vmlal.s16 q12, d16, %f[ker2][2]
\n
"
"vmlal.s16 q14, d16, %f[ker2][1]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker1][0]
\n
"
"vmlal.s16 q14, d20, %f[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker1][1]
\n
"
"vmlal.s16 q14, d20, %f[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker1][2]
\n
"
"vmlal.s16 q14, d20, %f[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker1][3]
\n
"
"vmlal.s16 q14, d20, %f[ker0][3]
\n
"
"vld1.s8 {d10}, [%[input_ptr3]], r0
\n
"
"vld1.s8 {d12}, [%[input_ptr4]], r0
\n
"
"vld1.s8 {d14}, [%[input_ptr5]], r0
\n
"
"vmovl.s8 q8, d10
\n
"
"vmlal.s16 q12, d16, %f[ker2][3]
\n
"
"vmlal.s16 q14, d16, %f[ker2][2]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %f[ker1][0]
\n
"
"vmlal.s16 q14, d20, %e[ker1][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %f[ker1][1]
\n
"
"vmlal.s16 q14, d20, %e[ker1][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %f[ker1][2]
\n
"
"vmlal.s16 q14, d20, %e[ker1][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %f[ker1][3]
\n
"
"vmlal.s16 q14, d20, %e[ker1][3]
\n
"
"vmovl.s8 q8, d12
\n
"
"vmlal.s16 q12, d16, %e[ker3][0]
\n
"
"vmlal.s16 q14, d16, %f[ker2][3]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker2][0]
\n
"
"vmlal.s16 q14, d20, %f[ker1][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker2][1]
\n
"
"vmlal.s16 q14, d20, %f[ker1][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker2][2]
\n
"
"vmlal.s16 q14, d20, %f[ker1][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker2][3]
\n
"
"vmlal.s16 q14, d20, %f[ker1][3]
\n
"
"vmovl.s8 q8, d14
\n
"
"vmlal.s16 q14, d16, %e[ker3][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q14, d20, %e[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q14, d20, %e[ker2][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q14, d20, %e[ker2][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q14, d20, %e[ker2][3]
\n
"
// restore output
"vst1.32 {d24-d25}, [%[output_ptr0]]!
\n
"
"vst1.32 {d28-d29}, [%[output_ptr1]]!
\n
"
"sub %[remain], #4
\n
"
"start_remain_%=:
\n
"
"cmp %[remain], #0
\n
"
"ble end_%=
\n
"
"mov r0, %[remain]
\n
"
"vld1.s8 {d10}, [%[input_ptr0]], r0
\n
"
"vld1.s8 {d12}, [%[input_ptr1]], r0
\n
"
"vld1.s8 {d14}, [%[input_ptr2]], r0
\n
"
"vmovl.s8 q8, d10
\n
"
"vmull.s16 q12, d16, %f[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker0][3]
\n
"
"vmovl.s8 q8, d12
\n
"
"vmlal.s16 q12, d16, %f[ker2][1]
\n
"
"vmull.s16 q14, d16, %f[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %f[ker0][0]
\n
"
"vmlal.s16 q14, d20, %e[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %f[ker0][1]
\n
"
"vmlal.s16 q14, d20, %e[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %f[ker0][2]
\n
"
"vmlal.s16 q14, d20, %e[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %f[ker0][3]
\n
"
"vmlal.s16 q14, d20, %e[ker0][3]
\n
"
"vmovl.s8 q8, d14
\n
"
"vmlal.s16 q12, d16, %f[ker2][2]
\n
"
"vmlal.s16 q14, d16, %f[ker2][1]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker1][0]
\n
"
"vmlal.s16 q14, d20, %f[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker1][1]
\n
"
"vmlal.s16 q14, d20, %f[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker1][2]
\n
"
"vmlal.s16 q14, d20, %f[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker1][3]
\n
"
"vmlal.s16 q14, d20, %f[ker0][3]
\n
"
"vld1.s8 {d10}, [%[input_ptr3]], r0
\n
"
"vld1.s8 {d12}, [%[input_ptr4]], r0
\n
"
"vld1.s8 {d14}, [%[input_ptr5]], r0
\n
"
"vmovl.s8 q8, d10
\n
"
"vmlal.s16 q12, d16, %f[ker2][3]
\n
"
"vmlal.s16 q14, d16, %f[ker2][2]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %f[ker1][0]
\n
"
"vmlal.s16 q14, d20, %e[ker1][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %f[ker1][1]
\n
"
"vmlal.s16 q14, d20, %e[ker1][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %f[ker1][2]
\n
"
"vmlal.s16 q14, d20, %e[ker1][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %f[ker1][3]
\n
"
"vmlal.s16 q14, d20, %e[ker1][3]
\n
"
"vmovl.s8 q8, d12
\n
"
"vmlal.s16 q12, d16, %e[ker3][0]
\n
"
"vmlal.s16 q14, d16, %f[ker2][3]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker2][0]
\n
"
"vmlal.s16 q14, d20, %f[ker1][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker2][1]
\n
"
"vmlal.s16 q14, d20, %f[ker1][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker2][2]
\n
"
"vmlal.s16 q14, d20, %f[ker1][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker2][3]
\n
"
"vmlal.s16 q14, d20, %f[ker1][3]
\n
"
"vmovl.s8 q8, d14
\n
"
"vmlal.s16 q14, d16, %e[ker3][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q14, d20, %e[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q14, d20, %e[ker2][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q14, d20, %e[ker2][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q14, d20, %e[ker2][3]
\n
"
"cmp %[remain], #2
\n
"
"blt store_2h1w_%=
\n
"
"vst1.32 {d24}, [%[output_ptr0]]!
\n
"
"vst1.32 {d28}, [%[output_ptr1]]!
\n
"
"cmp %[remain], #3
\n
"
"blt end_%=
\n
"
"vst1.32 {d25[0]}, [%[output_ptr0]]!
\n
"
"vst1.32 {d29[0]}, [%[output_ptr1]]!
\n
"
"b end_%=
\n
"
"store_2h1w_%=:
\n
"
"vst1.32 {d24[0]}, [%[output_ptr0]]!
\n
"
"vst1.32 {d28[0]}, [%[output_ptr1]]!
\n
"
"end_%=:
\n
"
:
[
input_ptr0
]
"+r"
(
input_ptr0
),
[
input_ptr1
]
"+r"
(
input_ptr1
),
[
input_ptr2
]
"+r"
(
input_ptr2
),
[
input_ptr3
]
"+r"
(
input_ptr3
),
[
input_ptr4
]
"+r"
(
input_ptr4
),
[
input_ptr5
]
"+r"
(
input_ptr5
),
[
output_ptr0
]
"+r"
(
output_ptr0
),
[
output_ptr1
]
"+r"
(
output_ptr1
),
[
loop
]
"+r"
(
loop
),
[
remain
]
"+r"
(
w_remain
)
:
[
ker0
]
"w"
(
_ker0
),
[
ker1
]
"w"
(
_ker1
),
[
ker2
]
"w"
(
_ker2
),
[
ker3
]
"w"
(
_ker3
)
:
"cc"
,
"memory"
,
"q4"
,
"q5"
,
"q6"
,
"q7"
,
"q8"
,
"q9"
,
"q10"
,
"q11"
,
"q12"
,
"q13"
,
"q14"
,
"q15"
,
"r0"
);
// pad right
if
(
padding_w
)
{
int16x4_t
row0
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr0
)));
int16x4_t
row1
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr1
)));
int16x4_t
row2
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr2
)));
int16x4_t
row3
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr3
)));
int16x4_t
row4
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr4
)));
int16x4_t
row5
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr5
)));
int16x4_t
zero
=
vdup_n_s16
(
0
);
int32x4_t
acc0
,
acc1
;
for
(
int
w
=
valid_w_end
;
w
<
output_w
;
++
w
)
{
int
padding
=
w
+
5
-
(
padding_w
+
input_w
);
if
(
padding
>=
5
)
{
*
output_ptr0
=
0.
f
;
*
output_ptr1
=
0.
f
;
}
else
{
int
iw
=
w
-
valid_w_end
;
int32_t
sum0
=
input_ptr0
[
iw
]
*
filter_ptr0
[
0
]
+
input_ptr1
[
iw
]
*
filter_ptr1
[
0
]
+
input_ptr2
[
iw
]
*
filter_ptr2
[
0
]
+
input_ptr3
[
iw
]
*
filter_ptr3
[
0
]
+
input_ptr4
[
iw
]
*
filter_ptr4
[
0
];
int32_t
sum1
=
input_ptr1
[
iw
]
*
filter_ptr0
[
0
]
+
input_ptr2
[
iw
]
*
filter_ptr1
[
0
]
+
input_ptr3
[
iw
]
*
filter_ptr2
[
0
]
+
input_ptr4
[
iw
]
*
filter_ptr3
[
0
]
+
input_ptr5
[
iw
]
*
filter_ptr4
[
0
];
row0
=
vext_s16
(
row0
,
zero
,
1
);
row1
=
vext_s16
(
row1
,
zero
,
1
);
row2
=
vext_s16
(
row2
,
zero
,
1
);
row3
=
vext_s16
(
row3
,
zero
,
1
);
row4
=
vext_s16
(
row4
,
zero
,
1
);
row5
=
vext_s16
(
row5
,
zero
,
1
);
acc0
=
vmull_s16
(
row0
,
_ker
[
0
]);
acc0
=
vmlal_s16
(
acc0
,
row1
,
_ker
[
1
]);
acc0
=
vmlal_s16
(
acc0
,
row2
,
_ker
[
2
]);
acc0
=
vmlal_s16
(
acc0
,
row3
,
_ker
[
3
]);
acc0
=
vmlal_s16
(
acc0
,
row4
,
_ker
[
4
]);
acc1
=
vmull_s16
(
row1
,
_ker
[
0
]);
acc1
=
vmlal_s16
(
acc1
,
row2
,
_ker
[
1
]);
acc1
=
vmlal_s16
(
acc1
,
row3
,
_ker
[
2
]);
acc1
=
vmlal_s16
(
acc1
,
row4
,
_ker
[
3
]);
acc1
=
vmlal_s16
(
acc1
,
row5
,
_ker
[
4
]);
acc0
=
vpaddq_s32
(
acc0
,
acc1
);
int32x2_t
sum
=
vpadd_s32
(
vget_low_s32
(
acc0
),
vget_high_s32
(
acc0
));
sum0
+=
vget_lane_s32
(
sum
,
0
);
sum1
+=
vget_lane_s32
(
sum
,
1
);
*
output_ptr0
=
sum0
;
*
output_ptr1
=
sum1
;
}
output_ptr0
++
;
output_ptr1
++
;
}
}
}
// remain height
int
start_h
=
valid_h_start
+
(
valid_h
&
0xfffe
);
if
(
start_h
<
valid_h_end
)
{
const
int8_t
*
input_ptr0
=
input_ptr
+
(
start_h
-
padding_h
)
*
input_w
;
const
int8_t
*
input_ptr1
=
input_ptr0
+
input_w
;
const
int8_t
*
input_ptr2
=
input_ptr1
+
input_w
;
const
int8_t
*
input_ptr3
=
input_ptr2
+
input_w
;
const
int8_t
*
input_ptr4
=
input_ptr3
+
input_w
;
int32_t
*
output_ptr0
=
output_ptr
+
start_h
*
output_w
;
// pad left
if
(
padding_w
)
{
int16x4_t
row0
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr0
)));
int16x4_t
row1
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr1
)));
int16x4_t
row2
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr2
)));
int16x4_t
row3
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr3
)));
int16x4_t
row4
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr4
)));
int16x4_t
zero
=
vdup_n_s16
(
0
);
int32x4_t
acc
;
for
(
int
w
=
valid_w_start
-
1
;
w
>=
0
;
--
w
)
{
int
padding
=
padding_w
-
w
;
if
(
padding
>=
5
)
{
output_ptr0
[
w
]
=
0
;
}
else
{
acc
=
vmull_s16
(
row0
,
_ker
[
0
]);
acc
=
vmlal_s16
(
acc
,
row1
,
_ker
[
1
]);
acc
=
vmlal_s16
(
acc
,
row2
,
_ker
[
2
]);
acc
=
vmlal_s16
(
acc
,
row3
,
_ker
[
3
]);
acc
=
vmlal_s16
(
acc
,
row4
,
_ker
[
4
]);
int32x2_t
sum
=
vpadd_s32
(
vget_low_s32
(
acc
),
vget_high_s32
(
acc
));
sum
=
vpadd_s32
(
sum
,
sum
);
vst1_lane_s32
(
output_ptr0
+
w
,
sum
,
0
);
row0
=
vext_s16
(
zero
,
row0
,
3
);
row1
=
vext_s16
(
zero
,
row1
,
3
);
row2
=
vext_s16
(
zero
,
row2
,
3
);
row3
=
vext_s16
(
zero
,
row3
,
3
);
row4
=
vext_s16
(
zero
,
row4
,
3
);
}
}
output_ptr0
+=
valid_w_start
;
}
// valid
int
loop
=
output_w_tiles
;
int
w_remain
=
output_w_remain
;
asm
volatile
(
"cmp %[loop], #0
\n
"
"ble start_remain4_%=
\n
"
"mov r0, #8
\n
"
"loop_1h8w_%=:
\n
"
"vld1.s8 {d10-d11}, [%[input_ptr0]], r0
\n
"
"vld1.s8 {d12-d13}, [%[input_ptr1]], r0
\n
"
"vld1.s8 {d14-d15}, [%[input_ptr2]], r0
\n
"
"vmovl.s8 q8, d10
\n
"
"vmovl.s8 q9, d11
\n
"
"vmull.s16 q12, d16, %f[ker2][0]
\n
"
"vmull.s16 q13, d17, %f[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker0][0]
\n
"
"vmlal.s16 q13, d21, %e[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker0][1]
\n
"
"vmlal.s16 q13, d21, %e[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker0][2]
\n
"
"vmlal.s16 q13, d21, %e[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker0][3]
\n
"
"vmlal.s16 q13, d21, %e[ker0][3]
\n
"
"vmovl.s8 q8, d12
\n
"
"vmovl.s8 q9, d13
\n
"
"vmlal.s16 q12, d16, %f[ker2][1]
\n
"
"vmlal.s16 q13, d17, %f[ker2][1]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %f[ker0][0]
\n
"
"vmlal.s16 q13, d21, %f[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %f[ker0][1]
\n
"
"vmlal.s16 q13, d21, %f[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %f[ker0][2]
\n
"
"vmlal.s16 q13, d21, %f[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %f[ker0][3]
\n
"
"vmlal.s16 q13, d21, %f[ker0][3]
\n
"
"vmovl.s8 q8, d14
\n
"
"vmovl.s8 q9, d15
\n
"
"vmlal.s16 q12, d16, %f[ker2][2]
\n
"
"vmlal.s16 q13, d17, %f[ker2][2]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker1][0]
\n
"
"vmlal.s16 q13, d21, %e[ker1][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker1][1]
\n
"
"vmlal.s16 q13, d21, %e[ker1][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker1][2]
\n
"
"vmlal.s16 q13, d21, %e[ker1][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker1][3]
\n
"
"vmlal.s16 q13, d21, %e[ker1][3]
\n
"
"vld1.s8 {d10-d11}, [%[input_ptr3]], r0
\n
"
"vld1.s8 {d12-d13}, [%[input_ptr4]], r0
\n
"
"vmovl.s8 q8, d10
\n
"
"vmovl.s8 q9, d11
\n
"
"vmlal.s16 q12, d16, %f[ker2][3]
\n
"
"vmlal.s16 q13, d17, %f[ker2][3]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %f[ker1][0]
\n
"
"vmlal.s16 q13, d21, %f[ker1][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %f[ker1][1]
\n
"
"vmlal.s16 q13, d21, %f[ker1][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %f[ker1][2]
\n
"
"vmlal.s16 q13, d21, %f[ker1][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %f[ker1][3]
\n
"
"vmlal.s16 q13, d21, %f[ker1][3]
\n
"
"vmovl.s8 q8, d12
\n
"
"vmovl.s8 q9, d13
\n
"
"vmlal.s16 q12, d16, %e[ker3][0]
\n
"
"vmlal.s16 q13, d17, %e[ker3][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker2][0]
\n
"
"vmlal.s16 q13, d21, %e[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker2][1]
\n
"
"vmlal.s16 q13, d21, %e[ker2][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker2][2]
\n
"
"vmlal.s16 q13, d21, %e[ker2][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker2][3]
\n
"
"vmlal.s16 q13, d21, %e[ker2][3]
\n
"
// restore output
"vst1.32 {q12-q13}, [%[output_ptr0]]!
\n
"
"subs %[loop], #1
\n
"
"bne loop_1h8w_%=
\n
"
"start_remain4_%=:
\n
"
"cmp %[remain], #4
\n
"
"blt start_remain_%=
\n
"
"mov r0, #4
\n
"
"vld1.s8 {d10}, [%[input_ptr0]], r0
\n
"
"vld1.s8 {d12}, [%[input_ptr1]], r0
\n
"
"vld1.s8 {d14}, [%[input_ptr2]], r0
\n
"
"vmovl.s8 q8, d10
\n
"
"vmull.s16 q12, d16, %f[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker0][3]
\n
"
"vmovl.s8 q8, d12
\n
"
"vmlal.s16 q12, d16, %f[ker2][1]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %f[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %f[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %f[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %f[ker0][3]
\n
"
"vmovl.s8 q8, d14
\n
"
"vmlal.s16 q12, d16, %f[ker2][2]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker1][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker1][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker1][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker1][3]
\n
"
"vld1.s8 {d10}, [%[input_ptr3]], r0
\n
"
"vld1.s8 {d12}, [%[input_ptr4]], r0
\n
"
"vmovl.s8 q8, d10
\n
"
"vmlal.s16 q12, d16, %f[ker2][3]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %f[ker1][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %f[ker1][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %f[ker1][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %f[ker1][3]
\n
"
"vmovl.s8 q8, d12
\n
"
"vmlal.s16 q12, d16, %e[ker3][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker2][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker2][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker2][3]
\n
"
// restore output
"vst1.32 {d24-d25}, [%[output_ptr0]]!
\n
"
"sub %[remain], #4
\n
"
"start_remain_%=:
\n
"
"cmp %[remain], #0
\n
"
"ble end_%=
\n
"
"mov r0, %[remain]
\n
"
"vld1.s8 {d10}, [%[input_ptr0]], r0
\n
"
"vld1.s8 {d12}, [%[input_ptr1]], r0
\n
"
"vld1.s8 {d14}, [%[input_ptr2]], r0
\n
"
"vmovl.s8 q8, d10
\n
"
"vmull.s16 q12, d16, %f[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker0][3]
\n
"
"vmovl.s8 q8, d12
\n
"
"vmlal.s16 q12, d16, %f[ker2][1]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %f[ker0][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %f[ker0][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %f[ker0][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %f[ker0][3]
\n
"
"vmovl.s8 q8, d14
\n
"
"vmlal.s16 q12, d16, %f[ker2][2]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker1][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker1][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker1][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker1][3]
\n
"
"vld1.s8 {d10}, [%[input_ptr3]], r0
\n
"
"vld1.s8 {d12}, [%[input_ptr4]], r0
\n
"
"vmovl.s8 q8, d10
\n
"
"vmlal.s16 q12, d16, %f[ker2][3]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %f[ker1][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %f[ker1][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %f[ker1][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %f[ker1][3]
\n
"
"vmovl.s8 q8, d12
\n
"
"vmlal.s16 q12, d16, %e[ker3][0]
\n
"
"vext.s16 q10, q8, q9, #1
\n
"
"vmlal.s16 q12, d20, %e[ker2][0]
\n
"
"vext.s16 q10, q8, q9, #2
\n
"
"vmlal.s16 q12, d20, %e[ker2][1]
\n
"
"vext.s16 q10, q8, q9, #3
\n
"
"vmlal.s16 q12, d20, %e[ker2][2]
\n
"
"vext.s16 q10, q8, q9, #4
\n
"
"vmlal.s16 q12, d20, %e[ker2][3]
\n
"
"cmp %[remain], #2
\n
"
"blt store_1h1w_%=
\n
"
"vst1.32 {d24}, [%[output_ptr0]]!
\n
"
"cmp %[remain], #3
\n
"
"blt end_%=
\n
"
"vst1.32 {d25[0]}, [%[output_ptr0]]!
\n
"
"b end_%=
\n
"
"store_1h1w_%=:
\n
"
"vst1.32 {d24[0]}, [%[output_ptr0]]!
\n
"
"end_%=:
\n
"
:
[
input_ptr0
]
"+r"
(
input_ptr0
),
[
input_ptr1
]
"+r"
(
input_ptr1
),
[
input_ptr2
]
"+r"
(
input_ptr2
),
[
input_ptr3
]
"+r"
(
input_ptr3
),
[
input_ptr4
]
"+r"
(
input_ptr4
),
[
output_ptr0
]
"+r"
(
output_ptr0
),
[
loop
]
"+r"
(
loop
),
[
remain
]
"+r"
(
w_remain
)
:
[
ker0
]
"w"
(
_ker0
),
[
ker1
]
"w"
(
_ker1
),
[
ker2
]
"w"
(
_ker2
),
[
ker3
]
"w"
(
_ker3
)
:
"cc"
,
"memory"
,
"q4"
,
"q5"
,
"q6"
,
"q7"
,
"q8"
,
"q9"
,
"q10"
,
"q11"
,
"q12"
,
"q13"
,
"q14"
,
"q15"
,
"r0"
);
// pad right
if
(
padding_w
)
{
int16x4_t
row0
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr0
)));
int16x4_t
row1
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr1
)));
int16x4_t
row2
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr2
)));
int16x4_t
row3
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr3
)));
int16x4_t
row4
=
vget_low_s16
(
vmovl_s8
(
vld1_s8
(
input_ptr4
)));
int16x4_t
zero
=
vdup_n_s16
(
0
);
int32x4_t
acc
;
for
(
int
w
=
valid_w_end
;
w
<
output_w
;
++
w
)
{
int
padding
=
w
+
5
-
(
padding_w
+
input_w
);
if
(
padding
>=
5
)
{
*
output_ptr0
=
0
;
}
else
{
int
iw
=
w
-
valid_w_end
;
int32_t
sum0
=
input_ptr0
[
iw
]
*
filter_ptr0
[
0
]
+
input_ptr1
[
iw
]
*
filter_ptr1
[
0
]
+
input_ptr2
[
iw
]
*
filter_ptr2
[
0
]
+
input_ptr3
[
iw
]
*
filter_ptr3
[
0
]
+
input_ptr4
[
iw
]
*
filter_ptr4
[
0
];
row0
=
vext_s16
(
row0
,
zero
,
1
);
row1
=
vext_s16
(
row1
,
zero
,
1
);
row2
=
vext_s16
(
row2
,
zero
,
1
);
row3
=
vext_s16
(
row3
,
zero
,
1
);
row4
=
vext_s16
(
row4
,
zero
,
1
);
acc
=
vmull_s16
(
row0
,
_ker
[
0
]);
acc
=
vmlal_s16
(
acc
,
row1
,
_ker
[
1
]);
acc
=
vmlal_s16
(
acc
,
row2
,
_ker
[
2
]);
acc
=
vmlal_s16
(
acc
,
row3
,
_ker
[
3
]);
acc
=
vmlal_s16
(
acc
,
row4
,
_ker
[
4
]);
int32x2_t
sum
=
vpadd_s32
(
vget_low_s32
(
acc
),
vget_high_s32
(
acc
));
sum
=
vpadd_s32
(
sum
,
sum
);
sum0
+=
vget_lane_s32
(
sum
,
0
);
*
output_ptr0
=
sum0
;
}
output_ptr0
++
;
}
}
}
// pad bottom
for
(
int
h
=
valid_h_end
;
h
<
output_h
;
++
h
)
{
DepthwiseConv5x5NormalRow
<
1
,
1
>
(
input_ptr
,
filter_ptr
,
h
,
input_h
,
input_w
,
padding_h
,
padding_w
,
output_w
,
output_ptr
,
_ker
,
kernel
);
}
}
}
template
<
>
void
DepthwiseConv5x5S2
<
int8_t
,
int32_t
>
(
const
framework
::
Tensor
&
input
,
const
framework
::
Tensor
&
filter
,
const
std
::
vector
<
int
>
&
paddings
,
framework
::
Tensor
*
output
)
{}
}
// namespace math
}
// namespace operators
}
// namespace paddle_mobile
#endif // __ARM_NEON__
src/operators/math/gemm.cpp
浏览文件 @
91b8d2be
...
...
@@ -3150,9 +3150,11 @@ void Gemm::SgemmWithPRelu(int m, int n, int k, const float *A, int lda,
void
Gemm
::
Sgemm_omp
(
int
m
,
int
n
,
int
k
,
float
alpha
,
const
float
*
A
,
int
lda
,
const
float
*
B
,
int
ldb
,
float
beta
,
float
*
C
,
int
ldc
,
bool
relu
,
float
*
bias
)
{
#ifndef __aarch64__
if
(
m
==
1
&&
bias
==
nullptr
)
{
return
VectorKernel
(
m
,
n
,
k
,
alpha
,
A
,
lda
,
B
,
ldb
,
beta
,
C
,
ldc
,
relu
);
}
#endif // __aarch64__
#ifdef _OPENMP
int
max_threads
=
omp_get_max_threads
();
#else
...
...
src/operators/math/pooling2x2.cpp
浏览文件 @
91b8d2be
...
...
@@ -19,6 +19,8 @@ limitations under the License. */
#include <arm_neon.h>
#include "operators/math/pooling.h"
// TODO(hjchen2): Optimize Pooling2x2NormalRow and use inline assembly
namespace
paddle_mobile
{
namespace
operators
{
namespace
math
{
...
...
@@ -60,7 +62,6 @@ struct Pooling2x2NormalRowLoadInput<P, 2> {
}
};
// TODO(hjchen2): To optimize Pooling2x2NormalRow
template
<
PoolingType
P
,
int
Stride
>
inline
void
Pooling2x2NormalRow
(
const
float
*
input
,
const
int
h_output
,
const
int
input_h
,
const
int
input_w
,
...
...
src/operators/op_param.h
浏览文件 @
91b8d2be
...
...
@@ -424,10 +424,10 @@ class ConvParam : public OpParam {
EXEC_DEPTHWISE3x3_FLOAT
,
EXEC_WINOGRAD3X3_FLOAT
,
EXEC_WINOGRAD5X5_FLOAT
,
EXEC_DEPTHWISE5x5S1_FLOAT
,
EXEC_DEPTHWISE5x5S2_FLOAT
,
EXEC_DEPTHWISE5x5_FLOAT
,
EXEC_GEMM_INT8
,
EXEC_DEPTHWISE3x3_INT8
,
EXEC_DEPTHWISE5x5_INT8
,
};
ExecMode
&
ExecMode
()
const
{
return
exec_mode_
;
}
...
...
test/operators/test_conv_op.cpp
浏览文件 @
91b8d2be
...
...
@@ -165,14 +165,12 @@ int TestConvOp(int in_channels, int in_height, int in_width, int out_channels,
auto
filter
=
filter_var
->
template
GetMutable
<
framework
::
LoDTensor
>();
SetupTensor
<
Itype
>
(
filter
,
filter_shape
,
-
20
,
20
);
for
(
int
i
=
0
;
i
<
input
->
numel
();
++
i
)
{
DLOG
<<
"input["
<<
i
<<
"] = "
<<
static_cast
<
int
>
(
input
->
data
<
int8_t
>
()[
i
]);
}
for
(
int
i
=
0
;
i
<
filter
->
numel
();
++
i
)
{
DLOG
<<
"filter["
<<
i
<<
"] = "
<<
static_cast
<
int
>
(
filter
->
data
<
int8_t
>
()[
i
]);
}
// for (int i = 0; i < input->numel(); ++i) {
// DLOG << "input[" << i << "] = " << float(input->data<Itype>()[i]);
// }
// for (int i = 0; i < filter->numel(); ++i) {
// DLOG << "filter[" << i << "] = " << float(filter->data<Itype>()[i]);
// }
auto
output_var
=
scope
.
get
()
->
Var
(
"output"
);
framework
::
AttributeMap
attrs
;
...
...
@@ -198,18 +196,12 @@ int TestConvOp(int in_channels, int in_height, int in_width, int out_channels,
// (ts_end.tv_nsec - ts_begin.tv_nsec) / 1e6;
// LOG(kLOG_INFO) << "elapsed: " << elapsed / 10.0 << " ms";
int
kernel_extent_h
=
dilation_h
*
(
kernel_h
-
1
)
+
1
;
int
kernel_extent_w
=
dilation_w
*
(
kernel_w
-
1
)
+
1
;
int
output_h
=
(
input_h
+
2
*
pad_h
-
kernel_extent_h
)
/
stride_h
+
1
;
int
output_w
=
(
input_w
+
2
*
pad_w
-
kernel_extent_w
)
/
stride_w
+
1
;
auto
output_shape
=
framework
::
make_ddim
(
std
::
vector
<
int
>
({
batch_size
,
output_c
,
output_h
,
output_w
}));
// compare results
auto
*
output
=
output_var
->
template
Get
<
framework
::
LoDTensor
>();
framework
::
Tensor
output_cmp
;
output_cmp
.
mutable_data
<
Otype
>
(
output
_shape
);
output_cmp
.
mutable_data
<
Otype
>
(
output
->
dims
()
);
conv2d
<
Itype
,
Otype
>
(
input
,
filter
,
attrs
,
&
output_cmp
);
// compare results
auto
output
=
output_var
->
template
Get
<
framework
::
LoDTensor
>();
const
Otype
*
output_data
=
output
->
data
<
Otype
>
();
Otype
*
output_cmp_data
=
output_cmp
.
data
<
Otype
>
();
for
(
int
i
=
0
;
i
<
output
->
numel
();
++
i
)
{
...
...
@@ -285,96 +277,39 @@ int main(int argc, char *argv[]) {
paddle_mobile
::
TestConvOp
<
int8_t
,
int32_t
,
3
,
5
,
2
>
(
in_channels
,
in_height
,
in_width
,
out_channels
,
groups
);
// // kernel = 7, pad = 0, stride = 2
// LOG(paddle_mobile::kLOG_INFO) << "int8, kernel=7, pad=0, stride=2";
// paddle_mobile::TestConvOp<int8_t, int32_t, 7, 0, 2>(in_channels,
// in_height,
// in_width,
// out_channels, groups);
// // kernel = 7, pad = 1, stride = 2
// LOG(paddle_mobile::kLOG_INFO) << "int8, kernel=7, pad=1, stride=2";
// paddle_mobile::TestConvOp<int8_t, int32_t, 7, 1, 2>(in_channels,
// in_height,
// in_width,
// out_channels, groups);
// // kernel = 7, pad = 3, stride = 2
// LOG(paddle_mobile::kLOG_INFO) << "int8, kernel=7, pad=3, stride=2";
// paddle_mobile::TestConvOp<int8_t, int32_t, 7, 3, 2>(in_channels,
// in_height,
// in_width,
// out_channels, groups);
// // kernel = 7, pad = 0, stride = 1
// LOG(paddle_mobile::kLOG_INFO) << "int8, kernel=7, pad=0, stride=1";
// paddle_mobile::TestConvOp<int8_t, int32_t, 7, 0, 1>(in_channels,
// in_height,
// in_width,
// out_channels, groups);
// // kernel = 7, pad = 1, stride = 1
// LOG(paddle_mobile::kLOG_INFO) << "int8, kernel=7, pad=1, stride=1";
// paddle_mobile::TestConvOp<int8_t, int32_t, 7, 1, 1>(in_channels,
// in_height,
// in_width,
// out_channels, groups);
// // kernel = 7, pad = 3, stride = 1
// LOG(paddle_mobile::kLOG_INFO) << "int8, kernel=7, pad=3, stride=1";
// paddle_mobile::TestConvOp<int8_t, int32_t, 7, 3, 1>(in_channels,
// in_height,
// in_width,
// out_channels, groups);
// // kernel = 7, pad = 5, stride = 3
// LOG(paddle_mobile::kLOG_INFO) << "int8, kernel=7, pad=5, stride=3";
// paddle_mobile::TestConvOp<int8_t, int32_t, 7, 5, 3>(in_channels,
// in_height,
// in_width,
// out_channels, groups);
// // kernel = 7, pad = 3, stride = 4
// LOG(paddle_mobile::kLOG_INFO) << "int8, kernel=7, pad=3, stride=4";
// paddle_mobile::TestConvOp<int8_t, int32_t, 7, 3, 4>(in_channels,
// in_height,
// in_width,
// out_channels, groups);
// // kernel = 3, pad = 0, stride = 1
// LOG(paddle_mobile::kLOG_INFO) << "int8, kernel=3, pad=0, stride=1";
// paddle_mobile::TestConvOp<int8_t, int32_t, 3, 0, 1>(in_channels,
// in_height,
// in_width,
// out_channels, groups);
// // kernel = 3, pad = 0, stride = 1
// LOG(paddle_mobile::kLOG_INFO) << "float, kernel=3, pad=0, stride=1";
// paddle_mobile::TestConvOp<float, float, 3, 0, 1>(in_channels, in_height,
// in_width, out_channels,
// groups);
// // kernel = 3, pad = 1, stride = 1
// LOG(paddle_mobile::kLOG_INFO) << "int8, kernel=3, pad=1, stride=1";
// paddle_mobile::TestConvOp<int8_t, int32_t, 3, 1, 1>(in_channels,
// in_height,
// in_width,
// out_channels, groups);
// // kernel = 3, pad = 1, stride = 1
// LOG(paddle_mobile::kLOG_INFO) << "float, kernel=3, pad=1, stride=1";
// paddle_mobile::TestConvOp<float, float, 3, 1, 1>(in_channels, in_height,
// in_width, out_channels,
// groups);
// // kernel = 5, pad = 0, stride = 1
// LOG(paddle_mobile::kLOG_INFO) << "int8, kernel=5, pad=0, stride=1";
// paddle_mobile::TestConvOp<int8_t, int32_t, 5, 0, 1>(in_channels,
// in_height,
// in_width,
// out_channels, groups);
// // kernel = 5, pad = 0, stride = 1
// LOG(paddle_mobile::kLOG_INFO) << "float, kernel=5, pad=0, stride=1";
// paddle_mobile::TestConvOp<float, float, 5, 0, 1>(in_channels, in_height,
// in_width, out_channels,
// groups);
// // kernel = 5, pad = 2, stride = 1
// LOG(paddle_mobile::kLOG_INFO) << "int8, kernel=5, pad=2, stride=1";
// paddle_mobile::TestConvOp<int8_t, int32_t, 5, 2, 1>(in_channels,
// in_height,
// in_width,
// out_channels, groups);
// // kernel = 5, pad = 2, stride = 1
// LOG(paddle_mobile::kLOG_INFO) << "float, kernel=5, pad=2, stride=1";
// paddle_mobile::TestConvOp<float, float, 5, 2, 1>(in_channels, in_height,
// in_width, out_channels,
// groups);
// kernel = 5, pad = 0, stride = 1
LOG
(
paddle_mobile
::
kLOG_INFO
)
<<
"float, kernel=5, pad=0, stride=1"
;
paddle_mobile
::
TestConvOp
<
float
,
float
,
5
,
0
,
1
>
(
in_channels
,
in_height
,
in_width
,
out_channels
,
groups
);
// kernel = 5, pad = 1, stride = 1
LOG
(
paddle_mobile
::
kLOG_INFO
)
<<
"float, kernel=5, pad=1, stride=1"
;
paddle_mobile
::
TestConvOp
<
float
,
float
,
5
,
1
,
1
>
(
in_channels
,
in_height
,
in_width
,
out_channels
,
groups
);
// kernel = 5, pad = 2, stride = 1
LOG
(
paddle_mobile
::
kLOG_INFO
)
<<
"float, kernel=5, pad=2, stride=1"
;
paddle_mobile
::
TestConvOp
<
float
,
float
,
5
,
2
,
1
>
(
in_channels
,
in_height
,
in_width
,
out_channels
,
groups
);
// kernel = 5, pad = 5, stride = 1
LOG
(
paddle_mobile
::
kLOG_INFO
)
<<
"float, kernel=5, pad=5, stride=1"
;
paddle_mobile
::
TestConvOp
<
float
,
float
,
5
,
5
,
1
>
(
in_channels
,
in_height
,
in_width
,
out_channels
,
groups
);
// kernel = 5, pad = 0, stride = 1
LOG
(
paddle_mobile
::
kLOG_INFO
)
<<
"int8, kernel=5, pad=0, stride=1"
;
paddle_mobile
::
TestConvOp
<
int8_t
,
int32_t
,
5
,
0
,
1
>
(
in_channels
,
in_height
,
in_width
,
out_channels
,
groups
);
// kernel = 5, pad = 1, stride = 1
LOG
(
paddle_mobile
::
kLOG_INFO
)
<<
"int8, kernel=5, pad=1, stride=1"
;
paddle_mobile
::
TestConvOp
<
int8_t
,
int32_t
,
5
,
1
,
1
>
(
in_channels
,
in_height
,
in_width
,
out_channels
,
groups
);
// kernel = 5, pad = 2, stride = 1
LOG
(
paddle_mobile
::
kLOG_INFO
)
<<
"int8, kernel=5, pad=2, stride=1"
;
paddle_mobile
::
TestConvOp
<
int8_t
,
int32_t
,
5
,
2
,
1
>
(
in_channels
,
in_height
,
in_width
,
out_channels
,
groups
);
// kernel = 5, pad = 5, stride = 1
LOG
(
paddle_mobile
::
kLOG_INFO
)
<<
"int8, kernel=5, pad=5, stride=1"
;
paddle_mobile
::
TestConvOp
<
int8_t
,
int32_t
,
5
,
5
,
1
>
(
in_channels
,
in_height
,
in_width
,
out_channels
,
groups
);
return
0
;
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录