Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
马赛克医院
Paddle
提交
be2ac9cc
P
Paddle
项目概览
马赛克医院
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
be2ac9cc
编写于
3月 29, 2020
作者:
W
Wojciech Uss
提交者:
GitHub
3月 29, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
separated QAT1 and QAT2 (#23284)
上级
58615a62
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
283 addition
and
260 deletion
+283
-260
python/paddle/fluid/contrib/slim/quantization/__init__.py
python/paddle/fluid/contrib/slim/quantization/__init__.py
+6
-3
python/paddle/fluid/contrib/slim/quantization/qat2_int8_mkldnn_pass.py
.../fluid/contrib/slim/quantization/qat2_int8_mkldnn_pass.py
+1
-257
python/paddle/fluid/contrib/slim/quantization/qat_int8_mkldnn_pass.py
...e/fluid/contrib/slim/quantization/qat_int8_mkldnn_pass.py
+276
-0
未找到文件。
python/paddle/fluid/contrib/slim/quantization/__init__.py
浏览文件 @
be2ac9cc
...
...
@@ -20,12 +20,15 @@ from . import quantization_strategy
from
.quantization_strategy
import
*
from
.
import
mkldnn_post_training_strategy
from
.mkldnn_post_training_strategy
import
*
from
.
import
quantization_mkldnn_pass
from
.quantization_mkldnn_pass
import
*
from
.
import
qat_int8_mkldnn_pass
from
.qat_int8_mkldnn_pass
import
*
from
.
import
qat2_int8_mkldnn_pass
from
.qat2_int8_mkldnn_pass
import
*
from
.
import
post_training_quantization
from
.post_training_quantization
import
*
__all__
=
quantization_pass
.
__all__
+
quantization_strategy
.
__all__
__all__
+=
mkldnn_post_training_strategy
.
__all__
__all__
+=
quantization_mkldnn_pass
.
__all__
__all__
+=
qat_int8_mkldnn_pass
.
__all__
__all__
+=
qat2_int8_mkldnn_pass
.
__all__
__all__
+=
post_training_quantization
.
__all__
python/paddle/fluid/contrib/slim/quantization/q
uantization
_mkldnn_pass.py
→
python/paddle/fluid/contrib/slim/quantization/q
at2_int8
_mkldnn_pass.py
浏览文件 @
be2ac9cc
...
...
@@ -17,263 +17,7 @@ from .... import core
from
....framework
import
IrGraph
from
....framework
import
IrNode
__all__
=
[
'QatInt8MkldnnPass'
,
'Qat2Int8MkldnnPass'
]
class
QatInt8MkldnnPass
(
object
):
"""
Convert QuantizationFreezePass generated IrGraph to MKL-DNN supported INT8
IrGraph. Following transformations did in this pass:
1. Convert int8 range weights with float32 data type, which are generated by
the QuantizationFreezePass, to float32 range weights with float32 data type
by using the corresponding scales. This conversion is because MKL-DNN INT8
conv2d kernel and mul kernel now only support float32 weights input, hence
weights quantization will happen inside the conv2d and mul INT8 kernel.
2. Create the new conv2d or mul op with the converted weights and link its output
to fake_dequantize_abs_max op's output and set conv2d's attribute "force_fp32
_output" as true
3. Transform fake_quantize_xx op to quantize op
4. Remove fake_dequantize_abs_max op
"""
def
__init__
(
self
,
_scope
=
None
,
_place
=
None
):
"""
Args:
scope(fluid.Scope): scope is used to initialize the new parameters.
place(fluid.CPUPlace): place is used to initialize the new parameters.
Examples:
.. code-block:: python
# The original graph will be rewrite.
import paddle.fluid as fluid
from paddle.fluid.contrib.slim.quantization
\
import QatInt8MkldnnPass
from paddle.fluid.framework import IrGraph
from paddle.fluid import core
graph = IrGraph(core.Graph(fluid.Program().desc), for_test=False)
place = fluid.CPUPlace()
mkldnn_pass = QatInt8MkldnnPass(fluid.global_scope(),
place)
mkldnn_pass.apply(graph)
"""
self
.
_scope
=
_scope
self
.
_place
=
_place
self
.
_quantize_type
=
[
'fake_quantize_moving_average_abs_max'
,
'fake_quantize_range_abs_max'
]
self
.
_dequantize_type
=
[
'fake_dequantize_max_abs'
]
self
.
_quantize_dequantize_type
=
[
'fake_quantize_dequantize_moving_average_abs_max'
]
self
.
_quantizable_ops
=
[
'conv2d'
,
'depthwise_conv2d'
,
'mul'
]
self
.
_conv_ops
=
[
'conv2d'
,
'depthwise_conv2d'
]
self
.
_pool_ops
=
[
'pool2d'
]
self
.
_in_scale
=
{}
self
.
_max_range
=
{}
self
.
_new_output
=
{}
self
.
_s8_max
=
127
def
apply
(
self
,
graph
):
"""
Quantize the graph for running MKL-DNN INT8 inference. According
to activation quantization type, the graph will transform fake
quantize ops to quantize ops and remove the fake dequantize ops.
Args:
graph(IrGraph): the applied graph.
"""
assert
isinstance
(
graph
,
IrGraph
),
'graph must be the instance of IrGraph.'
ops
=
graph
.
all_op_nodes
()
persistable_vars
=
[
p
.
name
()
for
p
in
graph
.
all_persistable_nodes
()]
# Collect the _in_scales and _max_range to calculate the new scales for MKL-DNN
# INT8 conv2d and mul
for
op_node
in
ops
:
if
op_node
.
name
()
in
self
.
_dequantize_type
:
input_name
=
op_node
.
input
(
"X"
)[
0
]
scale_name
=
op_node
.
input
(
"Scale"
)[
0
]
self
.
_in_scale
[
input_name
]
=
self
.
_load_param
(
self
.
_scope
,
scale_name
)[
0
]
self
.
_max_range
[
input_name
]
=
op_node
.
op
().
attr
(
"max_range"
)
self
.
_new_output
[
input_name
]
=
op_node
.
output
(
"Out"
)[
0
]
if
op_node
.
name
()
in
self
.
_quantize_dequantize_type
:
inputs
=
op_node
.
op
().
input_names
()
attrs
=
op_node
.
op
().
attr_names
()
input_name
=
op_node
.
input
(
"X"
)[
0
]
scale_name
=
op_node
.
input
(
"InScale"
)[
0
]
self
.
_in_scale
[
input_name
]
=
self
.
_load_param
(
self
.
_scope
,
scale_name
)[
0
]
# self._max_range[input_name] = op_node.op().attr("max_range")
self
.
_new_output
[
input_name
]
=
op_node
.
output
(
"Out"
)[
0
]
for
op_node
in
ops
:
if
op_node
.
name
()
in
self
.
_quantizable_ops
:
if
op_node
.
name
()
in
self
.
_conv_ops
:
self
.
_transform_to_conv_mkldnn
(
graph
,
op_node
)
elif
op_node
.
name
()
in
self
.
_pool_ops
:
self
.
_transform_to_pool_mkldnn
(
graph
,
op_node
)
else
:
self
.
_transform_to_mul_mkldnn
(
graph
,
op_node
)
elif
op_node
.
name
()
in
self
.
_quantize_type
:
self
.
_transform_to_quantize_mkldnn
(
graph
,
op_node
)
elif
op_node
.
name
()
in
self
.
_dequantize_type
:
self
.
_remove_fake_dequantize_op
(
graph
,
op_node
)
self
.
_remove_unused_var_nodes
(
graph
)
return
graph
def
_transform_to_pool_mkldnn
(
self
,
graph
,
op
):
output_name
=
op
.
output
(
"Out"
)[
0
]
input_name
=
op
.
input
(
"X"
)[
0
]
def
_transform_to_conv_mkldnn
(
self
,
graph
,
op_node
):
weight_name
=
op_node
.
input
(
"Filter"
)[
0
]
output_name
=
op_node
.
output
(
"Output"
)[
0
]
# Convert int8 range weights to fp32 range weights
weight
=
self
.
_load_param
(
self
.
_scope
,
weight_name
)
w_fp32
=
np
.
divide
(
np
.
multiply
(
weight
,
self
.
_s8_max
),
self
.
_max_range
[
output_name
])
w_fp32
=
w_fp32
.
reshape
(
weight
.
shape
)
self
.
_restore_var
(
weight_name
,
w_fp32
)
input_var_node
=
graph
.
_find_node_by_name
(
op_node
.
inputs
,
op_node
.
input
(
"Input"
)[
0
])
weight_var_node
=
graph
.
_find_node_by_name
(
op_node
.
inputs
,
weight_name
)
# Set fake_dequantize_abs_max's output as new output of conv2d
output_var_node
=
graph
.
_find_node_by_name
(
graph
.
all_var_nodes
(),
self
.
_new_output
[
output_name
])
attrs
=
{
name
:
op_node
.
op
().
attr
(
name
)
for
name
in
op_node
.
op
().
attr_names
()
}
conv_op_node
=
graph
.
create_op_node
(
op_type
=
'conv2d'
,
attrs
=
attrs
,
inputs
=
{
'Input'
:
input_var_node
,
'Filter'
:
weight_var_node
},
outputs
=
{
'Output'
:
output_var_node
})
# Based on the QAT's scales to calculate the scales of MKL-DNN INT8 conv2d
scale_in
=
self
.
_s8_max
/
self
.
_in_scale
[
output_name
]
scale_w
=
[]
scale_w
=
[
self
.
_max_range
[
output_name
]
/
self
.
_s8_max
]
conv_op_node
.
set_attr
(
"Scale_weights"
,
scale_w
)
conv_op_node
.
set_attr
(
"Scale_in"
,
scale_in
)
conv_op_node
.
set_attr
(
"Scale_out"
,
1.0
)
conv_op_node
.
set_attr
(
"use_mkldnn"
,
1
)
conv_op_node
.
set_attr
(
"force_fp32_output"
,
1
)
graph
.
link_to
(
input_var_node
,
conv_op_node
)
graph
.
link_to
(
weight_var_node
,
conv_op_node
)
graph
.
link_to
(
conv_op_node
,
output_var_node
)
graph
.
safe_remove_nodes
(
op_node
)
def
_transform_to_mul_mkldnn
(
self
,
graph
,
op_node
):
# For MKL-DNN INT8 mul, input Y should be the weights
weight_name
=
op_node
.
input
(
"Y"
)[
0
]
output_name
=
op_node
.
output
(
"Out"
)[
0
]
# Convert int8 range weights to fp32 range weights
weight
=
self
.
_load_param
(
self
.
_scope
,
weight_name
)
w_fp32
=
np
.
divide
(
np
.
multiply
(
weight
,
self
.
_s8_max
),
self
.
_max_range
[
output_name
])
w_fp32
=
w_fp32
.
reshape
(
weight
.
shape
)
self
.
_restore_var
(
weight_name
,
w_fp32
)
input_var_node
=
graph
.
_find_node_by_name
(
op_node
.
inputs
,
op_node
.
input
(
"X"
)[
0
])
weight_var_node
=
graph
.
_find_node_by_name
(
op_node
.
inputs
,
weight_name
)
# Set fake_dequantize_abs_max's output as new output of mul
output_var_node
=
graph
.
_find_node_by_name
(
graph
.
all_var_nodes
(),
self
.
_new_output
[
output_name
])
attrs
=
{
name
:
op_node
.
op
().
attr
(
name
)
for
name
in
op_node
.
op
().
attr_names
()
}
mul_op_node
=
graph
.
create_op_node
(
op_type
=
'mul'
,
attrs
=
attrs
,
inputs
=
{
'X'
:
input_var_node
,
'Y'
:
weight_var_node
},
outputs
=
{
'Out'
:
output_var_node
})
# Based on the QAT's scales to calculate MKL-DNN INT8 mul's scales
scale_in
=
self
.
_s8_max
/
self
.
_in_scale
[
output_name
]
scale_w
=
[]
scale_w
=
[
self
.
_max_range
[
output_name
]
/
self
.
_s8_max
]
mul_op_node
.
set_attr
(
"scale_y"
,
scale_w
)
mul_op_node
.
set_attr
(
"scale_x"
,
scale_in
)
mul_op_node
.
set_attr
(
"scale_out"
,
1.0
)
mul_op_node
.
set_attr
(
"use_mkldnn"
,
1
)
mul_op_node
.
set_attr
(
"force_fp32_output"
,
1
)
graph
.
link_to
(
input_var_node
,
mul_op_node
)
graph
.
link_to
(
weight_var_node
,
mul_op_node
)
graph
.
link_to
(
mul_op_node
,
output_var_node
)
graph
.
safe_remove_nodes
(
op_node
)
def
_transform_to_quantize_mkldnn
(
self
,
graph
,
op_node
):
"""
Transform fake_quantize_xx op to quantize mkldnn op in the graph.
"""
input_var_node
=
graph
.
_find_node_by_name
(
op_node
.
inputs
,
op_node
.
input
(
"X"
)[
0
])
output_var_node
=
graph
.
_find_node_by_name
(
op_node
.
outputs
,
op_node
.
output
(
"Out"
)[
0
])
scale_in
=
self
.
_s8_max
/
self
.
_load_param
(
self
.
_scope
,
op_node
.
input
(
"InScale"
)[
0
])[
0
]
quant_op_node
=
graph
.
create_op_node
(
op_type
=
'quantize'
,
attrs
=
{
'data_format'
:
'MKLDNNLAYOUT'
,
'use_mkldnn'
:
1
,
'Scale'
:
scale_in
,
'is_negative_input'
:
1
},
inputs
=
{
'Input'
:
input_var_node
},
outputs
=
{
'Output'
:
output_var_node
})
graph
.
link_to
(
input_var_node
,
quant_op_node
)
graph
.
link_to
(
quant_op_node
,
output_var_node
)
graph
.
safe_remove_nodes
(
op_node
)
def
_remove_fake_dequantize_op
(
self
,
graph
,
op_node
):
input_var_node
=
graph
.
_find_node_by_name
(
op_node
.
inputs
,
op_node
.
input
(
"X"
)[
0
])
graph
.
safe_remove_nodes
(
op_node
)
def
_load_param
(
self
,
scope
,
param_name
):
return
np
.
array
(
scope
.
find_var
(
param_name
).
get_tensor
())
def
_restore_var
(
self
,
name
,
array
):
tensor
=
self
.
_scope
.
find_var
(
name
).
get_tensor
()
tensor
.
set
(
array
,
self
.
_place
)
def
_remove_unused_var_nodes
(
self
,
graph
):
all_used_vars
=
set
()
ops
=
graph
.
all_op_nodes
()
for
op_node
in
ops
:
for
input_node
in
op_node
.
inputs
:
all_used_vars
.
add
(
input_node
)
for
output_node
in
op_node
.
outputs
:
all_used_vars
.
add
(
output_node
)
all_used_vars
=
{
n
.
node
for
n
in
all_used_vars
}
all_unused_vars
=
{
n
for
n
in
filter
(
lambda
node
:
node
.
node
not
in
all_used_vars
,
graph
.
all_var_nodes
())
}
graph
.
safe_remove_nodes
(
all_unused_vars
)
__all__
=
[
'Qat2Int8MkldnnPass'
]
class
Qat2Int8MkldnnPass
(
object
):
...
...
python/paddle/fluid/contrib/slim/quantization/qat_int8_mkldnn_pass.py
0 → 100644
浏览文件 @
be2ac9cc
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import
numpy
as
np
from
....
import
core
from
....framework
import
IrGraph
from
....framework
import
IrNode
__all__
=
[
'QatInt8MkldnnPass'
]
class
QatInt8MkldnnPass
(
object
):
"""
Convert QuantizationFreezePass generated IrGraph to MKL-DNN supported INT8
IrGraph. Following transformations did in this pass:
1. Convert int8 range weights with float32 data type, which are generated by
the QuantizationFreezePass, to float32 range weights with float32 data type
by using the corresponding scales. This conversion is because MKL-DNN INT8
conv2d kernel and mul kernel now only support float32 weights input, hence
weights quantization will happen inside the conv2d and mul INT8 kernel.
2. Create the new conv2d or mul op with the converted weights and link its output
to fake_dequantize_abs_max op's output and set conv2d's attribute "force_fp32
_output" as true
3. Transform fake_quantize_xx op to quantize op
4. Remove fake_dequantize_abs_max op
"""
def
__init__
(
self
,
_scope
=
None
,
_place
=
None
):
"""
Args:
scope(fluid.Scope): scope is used to initialize the new parameters.
place(fluid.CPUPlace): place is used to initialize the new parameters.
Examples:
.. code-block:: python
# The original graph will be rewrite.
import paddle.fluid as fluid
from paddle.fluid.contrib.slim.quantization
\
import QatInt8MkldnnPass
from paddle.fluid.framework import IrGraph
from paddle.fluid import core
graph = IrGraph(core.Graph(fluid.Program().desc), for_test=False)
place = fluid.CPUPlace()
mkldnn_pass = QatInt8MkldnnPass(fluid.global_scope(),
place)
mkldnn_pass.apply(graph)
"""
self
.
_scope
=
_scope
self
.
_place
=
_place
self
.
_quantize_type
=
[
'fake_quantize_moving_average_abs_max'
,
'fake_quantize_range_abs_max'
]
self
.
_dequantize_type
=
[
'fake_dequantize_max_abs'
]
self
.
_quantize_dequantize_type
=
[
'fake_quantize_dequantize_moving_average_abs_max'
]
self
.
_quantizable_ops
=
[
'conv2d'
,
'depthwise_conv2d'
,
'mul'
]
self
.
_conv_ops
=
[
'conv2d'
,
'depthwise_conv2d'
]
self
.
_pool_ops
=
[
'pool2d'
]
self
.
_in_scale
=
{}
self
.
_max_range
=
{}
self
.
_new_output
=
{}
self
.
_s8_max
=
127
def
apply
(
self
,
graph
):
"""
Quantize the graph for running MKL-DNN INT8 inference. According
to activation quantization type, the graph will transform fake
quantize ops to quantize ops and remove the fake dequantize ops.
Args:
graph(IrGraph): the applied graph.
"""
assert
isinstance
(
graph
,
IrGraph
),
'graph must be the instance of IrGraph.'
ops
=
graph
.
all_op_nodes
()
persistable_vars
=
[
p
.
name
()
for
p
in
graph
.
all_persistable_nodes
()]
# Collect the _in_scales and _max_range to calculate the new scales for MKL-DNN
# INT8 conv2d and mul
for
op_node
in
ops
:
if
op_node
.
name
()
in
self
.
_dequantize_type
:
input_name
=
op_node
.
input
(
"X"
)[
0
]
scale_name
=
op_node
.
input
(
"Scale"
)[
0
]
self
.
_in_scale
[
input_name
]
=
self
.
_load_param
(
self
.
_scope
,
scale_name
)[
0
]
self
.
_max_range
[
input_name
]
=
op_node
.
op
().
attr
(
"max_range"
)
self
.
_new_output
[
input_name
]
=
op_node
.
output
(
"Out"
)[
0
]
if
op_node
.
name
()
in
self
.
_quantize_dequantize_type
:
inputs
=
op_node
.
op
().
input_names
()
attrs
=
op_node
.
op
().
attr_names
()
input_name
=
op_node
.
input
(
"X"
)[
0
]
scale_name
=
op_node
.
input
(
"InScale"
)[
0
]
self
.
_in_scale
[
input_name
]
=
self
.
_load_param
(
self
.
_scope
,
scale_name
)[
0
]
# self._max_range[input_name] = op_node.op().attr("max_range")
self
.
_new_output
[
input_name
]
=
op_node
.
output
(
"Out"
)[
0
]
for
op_node
in
ops
:
if
op_node
.
name
()
in
self
.
_quantizable_ops
:
if
op_node
.
name
()
in
self
.
_conv_ops
:
self
.
_transform_to_conv_mkldnn
(
graph
,
op_node
)
elif
op_node
.
name
()
in
self
.
_pool_ops
:
self
.
_transform_to_pool_mkldnn
(
graph
,
op_node
)
else
:
self
.
_transform_to_mul_mkldnn
(
graph
,
op_node
)
elif
op_node
.
name
()
in
self
.
_quantize_type
:
self
.
_transform_to_quantize_mkldnn
(
graph
,
op_node
)
elif
op_node
.
name
()
in
self
.
_dequantize_type
:
self
.
_remove_fake_dequantize_op
(
graph
,
op_node
)
self
.
_remove_unused_var_nodes
(
graph
)
return
graph
def
_transform_to_pool_mkldnn
(
self
,
graph
,
op
):
output_name
=
op
.
output
(
"Out"
)[
0
]
input_name
=
op
.
input
(
"X"
)[
0
]
def
_transform_to_conv_mkldnn
(
self
,
graph
,
op_node
):
weight_name
=
op_node
.
input
(
"Filter"
)[
0
]
output_name
=
op_node
.
output
(
"Output"
)[
0
]
# Convert int8 range weights to fp32 range weights
weight
=
self
.
_load_param
(
self
.
_scope
,
weight_name
)
w_fp32
=
np
.
divide
(
np
.
multiply
(
weight
,
self
.
_s8_max
),
self
.
_max_range
[
output_name
])
w_fp32
=
w_fp32
.
reshape
(
weight
.
shape
)
self
.
_restore_var
(
weight_name
,
w_fp32
)
input_var_node
=
graph
.
_find_node_by_name
(
op_node
.
inputs
,
op_node
.
input
(
"Input"
)[
0
])
weight_var_node
=
graph
.
_find_node_by_name
(
op_node
.
inputs
,
weight_name
)
# Set fake_dequantize_abs_max's output as new output of conv2d
output_var_node
=
graph
.
_find_node_by_name
(
graph
.
all_var_nodes
(),
self
.
_new_output
[
output_name
])
attrs
=
{
name
:
op_node
.
op
().
attr
(
name
)
for
name
in
op_node
.
op
().
attr_names
()
}
conv_op_node
=
graph
.
create_op_node
(
op_type
=
'conv2d'
,
attrs
=
attrs
,
inputs
=
{
'Input'
:
input_var_node
,
'Filter'
:
weight_var_node
},
outputs
=
{
'Output'
:
output_var_node
})
# Based on the QAT's scales to calculate the scales of MKL-DNN INT8 conv2d
scale_in
=
self
.
_s8_max
/
self
.
_in_scale
[
output_name
]
scale_w
=
[]
scale_w
=
[
self
.
_max_range
[
output_name
]
/
self
.
_s8_max
]
conv_op_node
.
set_attr
(
"Scale_weights"
,
scale_w
)
conv_op_node
.
set_attr
(
"Scale_in"
,
scale_in
)
conv_op_node
.
set_attr
(
"Scale_out"
,
1.0
)
conv_op_node
.
set_attr
(
"use_mkldnn"
,
1
)
conv_op_node
.
set_attr
(
"force_fp32_output"
,
1
)
graph
.
link_to
(
input_var_node
,
conv_op_node
)
graph
.
link_to
(
weight_var_node
,
conv_op_node
)
graph
.
link_to
(
conv_op_node
,
output_var_node
)
graph
.
safe_remove_nodes
(
op_node
)
def
_transform_to_mul_mkldnn
(
self
,
graph
,
op_node
):
# For MKL-DNN INT8 mul, input Y should be the weights
weight_name
=
op_node
.
input
(
"Y"
)[
0
]
output_name
=
op_node
.
output
(
"Out"
)[
0
]
# Convert int8 range weights to fp32 range weights
weight
=
self
.
_load_param
(
self
.
_scope
,
weight_name
)
w_fp32
=
np
.
divide
(
np
.
multiply
(
weight
,
self
.
_s8_max
),
self
.
_max_range
[
output_name
])
w_fp32
=
w_fp32
.
reshape
(
weight
.
shape
)
self
.
_restore_var
(
weight_name
,
w_fp32
)
input_var_node
=
graph
.
_find_node_by_name
(
op_node
.
inputs
,
op_node
.
input
(
"X"
)[
0
])
weight_var_node
=
graph
.
_find_node_by_name
(
op_node
.
inputs
,
weight_name
)
# Set fake_dequantize_abs_max's output as new output of mul
output_var_node
=
graph
.
_find_node_by_name
(
graph
.
all_var_nodes
(),
self
.
_new_output
[
output_name
])
attrs
=
{
name
:
op_node
.
op
().
attr
(
name
)
for
name
in
op_node
.
op
().
attr_names
()
}
mul_op_node
=
graph
.
create_op_node
(
op_type
=
'mul'
,
attrs
=
attrs
,
inputs
=
{
'X'
:
input_var_node
,
'Y'
:
weight_var_node
},
outputs
=
{
'Out'
:
output_var_node
})
# Based on the QAT's scales to calculate MKL-DNN INT8 mul's scales
scale_in
=
self
.
_s8_max
/
self
.
_in_scale
[
output_name
]
scale_w
=
[]
scale_w
=
[
self
.
_max_range
[
output_name
]
/
self
.
_s8_max
]
mul_op_node
.
set_attr
(
"scale_y"
,
scale_w
)
mul_op_node
.
set_attr
(
"scale_x"
,
scale_in
)
mul_op_node
.
set_attr
(
"scale_out"
,
1.0
)
mul_op_node
.
set_attr
(
"use_mkldnn"
,
1
)
mul_op_node
.
set_attr
(
"force_fp32_output"
,
1
)
graph
.
link_to
(
input_var_node
,
mul_op_node
)
graph
.
link_to
(
weight_var_node
,
mul_op_node
)
graph
.
link_to
(
mul_op_node
,
output_var_node
)
graph
.
safe_remove_nodes
(
op_node
)
def
_transform_to_quantize_mkldnn
(
self
,
graph
,
op_node
):
"""
Transform fake_quantize_xx op to quantize mkldnn op in the graph.
"""
input_var_node
=
graph
.
_find_node_by_name
(
op_node
.
inputs
,
op_node
.
input
(
"X"
)[
0
])
output_var_node
=
graph
.
_find_node_by_name
(
op_node
.
outputs
,
op_node
.
output
(
"Out"
)[
0
])
scale_in
=
self
.
_s8_max
/
self
.
_load_param
(
self
.
_scope
,
op_node
.
input
(
"InScale"
)[
0
])[
0
]
quant_op_node
=
graph
.
create_op_node
(
op_type
=
'quantize'
,
attrs
=
{
'data_format'
:
'MKLDNNLAYOUT'
,
'use_mkldnn'
:
1
,
'Scale'
:
scale_in
,
'is_negative_input'
:
1
},
inputs
=
{
'Input'
:
input_var_node
},
outputs
=
{
'Output'
:
output_var_node
})
graph
.
link_to
(
input_var_node
,
quant_op_node
)
graph
.
link_to
(
quant_op_node
,
output_var_node
)
graph
.
safe_remove_nodes
(
op_node
)
def
_remove_fake_dequantize_op
(
self
,
graph
,
op_node
):
input_var_node
=
graph
.
_find_node_by_name
(
op_node
.
inputs
,
op_node
.
input
(
"X"
)[
0
])
graph
.
safe_remove_nodes
(
op_node
)
def
_load_param
(
self
,
scope
,
param_name
):
return
np
.
array
(
scope
.
find_var
(
param_name
).
get_tensor
())
def
_restore_var
(
self
,
name
,
array
):
tensor
=
self
.
_scope
.
find_var
(
name
).
get_tensor
()
tensor
.
set
(
array
,
self
.
_place
)
def
_remove_unused_var_nodes
(
self
,
graph
):
all_used_vars
=
set
()
ops
=
graph
.
all_op_nodes
()
for
op_node
in
ops
:
for
input_node
in
op_node
.
inputs
:
all_used_vars
.
add
(
input_node
)
for
output_node
in
op_node
.
outputs
:
all_used_vars
.
add
(
output_node
)
all_used_vars
=
{
n
.
node
for
n
in
all_used_vars
}
all_unused_vars
=
{
n
for
n
in
filter
(
lambda
node
:
node
.
node
not
in
all_used_vars
,
graph
.
all_var_nodes
())
}
graph
.
safe_remove_nodes
(
all_unused_vars
)
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录