Skip to content

X
X2Paddle

PaddlePaddle / X2Paddle
大约 1 年 前同步成功

通知 328
Star 698
Fork 167
X
X2Paddle

体验新版 GitCode，发现更多精彩内容 >>

提交 a2189f02 编写于 作者: M Macrobull
浏览文件
操作

fix validation precision

上级 414a5432
隐藏空白更改
内联 并排
Showing with 212 addition and 130 deletion
onnx2fluid/examples/convert_data_npz_0.py
浏览文件 @ a2189f02
......@@ -11,15 +11,32 @@ import numpy as np
from collections import OrderedDict as Dict
def _make_var_name(name):
"""
make a valid variable name in Python code
"""
if name == '':
return '_'
if name[0].isdigit():
return 'var_' + name
for s in ' *?\\/-:':
name = name.replace(s, '_')
if name.startswith('_'):
name = 'var' + name
return name
fn = sys.argv[1]
input_names = sys.argv[2].split(':')
output_name = sys.argv[3].split(':')
data = np.load(fn)
data = np.load(fn, encoding='bytes')
input_data = data['inputs']
output_data = data['outputs']
inputs = Dict(zip(input_names, [input_data]))
outputs = Dict(zip(output_name, [output_data]))
inputs = Dict(zip(map(_make_var_name, input_names), [input_data]))
outputs = Dict(zip(map(_make_var_name, output_name), [output_data]))
np.savez(fn, inputs=inputs, outputs=outputs) # overwrite
onnx2fluid/examples/convert_data_pb_0.py
浏览文件 @ a2189f02
......@@ -16,6 +16,23 @@ import onnx.numpy_helper as numpy_helper
from collections import OrderedDict as Dict
from glob import glob
def _make_var_name(name):
"""
make a valid variable name in Python code
"""
if name == '':
return '_'
if name[0].isdigit():
return 'var_' + name
for s in ' *?\\/-:':
name = name.replace(s, '_')
if name.startswith('_'):
name = 'var' + name
return name
data_dir = os.path.dirname(sys.argv[1])
input_names = sys.argv[2].split(':')
output_name = sys.argv[3].split(':')
......@@ -36,7 +53,7 @@ for fn in glob(os.path.join(data_dir, 'output_*.pb')):
tensor.ParseFromString(f.read())
outputs.append(numpy_helper.to_array(tensor))
inputs = Dict(zip(input_names, inputs))
outputs = Dict(zip(output_name, outputs))
inputs = Dict(zip(map(_make_var_name, input_names), inputs))
outputs = Dict(zip(map(_make_var_name, output_name), outputs))
np.savez(data_dir, inputs=inputs, outputs=outputs)
onnx2fluid/examples/onnx_model_zoo.sh
浏览文件 @ a2189f02
#! /usr/bin/env sh
get_url="aria2c -c -s8 -x8"
# setopt SH_WORD_SPLIT # if zsh
base_url="https://s3.amazonaws.com/download.onnx/models/opset_9/"
flags="-e -o /tmp/export/"
convert_flags="-e -o /tmp/export/"
validate_flags1="/tmp/export/model.py"
validate_flags2="/tmp/export/__model__"
# alias http_get="wget -c" # if no aria2
alias http_get="aria2c -c -s8 -x8"
# alias python="python3" # if ...
bvlc_alexnet()
{
......@@ -10,21 +17,24 @@ bvlc_alexnet()
fn_tar="$bn_tar.tar.gz"
fn_model="$bn_tar/model.onnx"
$get_url "$base_url$fn_tar"
http_get "$base_url$fn_tar"
echo "extracting ..."
tar xf "$fn_tar"
for npz in $bn_tar/*.npz
python -m onnx2fluid $convert_flags "$fn_model"
for npz in "$bn_tar"/*.npz
do
echo "converting $npz ..."
python convert_data_npz_0.py "$npz" "data_0" "prob_1"
python -m onnx2fluid $flags "$fn_model" -t $npz
python convert_data_npz_0.py "$npz" data_0 prob_1
python -m onnx2fluid.validation $validate_flags1 -t "$npz"
python -m onnx2fluid.validation $validate_flags2 -t "$npz"
done
for pb_dir in $bn_tar/*/
for pb_dir in "$bn_tar"/*/
do
echo "converting $pb_dir ..."
python convert_data_pb_0.py "$pb_dir" "data_0" "prob_1"
python -m onnx2fluid $flags "$fn_model" -t $(dirname "$pb_dir/x").npz
python convert_data_pb_0.py "$pb_dir" data_0 prob_1
python -m onnx2fluid.validation $validate_flags1 -t $(dirname "$pb_dir/x").npz
python -m onnx2fluid.validation $validate_flags2 -t $(dirname "$pb_dir/x").npz
done
}
......@@ -34,15 +44,17 @@ bvlc_googlenet()
fn_tar="$bn_tar.tar.gz"
fn_model="$bn_tar/model.onnx"
$get_url "$base_url$fn_tar"
http_get "$base_url$fn_tar"
echo "extracting ..."
tar xf "$fn_tar"
for pb_dir in $bn_tar/*/
python -m onnx2fluid $convert_flags "$fn_model"
for pb_dir in "$bn_tar"/*/
do
echo "converting $pb_dir"
python convert_data_pb_0.py "$pb_dir" "data_0" "prob_1"
python -m onnx2fluid $flags "$fn_model" -t $(dirname "$pb_dir/x").npz
python convert_data_pb_0.py "$pb_dir" data_0 prob_1
python -m onnx2fluid.validation $validate_flags1 -t $(dirname "$pb_dir/x").npz
python -m onnx2fluid.validation $validate_flags2 -t $(dirname "$pb_dir/x").npz
done
}
......@@ -52,15 +64,17 @@ bvlc_reference_caffenet()
fn_tar="$bn_tar.tar.gz"
fn_model="$bn_tar/model.onnx"
$get_url "$base_url$fn_tar"
http_get "$base_url$fn_tar"
echo "extracting ..."
tar xf "$fn_tar"
for pb_dir in $bn_tar/*/
python -m onnx2fluid $convert_flags "$fn_model"
for pb_dir in "$bn_tar"/*/
do
echo "converting $pb_dir"
python convert_data_pb_0.py "$pb_dir" "data_0" "prob_1"
python -m onnx2fluid $flags "$fn_model" -t $(dirname "$pb_dir/x").npz
python convert_data_pb_0.py "$pb_dir" data_0 prob_1
python -m onnx2fluid.validation $validate_flags1 -t $(dirname "$pb_dir/x").npz
python -m onnx2fluid.validation $validate_flags2 -t $(dirname "$pb_dir/x").npz
done
}
......@@ -70,15 +84,17 @@ bvlc_reference_rcnn_ilsvrc13()
fn_tar="$bn_tar.tar.gz"
fn_model="$bn_tar/model.onnx"
$get_url "$base_url$fn_tar"
http_get "$base_url$fn_tar"
echo "extracting ..."
tar xf "$fn_tar"
for pb_dir in $bn_tar/*/
python -m onnx2fluid $convert_flags "$fn_model"
for pb_dir in "$bn_tar"/*/
do
echo "converting $pb_dir"
python convert_data_pb_0.py "$pb_dir" "data_0" "fc_rcnn_1"
python -m onnx2fluid $flags "$fn_model" -t $(dirname "$pb_dir/x").npz
python convert_data_pb_0.py "$pb_dir" data_0 fc-rcnn_1
python -m onnx2fluid.validation $validate_flags1 -t $(dirname "$pb_dir/x").npz
python -m onnx2fluid.validation $validate_flags2 -t $(dirname "$pb_dir/x").npz
done
}
......@@ -88,22 +104,24 @@ inception_v1()
fn_tar="$bn_tar.tar.gz"
fn_model="$bn_tar/model.onnx"
$get_url "$base_url$fn_tar"
http_get "$base_url$fn_tar"
echo "extracting ..."
tar xf "$fn_tar"
for npz in $bn_tar/*.npz
python -m onnx2fluid $convert_flags "$fn_model"
for npz in "$bn_tar"/*.npz
do
echo "converting $npz ..."
python convert_data_npz_0.py "$npz" "data_0" "prob_1"
python -m onnx2fluid $flags "$fn_model" -t $npz
python convert_data_npz_0.py "$npz" data_0 prob_1
python -m onnx2fluid.validation $validate_flags1 -t "$npz"
python -m onnx2fluid.validation $validate_flags2 -t "$npz"
done
for pb_dir in $bn_tar/*/
for pb_dir in "$bn_tar"/*/
do
echo "converting $pb_dir"
python convert_data_pb_0.py "$pb_dir" "data_0" "prob_1"
python -m onnx2fluid $flags "$fn_model" -t $(dirname "$pb_dir/x").npz
echo "converting $pb_dir ..."
python convert_data_pb_0.py "$pb_dir" data_0 prob_1
python -m onnx2fluid.validation $validate_flags1 -t $(dirname "$pb_dir/x").npz
python -m onnx2fluid.validation $validate_flags2 -t $(dirname "$pb_dir/x").npz
done
}
......@@ -113,22 +131,24 @@ inception_v2()
fn_tar="$bn_tar.tar.gz"
fn_model="$bn_tar/model.onnx"
$get_url "$base_url$fn_tar"
http_get "$base_url$fn_tar"
echo "extracting ..."
tar xf "$fn_tar"
for npz in $bn_tar/*.npz
python -m onnx2fluid $convert_flags "$fn_model"
for npz in "$bn_tar"/*.npz
do
echo "converting $npz ..."
python convert_data_npz_0.py "$npz" "data_0" "prob_1"
python -m onnx2fluid $flags "$fn_model" -t $npz
python convert_data_npz_0.py "$npz" data_0 prob_1
python -m onnx2fluid.validation $validate_flags1 -t "$npz"
python -m onnx2fluid.validation $validate_flags2 -t "$npz"
done
for pb_dir in $bn_tar/*/
for pb_dir in "$bn_tar"/*/
do
echo "converting $pb_dir"
python convert_data_pb_0.py "$pb_dir" "data_0" "prob_1"
python -m onnx2fluid $flags "$fn_model" -t $(dirname "$pb_dir/x").npz
echo "converting $pb_dir ..."
python convert_data_pb_0.py "$pb_dir" data_0 prob_1
python -m onnx2fluid.validation $validate_flags1 -t $(dirname "$pb_dir/x").npz
python -m onnx2fluid.validation $validate_flags2 -t $(dirname "$pb_dir/x").npz
done
}
......@@ -138,22 +158,24 @@ resnet50()
fn_tar="$bn_tar.tar.gz"
fn_model="$bn_tar/model.onnx"
$get_url "$base_url$fn_tar"
http_get "$base_url$fn_tar"
echo "extracting ..."
tar xf "$fn_tar"
for npz in $bn_tar/*.npz
python -m onnx2fluid $convert_flags "$fn_model"
for npz in "$bn_tar"/*.npz
do
echo "converting $npz ..."
python convert_data_npz_0.py "$npz" "gpu_0/data_0" "gpu_0/softmaxout_1"
python -m onnx2fluid $flags "$fn_model" -t $npz
python convert_data_npz_0.py "$npz" gpu_0/data_0 gpu_0/softmaxout_1
python -m onnx2fluid.validation $validate_flags1 -t "$npz"
python -m onnx2fluid.validation $validate_flags2 -t "$npz"
done
for pb_dir in $bn_tar/*/
for pb_dir in "$bn_tar"/*/
do
echo "converting $pb_dir"
python convert_data_pb_0.py "$pb_dir" "gpu_0/data_0" "gpu_0/softmaxout_1"
python -m onnx2fluid $flags "$fn_model" -t $(dirname "$pb_dir/x").npz
echo "converting $pb_dir ..."
python convert_data_pb_0.py "$pb_dir" gpu_0/data_0 gpu_0/softmaxout_1
python -m onnx2fluid.validation $validate_flags1 -t $(dirname "$pb_dir/x").npz
python -m onnx2fluid.validation $validate_flags2 -t $(dirname "$pb_dir/x").npz
done
}
......@@ -163,15 +185,17 @@ shufflenet()
fn_tar="$bn_tar.tar.gz"
fn_model="$bn_tar/model.onnx"
$get_url "$base_url$fn_tar"
http_get "$base_url$fn_tar"
echo "extracting ..."
tar xf "$fn_tar"
for pb_dir in $bn_tar/*/
python -m onnx2fluid $convert_flags "$fn_model"
for pb_dir in "$bn_tar"/*/
do
echo "converting $pb_dir"
python convert_data_pb_0.py "$pb_dir" "gpu_0/data_0" "gpu_0/softmaxout_1"
python -m onnx2fluid $flags "$fn_model" -t $(dirname "$pb_dir/x").npz
echo "converting $pb_dir ..."
python convert_data_pb_0.py "$pb_dir" gpu_0/data_0 gpu_0/softmaxout_1
python -m onnx2fluid.validation $validate_flags1 -t $(dirname "$pb_dir/x").npz
python -m onnx2fluid.validation $validate_flags2 -t $(dirname "$pb_dir/x").npz
done
}
......@@ -181,15 +205,17 @@ squeezenet()
fn_tar="$bn_tar.tar.gz"
fn_model="$bn_tar/model.onnx"
$get_url "$base_url$fn_tar"
http_get "$base_url$fn_tar"
echo "extracting ..."
tar xf "$fn_tar"
for pb_dir in $bn_tar/*/
python -m onnx2fluid $convert_flags "$fn_model"
for pb_dir in "$bn_tar"/*/
do
echo "converting $pb_dir"
python convert_data_pb_0.py "$pb_dir" "data_0" "softmaxout_1"
python -m onnx2fluid $flags "$fn_model" -t $(dirname "$pb_dir/x").npz
python convert_data_pb_0.py "$pb_dir" data_0 softmaxout_1
python -m onnx2fluid.validation $validate_flags1 -t $(dirname "$pb_dir/x").npz
python -m onnx2fluid.validation $validate_flags2 -t $(dirname "$pb_dir/x").npz
done
}
......@@ -199,15 +225,17 @@ tiny_yolov2()
fn_tar="$bn_tar.tar.gz"
fn_model="$bn_tar/model.onnx"
$get_url "https://onnxzoo.blob.core.windows.net/models/opset_8/tiny_yolov2/$fn_tar"
http_get "https://onnxzoo.blob.core.windows.net/models/opset_8/tiny_yolov2/$fn_tar"
echo "extracting ..."
tar xf "$fn_tar"
for pb_dir in $bn_tar/*/
python -m onnx2fluid $convert_flags "$fn_model" -xy
for pb_dir in "$bn_tar"/*/
do
echo "converting $pb_dir"
python convert_data_pb_0.py "$pb_dir" "image" "grid"
python -m onnx2fluid $flags "$fn_model" -t $(dirname "$pb_dir/x").npz -x
python convert_data_pb_0.py "$pb_dir" image grid
python -m onnx2fluid.validation $validate_flags1 -t $(dirname "$pb_dir/x").npz
python -m onnx2fluid.validation $validate_flags2 -t $(dirname "$pb_dir/x").npz
done
}
......@@ -217,15 +245,17 @@ vgg19()
fn_tar="$bn_tar.tar.gz"
fn_model="$bn_tar/model.onnx"
$get_url "$base_url$fn_tar"
http_get "$base_url$fn_tar"
echo "extracting ..."
tar xf "$fn_tar"
for pb_dir in $bn_tar/*/
python -m onnx2fluid $convert_flags "$fn_model"
for pb_dir in "$bn_tar"/*/
do
echo "converting $pb_dir"
python convert_data_pb_0.py "$pb_dir" "data_0" "prob_1"
python -m onnx2fluid $flags "$fn_model" -t $(dirname "$pb_dir/x").npz
python convert_data_pb_0.py "$pb_dir" data_0 prob_1
python -m onnx2fluid.validation $validate_flags1 -t $(dirname "$pb_dir/x").npz
python -m onnx2fluid.validation $validate_flags2 -t $(dirname "$pb_dir/x").npz
done
}
......@@ -235,15 +265,17 @@ zfnet512()
fn_tar="$bn_tar.tar.gz"
fn_model="$bn_tar/model.onnx"
$get_url "$base_url$fn_tar"
http_get "$base_url$fn_tar"
echo "extracting ..."
tar xf "$fn_tar"
for pb_dir in $bn_tar/*/
python -m onnx2fluid $convert_flags "$fn_model"
for pb_dir in "$bn_tar"/*/
do
echo "converting $pb_dir"
python convert_data_pb_0.py "$pb_dir" "gpu_0/data_0" "gpu_0/softmax_1"
python -m onnx2fluid $flags "$fn_model" -t $(dirname "$pb_dir/x").npz
python convert_data_pb_0.py "$pb_dir" gpu_0/data_0 gpu_0/softmax_1
python -m onnx2fluid.validation $validate_flags1 -t $(dirname "$pb_dir/x").npz
python -m onnx2fluid.validation $validate_flags2 -t $(dirname "$pb_dir/x").npz
done
}
......
onnx2fluid/onnx2fluid/__main__.py
浏览文件 @ a2189f02
......@@ -89,7 +89,7 @@ parser.add_argument(
'--precision',
'-p',
type=int,
default=4,
default=3,
help='assertion decimal for validation',
)
args = parser.parse_args()
......
onnx2fluid/onnx2fluid/cmdline.py
浏览文件 @ a2189f02
......@@ -84,13 +84,13 @@ def main(**kwargs):
# in fact fluid can not fully clear the context
# continuous validation may be inaccurate
precision = 10**-kwargs.get('precision', 4)
decimal = kwargs.get('precision', 3)
logger.info('starting validation on desc ...')
passed &= validate(
shutil.os.path.join(save_dir, '__model__'),
golden_data_filename,
precision=precision,
decimal=decimal,
)
logger.info('starting validation on code ...')
......@@ -98,7 +98,7 @@ def main(**kwargs):
shutil.os.path.join(save_dir, model_basename),
golden_data_filename,
model_func_name=model_func_name,
precision=precision,
decimal=decimal,
save_inference_model=debug, # this overwrite desc file for test
)
......@@ -131,12 +131,13 @@ if __name__ == '__main__':
level=logging.DEBUG,
)
# main(model=['../examples/t5.onnx'],
# output_dir='/tmp/export/',
# embed_params=False,
# pedantic=False,
# test_data='../examples/t5.npz',
# debug=True)
main(
model=['../examples/t1.onnx'],
output_dir='/tmp/export/',
embed_params=False,
pedantic=False,
test_data='../examples/t1.npz',
debug=True)
main(
model=['../examples/inception_v2/model.onnx'],
......
onnx2fluid/onnx2fluid/symbolic.py
浏览文件 @ a2189f02
......@@ -250,20 +250,17 @@ def _default(prog, op_type, inputs, outputs, attrs, *args, name='', **kwargs):
arg_name,
))
for var_out in var_outs:
prog.VarDesc(var_out)
# dummy var_out
num_vars = len(var_outs)
num_args = len(fluid_output_args)
if num_vars < num_args:
assert fill_name_field, 'name required to name dummy output variables'
for idx_out in range(num_vars, num_args):
var_out = _make_var_name(name + '.' +
fluid_output_args[idx_out].lower())
var_out = name + '.' + fluid_output_args[idx_out] # dummy output
var_outs.append(var_out)
prog.VarDesc(var_out)
for var_out in var_outs:
prog.VarDesc(var_out)
prog.OpDesc(fluid_op, (var_inps, *fluid_input_args),
(var_outs, *fluid_output_args), fluid_attrs)
......@@ -301,7 +298,7 @@ def _pad_if_asymmetric(prog, pads, val_name, value_infos): # pads: SSEE
if symmetric:
return pads[:ndims], val_name
val_padded = val_name + '_padded'
val_padded = val_name + '_padded' # explicit variable
prog.Op(
'',
'Pad',
......@@ -637,6 +634,8 @@ def BatchNormalization(prog,
val_y, = outputs
var_x = _make_var_name(val_x)
var_y = _make_var_name(val_y)
var_saved_mean = name + '.saved_mean' # dummy output
var_saved_variance = name + '.saved_variance' # dummy output
# interpretation
fluid_op = 'batch_norm'
......@@ -645,10 +644,10 @@ def BatchNormalization(prog,
name_attr = ', name={}'.format(repr(name)) if name else ''
if embed_params:
assert name != ''
var_scale = '{}.w_0'.format(name)
var_b = '{}.b_0'.format(name)
var_mean = '{}.w_1'.format(name)
var_var = '{}.w_2'.format(name)
var_scale = name + '.w_0'
var_b = name + '.b_0'
var_mean = name + '.w_1'
var_var = name + '.w_2'
value_infos[val_scale].setdefault('embeded_as', []).append(var_scale)
value_infos[val_b].setdefault('embeded_as', []).append(var_b)
value_infos[val_mean].setdefault('embeded_as', []).append(var_mean)
......@@ -663,8 +662,6 @@ def BatchNormalization(prog,
', moving_mean_name={}, moving_variance_name={}').format(
repr(var_scale), repr(var_b), repr(var_mean),
repr(var_var))
var_saved_mean = '{}.saved_mean'.format(name) # dropped var
var_saved_variance = '{}.saved_variance'.format(name) # dropped var
# generationvalue_infos
prog.Code('{} = layers.{}({}, is_test=True, data_layout="NCHW"'
......@@ -795,7 +792,7 @@ def Constant(prog, inputs, outputs, attrs, value_infos, *args, **kwargs):
# dtype = np.dtype('float32') # HINT: force to float32
shape = attrs.get('shape', None) # additional, maybe var_name
shape = attrs.get('shape', None) #
if shape is None:
shape = _shape_or_none(value_infos, val_output)
if shape is None:
......@@ -807,6 +804,7 @@ def Constant(prog, inputs, outputs, attrs, value_infos, *args, **kwargs):
# generation
if value.size == 1: # scalar
value = value[0]
fluid_op = 'fill_constant'
prog.Code('{} = layers.{}(shape={}, dtype={}, value={})'.format(
var_output,
......@@ -814,12 +812,22 @@ def Constant(prog, inputs, outputs, attrs, value_infos, *args, **kwargs):
# attrs
shape,
repr(dtype.name),
value[0], # shape can be list or var_name
value,
))
value_infos[val_output]['const_value'] = value[0]
value_infos[val_output]['const_value'] = value
prog.VarDesc(var_output)
prog.OpDesc(
fluid_op,
([], ),
([var_output], 'Out'),
dict(
shape=shape,
dtype=dtype.name,
value=value,
),
)
else: # list parameter -> const_value
prog.Code('{} = {}'.format(
prog.Code('# {} = {} # passed directly as literal'.format(
var_output,
value.tolist(),
))
......@@ -834,6 +842,7 @@ def ConstantOfShape(prog, inputs, outputs, attrs, value_infos, *args, **kwargs):
# I/O
val_shape, = inputs
val_output, = outputs
var_shape = _make_var_name(val_shape)
shape = _const_weight_or_none(value_infos, val_shape)
if shape is None:
......@@ -843,8 +852,9 @@ def ConstantOfShape(prog, inputs, outputs, attrs, value_infos, *args, **kwargs):
'this is not supported')
dtype = attrs['value'].dtype
attrs = attrs.copy()
attrs.update(dict(shape=shape, dtype=dtype)) # pass var_name
attrs.update(dict(shape=shape, dtype=dtype)) # pass const
prog.Code('# shape:{}={} # const as literal'.format(var_shape, shape))
prog.Op(
'',
'Constant',
......@@ -898,10 +908,10 @@ def Conv(prog,
name_attr = ', name={}'.format(repr(name)) if name else ''
if embed_params:
assert name != ''
var_w = '{}.w_0'.format(name)
var_w = name + '.w_0'
value_infos[val_w].setdefault('embeded_as', []).append(var_w)
if has_bias:
var_b = '{}.b_0'.format(name)
var_b = name + '.b_0'
value_infos[val_b].setdefault('embeded_as', []).append(var_b)
param_attr = ''
else:
......@@ -935,7 +945,7 @@ def Conv(prog,
param_attr,
name_attr,
))
var_conv = _make_var_name(name + '.conv') # hidden variable
var_conv = name + '.conv' # hidden variable
prog.OpDesc(
fluid_op,
([var_x, var_w], 'Input', 'Filter'), # , 'Bias', 'ResidualData'
......@@ -951,8 +961,8 @@ def Conv(prog,
prog.IntermediateOp(
'',
'Add',
[var_conv, var_b],
[var_y], # var
[var_conv, var_b], #
[val_y],
dict(axis=1),
value_infos=value_infos,
name=(name + '.bias'),
......@@ -1007,10 +1017,10 @@ def ConvTranspose(prog,
name_attr = ', name={}'.format(repr(name)) if name else ''
if embed_params:
assert name != ''
var_w = '{}.w_0'.format(name)
var_w = name + '.w_0'
value_infos[val_w].setdefault('embeded_as', []).append(var_w)
if has_bias:
var_b = '{}.b_0'.format(name)
var_b = name + '.b_0'
value_infos[val_b].setdefault('embeded_as', []).append(var_b)
param_attr = ''
else:
......@@ -1045,7 +1055,7 @@ def ConvTranspose(prog,
param_attr,
name_attr,
))
var_conv = _make_var_name(name + '.conv') # hidden variable
var_conv = name + '.conv' # hidden variable
prog.OpDesc(
fluid_op,
([var_x, var_w], 'Input', 'Filter'), # , 'Bias', 'ResidualData'
......@@ -1062,8 +1072,8 @@ def ConvTranspose(prog,
prog.IntermediateOp(
'',
'Add',
[var_conv, var_b],
[var_y], # var
[var_conv, var_b], #
[val_y],
dict(axis=1),
value_infos=value_infos,
name=(name + '.bias'),
......@@ -1414,14 +1424,14 @@ def Reshape(prog, inputs, outputs, attrs, value_infos, name, *args, **kwargs):
val_data, val_shape = inputs
val_reshaped, = outputs
var_data = _make_var_name(val_data)
var_shape = _make_var_name(val_shape)
var_reshaped = _make_var_name(val_reshaped)
# interpretation
var_shape = _make_var_name(val_shape) # for code
shape = _const_weight_or_none(value_infos, val_shape)
is_const_shape = shape and 'const_value' in value_infos[val_shape]
if shape is None:
shape = _shape_or_none(value_infos, var_reshaped)
shape = _shape_or_none(value_infos, val_reshaped)
assert shape is not None, (
'given shape is neither const value nor deductible from output, '
'this is not supported')
......@@ -1429,6 +1439,7 @@ def Reshape(prog, inputs, outputs, attrs, value_infos, name, *args, **kwargs):
name_attr = ', name={}'.format(repr(name)) if name else ''
# generation
prog.Code('# shape:{}={} # const as literal'.format(var_shape, shape))
if is_const_shape:
prog.Code('{} = layers.{}({}'
', shape={}'
......@@ -1437,16 +1448,17 @@ def Reshape(prog, inputs, outputs, attrs, value_infos, name, *args, **kwargs):
fluid_op,
var_data,
# attrs
var_shape,
shape,
name_attr,
))
else:
var_shape_int32 = var_shape + '_int32'
val_shape_int32 = val_shape + '_int32' # explicit variable
var_shape_int32 = _make_var_name(val_shape_int32)
prog.Op(
'',
'Cast',
[var_shape],
[var_shape_int32], # var
[val_shape],
[val_shape_int32], # var
dict(to=np.dtype('int32')), # use np.dtype
value_infos=value_infos,
name=(name + '_cast'),
......@@ -1464,7 +1476,7 @@ def Reshape(prog, inputs, outputs, attrs, value_infos, name, *args, **kwargs):
name_attr,
))
fluid_op = 'reshape2'
var_xshape = _make_var_name(name + '.xshape')
var_xshape = name + '.xshape' # dummy output
prog.VarDesc(var_reshaped)
prog.VarDesc(var_xshape)
if is_const_shape:
......@@ -1580,6 +1592,7 @@ def Tile(prog, inputs, outputs, attrs, value_infos, name='', *args, **kwargs):
val_input, val_repeats = inputs
val_output, = outputs
var_input = _make_var_name(val_input)
var_repeats = _make_var_name(val_repeats)
var_output = _make_var_name(val_output)
# interpretation
......@@ -1589,6 +1602,7 @@ def Tile(prog, inputs, outputs, attrs, value_infos, name='', *args, **kwargs):
name_attr = ', name={}'.format(repr(name)) if name else ''
# generation
prog.Code('# repeats:{}={} # const as literal'.format(var_repeats, repeats))
prog.Code('{} = layers.{}({}'
', expand_times={}'
'{})'
......
onnx2fluid/onnx2fluid/validation.py
浏览文件 @ a2189f02
......@@ -37,7 +37,7 @@ def _ensure_list(obj):
def validate(fluid_model_filename,
golden_data_filename,
model_func_name='inference',
precision=1e-4,
decimal=3,
save_inference_model=False):
"""
inferece the converted Paddle fluid model, validate with given golden data
......@@ -91,16 +91,17 @@ def validate(fluid_model_filename,
# load data
logger.info('using golden data %s', golden_data_filename)
if golden_data_filename.endswith('.npz'):
test_data = np.load(golden_data_filename)
test_data = np.load(golden_data_filename, encoding='bytes')
input_data = test_data['inputs'].tolist()
output_data = test_data['outputs'].tolist()
else:
test_data = np.load(golden_data_filename).tolist()
test_data = np.load(golden_data_filename, encoding='bytes').tolist()
input_data = test_data['inputs']
output_data = test_data['outputs']
input_data = _flatten_dict(input_data)
output_data = _flatten_dict(output_data)
logger.info('found %d I/O golden data, starting test ...', len(test_data))
logger.info('found %d I/O golden data, starting test ...',
len(input_data) + len(output_data))
# DEBUG: reload test for python code
if basename.endswith('.py') and save_inference_model:
......@@ -124,7 +125,7 @@ def validate(fluid_model_filename,
for (name, truth), output in zip(output_data.items(), outputs):
logger.info('testing output {} ...'.format(name))
try:
np.testing.assert_almost_equal(output, truth, decimal=precision)
np.testing.assert_almost_equal(output, truth, decimal=decimal)
except AssertionError as e:
passed = False
logger.error('failed: %s\n', e)
......@@ -165,7 +166,7 @@ if __name__ == '__main__':
'--precision',
'-p',
type=int,
default=4,
default=3,
help='assertion decimal for validation',
)
args = parser.parse_args()
......@@ -177,10 +178,10 @@ if __name__ == '__main__':
debug = args.debug
fluid_model_filename = args.model[0]
golden_data_filename = args.test_data
precision = args.precision
decimal = args.precision
validate(
fluid_model_filename,
golden_data_filename,
precision=precision,
decimal=decimal,
save_inference_model=debug)
onnx2fluid/onnx2fluid/writer.py
浏览文件 @ a2189f02
......@@ -328,7 +328,7 @@ class Writer(object):
else:
var_name = make_var_name(name)
attr_name = make_attr_name(name)
prog.Code('# parameter: {}'.format(name))
prog.Code('# parameter {}: {}'.format(name, var_name))
prog.Code('{} = ParamAttr(name={})' # , trainable=True
.format(attr_name, repr(var_name)))
prog.Code(
......@@ -355,13 +355,13 @@ class Writer(object):
if remove_batch:
shape = shape[1:]
prog.Code('# input: {}'.format(name))
prog.Code('# input {}: {}'.format(name, var_name))
prog.Code((
'{} = layers.data(name={}, shape={}, dtype={}, '
'append_batch_size={})' # , stop_gradient=True
).format(
var_name,
repr(name),
repr(var_name),
shape,
repr(value_info['dtype'].name),
remove_batch,
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或