diff --git a/onnx2fluid/examples/convert_data_npz_0.py b/onnx2fluid/examples/convert_data_npz_0.py index f3eef275bda3079ff97214f15a7ff947e002f517..bd23f527a2f33981496aff1569be0ecfa5b77141 100644 --- a/onnx2fluid/examples/convert_data_npz_0.py +++ b/onnx2fluid/examples/convert_data_npz_0.py @@ -31,11 +31,17 @@ def _make_var_name(name): fn = sys.argv[1] input_names = sys.argv[2].split(':') output_name = sys.argv[3].split(':') +squeeze_data = len(sys.argv) > 4 data = np.load(fn, encoding='bytes') input_data = data['inputs'] output_data = data['outputs'] +while squeeze_data and input_data.ndim > 4 and input_data.shape[0] == 1: + input_data = input_data.squeeze(0) +while squeeze_data and output_data.ndim > 2 and output_data.shape[0] == 1: + output_data = output_data.squeeze(0) + inputs = Dict(zip(map(_make_var_name, input_names), [input_data])) outputs = Dict(zip(map(_make_var_name, output_name), [output_data])) diff --git a/onnx2fluid/examples/convert_data_pb_0.py b/onnx2fluid/examples/convert_data_pb_0.py index ae211494403b621b3334ea41990fa3ffcba3883b..fefb2711545358322604708dcedd29c149113542 100644 --- a/onnx2fluid/examples/convert_data_pb_0.py +++ b/onnx2fluid/examples/convert_data_pb_0.py @@ -36,6 +36,7 @@ def _make_var_name(name): data_dir = os.path.dirname(sys.argv[1]) input_names = sys.argv[2].split(':') output_name = sys.argv[3].split(':') +squeeze_data = len(sys.argv) > 4 # Load inputs inputs = [] @@ -43,7 +44,10 @@ for fn in glob(os.path.join(data_dir, 'input_*.pb')): tensor = onnx.TensorProto() with open(fn, 'rb') as f: tensor.ParseFromString(f.read()) - inputs.append(numpy_helper.to_array(tensor)) + tensor = numpy_helper.to_array(tensor) + while squeeze_data and tensor.ndim > 4 and tensor.shape[0] == 1: + tensor = tensor.squeeze(0) + inputs.append(tensor) # Load outputs outputs = [] @@ -51,7 +55,10 @@ for fn in glob(os.path.join(data_dir, 'output_*.pb')): tensor = onnx.TensorProto() with open(fn, 'rb') as f: tensor.ParseFromString(f.read()) - outputs.append(numpy_helper.to_array(tensor)) + tensor = numpy_helper.to_array(tensor) + while squeeze_data and tensor.ndim > 2 and tensor.shape[0] == 1: + tensor = tensor.squeeze(0) + outputs.append(tensor) inputs = Dict(zip(map(_make_var_name, input_names), inputs)) outputs = Dict(zip(map(_make_var_name, output_name), outputs)) diff --git a/onnx2fluid/examples/onnx_model_zoo.sh b/onnx2fluid/examples/onnx_model_zoo.sh index f9c7e9fd1f7fa57e25594d30f42eea27b18b594e..bc47156ace8750ee30c7534308d5de3609c9fe4a 100755 --- a/onnx2fluid/examples/onnx_model_zoo.sh +++ b/onnx2fluid/examples/onnx_model_zoo.sh @@ -18,6 +18,7 @@ bvlc_alexnet() fn_model="$bn_tar/model.onnx" http_get "$base_url$fn_tar" + rm -rf "$bn_tar/" echo "extracting ..." tar xf "$fn_tar" @@ -25,7 +26,7 @@ bvlc_alexnet() for npz in "$bn_tar"/*.npz do echo "converting $npz ..." - python convert_data_npz_0.py "$npz" data_0 prob_1 + python convert_data_npz_0.py "$npz" data_0 prob_1 -s python -m onnx2fluid.validation $validate_flags1 -t "$npz" python -m onnx2fluid.validation $validate_flags2 -t "$npz" done @@ -45,6 +46,7 @@ bvlc_googlenet() fn_model="$bn_tar/model.onnx" http_get "$base_url$fn_tar" + rm -rf "$bn_tar/" echo "extracting ..." tar xf "$fn_tar" @@ -65,6 +67,7 @@ bvlc_reference_caffenet() fn_model="$bn_tar/model.onnx" http_get "$base_url$fn_tar" + rm -rf "$bn_tar/" echo "extracting ..." tar xf "$fn_tar" @@ -85,6 +88,7 @@ bvlc_reference_rcnn_ilsvrc13() fn_model="$bn_tar/model.onnx" http_get "$base_url$fn_tar" + rm -rf "$bn_tar/" echo "extracting ..." tar xf "$fn_tar" @@ -93,8 +97,8 @@ bvlc_reference_rcnn_ilsvrc13() do echo "converting $pb_dir" python convert_data_pb_0.py "$pb_dir" data_0 fc-rcnn_1 - python -m onnx2fluid.validation $validate_flags1 -t $(dirname "$pb_dir/x").npz - python -m onnx2fluid.validation $validate_flags2 -t $(dirname "$pb_dir/x").npz + python -m onnx2fluid.validation $validate_flags1 -t $(dirname "$pb_dir/x").npz -p 0 + python -m onnx2fluid.validation $validate_flags2 -t $(dirname "$pb_dir/x").npz -p 0 done } @@ -105,6 +109,7 @@ inception_v1() fn_model="$bn_tar/model.onnx" http_get "$base_url$fn_tar" + rm -rf "$bn_tar/" echo "extracting ..." tar xf "$fn_tar" @@ -112,7 +117,7 @@ inception_v1() for npz in "$bn_tar"/*.npz do echo "converting $npz ..." - python convert_data_npz_0.py "$npz" data_0 prob_1 + python convert_data_npz_0.py "$npz" data_0 prob_1 -s python -m onnx2fluid.validation $validate_flags1 -t "$npz" python -m onnx2fluid.validation $validate_flags2 -t "$npz" done @@ -132,6 +137,7 @@ inception_v2() fn_model="$bn_tar/model.onnx" http_get "$base_url$fn_tar" + rm -rf "$bn_tar/" echo "extracting ..." tar xf "$fn_tar" @@ -139,7 +145,7 @@ inception_v2() for npz in "$bn_tar"/*.npz do echo "converting $npz ..." - python convert_data_npz_0.py "$npz" data_0 prob_1 + python convert_data_npz_0.py "$npz" data_0 prob_1 -s python -m onnx2fluid.validation $validate_flags1 -t "$npz" python -m onnx2fluid.validation $validate_flags2 -t "$npz" done @@ -159,6 +165,7 @@ resnet50() fn_model="$bn_tar/model.onnx" http_get "$base_url$fn_tar" + rm -rf "$bn_tar/" echo "extracting ..." tar xf "$fn_tar" @@ -166,7 +173,7 @@ resnet50() for npz in "$bn_tar"/*.npz do echo "converting $npz ..." - python convert_data_npz_0.py "$npz" gpu_0/data_0 gpu_0/softmaxout_1 + python convert_data_npz_0.py "$npz" gpu_0/data_0 gpu_0/softmaxout_1 -s python -m onnx2fluid.validation $validate_flags1 -t "$npz" python -m onnx2fluid.validation $validate_flags2 -t "$npz" done @@ -186,6 +193,7 @@ shufflenet() fn_model="$bn_tar/model.onnx" http_get "$base_url$fn_tar" + rm -rf "$bn_tar/" echo "extracting ..." tar xf "$fn_tar" @@ -206,6 +214,7 @@ squeezenet() fn_model="$bn_tar/model.onnx" http_get "$base_url$fn_tar" + rm -rf "$bn_tar/" echo "extracting ..." tar xf "$fn_tar" @@ -226,6 +235,7 @@ tiny_yolov2() fn_model="$bn_tar/model.onnx" http_get "https://onnxzoo.blob.core.windows.net/models/opset_8/tiny_yolov2/$fn_tar" + rm -rf "$bn_tar/" echo "extracting ..." tar xf "$fn_tar" @@ -246,6 +256,7 @@ vgg19() fn_model="$bn_tar/model.onnx" http_get "$base_url$fn_tar" + rm -rf "$bn_tar/" echo "extracting ..." tar xf "$fn_tar" @@ -266,6 +277,7 @@ zfnet512() fn_model="$bn_tar/model.onnx" http_get "$base_url$fn_tar" + rm -rf "$bn_tar/" echo "extracting ..." tar xf "$fn_tar" @@ -288,7 +300,7 @@ inception_v1 inception_v2 resnet50 shufflenet -squeezenet -tiny_yolov2 # not supported +squeezenet # softmax bug +# tiny_yolov2 # not supported vgg19 zfnet512 diff --git a/onnx2fluid/onnx2fluid/cmdline.py b/onnx2fluid/onnx2fluid/cmdline.py index 2aa32535a9bff0f3848592d0332654647a631b7c..2067dca55d3c43c2ef48bfd8635080670b1614ad 100644 --- a/onnx2fluid/onnx2fluid/cmdline.py +++ b/onnx2fluid/onnx2fluid/cmdline.py @@ -49,7 +49,8 @@ def main(**kwargs): basepath, _ = shutil.os.path.splitext(filename) save_dir = kwargs.get('output_dir', '') # model.onnx -> model/ - save_dir = (save_dir.rstrip('/') if save_dir else basepath) + '/' + save_dir = (save_dir.rstrip(shutil.os.sep) + if save_dir else basepath) + shutil.os.sep model_basename = DEFAULT_MODEL_MODULE + '.py' model_func_name = DEFAULT_MODEL_FUNC embed_params = kwargs.get('embed_params', False) @@ -109,7 +110,7 @@ def main(**kwargs): # create zip file if archive is not None: if archive == '': - archive = save_dir.rstrip('/') + '.zip' + archive = save_dir.rstrip(shutil.os.sep) + '.zip' logger.info('compressing file to %s ...', archive) shutil.sys.stderr.write('\n') shutil.sys.stderr.flush() diff --git a/onnx2fluid/onnx2fluid/symbolic.py b/onnx2fluid/onnx2fluid/symbolic.py index 5e0c82757fbe1a407b0a152adca94862f9c5db16..c2b48e18dc7236828aae7c9861c58a48812859a5 100644 --- a/onnx2fluid/onnx2fluid/symbolic.py +++ b/onnx2fluid/onnx2fluid/symbolic.py @@ -69,6 +69,7 @@ DEFAULT_OP_MAPPING = { 'Sin': ['sin', ['X'], ['Out']], 'Squeeze': ['squeeze', ['X'], ['Out']], # attrs bypassed, FIXME: emit squeeze2 'Softplus': ['softplus', ['X'], ['Out']], + # FIXME: default axis = -1, reshape required before and after 'Softmax': ['softmax', ['X'], ['Out'], dict(axis='')], 'Softsign': ['softsign', ['X'], ['Out']], 'Sqrt': ['sqrt', ['X'], ['Out']], @@ -799,7 +800,7 @@ def Constant(prog, inputs, outputs, attrs, value_infos, *args, **kwargs): shape = list(value.shape) _logger.warning( 'in (Constant -> %s): ' - 'shape of %s not inferred, ' + 'attribute "shape" of %s not inferred, ' 'using value as 1-D tensor may lead to fails', outputs, val_output) # generation @@ -1152,7 +1153,7 @@ def Gemm(prog, inputs, outputs, attrs, value_infos, name, *args, **kwargs): vm_dtype = np.dtype('float32') _logger.warning( 'in %s(%s -> Gemm -> %s): ' - 'beta seems to be an interger, ' + 'attribute "beta" seems to be an interger, ' 'however dtype can not be inferred, ' 'still use float32', name, inputs, outputs) beta = np.dtype(vm_dtype).type(beta) @@ -1432,9 +1433,17 @@ def Reshape(prog, inputs, outputs, attrs, value_infos, name, *args, **kwargs): is_const_shape = shape and 'const_value' in value_infos[val_shape] if shape is None: shape = _shape_or_none(value_infos, val_reshaped) - assert shape is not None, ( - 'given shape is neither const value nor deductible from output, ' - 'this is not supported') + + +# assert shape is not None, ('given shape is neither const value nor deductible from output, ' +# 'this is not supported') + if shape is None: + shape = [1, -1] # who knows + _logger.warning( + 'in %s(%s -> Reshape -> %s): ' + 'input "shape" not inferred, use [1, -1] as dummy value, ' + 'the behavior of Paddle fluid maybe undefined', name, inputs, + outputs) fluid_op = 'reshape' name_attr = ', name={}'.format(repr(name)) if name else '' @@ -1574,6 +1583,7 @@ def Sum(prog, inputs, outputs, *args, **kwargs): '[' + ', '.join(var_inps) + ']', # attrs )) + fluid_op = 'sum' prog.VarDesc(var_sum) prog.OpDesc( fluid_op,