diff --git a/tests/onnx/auto_scan_test.py b/tests/onnx/auto_scan_test.py new file mode 100644 index 0000000000000000000000000000000000000000..21b1774e971003f21271b353fa89379ce2841fa9 --- /dev/null +++ b/tests/onnx/auto_scan_test.py @@ -0,0 +1,217 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import unittest +import os +import time +import logging +import paddle + +import hypothesis +from hypothesis import given, settings, seed, reproduce_failure +import hypothesis.strategies as st +from onnxbase import ONNXConverter, randtool +from itertools import product +import copy +from inspect import isfunction + +paddle.set_device("cpu") + +logging.basicConfig(level=logging.INFO, format="%(message)s") + +settings.register_profile( + "ci", + max_examples=100, + suppress_health_check=hypothesis.HealthCheck.all(), + deadline=None, + print_blob=True, + derandomize=True, + report_multiple_bugs=False) +settings.register_profile( + "dev", + max_examples=1000, + suppress_health_check=hypothesis.HealthCheck.all(), + deadline=None, + print_blob=True, + derandomize=True, + report_multiple_bugs=False) +if float(os.getenv('TEST_NUM_PERCENT_CASES', default='1.0')) < 1 or \ + os.getenv('HYPOTHESIS_TEST_PROFILE', 'dev') == 'ci': + settings.load_profile("ci") +else: + settings.load_profile("dev") + + +class OPConvertAutoScanTest(unittest.TestCase): + def __init__(self, *args, **kwargs): + super(OPConvertAutoScanTest, self).__init__(*args, **kwargs) + np.random.seed(1024) + paddle.enable_static() + self.num_ran_models = 0 + + def run_and_statis(self, + max_examples=100, + opset_version=[7, 9, 15], + reproduce=None, + min_success_num=25, + max_duration=-1): + if os.getenv("CE_STAGE", "OFF") == "ON": + max_examples *= 10 + min_success_num *= 10 + # while at ce phase, there's no limit on time + max_duration = -1 + start_time = time.time() + settings.register_profile( + "ci", + max_examples=max_examples, + suppress_health_check=hypothesis.HealthCheck.all(), + deadline=None, + print_blob=True, + derandomize=True, + report_multiple_bugs=False, ) + settings.load_profile("ci") + + def sample_convert_generator(draw): + return self.sample_convert_config(draw) + + def run_test(configs): + return self.run_test(configs=configs) + + generator = st.composite(sample_convert_generator) + loop_func = given(generator())(run_test) + if reproduce is not None: + loop_func = reproduce(loop_func) + logging.info("Start to running test of {}".format(type(self))) + + paddle.disable_static() + loop_func() + + logging.info( + "===================Statistical Information===================") + logging.info("Number of Generated Programs: {}".format( + self.num_ran_models)) + successful_ran_programs = int(self.num_ran_models) + if successful_ran_programs < min_success_num: + logging.warning("satisfied_programs = ran_programs") + logging.error( + "At least {} programs need to ran successfully, but now only about {} programs satisfied.". + format(min_success_num, successful_ran_programs)) + assert False + used_time = time.time() - start_time + logging.info("Used time: {} s".format(round(used_time, 2))) + if max_duration > 0 and used_time > max_duration: + logging.error( + "The duration exceeds {} seconds, if this is neccessary, try to set a larger number for parameter `max_duration`.". + format(max_duration)) + assert False + + def run_test(self, configs): + config, attrs = configs + logging.info("Run configs: {}".format(config)) + logging.info("Run attrs: {}".format(attrs)) + + assert "op_names" in config.keys( + ), "config must include op_names in dict keys" + assert "test_data_shapes" in config.keys( + ), "config must include test_data_shapes in dict keys" + assert "test_data_types" in config.keys( + ), "config must include test_data_types in dict keys" + assert "opset_version" in config.keys( + ), "config must include opset_version in dict keys" + assert "inputs_name" in config.keys( + ), "config must include inputs_name in dict keys" + assert "outputs_name" in config.keys( + ), "config must include outputs_name in dict keys" + assert "inputs_shape" in config.keys( + ), "config must include inputs_shape in dict keys" + assert "outputs_shape" in config.keys( + ), "config must include outputs_shape in dict keys" + assert "outputs_dtype" in config.keys( + ), "config must include outputs_dtype in dict keys" + + op_names = config["op_names"] + test_data_shapes = config["test_data_shapes"] + test_data_types = config["test_data_types"] + opset_version = config["opset_version"] + inputs_name = config["inputs_name"] + outputs_name = config["outputs_name"] + inputs_shape = config["inputs_shape"] + outputs_shape = config["outputs_shape"] + outputs_dtype = config["outputs_dtype"] + + use_gpu = True + if "use_gpu" in config.keys(): + use_gpu = config["use_gpu"] + + self.num_ran_models += 1 + + if not isinstance(op_names, (tuple, list)): + op_names = [op_names] + if not isinstance(opset_version[0], (tuple, list)): + opset_version = [opset_version] + if len(opset_version) == 1 and len(op_names) != len(opset_version): + opset_version = opset_version * len(op_names) + + input_type_list = None + if len(test_data_types) > 1: + input_type_list = list(product(*test_data_types)) + elif len(test_data_types) == 1: + if isinstance(test_data_types[0], str): + input_type_list = [test_data_types[0]] + else: + input_type_list = test_data_types + elif len(test_data_types) == 0: + input_type_list = [["float32"] * len(test_data_shapes)] + + delta = 1e-5 + rtol = 1e-5 + if "delta" in config.keys(): + delta = config["delta"] + if "rtol" in config.keys(): + rtol = config["rtol"] + + for i in range(len(op_names)): + obj = ONNXConverter(op_names[i], opset_version[i], op_names[i], + inputs_name, outputs_name, inputs_shape, + outputs_shape, outputs_dtype, delta, rtol, + use_gpu, attrs) + for input_type in input_type_list: + input_data = list() + for j, shape in enumerate(test_data_shapes): + # Determine whether it is a user-defined data generation function + if isfunction(shape): + data = shape() + data = data.astype(input_type[j]) + input_data.append(data) + continue + if input_type[j].count('int') > 0: + input_data.append( + randtool("int", -20, 20, shape).astype(input_type[ + j])) + elif input_type[j].count('bool') > 0: + input_data.append( + randtool("bool", -2, 2, shape).astype(input_type[ + j])) + else: + input_data.append( + randtool("float", -2, 2, shape).astype(input_type[ + j])) + obj.set_input_data("input_data", tuple(input_data)) + logging.info("Now Run >>> dtype: {}, op_name: {}".format( + input_type, op_names[i])) + obj.run() + if len(input_type_list) == 0: + obj.run() + logging.info("Run Successfully!") diff --git a/tests/onnx/onnxbase.py b/tests/onnx/onnxbase.py new file mode 100644 index 0000000000000000000000000000000000000000..3cfac0ddda16c4345af803a66cad4e944f89a8a8 --- /dev/null +++ b/tests/onnx/onnxbase.py @@ -0,0 +1,275 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License" +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import numpy as np +import logging +import paddle +import onnx +from onnx import helper +from onnx import TensorProto +from onnxruntime import InferenceSession + +DTYPE_ONNX_STR_MAP = { + 'float32': TensorProto.FLOAT, + 'float64': TensorProto.DOUBLE, + 'int16': TensorProto.INT16, + 'int32': TensorProto.INT32, + 'int64': TensorProto.INT64, + 'bool': TensorProto.BOOL, +} + + +def compare(result, expect, delta=1e-10, rtol=1e-10): + """ + 比较函数 + :param result: 输入值 + :param expect: 输出值 + :param delta: 误差值 + :return: + """ + if type(result) == np.ndarray: + if type(expect) == list: + expect = expect[0] + expect = np.array(expect) + res = np.allclose(result, expect, atol=delta, rtol=rtol, equal_nan=True) + # 出错打印错误数据 + if res is False: + if result.dtype == np.bool_: + diff = abs(result.astype("int32") - expect.astype("int32")) + else: + diff = abs(result - expect) + logging.error("Output has diff! max diff: {}".format(np.amax(diff))) + if result.dtype != expect.dtype: + logging.error( + "Different output data types! res type is: {}, and expect type is: {}". + format(result.dtype, expect.dtype)) + assert res + assert result.shape == expect.shape, "result.shape: {} != expect.shape: {}".format( + result.shape, expect.shape) + assert result.dtype == expect.dtype, "result.dtype: {} != expect.dtype: {}".format( + result.dtype, expect.dtype) + elif isinstance(result, (list, tuple)) and len(result) > 1: + for i in range(len(result)): + if isinstance(result[i], (np.generic, np.ndarray)): + compare(result[i], expect[i], delta, rtol) + else: + compare(result[i].numpy(), expect[i], delta, rtol) + elif len(result) == 1: + compare(result[0], expect[0], delta, rtol) + + +def randtool(dtype, low, high, shape): + """ + np random tools + """ + if dtype == "int": + return np.random.randint(low, high, shape) + + elif dtype == "float": + return low + (high - low) * np.random.random(shape) + + elif dtype == "bool": + return np.random.randint(low, high, shape).astype("bool") + + +class ONNXConverter(object): + """ + onnx model transfer to paddle + """ + + def __init__(self, + file_name, + ver_list, + op_type=[], + inputs_name=[], + outputs_name=[], + inputs_shape=[], + outputs_shape=[], + outputs_dtype=[], + delta=1e-5, + rtol=1e-5, + use_gpu=True, + attrs=[]): + self.op_type = op_type + assert isinstance(self.op_type, + str), "The dtype of op_type must be string!" + self.seed = 33 + np.random.seed(self.seed) + paddle.seed(self.seed) + if use_gpu and paddle.device.is_compiled_with_cuda() is True: + self.places = ['gpu'] + else: + self.places = ['cpu'] + self.name = file_name + self._version = ver_list + self.pwd = os.getcwd() + self.delta = delta + self.rtol = rtol + self.static = False + self.kwargs_dict = {"input_data": ()} + self.input_feed = {} + self.inputs_dtype = [] + self.inputs_name = inputs_name + self.outputs_name = outputs_name + self.inputs_shape = inputs_shape + self.outputs_shape = outputs_shape + self.outputs_dtype = outputs_dtype + self.attrs = attrs + + def set_input_data(self, group_name, *args): + """ + set input data + """ + self.kwargs_dict[group_name] = args + if isinstance(self.kwargs_dict[group_name][0], tuple): + self.kwargs_dict[group_name] = self.kwargs_dict[group_name][0] + + i = 0 + for in_data in self.kwargs_dict[group_name]: + if isinstance(in_data, list): + for data in in_data: + self.inputs_dtype.append(str(data.dtype)) + self.input_feed[self.inputs_name[i]] = data + i += 1 + else: + if isinstance(in_data, tuple): + in_data = in_data[0] + self.inputs_dtype.append(str(in_data.dtype)) + self.input_feed[self.inputs_name[i]] = in_data + i += 1 + + def _mkdir(self): + """ + make dir to save all + """ + save_path = os.path.join(self.pwd, self.name) + if not os.path.exists(save_path): + os.mkdir(save_path) + + def _onnx_to_paddle(self, ver): + """ + convert onnx to paddle + """ + from x2paddle.convert import onnx2paddle + onnx_path = os.path.join(self.pwd, self.name, + self.name + '_' + str(ver) + '.onnx') + paddle_path = os.path.join(self.pwd, self.name, + self.name + '_' + str(ver) + '_paddle') + onnx2paddle(onnx_path, paddle_path, convert_to_lite=False) + + def _mk_paddle_res(self, ver): + """ + make paddle res + """ + paddle_path = os.path.join( + self.pwd, self.name, + self.name + '_' + str(ver) + '_paddle/inference_model/model') + paddle.disable_static() + # run + model = paddle.jit.load(paddle_path) + paddle_feed = list() + + for i in range(len(self.input_feed)): + paddle_feed.append(self.input_feed[self.inputs_name[i]]) + result = model(*paddle_feed) + # get paddle outputs + if isinstance(result, (tuple, list)): + result = tuple(out.numpy() for out in result) + else: + result = (result.numpy(), ) + print("paddle result:", result[0].shape) + return result + + def _mk_onnx_res(self, ver): + """ + make onnx res + """ + sess = InferenceSession( + os.path.join(self.pwd, self.name, self.name + '_' + str(ver) + + '.onnx')) + ort_outs = sess.run(output_names=None, input_feed=self.input_feed) + print("onnx result:", ort_outs[0].shape) + return ort_outs + + def set_onnx_inputs(self): + graph_inputs = list() + for i in range(len(self.inputs_name)): + graph_inputs.append( + helper.make_tensor_value_info(self.inputs_name[ + i], DTYPE_ONNX_STR_MAP[self.inputs_dtype[i]], + self.inputs_shape[i])) + + return graph_inputs + + def set_onnx_outputs(self): + graph_outputs = list() + for i in range(len(self.outputs_name)): + graph_outputs.append( + helper.make_tensor_value_info(self.outputs_name[ + i], DTYPE_ONNX_STR_MAP[self.outputs_dtype[i][0]], + self.outputs_shape[i])) + + return graph_outputs + + def _mk_onnx_graph(self, ver): + """ + make onnx graph + """ + node = onnx.helper.make_node( + self.op_type, + inputs=self.inputs_name, + outputs=self.outputs_name, + **self.attrs, ) + graph_inputs = self.set_onnx_inputs() + graph_outputs = self.set_onnx_outputs() + graph = helper.make_graph( + [node], + self.name, + graph_inputs, # graph inputs + graph_outputs, # graph outputs + ) + opset_imports = [helper.make_opsetid("", ver)] + model = helper.make_model( + graph, producer_name='onnx-example', opset_imports=opset_imports) + onnx.save(model, + os.path.join(self.pwd, self.name, + self.name + '_' + str(ver) + '.onnx')) + onnx.checker.check_model(model) + + def run(self): + """ + 1. make onnx model + 2. convert onnx to paddle + 3. use onnx to make res + 4. compare diff + """ + self._mkdir() + for place in self.places: + paddle.set_device(place) + onnx_res = {} + paddle_res = {} + # export onnx models and make onnx res + for v in self._version: + self._mk_onnx_graph(ver=v) + self._onnx_to_paddle(ver=v) + onnx_res[str(v)] = self._mk_onnx_res(ver=v) + paddle_res[str(v)] = self._mk_paddle_res(ver=v) + + for v in self._version: + compare( + onnx_res[str(v)], + paddle_res[str(v)], + delta=self.delta, + rtol=self.rtol) diff --git a/tests/onnx/test_auto_scan_conv.py b/tests/onnx/test_auto_scan_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..d738f065c32beac1fa0fa778f258b0200c3ef0c3 --- /dev/null +++ b/tests/onnx/test_auto_scan_conv.py @@ -0,0 +1,144 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License" +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from auto_scan_test import OPConvertAutoScanTest +from hypothesis import reproduce_failure +import hypothesis.strategies as st +import onnx +from onnx import helper +from onnx import TensorProto +import numpy as np +import unittest + + +class TestConvConvert(OPConvertAutoScanTest): + """ + api: paddle.nn.Conv2d + OPset version: 9 + 1.OPset version需要根据op_mapper中定义的version来设置。 + 2.测试中所有OP对应升级到Opset version 15。 + """ + + def sample_convert_config(self, draw): + input_shape = draw( + st.lists( + st.integers( + min_value=20, max_value=30), min_size=4, max_size=4)) + + kernel_size = draw( + st.lists( + st.integers( + min_value=1, max_value=7), min_size=4, max_size=4)) + + data_format = "NCHW" + + groups = draw(st.integers(min_value=1, max_value=4)) + muti1 = draw(st.integers(min_value=1, max_value=4)) + kernel_size[0] = groups * muti1 + input_shape[1] = kernel_size[1] * groups + + strides = draw( + st.lists( + st.integers( + min_value=1, max_value=5), min_size=1, max_size=2)) + if len(strides) == 1: + strides = strides[0] + if strides > kernel_size[2]: + strides = kernel_size[2] + if strides > kernel_size[3]: + strides = kernel_size[3] + strides = [strides, strides] + else: + if strides[0] > kernel_size[2]: + strides[0] = kernel_size[2] + if strides[1] > kernel_size[3]: + strides[1] = kernel_size[3] + + if draw(st.booleans()): + auto_pad = "NOTSET" + padding = None + if draw(st.booleans()): + padding = draw( + st.lists( + st.integers( + min_value=1, max_value=5), + min_size=2, + max_size=2)) + padding = [0, 0] + padding + else: + padding = draw( + st.lists( + st.integers( + min_value=1, max_value=5), + min_size=4, + max_size=4)) + else: + auto_pad = draw( + st.sampled_from( + ["SAME_LOWER", "SAME_UPPER", "VALID", "NOTSET"])) + padding = None + + dilations = draw( + st.lists( + st.integers( + min_value=1, max_value=3), min_size=2, max_size=2)) + + config = { + "op_names": ["Conv"], + "test_data_shapes": [input_shape, kernel_size], + "test_data_types": [['float32'], ['float32']], + "inputs_shape": [[-1, input_shape[1], -1, -1], kernel_size], + "outputs_shape": [[-1, kernel_size[0], -1, -1]], + "outputs_dtype": [['float32']], + "opset_version": [7, 9], + "inputs_name": ["x", "W"], + "outputs_name": ["y"], + "delta": 1e-4, + "rtol": 1e-4 + } + + # Warning: + # 1、SAME_UPPER and SAME_LOWER does not yet support dynamic shapes + # 2、dilations only support 1 + if "SAME" in auto_pad: + dilations = [1, 1] + config["inputs_shape"] = [input_shape, kernel_size] + config["outputs_shape"] = [[ + input_shape[0], kernel_size[0], + int(input_shape[2] / strides[0]), + int(input_shape[3] / strides[1]) + ]] + + if not isinstance(dilations, (tuple, list)): + dilations = [dilations] + if not isinstance(strides, (tuple, list)): + strides = [strides] + + attrs = { + "auto_pad": auto_pad, + "dilations": dilations, + "group": groups, + "kernel_shape": kernel_size[2:], + "pads": padding, + "strides": strides, + } + + return (config, attrs) + + def test(self): + self.run_and_statis(max_examples=30) + + +if __name__ == "__main__": + unittest.main() diff --git a/x2paddle/op_mapper/onnx2paddle/opset9/opset.py b/x2paddle/op_mapper/onnx2paddle/opset9/opset.py index 4db1033b08edf9274773dcbb58d804a7f920128d..82beecc355ed607ac67b26ab83803eadd8925c5b 100755 --- a/x2paddle/op_mapper/onnx2paddle/opset9/opset.py +++ b/x2paddle/op_mapper/onnx2paddle/opset9/opset.py @@ -92,12 +92,15 @@ def _is_static_shape(shape): return True -def _get_same_padding(in_size, kernel_size, stride): +def _get_same_padding(in_size, kernel_size, stride, autopad): new_size = int(math.ceil(in_size * 1.0 / stride)) pad_size = (new_size - 1) * stride + kernel_size - in_size pad0 = int(pad_size / 2) pad1 = pad_size - pad0 - return [pad0, pad1] + if autopad == "SAME_UPPER": + return [pad0, pad1] + if autopad == "SAME_LOWER": + return [pad1, pad0] def print_mapping_info(func): @@ -1540,9 +1543,9 @@ class OpSet9(): if auto_pad == "SAME_UPPER" or auto_pad == "SAME_LOWER": input_shape = val_x.out_shapes[0] pad_h = _get_same_padding(input_shape[2], kernel_shape[0], - strides[0]) + strides[0], auto_pad) pad_w = _get_same_padding(input_shape[3], kernel_shape[1], - strides[1]) + strides[1], auto_pad) paddings = pad_h + pad_w op_name = name_generator("pool", self.nn_name2id) @@ -1967,9 +1970,9 @@ class OpSet9(): if auto_pad == "SAME_UPPER" or auto_pad == "SAME_LOWER": input_shape = val_x.out_shapes[0] pad_h = _get_same_padding(input_shape[2], kernel_shape[0], - strides[0]) + strides[0], auto_pad) pad_w = _get_same_padding(input_shape[3], kernel_shape[1], - strides[1]) + strides[1], auto_pad) paddings = pad_h + pad_w layer_attrs = { @@ -2196,13 +2199,15 @@ class OpSet9(): pads = node.get_attr('pads', [0] * (convnd * 2)) input_shape = val_x.out_shapes[0] - paddings, val_x = self._pad_if_asymmetric(node, pads, val_x) + paddings = np.array(pads).reshape((2, -1)).transpose().astype("int32") + paddings = paddings.flatten().tolist() if auto_pad == "SAME_UPPER" or auto_pad == "SAME_LOWER": + assert -1 not in input_shape, 'SAME_UPPER and SAME_LOWER does not yet support dynamic shapes' pad_h = _get_same_padding(input_shape[2], kernel_shape[0], - strides[0]) + strides[0], auto_pad) pad_w = _get_same_padding(input_shape[3], kernel_shape[1], - strides[1]) + strides[1], auto_pad) paddings = pad_h + pad_w layer_inputs = {'x': val_x if isinstance(val_x, str) else val_x.name}