diff --git a/python/paddle/fluid/tests/unittests/op_test.py b/python/paddle/fluid/tests/unittests/op_test.py index 304109339e0f6551c8e48bdbd4946126247f03cd..a0c8723323ce59cd05162fa5bdfb4e664f02a8d5 100644 --- a/python/paddle/fluid/tests/unittests/op_test.py +++ b/python/paddle/fluid/tests/unittests/op_test.py @@ -1317,7 +1317,239 @@ class OpTest(unittest.TestCase): check_dygraph=True, inplace_atol=None, check_eager=False): - self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs) + def find_imperative_actual(target_name, dygraph_outs, place): + for name in dygraph_outs: + if name == target_name: + return dygraph_outs[name][0] + var_list = dygraph_outs[name] + for i, var in enumerate(var_list): + if var.name == target_name: + return dygraph_outs[name][i] + self.assertTrue(False, "Found failed {} {}".format( + dygraph_outs.keys(), target_name)) + + def find_actual(target_name, fetch_list): + found = [ + i for i, var_name in enumerate(fetch_list) + if var_name == target_name + ] + self.assertTrue( + len(found) == 1, "Found {} {}".format(len(found), target_name)) + return found[0] + + class Checker(object): + """ base class for check with self.outputs. + currently don't support check between checkers. + """ + + def __init__(self, op_test, expect_dict): + """ expect_dict is the self.outputs + support : {str: [numpy]} and {str: [(str, numpy), (str, numpy)]} + """ + self.expects = expect_dict + self.checker_name = "checker" + self.op_test = op_test # stop the op_test object. + self.op_type = op_test.op_type + + def convert_uint16_to_float(self, actual_np, expect_np): + raise NotImplementedError("base class, not implement!") + + def calculate_output(self): + """ + judge whether convert current output and expect to uint16. + return True | False + """ + pass + + def _is_skip_name(self, name): + if name not in self.expects: + return True + if no_check_set is not None and name in no_check_set: + return True + return False + + def find_actual_value(self, name): + """ return: (actual_tensor(var_base), actual_numpy) + """ + raise NotImplementedError("base class, not implement!") + + def _compare_numpy(self, name, actual_np, expect_np): + self.op_test.assertTrue( + np.allclose( + actual_np, + expect_np, + atol=atol, + rtol=self.rtol if hasattr(self, 'rtol') else 1e-5, + equal_nan=equal_nan), + "Output (" + name + ") has diff at " + str(place) + " in " + + self.checker_name + " checker") + + def _compare_list(self, name, actual, expect): + """ if expect is a tuple, we need to compare list. + """ + raise NotImplementedError("base class, not implement!") + + def compare_single_output_with_expect(self, name, expect): + actual, actual_np = self.find_actual_value(name) + expect_np = expect[0] \ + if isinstance(expect, tuple) else expect + actual_np, expect_np = self.convert_uint16_to_float_ifneed( + actual_np, expect_np) + # NOTE(zhiqiu): np.allclose([], [1.]) returns True + # see details: https://stackoverflow.com/questions/38331703/why-does-numpys-broadcasting-sometimes-allow-comparing-arrays-of-different-leng + if expect_np.size == 0: + self.op_test.assertTrue(actual_np.size == 0) # }}} + self._compare_numpy(name, actual_np, expect_np) + if isinstance(expect, tuple): + self._compare_list(name, actual, expect) + + def compare_outputs_with_expects(self): + for out_name, out_dup in Operator.get_op_outputs(self.op_type): + if self._is_skip_name(out_name): continue + if out_dup: + # if self.output = {'name': [(subname, Tensor), (subname, Tensor)]} + sub_out = self.expects[out_name] + if not isinstance(sub_out, list): + raise AssertionError("sub_out type %s is not list", + type(sub_out)) + for item in sub_out: + sub_out_name, expect = item[0], item[1] + self.compare_single_output_with_expect(sub_out_name, + expect) + else: + expect = self.expects[out_name] + self.compare_single_output_with_expect(out_name, expect) + + def check(self): + """ + return None means ok, raise Error means failed. + + the main enter point of Checker class + """ + self.calculate_output() + self.compare_outputs_with_expects() + + class StaticChecker(Checker): + def calculate_output(self): + outs, fetch_list = self.op_test._calc_output( + place, no_check_set=no_check_set) + self.outputs = outs + self.fetch_list = fetch_list + + def find_actual_value(self, name): + idx = find_actual(name, self.fetch_list) + actual = self.outputs[idx] + actual_t = np.array(actual) + return actual, actual_t + + def convert_uint16_to_float_ifneed(self, actual_np, expect_np): + """ + judge whether convert current output and expect to uint16. + return True | False + """ + if actual_np.dtype == np.uint16 and expect_np.dtype in [ + np.float32, np.float64 + ]: + actual_np = convert_uint16_to_float(actual_np) + self.rtol = 1.e-2 + else: + self.rtol = 1.e-5 + if expect_np.dtype == np.uint16 and actual_np.dtype == np.uint16: + nonlocal atol + expect_np = convert_uint16_to_float(expect_np) + actual_np = convert_uint16_to_float(actual_np) + atol = max(atol, 0.03) + return actual_np, expect_np + + def _compare_list(self, name, actual, expect): + """ if expect is a tuple, we need to compare list. + """ + self.op_test.assertListEqual( + actual.recursive_sequence_lengths(), expect[1], + "Output (" + name + ") has different lod at " + str(place)) + + class DygraphChecker(Checker): + def calculate_output(self): + self.outputs = self.op_test._calc_dygraph_output( + place, no_check_set=no_check_set) + + def find_actual_value(self, name): + with fluid.dygraph.base.guard(place=place): + imperative_actual = find_imperative_actual( + name, self.outputs, place) + imperative_actual_t = np.array(imperative_actual.value() + .get_tensor()) + return imperative_actual, imperative_actual_t + + def convert_uint16_to_float_ifneed(self, actual_np, expect_np): + if self.op_test.is_bfloat16_op(): + if actual_np.dtype == np.uint16: + actual_np = convert_uint16_to_float(actual_np) + if expect_np.dtype == np.uint16: + expect_np = convert_uint16_to_float(expect_np) # }}} + return actual_np, expect_np + + def _compare_list(self, name, actual, expect): + """ if expect is a tuple, we need to compare list. + """ + with fluid.dygraph.base.guard(place=place): + self.op_test.assertListEqual( + actual.value().get_tensor() + .recursive_sequence_lengths(), expect[1], + "Output (" + name + ") has different lod at " + + str(place) + " in dygraph mode") + + def _compare_numpy(self, name, actual_np, expect_np): + if six.moves.reduce(lambda x, y: x * y, actual_np.shape, + 1) == 0 and six.moves.reduce( + lambda x, y: x * y, expect_np.shape, + 1) == 0: + pass + else: + self.op_test.assertTrue( + np.allclose( + actual_np, + expect_np, + atol=atol, + rtol=self.rtol if hasattr(self, 'rtol') else 1e-5, + equal_nan=equal_nan), + "Output (" + name + ") has diff at " + str(place) + + " in " + self.checker_name + " checker") + + class EagerChecker(DygraphChecker): + def calculate_output(self): + # we only check end2end api when check_eager=True + with _test_eager_guard(): + eager_dygraph_outs = self.op_test._calc_python_api_output( + place) + if eager_dygraph_outs is None: + # missing KernelSignature, fall back to eager middle output. + eager_dygraph_outs = self.op_test._calc_dygraph_output( + place, no_check_set=no_check_set) + self.outputs = eager_dygraph_outs + + def _compare_numpy(self, name, actual_np, expect_np): + with _test_eager_guard(): + super()._compare_numpy(name, actual_np, expect_np) + + def convert_uint16_to_float_ifneed(self, actual_np, expect_np): + with _test_eager_guard(): + return super().convert_uint16_to_float_ifneed(actual_np, + expect_np) + + def find_actual_value(self, name): + with _test_eager_guard(): + return super().find_actual_value(name) + + def _compare_list(self, name, actual, expect): + """ if expect is a tuple, we need to compare list. + """ + with _test_eager_guard(): + super()._compare_list(name, actual, expect) + +# set some flags by the combination of arguments. + + self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs) # {{{ if self.dtype == np.float64 and \ self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_OUTPUT_THRESHOLD_OP_LIST: atol = 0 @@ -1337,260 +1569,19 @@ class OpTest(unittest.TestCase): if no_check_set is not None: if self.op_type not in no_check_set_white_list.no_check_set_white_list: raise AssertionError( - "no_check_set of op %s must be set to None." % self.op_type) - + "no_check_set of op %s must be set to None." % + self.op_type) # }}} + static_checker = StaticChecker(self, self.outputs) + static_checker.check() + outs, fetch_list = static_checker.outputs, static_checker.fetch_list if check_dygraph: - if _in_eager_without_dygraph_check(): - _enable_legacy_dygraph() - dygraph_outs = self._calc_dygraph_output( - place, no_check_set=no_check_set) - _disable_legacy_dygraph() - else: - dygraph_outs = self._calc_dygraph_output( - place, no_check_set=no_check_set) + dygraph_checker = DygraphChecker(self, self.outputs) + dygraph_checker.check() + dygraph_outs = dygraph_checker.outputs if check_eager: - # we only check end2end api when check_eager=True - with fluid.dygraph.base.guard(place): - with _test_eager_guard(): - eager_dygraph_outs = self._calc_python_api_output(place) - if eager_dygraph_outs is None: - # missing KernelSignature, fall back to eager middle output. - eager_dygraph_outs = self._calc_dygraph_output( - place, no_check_set=no_check_set) - - outs, fetch_list = self._calc_output(place, no_check_set=no_check_set) - - for out_name, out_dup in Operator.get_op_outputs(self.op_type): - if out_name not in self.outputs: - continue - if no_check_set is not None and out_name in no_check_set: - continue - - def find_imperative_actual(target_name, dygraph_outs, place): - with fluid.dygraph.base.guard(place=place): - for name in dygraph_outs: - if name == target_name: - return dygraph_outs[name][0] - var_list = dygraph_outs[name] - for i, var in enumerate(var_list): - if var.name == target_name: - return dygraph_outs[name][i] - self.assertTrue(False, "Found failed {} {}".format( - dygraph_outs.keys(), target_name)) - - def find_actual(target_name, fetch_list): - found = [ - i for i, var_name in enumerate(fetch_list) - if var_name == target_name - ] - self.assertTrue( - len(found) == 1, "Found {} {}".format( - len(found), target_name)) - return found[0] - - if out_dup: - sub_out = self.outputs[out_name] - if not isinstance(sub_out, list): - raise AssertionError("sub_out type %s is not list", - type(sub_out)) - for item in sub_out: - sub_out_name, expect = item[0], item[1] - if check_dygraph: - if _in_eager_without_dygraph_check(): - _enable_legacy_dygraph() - imperative_actual = find_imperative_actual( - sub_out_name, dygraph_outs, place) - imperative_actual_t = np.array( - imperative_actual.value().get_tensor()) - _disable_legacy_dygraph() - else: - imperative_actual = find_imperative_actual( - sub_out_name, dygraph_outs, place) - imperative_actual_t = np.array( - imperative_actual.value().get_tensor()) - if check_eager: - with fluid.dygraph.base.guard(place): - with _test_eager_guard(): - eager_imperative_actual = find_imperative_actual( - sub_out_name, eager_dygraph_outs, place) - eager_imperative_actual_t = eager_imperative_actual.numpy( - ) - - idx = find_actual(sub_out_name, fetch_list) - actual = outs[idx] - actual_t = np.array(actual) - expect_t = expect[0] \ - if isinstance(expect, tuple) else expect - self.assertTrue( - np.allclose( - actual_t, expect_t, atol=atol, equal_nan=equal_nan), - "Output (" + sub_out_name + ") has diff at " + - str(place)) - if check_dygraph: - self.assertTrue( - np.allclose( - imperative_actual_t, - expect_t, - atol=atol, - equal_nan=equal_nan), - "Output (" + sub_out_name + ") has diff at " + - str(place) + " in dygraph mode") - if check_eager: - with fluid.dygraph.base.guard(place): - with _test_eager_guard(): - self.assertTrue( - np.allclose( - eager_imperative_actual_t, - expect_t, - atol=atol, - equal_nan=equal_nan), - "Output (" + sub_out_name + ") has diff at " - + str(place) + " in eager dygraph mode") - if isinstance(expect, tuple): - self.assertListEqual( - actual.recursive_sequence_lengths(), expect[1], - "Output (" + sub_out_name + - ") has different lod at " + str(place)) - if check_dygraph: - self.assertListEqual( - imperative_actual.value().get_tensor() - .recursive_sequence_lengths(), expect[1], - "Output (" + out_name + - ") has different lod at " + str(place) + - " in dygraph mode") - if check_eager: - with fluid.dygraph.base.guard(place): - with _test_eager_guard(): - self.assertListEqual( - eager_imperative_actual.value( - ).get_tensor() - .recursive_sequence_lengths(), - expect[1], "Output (" + out_name + - ") has different lod at " + str(place) + - " in eager dygraph mode") - else: - if check_dygraph: - - with fluid.dygraph.base.guard(place=place): - imperative_actual = find_imperative_actual( - out_name, dygraph_outs, place) - imperative_actual_t = np.array(imperative_actual.value() - .get_tensor()) - if check_eager: - with fluid.dygraph.base.guard(place): - with _test_eager_guard(): - eager_imperative_actual = find_imperative_actual( - out_name, eager_dygraph_outs, place) - eager_imperative_actual_t = eager_imperative_actual.numpy( - ) - - idx = find_actual(out_name, fetch_list) - actual = outs[idx] - actual_t = np.array(actual) - - expect = self.outputs[out_name] - expect_t = expect[0] if isinstance(expect, tuple) else expect - - # np.uint16 represents bfloat16 - if actual_t.dtype == np.uint16 and expect_t.dtype in [ - np.float32, np.float64 - ]: - actual_t = convert_uint16_to_float(actual_t) - rtol = 1.e-2 - else: - rtol = 1.e-5 - - if expect_t.dtype == np.uint16 and actual_t.dtype == np.uint16: - expect_t = convert_uint16_to_float(expect_t) - actual_t = convert_uint16_to_float(actual_t) - atol = max(atol, 0.03) - - # NOTE(zhiqiu): np.allclose([], [1.]) returns True - # see details: https://stackoverflow.com/questions/38331703/why-does-numpys-broadcasting-sometimes-allow-comparing-arrays-of-different-leng - if expect_t.size == 0: - self.assertTrue(actual_t.size == 0) - - self.assertTrue( - np.allclose( - actual_t, - expect_t, - atol=atol, - rtol=rtol, - equal_nan=equal_nan), - "Output (" + out_name + ") has diff at " + str(place) + - "\nExpect " + str(expect_t) + "\n" + "But Got" + - str(actual_t) + " in class " + self.__class__.__name__) - if check_dygraph: - if self.is_bfloat16_op(): - if imperative_actual_t.dtype == np.uint16: - imperative_actual_t = convert_uint16_to_float( - imperative_actual_t) - if expect_t.dtype == np.uint16: - expect_t = convert_uint16_to_float(expect_t) - if six.moves.reduce( - lambda x, y: x * y, imperative_actual_t.shape, - 1) == 0 and six.moves.reduce( - lambda x, y: x * y, expect_t.shape, 1) == 0: - pass - else: - self.assertTrue( - np.allclose( - imperative_actual_t, - expect_t, - atol=atol, - rtol=rtol, - equal_nan=equal_nan), - "Output (" + out_name + ") has diff at " + - str(place) + "\nExpect " + str(expect_t) + "\n" + - "But Got" + str(imperative_actual_t) + " in class " - + self.__class__.__name__) - if check_eager: - with fluid.dygraph.base.guard(place): - with _test_eager_guard(): - if self.is_bfloat16_op(): - if eager_imperative_actual_t.dtype == np.uint16: - eager_imperative_actual_t = convert_uint16_to_float( - eager_imperative_actual_t) - if expect_t.dtype == np.uint16: - expect_t = convert_uint16_to_float(expect_t) - if six.moves.reduce(lambda x, y: x * y, - eager_imperative_actual_t.shape, - 1) == 0 and six.moves.reduce( - lambda x, y: x * y, - expect_t.shape, 1) == 0: - pass - else: - self.assertTrue( - np.allclose( - eager_imperative_actual_t, - expect_t, - atol=atol, - rtol=rtol, - equal_nan=equal_nan), - "Output (" + out_name + ") has diff at " + - str(place) + "\nExpect " + str(expect_t) + - "\n" + "But Got" + - str(eager_imperative_actual_t) + - " in class " + self.__class__.__name__) - if isinstance(expect, tuple): - self.assertListEqual(actual.recursive_sequence_lengths(), - expect[1], "Output (" + out_name + - ") has different lod at " + str(place)) - if check_dygraph: - self.assertListEqual( - imperative_actual.value().get_tensor() - .recursive_sequence_lengths(), expect[1], - "Output (" + out_name + ") has different lod at " + - str(place) + " in eager dygraph mode") - if check_eager: - with fluid.dygraph.base.guard(place): - with _test_eager_guard(): - self.assertListEqual( - eager_imperative_actual.value().get_tensor() - .recursive_sequence_lengths(), expect[1], - "Output (" + out_name + - ") has different lod at " + str(place) + - " in eager dygraph mode") + eager_checker = EagerChecker(self, self.outputs) + eager_checker.check() + eager_dygraph_outs = eager_checker.outputs # Note(zhiqiu): inplace_atol should be only set when op doesn't ensure # computational consistency. @@ -1619,6 +1610,8 @@ class OpTest(unittest.TestCase): else: return outs, fetch_list +# }}} + def check_compile_vs_runtime(self, fetch_list, fetch_outs): def find_fetch_index(target_name, fetch_list): found = [