Skip to content

P
Paddle

PaddlePaddle / Paddle
大约 2 年 前同步成功

通知 2325
Star 20933
Fork 5424
P
Paddle
“bb6c627b41cf4c0730e504692544dfe23354dd28”上不存在“...Pipeline/imdb_model_ensemble/test_pipeline_server.py”
未验证 提交 37f914c8 编写于 作者: X xiongkun 提交者： GitHub
浏览文件
操作

[ Optest ] refactor optest check_output_with_place logic (#40928) 

* first version, maybe many errors

* refactor op_test

* fix compare list

* fix bg

* fix bugs
上级 afa0e82c
隐藏空白更改
内联 并排
Showing with 246 addition and 253 deletion
python/paddle/fluid/tests/unittests/op_test.py
浏览文件 @ 37f914c8
......@@ -1317,7 +1317,239 @@ class OpTest(unittest.TestCase):
check_dygraph=True,
inplace_atol=None,
check_eager=False):
self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs)
def find_imperative_actual(target_name, dygraph_outs, place):
for name in dygraph_outs:
if name == target_name:
return dygraph_outs[name][0]
var_list = dygraph_outs[name]
for i, var in enumerate(var_list):
if var.name == target_name:
return dygraph_outs[name][i]
self.assertTrue(False, "Found failed {} {}".format(
dygraph_outs.keys(), target_name))
def find_actual(target_name, fetch_list):
found = [
i for i, var_name in enumerate(fetch_list)
if var_name == target_name
]
self.assertTrue(
len(found) == 1, "Found {} {}".format(len(found), target_name))
return found[0]
class Checker(object):
""" base class for check with self.outputs.
currently don't support check between checkers.
"""
def __init__(self, op_test, expect_dict):
""" expect_dict is the self.outputs
support : {str: [numpy]} and {str: [(str, numpy), (str, numpy)]}
"""
self.expects = expect_dict
self.checker_name = "checker"
self.op_test = op_test # stop the op_test object.
self.op_type = op_test.op_type
def convert_uint16_to_float(self, actual_np, expect_np):
raise NotImplementedError("base class, not implement!")
def calculate_output(self):
"""
judge whether convert current output and expect to uint16.
return True | False
"""
pass
def _is_skip_name(self, name):
if name not in self.expects:
return True
if no_check_set is not None and name in no_check_set:
return True
return False
def find_actual_value(self, name):
""" return: (actual_tensor(var_base), actual_numpy)
"""
raise NotImplementedError("base class, not implement!")
def _compare_numpy(self, name, actual_np, expect_np):
self.op_test.assertTrue(
np.allclose(
actual_np,
expect_np,
atol=atol,
rtol=self.rtol if hasattr(self, 'rtol') else 1e-5,
equal_nan=equal_nan),
"Output (" + name + ") has diff at " + str(place) + " in " +
self.checker_name + " checker")
def _compare_list(self, name, actual, expect):
""" if expect is a tuple, we need to compare list.
"""
raise NotImplementedError("base class, not implement!")
def compare_single_output_with_expect(self, name, expect):
actual, actual_np = self.find_actual_value(name)
expect_np = expect[0] \
if isinstance(expect, tuple) else expect
actual_np, expect_np = self.convert_uint16_to_float_ifneed(
actual_np, expect_np)
# NOTE(zhiqiu): np.allclose([], [1.]) returns True
# see details: https://stackoverflow.com/questions/38331703/why-does-numpys-broadcasting-sometimes-allow-comparing-arrays-of-different-leng
if expect_np.size == 0:
self.op_test.assertTrue(actual_np.size == 0) # }}}
self._compare_numpy(name, actual_np, expect_np)
if isinstance(expect, tuple):
self._compare_list(name, actual, expect)
def compare_outputs_with_expects(self):
for out_name, out_dup in Operator.get_op_outputs(self.op_type):
if self._is_skip_name(out_name): continue
if out_dup:
# if self.output = {'name': [(subname, Tensor), (subname, Tensor)]}
sub_out = self.expects[out_name]
if not isinstance(sub_out, list):
raise AssertionError("sub_out type %s is not list",
type(sub_out))
for item in sub_out:
sub_out_name, expect = item[0], item[1]
self.compare_single_output_with_expect(sub_out_name,
expect)
else:
expect = self.expects[out_name]
self.compare_single_output_with_expect(out_name, expect)
def check(self):
"""
return None means ok, raise Error means failed.
the main enter point of Checker class
"""
self.calculate_output()
self.compare_outputs_with_expects()
class StaticChecker(Checker):
def calculate_output(self):
outs, fetch_list = self.op_test._calc_output(
place, no_check_set=no_check_set)
self.outputs = outs
self.fetch_list = fetch_list
def find_actual_value(self, name):
idx = find_actual(name, self.fetch_list)
actual = self.outputs[idx]
actual_t = np.array(actual)
return actual, actual_t
def convert_uint16_to_float_ifneed(self, actual_np, expect_np):
"""
judge whether convert current output and expect to uint16.
return True | False
"""
if actual_np.dtype == np.uint16 and expect_np.dtype in [
np.float32, np.float64
]:
actual_np = convert_uint16_to_float(actual_np)
self.rtol = 1.e-2
else:
self.rtol = 1.e-5
if expect_np.dtype == np.uint16 and actual_np.dtype == np.uint16:
nonlocal atol
expect_np = convert_uint16_to_float(expect_np)
actual_np = convert_uint16_to_float(actual_np)
atol = max(atol, 0.03)
return actual_np, expect_np
def _compare_list(self, name, actual, expect):
""" if expect is a tuple, we need to compare list.
"""
self.op_test.assertListEqual(
actual.recursive_sequence_lengths(), expect[1],
"Output (" + name + ") has different lod at " + str(place))
class DygraphChecker(Checker):
def calculate_output(self):
self.outputs = self.op_test._calc_dygraph_output(
place, no_check_set=no_check_set)
def find_actual_value(self, name):
with fluid.dygraph.base.guard(place=place):
imperative_actual = find_imperative_actual(
name, self.outputs, place)
imperative_actual_t = np.array(imperative_actual.value()
.get_tensor())
return imperative_actual, imperative_actual_t
def convert_uint16_to_float_ifneed(self, actual_np, expect_np):
if self.op_test.is_bfloat16_op():
if actual_np.dtype == np.uint16:
actual_np = convert_uint16_to_float(actual_np)
if expect_np.dtype == np.uint16:
expect_np = convert_uint16_to_float(expect_np) # }}}
return actual_np, expect_np
def _compare_list(self, name, actual, expect):
""" if expect is a tuple, we need to compare list.
"""
with fluid.dygraph.base.guard(place=place):
self.op_test.assertListEqual(
actual.value().get_tensor()
.recursive_sequence_lengths(), expect[1],
"Output (" + name + ") has different lod at " +
str(place) + " in dygraph mode")
def _compare_numpy(self, name, actual_np, expect_np):
if six.moves.reduce(lambda x, y: x * y, actual_np.shape,
1) == 0 and six.moves.reduce(
lambda x, y: x * y, expect_np.shape,
1) == 0:
pass
else:
self.op_test.assertTrue(
np.allclose(
actual_np,
expect_np,
atol=atol,
rtol=self.rtol if hasattr(self, 'rtol') else 1e-5,
equal_nan=equal_nan),
"Output (" + name + ") has diff at " + str(place) +
" in " + self.checker_name + " checker")
class EagerChecker(DygraphChecker):
def calculate_output(self):
# we only check end2end api when check_eager=True
with _test_eager_guard():
eager_dygraph_outs = self.op_test._calc_python_api_output(
place)
if eager_dygraph_outs is None:
# missing KernelSignature, fall back to eager middle output.
eager_dygraph_outs = self.op_test._calc_dygraph_output(
place, no_check_set=no_check_set)
self.outputs = eager_dygraph_outs
def _compare_numpy(self, name, actual_np, expect_np):
with _test_eager_guard():
super()._compare_numpy(name, actual_np, expect_np)
def convert_uint16_to_float_ifneed(self, actual_np, expect_np):
with _test_eager_guard():
return super().convert_uint16_to_float_ifneed(actual_np,
expect_np)
def find_actual_value(self, name):
with _test_eager_guard():
return super().find_actual_value(name)
def _compare_list(self, name, actual, expect):
""" if expect is a tuple, we need to compare list.
"""
with _test_eager_guard():
super()._compare_list(name, actual, expect)
# set some flags by the combination of arguments.
self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs) # {{{
if self.dtype == np.float64 and \
self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_OUTPUT_THRESHOLD_OP_LIST:
atol = 0
......@@ -1337,260 +1569,19 @@ class OpTest(unittest.TestCase):
if no_check_set is not None:
if self.op_type not in no_check_set_white_list.no_check_set_white_list:
raise AssertionError(
"no_check_set of op %s must be set to None." % self.op_type)
"no_check_set of op %s must be set to None." %
self.op_type) # }}}
static_checker = StaticChecker(self, self.outputs)
static_checker.check()
outs, fetch_list = static_checker.outputs, static_checker.fetch_list
if check_dygraph:
if _in_eager_without_dygraph_check():
_enable_legacy_dygraph()
dygraph_outs = self._calc_dygraph_output(
place, no_check_set=no_check_set)
_disable_legacy_dygraph()
else:
dygraph_outs = self._calc_dygraph_output(
place, no_check_set=no_check_set)
dygraph_checker = DygraphChecker(self, self.outputs)
dygraph_checker.check()
dygraph_outs = dygraph_checker.outputs
if check_eager:
# we only check end2end api when check_eager=True
with fluid.dygraph.base.guard(place):
with _test_eager_guard():
eager_dygraph_outs = self._calc_python_api_output(place)
if eager_dygraph_outs is None:
# missing KernelSignature, fall back to eager middle output.
eager_dygraph_outs = self._calc_dygraph_output(
place, no_check_set=no_check_set)
outs, fetch_list = self._calc_output(place, no_check_set=no_check_set)
for out_name, out_dup in Operator.get_op_outputs(self.op_type):
if out_name not in self.outputs:
continue
if no_check_set is not None and out_name in no_check_set:
continue
def find_imperative_actual(target_name, dygraph_outs, place):
with fluid.dygraph.base.guard(place=place):
for name in dygraph_outs:
if name == target_name:
return dygraph_outs[name][0]
var_list = dygraph_outs[name]
for i, var in enumerate(var_list):
if var.name == target_name:
return dygraph_outs[name][i]
self.assertTrue(False, "Found failed {} {}".format(
dygraph_outs.keys(), target_name))
def find_actual(target_name, fetch_list):
found = [
i for i, var_name in enumerate(fetch_list)
if var_name == target_name
]
self.assertTrue(
len(found) == 1, "Found {} {}".format(
len(found), target_name))
return found[0]
if out_dup:
sub_out = self.outputs[out_name]
if not isinstance(sub_out, list):
raise AssertionError("sub_out type %s is not list",
type(sub_out))
for item in sub_out:
sub_out_name, expect = item[0], item[1]
if check_dygraph:
if _in_eager_without_dygraph_check():
_enable_legacy_dygraph()
imperative_actual = find_imperative_actual(
sub_out_name, dygraph_outs, place)
imperative_actual_t = np.array(
imperative_actual.value().get_tensor())
_disable_legacy_dygraph()
else:
imperative_actual = find_imperative_actual(
sub_out_name, dygraph_outs, place)
imperative_actual_t = np.array(
imperative_actual.value().get_tensor())
if check_eager:
with fluid.dygraph.base.guard(place):
with _test_eager_guard():
eager_imperative_actual = find_imperative_actual(
sub_out_name, eager_dygraph_outs, place)
eager_imperative_actual_t = eager_imperative_actual.numpy(
)
idx = find_actual(sub_out_name, fetch_list)
actual = outs[idx]
actual_t = np.array(actual)
expect_t = expect[0] \
if isinstance(expect, tuple) else expect
self.assertTrue(
np.allclose(
actual_t, expect_t, atol=atol, equal_nan=equal_nan),
"Output (" + sub_out_name + ") has diff at " +
str(place))
if check_dygraph:
self.assertTrue(
np.allclose(
imperative_actual_t,
expect_t,
atol=atol,
equal_nan=equal_nan),
"Output (" + sub_out_name + ") has diff at " +
str(place) + " in dygraph mode")
if check_eager:
with fluid.dygraph.base.guard(place):
with _test_eager_guard():
self.assertTrue(
np.allclose(
eager_imperative_actual_t,
expect_t,
atol=atol,
equal_nan=equal_nan),
"Output (" + sub_out_name + ") has diff at "
+ str(place) + " in eager dygraph mode")
if isinstance(expect, tuple):
self.assertListEqual(
actual.recursive_sequence_lengths(), expect[1],
"Output (" + sub_out_name +
") has different lod at " + str(place))
if check_dygraph:
self.assertListEqual(
imperative_actual.value().get_tensor()
.recursive_sequence_lengths(), expect[1],
"Output (" + out_name +
") has different lod at " + str(place) +
" in dygraph mode")
if check_eager:
with fluid.dygraph.base.guard(place):
with _test_eager_guard():
self.assertListEqual(
eager_imperative_actual.value(
).get_tensor()
.recursive_sequence_lengths(),
expect[1], "Output (" + out_name +
") has different lod at " + str(place) +
" in eager dygraph mode")
else:
if check_dygraph:
with fluid.dygraph.base.guard(place=place):
imperative_actual = find_imperative_actual(
out_name, dygraph_outs, place)
imperative_actual_t = np.array(imperative_actual.value()
.get_tensor())
if check_eager:
with fluid.dygraph.base.guard(place):
with _test_eager_guard():
eager_imperative_actual = find_imperative_actual(
out_name, eager_dygraph_outs, place)
eager_imperative_actual_t = eager_imperative_actual.numpy(
)
idx = find_actual(out_name, fetch_list)
actual = outs[idx]
actual_t = np.array(actual)
expect = self.outputs[out_name]
expect_t = expect[0] if isinstance(expect, tuple) else expect
# np.uint16 represents bfloat16
if actual_t.dtype == np.uint16 and expect_t.dtype in [
np.float32, np.float64
]:
actual_t = convert_uint16_to_float(actual_t)
rtol = 1.e-2
else:
rtol = 1.e-5
if expect_t.dtype == np.uint16 and actual_t.dtype == np.uint16:
expect_t = convert_uint16_to_float(expect_t)
actual_t = convert_uint16_to_float(actual_t)
atol = max(atol, 0.03)
# NOTE(zhiqiu): np.allclose([], [1.]) returns True
# see details: https://stackoverflow.com/questions/38331703/why-does-numpys-broadcasting-sometimes-allow-comparing-arrays-of-different-leng
if expect_t.size == 0:
self.assertTrue(actual_t.size == 0)
self.assertTrue(
np.allclose(
actual_t,
expect_t,
atol=atol,
rtol=rtol,
equal_nan=equal_nan),
"Output (" + out_name + ") has diff at " + str(place) +
"\nExpect " + str(expect_t) + "\n" + "But Got" +
str(actual_t) + " in class " + self.__class__.__name__)
if check_dygraph:
if self.is_bfloat16_op():
if imperative_actual_t.dtype == np.uint16:
imperative_actual_t = convert_uint16_to_float(
imperative_actual_t)
if expect_t.dtype == np.uint16:
expect_t = convert_uint16_to_float(expect_t)
if six.moves.reduce(
lambda x, y: x * y, imperative_actual_t.shape,
1) == 0 and six.moves.reduce(
lambda x, y: x * y, expect_t.shape, 1) == 0:
pass
else:
self.assertTrue(
np.allclose(
imperative_actual_t,
expect_t,
atol=atol,
rtol=rtol,
equal_nan=equal_nan),
"Output (" + out_name + ") has diff at " +
str(place) + "\nExpect " + str(expect_t) + "\n" +
"But Got" + str(imperative_actual_t) + " in class "
+ self.__class__.__name__)
if check_eager:
with fluid.dygraph.base.guard(place):
with _test_eager_guard():
if self.is_bfloat16_op():
if eager_imperative_actual_t.dtype == np.uint16:
eager_imperative_actual_t = convert_uint16_to_float(
eager_imperative_actual_t)
if expect_t.dtype == np.uint16:
expect_t = convert_uint16_to_float(expect_t)
if six.moves.reduce(lambda x, y: x * y,
eager_imperative_actual_t.shape,
1) == 0 and six.moves.reduce(
lambda x, y: x * y,
expect_t.shape, 1) == 0:
pass
else:
self.assertTrue(
np.allclose(
eager_imperative_actual_t,
expect_t,
atol=atol,
rtol=rtol,
equal_nan=equal_nan),
"Output (" + out_name + ") has diff at " +
str(place) + "\nExpect " + str(expect_t) +
"\n" + "But Got" +
str(eager_imperative_actual_t) +
" in class " + self.__class__.__name__)
if isinstance(expect, tuple):
self.assertListEqual(actual.recursive_sequence_lengths(),
expect[1], "Output (" + out_name +
") has different lod at " + str(place))
if check_dygraph:
self.assertListEqual(
imperative_actual.value().get_tensor()
.recursive_sequence_lengths(), expect[1],
"Output (" + out_name + ") has different lod at " +
str(place) + " in eager dygraph mode")
if check_eager:
with fluid.dygraph.base.guard(place):
with _test_eager_guard():
self.assertListEqual(
eager_imperative_actual.value().get_tensor()
.recursive_sequence_lengths(), expect[1],
"Output (" + out_name +
") has different lod at " + str(place) +
" in eager dygraph mode")
eager_checker = EagerChecker(self, self.outputs)
eager_checker.check()
eager_dygraph_outs = eager_checker.outputs
# Note(zhiqiu): inplace_atol should be only set when op doesn't ensure
# computational consistency.
......@@ -1619,6 +1610,8 @@ class OpTest(unittest.TestCase):
else:
return outs, fetch_list
# }}}
def check_compile_vs_runtime(self, fetch_list, fetch_outs):
def find_fetch_index(target_name, fetch_list):
found = [
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或