未验证 提交 b5882c6e 编写于 作者: A Aurelius84 提交者: GitHub

[UnitTest]Enhance grep syntax to avoid random failed of test_dist_mnist_dgc_nccl (#32946)

* Enhance grep syntax to avoid random failed

* Enhance grep syntax to avoid random failed
上级 b8d493df
...@@ -25,12 +25,15 @@ flag_name = os.path.splitext(__file__)[0] ...@@ -25,12 +25,15 @@ flag_name = os.path.splitext(__file__)[0]
def count_of_sparse_all_reduce_calls(file_name): def count_of_sparse_all_reduce_calls(file_name):
cmd = 'grep sparse_all_reduce_op_handle ' + file_name + ' | grep in_numel | wc -l' # NOTE(Aurelius84): The log file contains some binary contents that causes error
# while `grep`. So we add `-a` to fix it.
# -a, --text equivalent to --binary-files=text, make binaries equivalent to text.
cmd = 'grep -a sparse_all_reduce_op_handle ' + file_name + ' | grep in_numel | wc -l'
child = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) child = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
result = child.communicate()[0] result = child.communicate()[0]
print('test_info: result = ' + str(result)) print('test_info: result = ' + str(result))
# note. in python3, result is b'num', != 'num' # NOTE: in python3, result is b'num', != 'num'
return int(result) return int(result)
...@@ -59,7 +62,7 @@ class TestDistMnistNCCL2DGC(TestDistBase): ...@@ -59,7 +62,7 @@ class TestDistMnistNCCL2DGC(TestDistBase):
# only 1 layer use dgc now, run_step=5, rampup_begin_step=2, so 1 * (5 - 2) = 3 # only 1 layer use dgc now, run_step=5, rampup_begin_step=2, so 1 * (5 - 2) = 3
# temp close this test. In python3 CI, the log is right, but the result # temp close this test. In python3 CI, the log is right, but the result
# has a problem, may be in multi process mode, log is not writed in time. # has a problem, may be in multi process mode, log is not written in time.
# self.assertEqual(result, 3) # self.assertEqual(result, 3)
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册