未验证 提交 73f1b471 编写于 作者: 武毅 提交者: GitHub

Merge pull request #8292 from typhoonzero/fix_book_ci_hung

Fix CI hung
...@@ -15,6 +15,8 @@ ...@@ -15,6 +15,8 @@
import numpy as np import numpy as np
import paddle.v2 as paddle import paddle.v2 as paddle
import paddle.v2.fluid as fluid import paddle.v2.fluid as fluid
import math
import sys
# need to fix random seed and training data to compare the loss # need to fix random seed and training data to compare the loss
# value accurately calculated by the default and the memory optimization # value accurately calculated by the default and the memory optimization
...@@ -63,4 +65,6 @@ for pass_id in range(PASS_NUM): ...@@ -63,4 +65,6 @@ for pass_id in range(PASS_NUM):
if avg_loss_value[0] < 10.0: if avg_loss_value[0] < 10.0:
exit(0) # if avg cost less than 10.0, we think our code is good. exit(0) # if avg cost less than 10.0, we think our code is good.
if math.isnan(float(avg_loss_value)):
sys.exit("got NaN loss, training failed.")
exit(1) exit(1)
...@@ -18,6 +18,8 @@ import sys ...@@ -18,6 +18,8 @@ import sys
import paddle.v2 as paddle import paddle.v2 as paddle
import paddle.v2.fluid as fluid import paddle.v2.fluid as fluid
import math
import sys
# need to fix random seed and training data to compare the loss # need to fix random seed and training data to compare the loss
# value accurately calculated by the default and the memory optimization # value accurately calculated by the default and the memory optimization
...@@ -152,7 +154,10 @@ for pass_id in range(PASS_NUM): ...@@ -152,7 +154,10 @@ for pass_id in range(PASS_NUM):
print("loss:" + str(loss) + " acc:" + str(acc) + " pass_acc:" + str( print("loss:" + str(loss) + " acc:" + str(acc) + " pass_acc:" + str(
pass_acc)) pass_acc))
# this model is slow, so if we can train two mini batch, we think it works properly. # this model is slow, so if we can train two mini batch, we think it works properly.
if i > 2: if i > 2:
exit(0) exit(0)
if math.isnan(float(loss)):
sys.exit("got NaN loss, training failed.")
i += 1 i += 1
exit(1) exit(1)
...@@ -19,6 +19,8 @@ import paddle.v2.fluid.core as core ...@@ -19,6 +19,8 @@ import paddle.v2.fluid.core as core
import paddle.v2.fluid.framework as framework import paddle.v2.fluid.framework as framework
import paddle.v2.fluid.layers as layers import paddle.v2.fluid.layers as layers
from paddle.v2.fluid.executor import Executor from paddle.v2.fluid.executor import Executor
import math
import sys
dict_size = 30000 dict_size = 30000
source_dict_dim = target_dict_dim = dict_size source_dict_dim = target_dict_dim = dict_size
...@@ -137,6 +139,8 @@ def main(): ...@@ -137,6 +139,8 @@ def main():
" avg_cost=" + str(avg_cost_val)) " avg_cost=" + str(avg_cost_val))
if batch_id > 2: if batch_id > 2:
exit(0) exit(0)
if math.isnan(float(avg_cost_val)):
sys.exit("got NaN loss, training failed.")
batch_id += 1 batch_id += 1
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册