From 0a7ae369f6d6a6be7b87d13834b8ff0a7e3614d1 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 9 Feb 2018 10:27:47 +0800 Subject: [PATCH] fix CI hung --- .../tests/book_memory_optimization/test_memopt_fit_a_line.py | 4 ++++ .../test_memopt_image_classification_train.py | 5 +++++ .../test_memopt_machine_translation.py | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py index 7ad5e2c594..045db8390c 100644 --- a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py +++ b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py @@ -15,6 +15,8 @@ import numpy as np import paddle.v2 as paddle import paddle.v2.fluid as fluid +import math +import sys # need to fix random seed and training data to compare the loss # value accurately calculated by the default and the memory optimization @@ -63,4 +65,6 @@ for pass_id in range(PASS_NUM): if avg_loss_value[0] < 10.0: exit(0) # if avg cost less than 10.0, we think our code is good. + if math.isnan(float(avg_loss_value)): + sys.exit("got NaN loss, training failed.") exit(1) diff --git a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py index 26673afd83..9fbb36d363 100644 --- a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py +++ b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py @@ -18,6 +18,8 @@ import sys import paddle.v2 as paddle import paddle.v2.fluid as fluid +import math +import sys # need to fix random seed and training data to compare the loss # value accurately calculated by the default and the memory optimization @@ -152,7 +154,10 @@ for pass_id in range(PASS_NUM): print("loss:" + str(loss) + " acc:" + str(acc) + " pass_acc:" + str( pass_acc)) # this model is slow, so if we can train two mini batch, we think it works properly. + if i > 2: exit(0) + if math.isnan(float(loss)): + sys.exit("got NaN loss, training failed.") i += 1 exit(1) diff --git a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py index ffd53e7a78..48abaa8d87 100644 --- a/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py +++ b/python/paddle/v2/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py @@ -19,6 +19,8 @@ import paddle.v2.fluid.core as core import paddle.v2.fluid.framework as framework import paddle.v2.fluid.layers as layers from paddle.v2.fluid.executor import Executor +import math +import sys dict_size = 30000 source_dict_dim = target_dict_dim = dict_size @@ -137,6 +139,8 @@ def main(): " avg_cost=" + str(avg_cost_val)) if batch_id > 2: exit(0) + if math.isnan(float(avg_cost_val)): + sys.exit("got NaN loss, training failed.") batch_id += 1 -- GitLab