From eca8a5795dbdb810e9fdfb675deb31b4955dd364 Mon Sep 17 00:00:00 2001 From: zlsh80826 Date: Thu, 12 May 2022 11:23:45 +0800 Subject: [PATCH] Use tempfile to place the temporary files (#42626) * Use tempfile to place the temporary files * Revise test_bert to use tempfile for temporary files * Use tempfile for test_transformer * Fix test_dataset file race --- .../unittests/dygraph_to_static/test_bert.py | 274 +++++++++--------- .../dygraph_to_static/test_build_strategy.py | 17 +- .../dygraph_to_static/test_resnet.py | 259 +++++++++-------- .../dygraph_to_static/test_transformer.py | 14 + .../fluid/tests/unittests/test_dataset.py | 16 +- .../test_dataset_consistency_inspection.py | 19 +- 6 files changed, 318 insertions(+), 281 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bert.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bert.py index a9e94ef09b9..db533e6379a 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bert.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_bert.py @@ -14,6 +14,7 @@ import os import time +import tempfile import unittest import numpy as np @@ -33,32 +34,118 @@ place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace( SEED = 2020 STEP_NUM = 10 PRINT_STEP = 2 -MODEL_SAVE_DIR = "./inference" -MODEL_SAVE_PREFIX = "./inference/bert" -MODEL_FILENAME = "bert" + INFER_MODEL_SUFFIX -PARAMS_FILENAME = "bert" + INFER_PARAMS_SUFFIX -DY_STATE_DICT_SAVE_PATH = "./bert.dygraph" - - -def train(bert_config, data_reader, to_static): - with fluid.dygraph.guard(place): - fluid.default_main_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - data_loader = fluid.io.DataLoader.from_generator( - capacity=50, iterable=True) - data_loader.set_batch_generator( - data_reader.data_generator(), places=place) - - bert = PretrainModelLayer( - config=bert_config, weight_sharing=False, use_fp16=False) - - optimizer = fluid.optimizer.Adam(parameter_list=bert.parameters()) - step_idx = 0 - speed_list = [] - for input_data in data_loader(): - src_ids, pos_ids, sent_ids, input_mask, mask_label, mask_pos, labels = input_data - next_sent_acc, mask_lm_loss, total_loss = bert( + + +class TestBert(unittest.TestCase): + def setUp(self): + self.bert_config = get_bert_config() + self.data_reader = get_feed_data_reader(self.bert_config) + self.temp_dir = tempfile.TemporaryDirectory() + self.model_save_dir = os.path.join(self.temp_dir.name, 'inference') + self.model_save_prefix = os.path.join(self.model_save_dir, 'bert') + self.model_filename = 'bert' + INFER_MODEL_SUFFIX + self.params_filename = 'bert' + INFER_PARAMS_SUFFIX + self.dy_state_dict_save_path = os.path.join(self.temp_dir.name, + 'bert.dygraph') + + def tearDown(self): + self.temp_dir.cleanup() + + def train(self, bert_config, data_reader, to_static): + with fluid.dygraph.guard(place): + fluid.default_main_program().random_seed = SEED + fluid.default_startup_program().random_seed = SEED + + data_loader = fluid.io.DataLoader.from_generator( + capacity=50, iterable=True) + data_loader.set_batch_generator( + data_reader.data_generator(), places=place) + + bert = PretrainModelLayer( + config=bert_config, weight_sharing=False, use_fp16=False) + + optimizer = fluid.optimizer.Adam(parameter_list=bert.parameters()) + step_idx = 0 + speed_list = [] + for input_data in data_loader(): + src_ids, pos_ids, sent_ids, input_mask, mask_label, mask_pos, labels = input_data + next_sent_acc, mask_lm_loss, total_loss = bert( + src_ids=src_ids, + position_ids=pos_ids, + sentence_ids=sent_ids, + input_mask=input_mask, + mask_label=mask_label, + mask_pos=mask_pos, + labels=labels) + total_loss.backward() + optimizer.minimize(total_loss) + bert.clear_gradients() + + acc = np.mean(np.array(next_sent_acc.numpy())) + loss = np.mean(np.array(total_loss.numpy())) + ppl = np.mean(np.exp(np.array(mask_lm_loss.numpy()))) + + if step_idx % PRINT_STEP == 0: + if step_idx == 0: + print("Step: %d, loss: %f, ppl: %f, next_sent_acc: %f" % + (step_idx, loss, ppl, acc)) + avg_batch_time = time.time() + else: + speed = PRINT_STEP / (time.time() - avg_batch_time) + speed_list.append(speed) + print( + "Step: %d, loss: %f, ppl: %f, next_sent_acc: %f, speed: %.3f steps/s" + % (step_idx, loss, ppl, acc, speed)) + avg_batch_time = time.time() + + step_idx += 1 + if step_idx == STEP_NUM: + if to_static: + fluid.dygraph.jit.save(bert, self.model_save_prefix) + else: + fluid.dygraph.save_dygraph(bert.state_dict(), + self.dy_state_dict_save_path) + break + return loss, ppl + + def train_dygraph(self, bert_config, data_reader): + program_translator.enable(False) + return self.train(bert_config, data_reader, False) + + def train_static(self, bert_config, data_reader): + program_translator.enable(True) + return self.train(bert_config, data_reader, True) + + def predict_static(self, data): + paddle.enable_static() + exe = fluid.Executor(place) + # load inference model + [inference_program, feed_target_names, + fetch_targets] = fluid.io.load_inference_model( + self.model_save_dir, + executor=exe, + model_filename=self.model_filename, + params_filename=self.params_filename) + pred_res = exe.run(inference_program, + feed=dict(zip(feed_target_names, data)), + fetch_list=fetch_targets) + + return pred_res + + def predict_dygraph(self, bert_config, data): + program_translator.enable(False) + with fluid.dygraph.guard(place): + bert = PretrainModelLayer( + config=bert_config, weight_sharing=False, use_fp16=False) + model_dict, _ = fluid.dygraph.load_dygraph( + self.dy_state_dict_save_path) + + bert.set_dict(model_dict) + bert.eval() + + input_vars = [fluid.dygraph.to_variable(x) for x in data] + src_ids, pos_ids, sent_ids, input_mask, mask_label, mask_pos, labels = input_vars + pred_res = bert( src_ids=src_ids, position_ids=pos_ids, sentence_ids=sent_ids, @@ -66,120 +153,33 @@ def train(bert_config, data_reader, to_static): mask_label=mask_label, mask_pos=mask_pos, labels=labels) - total_loss.backward() - optimizer.minimize(total_loss) - bert.clear_gradients() - - acc = np.mean(np.array(next_sent_acc.numpy())) - loss = np.mean(np.array(total_loss.numpy())) - ppl = np.mean(np.exp(np.array(mask_lm_loss.numpy()))) - - if step_idx % PRINT_STEP == 0: - if step_idx == 0: - print("Step: %d, loss: %f, ppl: %f, next_sent_acc: %f" % - (step_idx, loss, ppl, acc)) - avg_batch_time = time.time() - else: - speed = PRINT_STEP / (time.time() - avg_batch_time) - speed_list.append(speed) - print( - "Step: %d, loss: %f, ppl: %f, next_sent_acc: %f, speed: %.3f steps/s" - % (step_idx, loss, ppl, acc, speed)) - avg_batch_time = time.time() - - step_idx += 1 - if step_idx == STEP_NUM: - if to_static: - fluid.dygraph.jit.save(bert, MODEL_SAVE_PREFIX) - else: - fluid.dygraph.save_dygraph(bert.state_dict(), - DY_STATE_DICT_SAVE_PATH) - break - return loss, ppl - - -def train_dygraph(bert_config, data_reader): - program_translator.enable(False) - return train(bert_config, data_reader, False) - - -def train_static(bert_config, data_reader): - program_translator.enable(True) - return train(bert_config, data_reader, True) - - -def predict_static(data): - paddle.enable_static() - exe = fluid.Executor(place) - # load inference model - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model( - MODEL_SAVE_DIR, - executor=exe, - model_filename=MODEL_FILENAME, - params_filename=PARAMS_FILENAME) - pred_res = exe.run(inference_program, - feed=dict(zip(feed_target_names, data)), - fetch_list=fetch_targets) - - return pred_res - - -def predict_dygraph(bert_config, data): - program_translator.enable(False) - with fluid.dygraph.guard(place): - bert = PretrainModelLayer( - config=bert_config, weight_sharing=False, use_fp16=False) - model_dict, _ = fluid.dygraph.load_dygraph(DY_STATE_DICT_SAVE_PATH) - - bert.set_dict(model_dict) - bert.eval() - - input_vars = [fluid.dygraph.to_variable(x) for x in data] - src_ids, pos_ids, sent_ids, input_mask, mask_label, mask_pos, labels = input_vars - pred_res = bert( - src_ids=src_ids, - position_ids=pos_ids, - sentence_ids=sent_ids, - input_mask=input_mask, - mask_label=mask_label, - mask_pos=mask_pos, - labels=labels) - pred_res = [var.numpy() for var in pred_res] + pred_res = [var.numpy() for var in pred_res] - return pred_res - - -def predict_dygraph_jit(data): - with fluid.dygraph.guard(place): - bert = fluid.dygraph.jit.load(MODEL_SAVE_PREFIX) - bert.eval() - - src_ids, pos_ids, sent_ids, input_mask, mask_label, mask_pos, labels = data - pred_res = bert(src_ids, pos_ids, sent_ids, input_mask, mask_label, - mask_pos, labels) - pred_res = [var.numpy() for var in pred_res] - - return pred_res + return pred_res + def predict_dygraph_jit(self, data): + with fluid.dygraph.guard(place): + bert = fluid.dygraph.jit.load(self.model_save_prefix) + bert.eval() -def predict_analysis_inference(data): - output = PredictorTools(MODEL_SAVE_DIR, MODEL_FILENAME, PARAMS_FILENAME, - data) - out = output() - return out + src_ids, pos_ids, sent_ids, input_mask, mask_label, mask_pos, labels = data + pred_res = bert(src_ids, pos_ids, sent_ids, input_mask, mask_label, + mask_pos, labels) + pred_res = [var.numpy() for var in pred_res] + return pred_res -class TestBert(unittest.TestCase): - def setUp(self): - self.bert_config = get_bert_config() - self.data_reader = get_feed_data_reader(self.bert_config) + def predict_analysis_inference(self, data): + output = PredictorTools(self.model_save_dir, self.model_filename, + self.params_filename, data) + out = output() + return out def test_train(self): - static_loss, static_ppl = train_static(self.bert_config, - self.data_reader) - dygraph_loss, dygraph_ppl = train_dygraph(self.bert_config, - self.data_reader) + static_loss, static_ppl = self.train_static(self.bert_config, + self.data_reader) + dygraph_loss, dygraph_ppl = self.train_dygraph(self.bert_config, + self.data_reader) self.assertTrue( np.allclose(static_loss, dygraph_loss), msg="static_loss: {} \n dygraph_loss: {}".format(static_loss, @@ -193,10 +193,10 @@ class TestBert(unittest.TestCase): def verify_predict(self): for data in self.data_reader.data_generator()(): - dygraph_pred_res = predict_dygraph(self.bert_config, data) - static_pred_res = predict_static(data) - dygraph_jit_pred_res = predict_dygraph_jit(data) - predictor_pred_res = predict_analysis_inference(data) + dygraph_pred_res = self.predict_dygraph(self.bert_config, data) + static_pred_res = self.predict_static(data) + dygraph_jit_pred_res = self.predict_dygraph_jit(data) + predictor_pred_res = self.predict_analysis_inference(data) for dy_res, st_res, dy_jit_res, predictor_res in zip( dygraph_pred_res, static_pred_res, dygraph_jit_pred_res, diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_build_strategy.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_build_strategy.py index f7d469327a3..95ea5ad227e 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_build_strategy.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_build_strategy.py @@ -18,8 +18,7 @@ import unittest import numpy as np from paddle.jit import ProgramTranslator -from test_resnet import ResNet, train, predict_dygraph_jit -from test_resnet import predict_dygraph, predict_static, predict_analysis_inference +from test_resnet import ResNet, ResNetHelper program_translator = ProgramTranslator() @@ -31,20 +30,20 @@ class TestResnetWithPass(unittest.TestCase): self.build_strategy.fuse_bn_act_ops = True self.build_strategy.fuse_bn_add_act_ops = True self.build_strategy.enable_addto = True + self.resnet_helper = ResNetHelper() # NOTE: for enable_addto paddle.fluid.set_flags({"FLAGS_max_inplace_grad_add": 8}) def train(self, to_static): program_translator.enable(to_static) - - return train(to_static, self.build_strategy) + return self.resnet_helper.train(to_static, self.build_strategy) def verify_predict(self): image = np.random.random([1, 3, 224, 224]).astype('float32') - dy_pre = predict_dygraph(image) - st_pre = predict_static(image) - dy_jit_pre = predict_dygraph_jit(image) - predictor_pre = predict_analysis_inference(image) + dy_pre = self.resnet_helper.predict_dygraph(image) + st_pre = self.resnet_helper.predict_static(image) + dy_jit_pre = self.resnet_helper.predict_dygraph_jit(image) + predictor_pre = self.resnet_helper.predict_analysis_inference(image) self.assertTrue( np.allclose(dy_pre, st_pre), msg="dy_pre:\n {}\n, st_pre: \n{}.".format(dy_pre, st_pre)) @@ -69,7 +68,7 @@ class TestResnetWithPass(unittest.TestCase): paddle.fluid.set_flags({'FLAGS_use_mkldnn': True}) try: if paddle.fluid.core.is_compiled_with_mkldnn(): - train(True, self.build_strategy) + self.resnet_helper.train(True, self.build_strategy) finally: paddle.fluid.set_flags({'FLAGS_use_mkldnn': False}) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet.py index efb69b530ef..1a531c65bbf 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_resnet.py @@ -14,8 +14,10 @@ from __future__ import print_function +import os import math import time +import tempfile import unittest import numpy as np @@ -39,11 +41,6 @@ epoch_num = 1 place = fluid.CUDAPlace(0) if fluid.is_compiled_with_cuda() \ else fluid.CPUPlace() -MODEL_SAVE_DIR = "./inference" -MODEL_SAVE_PREFIX = "./inference/resnet" -MODEL_FILENAME = "resnet" + INFER_MODEL_SUFFIX -PARAMS_FILENAME = "resnet" + INFER_PARAMS_SUFFIX -DY_STATE_DICT_SAVE_PATH = "./resnet.dygraph" program_translator = ProgramTranslator() if fluid.is_compiled_with_cuda(): @@ -212,130 +209,148 @@ def reader_decorator(reader): return __reader__ -def train(to_static, build_strategy=None): - """ - Tests model decorated by `dygraph_to_static_output` in static mode. For users, the model is defined in dygraph mode and trained in static mode. - """ - with fluid.dygraph.guard(place): - np.random.seed(SEED) - paddle.seed(SEED) - paddle.framework.random._manual_program_seed(SEED) - - train_reader = paddle.batch( - reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), - batch_size=batch_size, - drop_last=True) - data_loader = fluid.io.DataLoader.from_generator( - capacity=5, iterable=True) - data_loader.set_sample_list_generator(train_reader) - - resnet = ResNet() - if to_static: - resnet = paddle.jit.to_static(resnet, build_strategy=build_strategy) - optimizer = optimizer_setting(parameter_list=resnet.parameters()) - - for epoch in range(epoch_num): - total_loss = 0.0 - total_acc1 = 0.0 - total_acc5 = 0.0 - total_sample = 0 - - for batch_id, data in enumerate(data_loader()): - start_time = time.time() - img, label = data - - pred = resnet(img) - loss = fluid.layers.cross_entropy(input=pred, label=label) - avg_loss = fluid.layers.mean(x=loss) - acc_top1 = fluid.layers.accuracy(input=pred, label=label, k=1) - acc_top5 = fluid.layers.accuracy(input=pred, label=label, k=5) - - avg_loss.backward() - optimizer.minimize(avg_loss) - resnet.clear_gradients() - - total_loss += avg_loss - total_acc1 += acc_top1 - total_acc5 += acc_top5 - total_sample += 1 - - end_time = time.time() - if batch_id % 2 == 0: - print( "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" % \ - ( epoch, batch_id, total_loss.numpy() / total_sample, \ - total_acc1.numpy() / total_sample, total_acc5.numpy() / total_sample, end_time-start_time)) - if batch_id == 10: - if to_static: - fluid.dygraph.jit.save(resnet, MODEL_SAVE_PREFIX) - else: - fluid.dygraph.save_dygraph(resnet.state_dict(), - DY_STATE_DICT_SAVE_PATH) - # avoid dataloader throw abort signaal - data_loader._reset() - break - - return total_loss.numpy() - - -def predict_dygraph(data): - program_translator.enable(False) - with fluid.dygraph.guard(place): - resnet = ResNet() - - model_dict, _ = fluid.dygraph.load_dygraph(DY_STATE_DICT_SAVE_PATH) - resnet.set_dict(model_dict) - resnet.eval() - - pred_res = resnet(fluid.dygraph.to_variable(data)) - - return pred_res.numpy() - - -def predict_static(data): - paddle.enable_static() - exe = fluid.Executor(place) - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model( - MODEL_SAVE_DIR, - executor=exe, - model_filename=MODEL_FILENAME, - params_filename=PARAMS_FILENAME) - - pred_res = exe.run(inference_program, - feed={feed_target_names[0]: data}, - fetch_list=fetch_targets) - - return pred_res[0] - - -def predict_dygraph_jit(data): - with fluid.dygraph.guard(place): - resnet = fluid.dygraph.jit.load(MODEL_SAVE_PREFIX) - resnet.eval() - - pred_res = resnet(data) - - return pred_res.numpy() - - -def predict_analysis_inference(data): - output = PredictorTools(MODEL_SAVE_DIR, MODEL_FILENAME, PARAMS_FILENAME, - [data]) - out = output() - return out +class ResNetHelper: + def __init__(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.model_save_dir = os.path.join(self.temp_dir.name, 'inference') + self.model_save_prefix = os.path.join(self.model_save_dir, 'resnet') + self.model_filename = 'resnet' + INFER_MODEL_SUFFIX + self.params_filename = 'resnet' + INFER_PARAMS_SUFFIX + self.dy_state_dict_save_path = os.path.join(self.temp_dir.name, + 'resnet.dygraph') + + def __del__(self): + self.temp_dir.cleanup() + + def train(self, to_static, build_strategy=None): + """ + Tests model decorated by `dygraph_to_static_output` in static mode. For users, the model is defined in dygraph mode and trained in static mode. + """ + with fluid.dygraph.guard(place): + np.random.seed(SEED) + paddle.seed(SEED) + paddle.framework.random._manual_program_seed(SEED) + + train_reader = paddle.batch( + reader_decorator(paddle.dataset.flowers.train(use_xmap=False)), + batch_size=batch_size, + drop_last=True) + data_loader = fluid.io.DataLoader.from_generator( + capacity=5, iterable=True) + data_loader.set_sample_list_generator(train_reader) + + resnet = ResNet() + if to_static: + resnet = paddle.jit.to_static( + resnet, build_strategy=build_strategy) + optimizer = optimizer_setting(parameter_list=resnet.parameters()) + + for epoch in range(epoch_num): + total_loss = 0.0 + total_acc1 = 0.0 + total_acc5 = 0.0 + total_sample = 0 + + for batch_id, data in enumerate(data_loader()): + start_time = time.time() + img, label = data + + pred = resnet(img) + loss = fluid.layers.cross_entropy(input=pred, label=label) + avg_loss = fluid.layers.mean(x=loss) + acc_top1 = fluid.layers.accuracy( + input=pred, label=label, k=1) + acc_top5 = fluid.layers.accuracy( + input=pred, label=label, k=5) + + avg_loss.backward() + optimizer.minimize(avg_loss) + resnet.clear_gradients() + + total_loss += avg_loss + total_acc1 += acc_top1 + total_acc5 += acc_top5 + total_sample += 1 + + end_time = time.time() + if batch_id % 2 == 0: + print( "epoch %d | batch step %d, loss %0.3f, acc1 %0.3f, acc5 %0.3f, time %f" % \ + ( epoch, batch_id, total_loss.numpy() / total_sample, \ + total_acc1.numpy() / total_sample, total_acc5.numpy() / total_sample, end_time-start_time)) + if batch_id == 10: + if to_static: + fluid.dygraph.jit.save(resnet, + self.model_save_prefix) + else: + fluid.dygraph.save_dygraph( + resnet.state_dict(), + self.dy_state_dict_save_path) + # avoid dataloader throw abort signaal + data_loader._reset() + break + + return total_loss.numpy() + + def predict_dygraph(self, data): + program_translator.enable(False) + with fluid.dygraph.guard(place): + resnet = ResNet() + + model_dict, _ = fluid.dygraph.load_dygraph( + self.dy_state_dict_save_path) + resnet.set_dict(model_dict) + resnet.eval() + + pred_res = resnet(fluid.dygraph.to_variable(data)) + + return pred_res.numpy() + + def predict_static(self, data): + paddle.enable_static() + exe = fluid.Executor(place) + [inference_program, feed_target_names, + fetch_targets] = fluid.io.load_inference_model( + self.model_save_dir, + executor=exe, + model_filename=self.model_filename, + params_filename=self.params_filename) + + pred_res = exe.run(inference_program, + feed={feed_target_names[0]: data}, + fetch_list=fetch_targets) + + return pred_res[0] + + def predict_dygraph_jit(self, data): + with fluid.dygraph.guard(place): + resnet = fluid.dygraph.jit.load(self.model_save_prefix) + resnet.eval() + + pred_res = resnet(data) + + return pred_res.numpy() + + def predict_analysis_inference(self, data): + output = PredictorTools(self.model_save_dir, self.model_filename, + self.params_filename, [data]) + out = output() + return out class TestResnet(unittest.TestCase): + def setUp(self): + self.resnet_helper = ResNetHelper() + def train(self, to_static): program_translator.enable(to_static) - return train(to_static) + return self.resnet_helper.train(to_static) def verify_predict(self): image = np.random.random([1, 3, 224, 224]).astype('float32') - dy_pre = predict_dygraph(image) - st_pre = predict_static(image) - dy_jit_pre = predict_dygraph_jit(image) - predictor_pre = predict_analysis_inference(image) + dy_pre = self.resnet_helper.predict_dygraph(image) + st_pre = self.resnet_helper.predict_static(image) + dy_jit_pre = self.resnet_helper.predict_dygraph_jit(image) + predictor_pre = self.resnet_helper.predict_analysis_inference(image) self.assertTrue( np.allclose(dy_pre, st_pre), msg="dy_pre:\n {}\n, st_pre: \n{}.".format(dy_pre, st_pre)) @@ -360,7 +375,7 @@ class TestResnet(unittest.TestCase): fluid.set_flags({'FLAGS_use_mkldnn': True}) try: if paddle.fluid.core.is_compiled_with_mkldnn(): - train(to_static=True) + self.resnet_helper.train(to_static=True) finally: fluid.set_flags({'FLAGS_use_mkldnn': False}) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py index 06f2c60dfae..c8fe3e39329 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_transformer.py @@ -15,6 +15,7 @@ import logging import os import time +import tempfile import unittest import numpy as np @@ -371,8 +372,21 @@ def predict_static(args, batch_generator): class TestTransformer(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDwon(self): + self.temp_dir.cleanup() + def prepare(self, mode='train'): args = util.ModelHyperParams() + args.save_dygraph_model_path = os.path.join( + self.temp_dir.name, args.save_dygraph_model_path) + args.save_static_model_path = os.path.join(self.temp_dir.name, + args.save_static_model_path) + args.inference_model_dir = os.path.join(self.temp_dir.name, + args.inference_model_dir) + args.output_file = os.path.join(self.temp_dir.name, args.output_file) batch_generator = util.get_feed_data_reader(args, mode) return args, batch_generator diff --git a/python/paddle/fluid/tests/unittests/test_dataset.py b/python/paddle/fluid/tests/unittests/test_dataset.py index 348945b73e1..5ef5a1016cc 100644 --- a/python/paddle/fluid/tests/unittests/test_dataset.py +++ b/python/paddle/fluid/tests/unittests/test_dataset.py @@ -24,6 +24,7 @@ import paddle.fluid.core as core import numpy as np import os import shutil +import tempfile import unittest @@ -82,12 +83,17 @@ class TestDataset(unittest.TestCase): """ Testcase for InMemoryDataset from create to run. """ - with open("test_run_with_dump_a.txt", "w") as f: + + temp_dir = tempfile.TemporaryDirectory() + dump_a_path = os.path.join(temp_dir.name, 'test_run_with_dump_a.txt') + dump_b_path = os.path.join(temp_dir.name, 'test_run_with_dump_b.txt') + + with open(dump_a_path, "w") as f: data = "1 a 1 a 1 1 2 3 3 4 5 5 5 5 1 1\n" data += "1 b 1 b 1 2 2 3 4 4 6 6 6 6 1 2\n" data += "1 c 1 c 1 3 2 3 5 4 7 7 7 7 1 3\n" f.write(data) - with open("test_run_with_dump_b.txt", "w") as f: + with open(dump_b_path, "w") as f: data = "1 d 1 d 1 4 2 3 3 4 5 5 5 5 1 4\n" data += "1 e 1 e 1 5 2 3 4 4 6 6 6 6 1 5\n" data += "1 f 1 f 1 6 2 3 5 4 7 7 7 7 1 6\n" @@ -110,8 +116,7 @@ class TestDataset(unittest.TestCase): parse_content=True, fea_eval=True, candidate_size=10000) - dataset.set_filelist( - ["test_run_with_dump_a.txt", "test_run_with_dump_b.txt"]) + dataset.set_filelist([dump_a_path, dump_b_path]) dataset.load_into_memory() dataset.local_shuffle() @@ -129,8 +134,7 @@ class TestDataset(unittest.TestCase): except Exception as e: self.assertTrue(False) - os.remove("./test_run_with_dump_a.txt") - os.remove("./test_run_with_dump_b.txt") + temp_dir.cleanup() def test_dataset_config(self): """ Testcase for dataset configuration. """ diff --git a/python/paddle/fluid/tests/unittests/test_dataset_consistency_inspection.py b/python/paddle/fluid/tests/unittests/test_dataset_consistency_inspection.py index 5911ada1817..911bee69e8b 100644 --- a/python/paddle/fluid/tests/unittests/test_dataset_consistency_inspection.py +++ b/python/paddle/fluid/tests/unittests/test_dataset_consistency_inspection.py @@ -25,6 +25,7 @@ import random import math import os import shutil +import tempfile import unittest import paddle.fluid.incubate.data_generator as dg @@ -282,7 +283,11 @@ class TestDataset(unittest.TestCase): """ Testcase for InMemoryDataset of consistency insepection of use_var_list and data_generator. """ - with open("test_run_with_dump_a.txt", "w") as f: + + temp_dir = tempfile.TemporaryDirectory() + dump_a_path = os.path.join(temp_dir.name, 'test_run_with_dump_a.txt') + + with open(dump_a_path, "w") as f: # data = "\n" # data += "\n" data = "2 1;1 9;20002001 20001240 20001860 20003611 20000723;20002001 20001240 20001860 20003611 20000723;0;40000001;20002001 20001240 20001860 20003611 20000157 20000723 20000070 20002616 20000157 20000005;20002001 20001240 20001860 20003611 20000157 20001776 20000070 20002616 20000157 20000005;20002001 20001240 20001860 20003611 20000723 20000070 20002001 20001240 20001860 20003611 20012788 20000157;20002001 20001240 20001860 20003611 20000623 20000251 20000157 20000723 20000070 20000001 20000057;20002640 20004695 20000157 20000723 20000070 20002001 20001240 20001860 20003611;20002001 20001240 20001860 20003611 20000157 20000723 20000070 20003519 20000005;20002001 20001240 20001860 20003611 20000157 20001776 20000070 20003519 20000005;20002001 20001240 20001860 20003611 20000723 20000070 20002001 20001240 20001860 20003611 20131464;20002001 20001240 20001860 20003611 20018820 20000157 20000723 20000070 20000001 20000057;20002640 20034154 20000723 20000070 20002001 20001240 20001860 20003611;10000200;10000200;10063938;10000008;10000177;20002001 20001240 20001860 20003611 20010833 20000210 20000500 20000401 20000251 20012198 20001023 20000157;20002001 20001240 20001860 20003611 20012396 20000500 20002513 20012198 20001023 20000157;10000123;30000004;0.623 0.233 0.290 0.208 0.354 49.000 0.000 0.000 0.000 -1.000 0.569 0.679 0.733 53 17 2 0;20002001 20001240 20001860 20003611 20000723;20002001 20001240 20001860 20003611 20000723;10000047;30000004;0.067 0.000 0.161 0.005 0.000 49.000 0.000 0.000 0.000 -1.000 0.000 0.378 0.043 0 6 0 0;20002001 20001240 20001860 20003611 20000157 20000723 20000070 20002616 20000157 20000005;20002001 20001240 20001860 20003611 20000157 20000723 20000070 20003519 20000005;10000200;30000001;0.407 0.111 0.196 0.095 0.181 49.000 0.000 0.000 0.000 -1.000 0.306 0.538 0.355 48 8 0 0;20002001 20001240 20001860 20003611 20000157 20001776 20000070 20002616 20000157 20000005;20002001 20001240 20001860 20003611 20000157 20001776 20000070 20003519 20000005;10000200;30000001;0.226 0.029 0.149 0.031 0.074 49.000 0.000 0.000 0.000 -1.000 0.220 0.531 0.286 26 6 0 0;20002001 20001240 20001860 20003611 20000723 20000070 20002001 20001240 20001860 20003611 20012788 20000157;20002001 20001240 20001860 20003611 20000723 20000070 20002001 20001240 20001860 20003611 20131464;10063938;30000001;0.250 0.019 0.138 0.012 0.027 49.000 0.000 0.000 0.000 -1.000 0.370 0.449 0.327 7 2 0 0;20002001 20001240 20001860 20003611 20000723;20002001 20001240 20001860 20003611 20000723;10000003;30000002;0.056 0.000 0.139 0.003 0.000 49.000 0.000 0.000 0.000 -1.000 0.000 0.346 0.059 15 3 0 0;20002001 20001240 20001860 20003611 20000623 20000251 20000157 20000723 20000070 20000001 20000057;20002001 20001240 20001860 20003611 20018820 20000157 20000723 20000070 20000001 20000057;10000008;30000001;0.166 0.004 0.127 0.001 0.004 49.000 0.000 0.000 0.000 -1.000 0.103 0.417 0.394 10 3 0 0;20002640 20004695 20000157 20000723 20000070 20002001 20001240 20001860 20003611;20002640 20034154 20000723 20000070 20002001 20001240 20001860 20003611;10000177;30000001;0.094 0.008 0.157 0.012 0.059 49.000 0.000 0.000 0.000 -1.000 0.051 0.382 0.142 21 0 0 0;20002001 20001240 20001860 20003611 20000157 20001776 20000070 20000157;20002001 20001240 20001860 20003611 20000157 20001776 20000070 20000157;10000134;30000001;0.220 0.016 0.181 0.037 0.098 49.000 0.000 0.000 0.000 -1.000 0.192 0.453 0.199 17 1 0 0;20002001 20001240 20001860 20003611 20002640 20004695 20000157 20000723 20000070 20002001 20001240 20001860 20003611;20002001 20001240 20001860 20003611 20002640 20034154 20000723 20000070 20002001 20001240 20001860 20003611;10000638;30000001;0.000 0.000 0.000 0.000 0.000 49.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0 0 0 0;\n" @@ -348,7 +353,7 @@ class TestDataset(unittest.TestCase): generator_class = CTRDataset(mode=0) try: dataset._check_use_var_with_data_generator( - slot_data, generator_class, "test_run_with_dump_a.txt") + slot_data, generator_class, dump_a_path) print("case 1: check passed!") except Exception as e: print("warning: catch expected error") @@ -360,7 +365,7 @@ class TestDataset(unittest.TestCase): generator_class = CTRDataset(mode=2) try: dataset._check_use_var_with_data_generator( - slot_data, generator_class, "test_run_with_dump_a.txt") + slot_data, generator_class, dump_a_path) except Exception as e: print("warning: case 2 catch expected error") print(e) @@ -371,7 +376,7 @@ class TestDataset(unittest.TestCase): generator_class = CTRDataset(mode=3) try: dataset._check_use_var_with_data_generator( - slot_data, generator_class, "test_run_with_dump_a.txt") + slot_data, generator_class, dump_a_path) except Exception as e: print("warning: case 3 catch expected error") print(e) @@ -382,7 +387,7 @@ class TestDataset(unittest.TestCase): generator_class = CTRDataset(mode=4) try: dataset._check_use_var_with_data_generator( - slot_data, generator_class, "test_run_with_dump_a.txt") + slot_data, generator_class, dump_a_path) except Exception as e: print("warning: case 4 catch expected error") print(e) @@ -393,13 +398,13 @@ class TestDataset(unittest.TestCase): generator_class = CTRDataset(mode=5) try: dataset._check_use_var_with_data_generator( - slot_data, generator_class, "test_run_with_dump_a.txt") + slot_data, generator_class, dump_a_path) except Exception as e: print("warning: case 5 catch expected error") print(e) print("========================================") - os.remove("./test_run_with_dump_a.txt") + temp_dir.cleanup() if __name__ == '__main__': -- GitLab