From 95f66c265dc3eba028d4a433f73602e2b087542d Mon Sep 17 00:00:00 2001 From: freeliuzc Date: Tue, 14 Jun 2022 11:30:45 +0800 Subject: [PATCH] Use tempfile to place all the temporary files. Modify some code structure. (#43376) --- .../unittests/test_directory_migration.py | 13 +- .../tests/unittests/test_imperative_deepcf.py | 208 +++++++++--------- .../tests/unittests/test_imperative_gnn.py | 1 + .../fluid/tests/unittests/test_input_spec.py | 14 +- .../tests/unittests/test_layout_autotune.py | 11 +- .../fluid/tests/unittests/test_optimizer.py | 21 +- .../unittests/test_traced_layer_err_msg.py | 18 +- 7 files changed, 166 insertions(+), 120 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_directory_migration.py b/python/paddle/fluid/tests/unittests/test_directory_migration.py index 727fcb28cc2..4cafd19d913 100644 --- a/python/paddle/fluid/tests/unittests/test_directory_migration.py +++ b/python/paddle/fluid/tests/unittests/test_directory_migration.py @@ -17,14 +17,22 @@ from __future__ import print_function import os import sys import time +import tempfile import subprocess import unittest + import numpy as np import paddle class TestDirectory(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def get_import_command(self, module): paths = module.split('.') if len(paths) == 1: @@ -77,7 +85,7 @@ class TestDirectory(unittest.TestCase): 'paddle.static.nn.spectral_norm', 'paddle.static.nn.embedding' ] - import_file = 'run_import_modules.py' + import_file = os.path.join(self.temp_dir.name, 'run_import_modules.py') with open(import_file, "w") as wb: for module in new_directory: @@ -137,7 +145,8 @@ class TestDirectory(unittest.TestCase): 'paddle.declarative.spectral_norm', 'paddle.declarative.embedding' ] - import_file = 'run_old_import_modules.py' + import_file = os.path.join(self.temp_dir.name, + 'run_old_import_modules.py') with open(import_file, "w") as wb: cmd_context_count = """ diff --git a/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py b/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py index 822a0fcc449..3da576045c5 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py @@ -26,13 +26,6 @@ from paddle.fluid.dygraph.base import to_variable from paddle.fluid.dygraph import Linear from paddle.fluid.framework import _test_eager_guard -# Can use Amusic dataset as the DeepCF describes. -DATA_PATH = os.environ.get('DATA_PATH', '') - -BATCH_SIZE = int(os.environ.get('BATCH_SIZE', 128)) -NUM_BATCHES = int(os.environ.get('NUM_BATCHES', 5)) -NUM_EPOCHES = int(os.environ.get('NUM_EPOCHES', 1)) - class DMF(fluid.Layer): @@ -129,84 +122,90 @@ class DeepCF(fluid.Layer): return prediction -def get_data(): - user_ids = [] - item_ids = [] - labels = [] - NUM_USERS = 100 - NUM_ITEMS = 1000 - matrix = np.zeros([NUM_USERS, NUM_ITEMS], dtype=np.float32) +class TestDygraphDeepCF(unittest.TestCase): - for uid in range(NUM_USERS): - for iid in range(NUM_ITEMS): - label = float(random.randint(1, 6) == 1) + def setUp(self): + # Can use Amusic dataset as the DeepCF describes. + self.data_path = os.environ.get('DATA_PATH', '') + + self.batch_size = int(os.environ.get('BATCH_SIZE', 128)) + self.num_batches = int(os.environ.get('NUM_BATCHES', 5)) + self.num_epoches = int(os.environ.get('NUM_EPOCHES', 1)) + + def get_data(self): + user_ids = [] + item_ids = [] + labels = [] + NUM_USERS = 100 + NUM_ITEMS = 1000 + matrix = np.zeros([NUM_USERS, NUM_ITEMS], dtype=np.float32) + + for uid in range(NUM_USERS): + for iid in range(NUM_ITEMS): + label = float(random.randint(1, 6) == 1) + user_ids.append(uid) + item_ids.append(iid) + labels.append(label) + matrix[uid, iid] = label + indices = np.arange(len(user_ids)) + np.random.shuffle(indices) + users_np = np.array(user_ids, dtype=np.int32)[indices] + items_np = np.array(item_ids, dtype=np.int32)[indices] + labels_np = np.array(labels, dtype=np.float32)[indices] + return np.expand_dims(users_np, -1), \ + np.expand_dims(items_np, -1), \ + np.expand_dims(labels_np, -1), NUM_USERS, NUM_ITEMS, matrix + + def load_data(self): + sys.stderr.write('loading from %s\n' % self.data_path) + likes = dict() + num_users = -1 + num_items = -1 + with open(self.data_path, 'r') as f: + for l in f.readlines(): + uid, iid, rating = [int(v) for v in l.split('\t')] + num_users = max(num_users, uid + 1) + num_items = max(num_items, iid + 1) + if float(rating) > 0.0: + likes[(uid, iid)] = 1.0 + + user_ids = [] + item_ids = [] + labels = [] + matrix = np.zeros([num_users, num_items], dtype=np.float32) + for uid, iid in likes.keys(): user_ids.append(uid) item_ids.append(iid) - labels.append(label) - matrix[uid, iid] = label - indices = np.arange(len(user_ids)) - np.random.shuffle(indices) - users_np = np.array(user_ids, dtype=np.int32)[indices] - items_np = np.array(item_ids, dtype=np.int32)[indices] - labels_np = np.array(labels, dtype=np.float32)[indices] - return np.expand_dims(users_np, -1), \ - np.expand_dims(items_np, -1), \ - np.expand_dims(labels_np, -1), NUM_USERS, NUM_ITEMS, matrix - - -def load_data(DATA_PATH): - sys.stderr.write('loading from %s\n' % DATA_PATH) - likes = dict() - num_users = -1 - num_items = -1 - with open(DATA_PATH, 'r') as f: - for l in f.readlines(): - uid, iid, rating = [int(v) for v in l.split('\t')] - num_users = max(num_users, uid + 1) - num_items = max(num_items, iid + 1) - if float(rating) > 0.0: - likes[(uid, iid)] = 1.0 - - user_ids = [] - item_ids = [] - labels = [] - matrix = np.zeros([num_users, num_items], dtype=np.float32) - for uid, iid in likes.keys(): - user_ids.append(uid) - item_ids.append(iid) - labels.append(1.0) - matrix[uid, iid] = 1.0 - - negative = 0 - while negative < 3: - nuid = random.randint(0, num_users - 1) - niid = random.randint(0, num_items - 1) - if (nuid, niid) not in likes: - negative += 1 - user_ids.append(nuid) - item_ids.append(niid) - labels.append(0.0) - - indices = np.arange(len(user_ids)) - np.random.shuffle(indices) - users_np = np.array(user_ids, dtype=np.int32)[indices] - items_np = np.array(item_ids, dtype=np.int32)[indices] - labels_np = np.array(labels, dtype=np.float32)[indices] - return np.expand_dims(users_np, -1), \ - np.expand_dims(items_np, -1), \ - np.expand_dims(labels_np, -1), num_users, num_items, matrix - - -class TestDygraphDeepCF(unittest.TestCase): + labels.append(1.0) + matrix[uid, iid] = 1.0 + + negative = 0 + while negative < 3: + nuid = random.randint(0, num_users - 1) + niid = random.randint(0, num_items - 1) + if (nuid, niid) not in likes: + negative += 1 + user_ids.append(nuid) + item_ids.append(niid) + labels.append(0.0) + + indices = np.arange(len(user_ids)) + np.random.shuffle(indices) + users_np = np.array(user_ids, dtype=np.int32)[indices] + items_np = np.array(item_ids, dtype=np.int32)[indices] + labels_np = np.array(labels, dtype=np.float32)[indices] + return np.expand_dims(users_np, -1), \ + np.expand_dims(items_np, -1), \ + np.expand_dims(labels_np, -1), num_users, num_items, matrix def test_deefcf(self): seed = 90 - if DATA_PATH: + if self.data_path: (users_np, items_np, labels_np, num_users, num_items, - matrix) = load_data(DATA_PATH) + matrix) = self.load_data() else: (users_np, items_np, labels_np, num_users, num_items, - matrix) = get_data() + matrix) = self.get_data() paddle.seed(seed) paddle.framework.random._manual_program_seed(seed) startup = fluid.Program() @@ -228,17 +227,19 @@ class TestDygraphDeepCF(unittest.TestCase): exe = fluid.Executor(fluid.CPUPlace( ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) exe.run(startup) - for e in range(NUM_EPOCHES): + for e in range(self.num_epoches): sys.stderr.write('epoch %d\n' % e) - for slice in range(0, BATCH_SIZE * NUM_BATCHES, BATCH_SIZE): - if slice + BATCH_SIZE >= users_np.shape[0]: + for slice in range(0, self.batch_size * self.num_batches, + self.batch_size): + if slice + self.batch_size >= users_np.shape[0]: break static_loss = exe.run( main, feed={ - users.name: users_np[slice:slice + BATCH_SIZE], - items.name: items_np[slice:slice + BATCH_SIZE], - labels.name: labels_np[slice:slice + BATCH_SIZE] + users.name: users_np[slice:slice + self.batch_size], + items.name: items_np[slice:slice + self.batch_size], + labels.name: + labels_np[slice:slice + self.batch_size] }, fetch_list=[loss])[0] sys.stderr.write('static loss %s\n' % static_loss) @@ -250,18 +251,20 @@ class TestDygraphDeepCF(unittest.TestCase): deepcf = DeepCF(num_users, num_items, matrix) adam = fluid.optimizer.AdamOptimizer( 0.01, parameter_list=deepcf.parameters()) - for e in range(NUM_EPOCHES): + for e in range(self.num_epoches): sys.stderr.write('epoch %d\n' % e) - for slice in range(0, BATCH_SIZE * NUM_BATCHES, BATCH_SIZE): - if slice + BATCH_SIZE >= users_np.shape[0]: + for slice in range(0, self.batch_size * self.num_batches, + self.batch_size): + if slice + self.batch_size >= users_np.shape[0]: break prediction = deepcf( - to_variable(users_np[slice:slice + BATCH_SIZE]), - to_variable(items_np[slice:slice + BATCH_SIZE])) + to_variable(users_np[slice:slice + self.batch_size]), + to_variable(items_np[slice:slice + self.batch_size])) loss = fluid.layers.reduce_sum( fluid.layers.log_loss( prediction, - to_variable(labels_np[slice:slice + BATCH_SIZE]))) + to_variable(labels_np[slice:slice + + self.batch_size]))) loss.backward() adam.minimize(loss) deepcf.clear_gradients() @@ -276,18 +279,20 @@ class TestDygraphDeepCF(unittest.TestCase): adam2 = fluid.optimizer.AdamOptimizer( 0.01, parameter_list=deepcf2.parameters()) fluid.set_flags({'FLAGS_sort_sum_gradient': True}) - for e in range(NUM_EPOCHES): + for e in range(self.num_epoches): sys.stderr.write('epoch %d\n' % e) - for slice in range(0, BATCH_SIZE * NUM_BATCHES, BATCH_SIZE): - if slice + BATCH_SIZE >= users_np.shape[0]: + for slice in range(0, self.batch_size * self.num_batches, + self.batch_size): + if slice + self.batch_size >= users_np.shape[0]: break prediction2 = deepcf2( - to_variable(users_np[slice:slice + BATCH_SIZE]), - to_variable(items_np[slice:slice + BATCH_SIZE])) + to_variable(users_np[slice:slice + self.batch_size]), + to_variable(items_np[slice:slice + self.batch_size])) loss2 = fluid.layers.reduce_sum( fluid.layers.log_loss( prediction2, - to_variable(labels_np[slice:slice + BATCH_SIZE]))) + to_variable(labels_np[slice:slice + + self.batch_size]))) loss2.backward() adam2.minimize(loss2) deepcf2.clear_gradients() @@ -306,19 +311,22 @@ class TestDygraphDeepCF(unittest.TestCase): adam = fluid.optimizer.AdamOptimizer( 0.01, parameter_list=deepcf.parameters()) - for e in range(NUM_EPOCHES): + for e in range(self.num_epoches): sys.stderr.write('epoch %d\n' % e) - for slice in range(0, BATCH_SIZE * NUM_BATCHES, BATCH_SIZE): - if slice + BATCH_SIZE >= users_np.shape[0]: + for slice in range(0, self.batch_size * self.num_batches, + self.batch_size): + if slice + self.batch_size >= users_np.shape[0]: break prediction = deepcf( - to_variable(users_np[slice:slice + BATCH_SIZE]), - to_variable(items_np[slice:slice + BATCH_SIZE])) + to_variable(users_np[slice:slice + + self.batch_size]), + to_variable(items_np[slice:slice + + self.batch_size])) loss = fluid.layers.reduce_sum( fluid.layers.log_loss( prediction, to_variable(labels_np[slice:slice + - BATCH_SIZE]))) + self.batch_size]))) loss.backward() adam.minimize(loss) deepcf.clear_gradients() diff --git a/python/paddle/fluid/tests/unittests/test_imperative_gnn.py b/python/paddle/fluid/tests/unittests/test_imperative_gnn.py index 6acab36221f..28d24f4b5b7 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_gnn.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_gnn.py @@ -177,4 +177,5 @@ class TestDygraphGNN(unittest.TestCase): if __name__ == '__main__': + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_input_spec.py b/python/paddle/fluid/tests/unittests/test_input_spec.py index f8f04229a4d..a076b69cc00 100644 --- a/python/paddle/fluid/tests/unittests/test_input_spec.py +++ b/python/paddle/fluid/tests/unittests/test_input_spec.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import unittest +import tempfile import numpy as np + import paddle import paddle.fluid as fluid from paddle.static import InputSpec @@ -160,6 +163,10 @@ class TestNetWithNonTensorSpec(unittest.TestCase): self.out_num = 16 self.x_spec = paddle.static.InputSpec([-1, 16], name='x') self.x = paddle.randn([4, 16]) + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() @classmethod def setUpClass(cls): @@ -182,7 +189,7 @@ class TestNetWithNonTensorSpec(unittest.TestCase): self.check_result(specs, 'list') def check_result(self, specs, path): - path = './net_non_tensor_' + path + path = os.path.join(self.temp_dir.name, './net_non_tensor_', path) net = NetWithNonTensorSpec(self.in_num, self.out_num) net.eval() @@ -218,7 +225,7 @@ class TestNetWithNonTensorSpec(unittest.TestCase): net = paddle.jit.to_static(net, input_spec=specs) net.eval() - path = './net_twice' + path = os.path.join(self.temp_dir.name, './net_twice') # NOTE: check input_specs_compatible new_specs = [self.x_spec, True, "bn", 10] @@ -264,6 +271,7 @@ class TestNetWithNonTensorSpecWithPrune(unittest.TestCase): self.y_spec = paddle.static.InputSpec([16], name='y') self.x = paddle.randn([4, 16]) self.y = paddle.randn([16]) + self.temp_dir = tempfile.TemporaryDirectory() @classmethod def setUpClass(cls): @@ -271,7 +279,7 @@ class TestNetWithNonTensorSpecWithPrune(unittest.TestCase): def test_non_tensor_with_prune(self): specs = [self.x_spec, self.y_spec, True] - path = './net_non_tensor_prune_' + path = os.path.join(self.temp_dir.name, './net_non_tensor_prune_') net = NetWithNonTensorSpecPrune(self.in_num, self.out_num) net.eval() diff --git a/python/paddle/fluid/tests/unittests/test_layout_autotune.py b/python/paddle/fluid/tests/unittests/test_layout_autotune.py index d29f47c8ab1..f17bffe3b86 100644 --- a/python/paddle/fluid/tests/unittests/test_layout_autotune.py +++ b/python/paddle/fluid/tests/unittests/test_layout_autotune.py @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import paddle +import os +import json +import tempfile import unittest +import warnings import numpy + +import paddle import paddle.nn.functional as F -import tempfile -import warnings -import json -import os from paddle.fluid.framework import _enable_legacy_dygraph _enable_legacy_dygraph() diff --git a/python/paddle/fluid/tests/unittests/test_optimizer.py b/python/paddle/fluid/tests/unittests/test_optimizer.py index b70b69ca97c..490167a8ff7 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer.py @@ -14,6 +14,8 @@ from __future__ import print_function +import os +import tempfile import unittest import paddle.fluid as fluid @@ -29,8 +31,6 @@ import paddle from paddle.io import Dataset import numpy -paddle.enable_static() - class TestOptimizer(unittest.TestCase): @@ -1279,6 +1279,12 @@ class TestMasterWeightSaveForFP16(unittest.TestCase): Master weights will be saved by optimizer::state_dict. ''' + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() + def check_with_opt_state_dict(self, use_save_load=True): paddle.seed(100) numpy.random.seed(100) @@ -1340,10 +1346,12 @@ class TestMasterWeightSaveForFP16(unittest.TestCase): optimizer.clear_grad(set_to_zero=False) if use_save_load and i == 5: - paddle.save(model.state_dict(), "model.pdparams") - paddle.save(optimizer.state_dict(), "opt.pdopt") - model.set_state_dict(paddle.load("model.pdparams")) - optimizer.set_state_dict(paddle.load("opt.pdopt")) + model_path = os.path.join(self.temp_dir.name, "model.pdparams") + optimizer_path = os.path.join(self.temp_dir.name, "opt.pdopt") + paddle.save(model.state_dict(), model_path) + paddle.save(optimizer.state_dict(), optimizer_path) + model.set_state_dict(paddle.load(model_path)) + optimizer.set_state_dict(paddle.load(optimizer_path)) return loss.numpy() @@ -1359,4 +1367,5 @@ class TestMasterWeightSaveForFP16(unittest.TestCase): if __name__ == '__main__': + paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py b/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py index a2ccfa925ed..59b652b1d3d 100644 --- a/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py +++ b/python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py @@ -13,12 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import numpy as np +import tempfile +import unittest + import paddle import paddle.fluid as fluid -import unittest import paddle.nn as nn -import os class SimpleFCLayer(nn.Layer): @@ -54,6 +56,10 @@ class TestTracedLayerErrMsg(unittest.TestCase): self.fc_size = 2 self.layer = self._train_simple_net() self.type_str = 'class' + self.temp_dir = tempfile.TemporaryDirectory() + + def tearDown(self): + self.temp_dir.cleanup() def test_trace_err(self): if fluid.framework.in_dygraph_mode(): @@ -122,7 +128,7 @@ class TestTracedLayerErrMsg(unittest.TestCase): dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace( self.layer, [in_x]) - path = './traced_layer_err_msg' + path = os.path.join(self.temp_dir.name, './traced_layer_err_msg') with self.assertRaises(TypeError) as e: traced_layer.save_inference_model([0]) self.assertEqual( @@ -193,11 +199,15 @@ class TestTracedLayerSaveInferenceModel(unittest.TestCase): """test save_inference_model will automaticlly create non-exist dir""" def setUp(self): - self.save_path = "./nonexist_dir/fc" + self.temp_dir = tempfile.TemporaryDirectory() + self.save_path = os.path.join(self.temp_dir.name, "./nonexist_dir/fc") import shutil if os.path.exists(os.path.dirname(self.save_path)): shutil.rmtree(os.path.dirname(self.save_path)) + def tearDown(self): + self.temp_dir.cleanup() + def test_mkdir_when_input_path_non_exist(self): if fluid.framework.in_dygraph_mode(): return -- GitLab