Use tempfile to place all the temporary files. Modify some code structure. (#43376)

python/paddle/fluid/tests/unittests/test_directory_migration.py

@@ -17,14 +17,22 @@ from __future__ import print_function
 import os
 import sys
 import time
+import tempfile
 import subprocess
 import unittest
+
 import numpy as np
 import paddle
 
 
 class TestDirectory(unittest.TestCase):
 
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
+
     def get_import_command(self, module):
         paths = module.split('.')
         if len(paths) == 1:
@@ -77,7 +85,7 @@ class TestDirectory(unittest.TestCase):
             'paddle.static.nn.spectral_norm', 'paddle.static.nn.embedding'
         ]
 
-        import_file = 'run_import_modules.py'
+        import_file = os.path.join(self.temp_dir.name, 'run_import_modules.py')
 
         with open(import_file, "w") as wb:
             for module in new_directory:
@@ -137,7 +145,8 @@ class TestDirectory(unittest.TestCase):
             'paddle.declarative.spectral_norm', 'paddle.declarative.embedding'
         ]
 
-        import_file = 'run_old_import_modules.py'
+        import_file = os.path.join(self.temp_dir.name,
+                                   'run_old_import_modules.py')
 
         with open(import_file, "w") as wb:
             cmd_context_count = """

python/paddle/fluid/tests/unittests/test_imperative_deepcf.py

@@ -26,13 +26,6 @@ from paddle.fluid.dygraph.base import to_variable
 from paddle.fluid.dygraph import Linear
 from paddle.fluid.framework import _test_eager_guard
 
-# Can use Amusic dataset as the DeepCF describes.
-DATA_PATH = os.environ.get('DATA_PATH', '')
-
-BATCH_SIZE = int(os.environ.get('BATCH_SIZE', 128))
-NUM_BATCHES = int(os.environ.get('NUM_BATCHES', 5))
-NUM_EPOCHES = int(os.environ.get('NUM_EPOCHES', 1))
-
 
 class DMF(fluid.Layer):
 
@@ -129,84 +122,90 @@ class DeepCF(fluid.Layer):
         return prediction
 
 
-def get_data():
-    user_ids = []
-    item_ids = []
-    labels = []
-    NUM_USERS = 100
-    NUM_ITEMS = 1000
-    matrix = np.zeros([NUM_USERS, NUM_ITEMS], dtype=np.float32)
+class TestDygraphDeepCF(unittest.TestCase):
 
-    for uid in range(NUM_USERS):
-        for iid in range(NUM_ITEMS):
-            label = float(random.randint(1, 6) == 1)
+    def setUp(self):
+        # Can use Amusic dataset as the DeepCF describes.
+        self.data_path = os.environ.get('DATA_PATH', '')
+
+        self.batch_size = int(os.environ.get('BATCH_SIZE', 128))
+        self.num_batches = int(os.environ.get('NUM_BATCHES', 5))
+        self.num_epoches = int(os.environ.get('NUM_EPOCHES', 1))
+
+    def get_data(self):
+        user_ids = []
+        item_ids = []
+        labels = []
+        NUM_USERS = 100
+        NUM_ITEMS = 1000
+        matrix = np.zeros([NUM_USERS, NUM_ITEMS], dtype=np.float32)
+
+        for uid in range(NUM_USERS):
+            for iid in range(NUM_ITEMS):
+                label = float(random.randint(1, 6) == 1)
+                user_ids.append(uid)
+                item_ids.append(iid)
+                labels.append(label)
+                matrix[uid, iid] = label
+        indices = np.arange(len(user_ids))
+        np.random.shuffle(indices)
+        users_np = np.array(user_ids, dtype=np.int32)[indices]
+        items_np = np.array(item_ids, dtype=np.int32)[indices]
+        labels_np = np.array(labels, dtype=np.float32)[indices]
+        return np.expand_dims(users_np, -1), \
+            np.expand_dims(items_np, -1), \
+            np.expand_dims(labels_np, -1), NUM_USERS, NUM_ITEMS, matrix
+
+    def load_data(self):
+        sys.stderr.write('loading from %s\n' % self.data_path)
+        likes = dict()
+        num_users = -1
+        num_items = -1
+        with open(self.data_path, 'r') as f:
+            for l in f.readlines():
+                uid, iid, rating = [int(v) for v in l.split('\t')]
+                num_users = max(num_users, uid + 1)
+                num_items = max(num_items, iid + 1)
+                if float(rating) > 0.0:
+                    likes[(uid, iid)] = 1.0
+
+        user_ids = []
+        item_ids = []
+        labels = []
+        matrix = np.zeros([num_users, num_items], dtype=np.float32)
+        for uid, iid in likes.keys():
             user_ids.append(uid)
             item_ids.append(iid)
-            labels.append(label)
-            matrix[uid, iid] = label
-    indices = np.arange(len(user_ids))
-    np.random.shuffle(indices)
-    users_np = np.array(user_ids, dtype=np.int32)[indices]
-    items_np = np.array(item_ids, dtype=np.int32)[indices]
-    labels_np = np.array(labels, dtype=np.float32)[indices]
-    return np.expand_dims(users_np, -1), \
-           np.expand_dims(items_np, -1), \
-           np.expand_dims(labels_np, -1), NUM_USERS, NUM_ITEMS, matrix
-
-
-def load_data(DATA_PATH):
-    sys.stderr.write('loading from %s\n' % DATA_PATH)
-    likes = dict()
-    num_users = -1
-    num_items = -1
-    with open(DATA_PATH, 'r') as f:
-        for l in f.readlines():
-            uid, iid, rating = [int(v) for v in l.split('\t')]
-            num_users = max(num_users, uid + 1)
-            num_items = max(num_items, iid + 1)
-            if float(rating) > 0.0:
-                likes[(uid, iid)] = 1.0
-
-    user_ids = []
-    item_ids = []
-    labels = []
-    matrix = np.zeros([num_users, num_items], dtype=np.float32)
-    for uid, iid in likes.keys():
-        user_ids.append(uid)
-        item_ids.append(iid)
-        labels.append(1.0)
-        matrix[uid, iid] = 1.0
-
-        negative = 0
-        while negative < 3:
-            nuid = random.randint(0, num_users - 1)
-            niid = random.randint(0, num_items - 1)
-            if (nuid, niid) not in likes:
-                negative += 1
-                user_ids.append(nuid)
-                item_ids.append(niid)
-                labels.append(0.0)
-
-    indices = np.arange(len(user_ids))
-    np.random.shuffle(indices)
-    users_np = np.array(user_ids, dtype=np.int32)[indices]
-    items_np = np.array(item_ids, dtype=np.int32)[indices]
-    labels_np = np.array(labels, dtype=np.float32)[indices]
-    return np.expand_dims(users_np, -1), \
-           np.expand_dims(items_np, -1), \
-           np.expand_dims(labels_np, -1), num_users, num_items, matrix
-
-
-class TestDygraphDeepCF(unittest.TestCase):
+            labels.append(1.0)
+            matrix[uid, iid] = 1.0
+
+            negative = 0
+            while negative < 3:
+                nuid = random.randint(0, num_users - 1)
+                niid = random.randint(0, num_items - 1)
+                if (nuid, niid) not in likes:
+                    negative += 1
+                    user_ids.append(nuid)
+                    item_ids.append(niid)
+                    labels.append(0.0)
+
+        indices = np.arange(len(user_ids))
+        np.random.shuffle(indices)
+        users_np = np.array(user_ids, dtype=np.int32)[indices]
+        items_np = np.array(item_ids, dtype=np.int32)[indices]
+        labels_np = np.array(labels, dtype=np.float32)[indices]
+        return np.expand_dims(users_np, -1), \
+            np.expand_dims(items_np, -1), \
+            np.expand_dims(labels_np, -1), num_users, num_items, matrix
 
     def test_deefcf(self):
         seed = 90
-        if DATA_PATH:
+        if self.data_path:
             (users_np, items_np, labels_np, num_users, num_items,
-             matrix) = load_data(DATA_PATH)
+             matrix) = self.load_data()
         else:
             (users_np, items_np, labels_np, num_users, num_items,
-             matrix) = get_data()
+             matrix) = self.get_data()
         paddle.seed(seed)
         paddle.framework.random._manual_program_seed(seed)
         startup = fluid.Program()
@@ -228,17 +227,19 @@ class TestDygraphDeepCF(unittest.TestCase):
             exe = fluid.Executor(fluid.CPUPlace(
             ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0))
             exe.run(startup)
-            for e in range(NUM_EPOCHES):
+            for e in range(self.num_epoches):
                 sys.stderr.write('epoch %d\n' % e)
-                for slice in range(0, BATCH_SIZE * NUM_BATCHES, BATCH_SIZE):
-                    if slice + BATCH_SIZE >= users_np.shape[0]:
+                for slice in range(0, self.batch_size * self.num_batches,
+                                   self.batch_size):
+                    if slice + self.batch_size >= users_np.shape[0]:
                         break
                     static_loss = exe.run(
                         main,
                         feed={
-                            users.name: users_np[slice:slice + BATCH_SIZE],
-                            items.name: items_np[slice:slice + BATCH_SIZE],
-                            labels.name: labels_np[slice:slice + BATCH_SIZE]
+                            users.name: users_np[slice:slice + self.batch_size],
+                            items.name: items_np[slice:slice + self.batch_size],
+                            labels.name:
+                            labels_np[slice:slice + self.batch_size]
                         },
                         fetch_list=[loss])[0]
                     sys.stderr.write('static loss %s\n' % static_loss)
@@ -250,18 +251,20 @@ class TestDygraphDeepCF(unittest.TestCase):
             deepcf = DeepCF(num_users, num_items, matrix)
             adam = fluid.optimizer.AdamOptimizer(
                 0.01, parameter_list=deepcf.parameters())
-            for e in range(NUM_EPOCHES):
+            for e in range(self.num_epoches):
                 sys.stderr.write('epoch %d\n' % e)
-                for slice in range(0, BATCH_SIZE * NUM_BATCHES, BATCH_SIZE):
-                    if slice + BATCH_SIZE >= users_np.shape[0]:
+                for slice in range(0, self.batch_size * self.num_batches,
+                                   self.batch_size):
+                    if slice + self.batch_size >= users_np.shape[0]:
                         break
                     prediction = deepcf(
-                        to_variable(users_np[slice:slice + BATCH_SIZE]),
-                        to_variable(items_np[slice:slice + BATCH_SIZE]))
+                        to_variable(users_np[slice:slice + self.batch_size]),
+                        to_variable(items_np[slice:slice + self.batch_size]))
                     loss = fluid.layers.reduce_sum(
                         fluid.layers.log_loss(
                             prediction,
-                            to_variable(labels_np[slice:slice + BATCH_SIZE])))
+                            to_variable(labels_np[slice:slice +
+                                                  self.batch_size])))
                     loss.backward()
                     adam.minimize(loss)
                     deepcf.clear_gradients()
@@ -276,18 +279,20 @@ class TestDygraphDeepCF(unittest.TestCase):
             adam2 = fluid.optimizer.AdamOptimizer(
                 0.01, parameter_list=deepcf2.parameters())
             fluid.set_flags({'FLAGS_sort_sum_gradient': True})
-            for e in range(NUM_EPOCHES):
+            for e in range(self.num_epoches):
                 sys.stderr.write('epoch %d\n' % e)
-                for slice in range(0, BATCH_SIZE * NUM_BATCHES, BATCH_SIZE):
-                    if slice + BATCH_SIZE >= users_np.shape[0]:
+                for slice in range(0, self.batch_size * self.num_batches,
+                                   self.batch_size):
+                    if slice + self.batch_size >= users_np.shape[0]:
                         break
                     prediction2 = deepcf2(
-                        to_variable(users_np[slice:slice + BATCH_SIZE]),
-                        to_variable(items_np[slice:slice + BATCH_SIZE]))
+                        to_variable(users_np[slice:slice + self.batch_size]),
+                        to_variable(items_np[slice:slice + self.batch_size]))
                     loss2 = fluid.layers.reduce_sum(
                         fluid.layers.log_loss(
                             prediction2,
-                            to_variable(labels_np[slice:slice + BATCH_SIZE])))
+                            to_variable(labels_np[slice:slice +
+                                                  self.batch_size])))
                     loss2.backward()
                     adam2.minimize(loss2)
                     deepcf2.clear_gradients()
@@ -306,19 +311,22 @@ class TestDygraphDeepCF(unittest.TestCase):
                 adam = fluid.optimizer.AdamOptimizer(
                     0.01, parameter_list=deepcf.parameters())
 
-                for e in range(NUM_EPOCHES):
+                for e in range(self.num_epoches):
                     sys.stderr.write('epoch %d\n' % e)
-                    for slice in range(0, BATCH_SIZE * NUM_BATCHES, BATCH_SIZE):
-                        if slice + BATCH_SIZE >= users_np.shape[0]:
+                    for slice in range(0, self.batch_size * self.num_batches,
+                                       self.batch_size):
+                        if slice + self.batch_size >= users_np.shape[0]:
                             break
                         prediction = deepcf(
-                            to_variable(users_np[slice:slice + BATCH_SIZE]),
-                            to_variable(items_np[slice:slice + BATCH_SIZE]))
+                            to_variable(users_np[slice:slice +
+                                                 self.batch_size]),
+                            to_variable(items_np[slice:slice +
+                                                 self.batch_size]))
                         loss = fluid.layers.reduce_sum(
                             fluid.layers.log_loss(
                                 prediction,
                                 to_variable(labels_np[slice:slice +
-                                                      BATCH_SIZE])))
+                                                      self.batch_size])))
                         loss.backward()
                         adam.minimize(loss)
                         deepcf.clear_gradients()

python/paddle/fluid/tests/unittests/test_imperative_gnn.py

@@ -177,4 +177,5 @@ class TestDygraphGNN(unittest.TestCase):
 
 
 if __name__ == '__main__':
+    paddle.enable_static()
     unittest.main()

python/paddle/fluid/tests/unittests/test_input_spec.py

@@ -12,8 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import unittest
+import tempfile
 import numpy as np
+
 import paddle
 import paddle.fluid as fluid
 from paddle.static import InputSpec
@@ -160,6 +163,10 @@ class TestNetWithNonTensorSpec(unittest.TestCase):
         self.out_num = 16
         self.x_spec = paddle.static.InputSpec([-1, 16], name='x')
         self.x = paddle.randn([4, 16])
+        self.temp_dir = tempfile.TemporaryDirectory()
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
 
     @classmethod
     def setUpClass(cls):
@@ -182,7 +189,7 @@ class TestNetWithNonTensorSpec(unittest.TestCase):
         self.check_result(specs, 'list')
 
     def check_result(self, specs, path):
-        path = './net_non_tensor_' + path
+        path = os.path.join(self.temp_dir.name, './net_non_tensor_', path)
 
         net = NetWithNonTensorSpec(self.in_num, self.out_num)
         net.eval()
@@ -218,7 +225,7 @@ class TestNetWithNonTensorSpec(unittest.TestCase):
         net = paddle.jit.to_static(net, input_spec=specs)
         net.eval()
 
-        path = './net_twice'
+        path = os.path.join(self.temp_dir.name, './net_twice')
 
         # NOTE: check input_specs_compatible
         new_specs = [self.x_spec, True, "bn", 10]
@@ -264,6 +271,7 @@ class TestNetWithNonTensorSpecWithPrune(unittest.TestCase):
         self.y_spec = paddle.static.InputSpec([16], name='y')
         self.x = paddle.randn([4, 16])
         self.y = paddle.randn([16])
+        self.temp_dir = tempfile.TemporaryDirectory()
 
     @classmethod
     def setUpClass(cls):
@@ -271,7 +279,7 @@ class TestNetWithNonTensorSpecWithPrune(unittest.TestCase):
 
     def test_non_tensor_with_prune(self):
         specs = [self.x_spec, self.y_spec, True]
-        path = './net_non_tensor_prune_'
+        path = os.path.join(self.temp_dir.name, './net_non_tensor_prune_')
 
         net = NetWithNonTensorSpecPrune(self.in_num, self.out_num)
         net.eval()

python/paddle/fluid/tests/unittests/test_layout_autotune.py

@@ -12,14 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import paddle
+import os
+import json
+import tempfile
 import unittest
+import warnings
 import numpy
+
+import paddle
 import paddle.nn.functional as F
-import tempfile
-import warnings
-import json
-import os
 from paddle.fluid.framework import _enable_legacy_dygraph
 
 _enable_legacy_dygraph()

python/paddle/fluid/tests/unittests/test_optimizer.py

@@ -14,6 +14,8 @@
 
 from __future__ import print_function
 
+import os
+import tempfile
 import unittest
 
 import paddle.fluid as fluid
@@ -29,8 +31,6 @@ import paddle
 from paddle.io import Dataset
 import numpy
 
-paddle.enable_static()
-
 
 class TestOptimizer(unittest.TestCase):
 
@@ -1279,6 +1279,12 @@ class TestMasterWeightSaveForFP16(unittest.TestCase):
     Master weights will be saved by optimizer::state_dict.
     '''
 
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
+
     def check_with_opt_state_dict(self, use_save_load=True):
         paddle.seed(100)
         numpy.random.seed(100)
@@ -1340,10 +1346,12 @@ class TestMasterWeightSaveForFP16(unittest.TestCase):
             optimizer.clear_grad(set_to_zero=False)
 
             if use_save_load and i == 5:
-                paddle.save(model.state_dict(), "model.pdparams")
-                paddle.save(optimizer.state_dict(), "opt.pdopt")
-                model.set_state_dict(paddle.load("model.pdparams"))
-                optimizer.set_state_dict(paddle.load("opt.pdopt"))
+                model_path = os.path.join(self.temp_dir.name, "model.pdparams")
+                optimizer_path = os.path.join(self.temp_dir.name, "opt.pdopt")
+                paddle.save(model.state_dict(), model_path)
+                paddle.save(optimizer.state_dict(), optimizer_path)
+                model.set_state_dict(paddle.load(model_path))
+                optimizer.set_state_dict(paddle.load(optimizer_path))
 
         return loss.numpy()
 
@@ -1359,4 +1367,5 @@ class TestMasterWeightSaveForFP16(unittest.TestCase):
 
 
 if __name__ == '__main__':
+    paddle.enable_static()
     unittest.main()

python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py

@@ -13,12 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import numpy as np
+import tempfile
+import unittest
+
 import paddle
 import paddle.fluid as fluid
-import unittest
 import paddle.nn as nn
-import os
 
 
 class SimpleFCLayer(nn.Layer):
@@ -54,6 +56,10 @@ class TestTracedLayerErrMsg(unittest.TestCase):
         self.fc_size = 2
         self.layer = self._train_simple_net()
         self.type_str = 'class'
+        self.temp_dir = tempfile.TemporaryDirectory()
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
 
     def test_trace_err(self):
         if fluid.framework.in_dygraph_mode():
@@ -122,7 +128,7 @@ class TestTracedLayerErrMsg(unittest.TestCase):
             dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace(
                 self.layer, [in_x])
 
-            path = './traced_layer_err_msg'
+            path = os.path.join(self.temp_dir.name, './traced_layer_err_msg')
             with self.assertRaises(TypeError) as e:
                 traced_layer.save_inference_model([0])
             self.assertEqual(
@@ -193,11 +199,15 @@ class TestTracedLayerSaveInferenceModel(unittest.TestCase):
     """test save_inference_model will automaticlly create non-exist dir"""
 
     def setUp(self):
-        self.save_path = "./nonexist_dir/fc"
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.save_path = os.path.join(self.temp_dir.name, "./nonexist_dir/fc")
         import shutil
         if os.path.exists(os.path.dirname(self.save_path)):
             shutil.rmtree(os.path.dirname(self.save_path))
 
+    def tearDown(self):
+        self.temp_dir.cleanup()
+
     def test_mkdir_when_input_path_non_exist(self):
         if fluid.framework.in_dygraph_mode():
             return