Use tempfile to place all the temporary files. Modify some code structure. (#43376)

python/paddle/fluid/tests/unittests/test_directory_migration.py

@@ -17,14 +17,22 @@ from __future__ import print_function
 import os
 import sys
 import time
+import tempfile
 import subprocess
 import unittest
+
 import numpy as np
 import paddle
 
 
 class TestDirectory(unittest.TestCase):
 
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
+
     def get_import_command(self, module):
         paths = module.split('.')
         if len(paths) == 1:
@@ -77,7 +85,7 @@ class TestDirectory(unittest.TestCase):
             'paddle.static.nn.spectral_norm', 'paddle.static.nn.embedding'
         ]
 
-        import_file = 'run_import_modules.py'
+        import_file = os.path.join(self.temp_dir.name, 'run_import_modules.py')
 
         with open(import_file, "w") as wb:
             for module in new_directory:
@@ -137,7 +145,8 @@ class TestDirectory(unittest.TestCase):
             'paddle.declarative.spectral_norm', 'paddle.declarative.embedding'
         ]
 
-        import_file = 'run_old_import_modules.py'
+        import_file = os.path.join(self.temp_dir.name,
+                                   'run_old_import_modules.py')
 
         with open(import_file, "w") as wb:
             cmd_context_count = """

python/paddle/fluid/tests/unittests/test_imperative_deepcf.py

@@ -26,13 +26,6 @@ from paddle.fluid.dygraph.base import to_variable
 from paddle.fluid.dygraph import Linear
 from paddle.fluid.framework import _test_eager_guard
 
-# Can use Amusic dataset as the DeepCF describes.
-DATA_PATH = os.environ.get('DATA_PATH', '')
-
-BATCH_SIZE = int(os.environ.get('BATCH_SIZE', 128))
-NUM_BATCHES = int(os.environ.get('NUM_BATCHES', 5))
-NUM_EPOCHES = int(os.environ.get('NUM_EPOCHES', 1))
-
 
 class DMF(fluid.Layer):
 
@@ -129,7 +122,17 @@ class DeepCF(fluid.Layer):
         return prediction
 
 
-def get_data():
+class TestDygraphDeepCF(unittest.TestCase):
+
+    def setUp(self):
+        # Can use Amusic dataset as the DeepCF describes.
+        self.data_path = os.environ.get('DATA_PATH', '')
+
+        self.batch_size = int(os.environ.get('BATCH_SIZE', 128))
+        self.num_batches = int(os.environ.get('NUM_BATCHES', 5))
+        self.num_epoches = int(os.environ.get('NUM_EPOCHES', 1))
+
+    def get_data(self):
         user_ids = []
         item_ids = []
         labels = []
@@ -153,13 +156,12 @@ def get_data():
             np.expand_dims(items_np, -1), \
             np.expand_dims(labels_np, -1), NUM_USERS, NUM_ITEMS, matrix
 
-
-def load_data(DATA_PATH):
-    sys.stderr.write('loading from %s\n' % DATA_PATH)
+    def load_data(self):
+        sys.stderr.write('loading from %s\n' % self.data_path)
         likes = dict()
         num_users = -1
         num_items = -1
-    with open(DATA_PATH, 'r') as f:
+        with open(self.data_path, 'r') as f:
             for l in f.readlines():
                 uid, iid, rating = [int(v) for v in l.split('\t')]
                 num_users = max(num_users, uid + 1)
@@ -196,17 +198,14 @@ def load_data(DATA_PATH):
             np.expand_dims(items_np, -1), \
             np.expand_dims(labels_np, -1), num_users, num_items, matrix
 
-
-class TestDygraphDeepCF(unittest.TestCase):
-
     def test_deefcf(self):
         seed = 90
-        if DATA_PATH:
+        if self.data_path:
             (users_np, items_np, labels_np, num_users, num_items,
-             matrix) = load_data(DATA_PATH)
+             matrix) = self.load_data()
         else:
             (users_np, items_np, labels_np, num_users, num_items,
-             matrix) = get_data()
+             matrix) = self.get_data()
         paddle.seed(seed)
         paddle.framework.random._manual_program_seed(seed)
         startup = fluid.Program()
@@ -228,17 +227,19 @@ class TestDygraphDeepCF(unittest.TestCase):
             exe = fluid.Executor(fluid.CPUPlace(
             ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0))
             exe.run(startup)
-            for e in range(NUM_EPOCHES):
+            for e in range(self.num_epoches):
                 sys.stderr.write('epoch %d\n' % e)
-                for slice in range(0, BATCH_SIZE * NUM_BATCHES, BATCH_SIZE):
-                    if slice + BATCH_SIZE >= users_np.shape[0]:
+                for slice in range(0, self.batch_size * self.num_batches,
+                                   self.batch_size):
+                    if slice + self.batch_size >= users_np.shape[0]:
                         break
                     static_loss = exe.run(
                         main,
                         feed={
-                            users.name: users_np[slice:slice + BATCH_SIZE],
-                            items.name: items_np[slice:slice + BATCH_SIZE],
-                            labels.name: labels_np[slice:slice + BATCH_SIZE]
+                            users.name: users_np[slice:slice + self.batch_size],
+                            items.name: items_np[slice:slice + self.batch_size],
+                            labels.name:
+                            labels_np[slice:slice + self.batch_size]
                         },
                         fetch_list=[loss])[0]
                     sys.stderr.write('static loss %s\n' % static_loss)
@@ -250,18 +251,20 @@ class TestDygraphDeepCF(unittest.TestCase):
             deepcf = DeepCF(num_users, num_items, matrix)
             adam = fluid.optimizer.AdamOptimizer(
                 0.01, parameter_list=deepcf.parameters())
-            for e in range(NUM_EPOCHES):
+            for e in range(self.num_epoches):
                 sys.stderr.write('epoch %d\n' % e)
-                for slice in range(0, BATCH_SIZE * NUM_BATCHES, BATCH_SIZE):
-                    if slice + BATCH_SIZE >= users_np.shape[0]:
+                for slice in range(0, self.batch_size * self.num_batches,
+                                   self.batch_size):
+                    if slice + self.batch_size >= users_np.shape[0]:
                         break
                     prediction = deepcf(
-                        to_variable(users_np[slice:slice + BATCH_SIZE]),
-                        to_variable(items_np[slice:slice + BATCH_SIZE]))
+                        to_variable(users_np[slice:slice + self.batch_size]),
+                        to_variable(items_np[slice:slice + self.batch_size]))
                     loss = fluid.layers.reduce_sum(
                         fluid.layers.log_loss(
                             prediction,
-                            to_variable(labels_np[slice:slice + BATCH_SIZE])))
+                            to_variable(labels_np[slice:slice +
+                                                  self.batch_size])))
                     loss.backward()
                     adam.minimize(loss)
                     deepcf.clear_gradients()
@@ -276,18 +279,20 @@ class TestDygraphDeepCF(unittest.TestCase):
             adam2 = fluid.optimizer.AdamOptimizer(
                 0.01, parameter_list=deepcf2.parameters())
             fluid.set_flags({'FLAGS_sort_sum_gradient': True})
-            for e in range(NUM_EPOCHES):
+            for e in range(self.num_epoches):
                 sys.stderr.write('epoch %d\n' % e)
-                for slice in range(0, BATCH_SIZE * NUM_BATCHES, BATCH_SIZE):
-                    if slice + BATCH_SIZE >= users_np.shape[0]:
+                for slice in range(0, self.batch_size * self.num_batches,
+                                   self.batch_size):
+                    if slice + self.batch_size >= users_np.shape[0]:
                         break
                     prediction2 = deepcf2(
-                        to_variable(users_np[slice:slice + BATCH_SIZE]),
-                        to_variable(items_np[slice:slice + BATCH_SIZE]))
+                        to_variable(users_np[slice:slice + self.batch_size]),
+                        to_variable(items_np[slice:slice + self.batch_size]))
                     loss2 = fluid.layers.reduce_sum(
                         fluid.layers.log_loss(
                             prediction2,
-                            to_variable(labels_np[slice:slice + BATCH_SIZE])))
+                            to_variable(labels_np[slice:slice +
+                                                  self.batch_size])))
                     loss2.backward()
                     adam2.minimize(loss2)
                     deepcf2.clear_gradients()
@@ -306,19 +311,22 @@ class TestDygraphDeepCF(unittest.TestCase):
                 adam = fluid.optimizer.AdamOptimizer(
                     0.01, parameter_list=deepcf.parameters())
 
-                for e in range(NUM_EPOCHES):
+                for e in range(self.num_epoches):
                     sys.stderr.write('epoch %d\n' % e)
-                    for slice in range(0, BATCH_SIZE * NUM_BATCHES, BATCH_SIZE):
-                        if slice + BATCH_SIZE >= users_np.shape[0]:
+                    for slice in range(0, self.batch_size * self.num_batches,
+                                       self.batch_size):
+                        if slice + self.batch_size >= users_np.shape[0]:
                             break
                         prediction = deepcf(
-                            to_variable(users_np[slice:slice + BATCH_SIZE]),
-                            to_variable(items_np[slice:slice + BATCH_SIZE]))
+                            to_variable(users_np[slice:slice +
+                                                 self.batch_size]),
+                            to_variable(items_np[slice:slice +
+                                                 self.batch_size]))
                         loss = fluid.layers.reduce_sum(
                             fluid.layers.log_loss(
                                 prediction,
                                 to_variable(labels_np[slice:slice +
-                                                      BATCH_SIZE])))
+                                                      self.batch_size])))
                         loss.backward()
                         adam.minimize(loss)
                         deepcf.clear_gradients()

python/paddle/fluid/tests/unittests/test_imperative_gnn.py

@@ -177,4 +177,5 @@ class TestDygraphGNN(unittest.TestCase):
 
 
 if __name__ == '__main__':
+    paddle.enable_static()
     unittest.main()

python/paddle/fluid/tests/unittests/test_input_spec.py

@@ -12,8 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import unittest
+import tempfile
 import numpy as np
+
 import paddle
 import paddle.fluid as fluid
 from paddle.static import InputSpec
@@ -160,6 +163,10 @@ class TestNetWithNonTensorSpec(unittest.TestCase):
         self.out_num = 16
         self.x_spec = paddle.static.InputSpec([-1, 16], name='x')
         self.x = paddle.randn([4, 16])
+        self.temp_dir = tempfile.TemporaryDirectory()
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
 
     @classmethod
     def setUpClass(cls):
@@ -182,7 +189,7 @@ class TestNetWithNonTensorSpec(unittest.TestCase):
         self.check_result(specs, 'list')
 
     def check_result(self, specs, path):
-        path = './net_non_tensor_' + path
+        path = os.path.join(self.temp_dir.name, './net_non_tensor_', path)
 
         net = NetWithNonTensorSpec(self.in_num, self.out_num)
         net.eval()
@@ -218,7 +225,7 @@ class TestNetWithNonTensorSpec(unittest.TestCase):
         net = paddle.jit.to_static(net, input_spec=specs)
         net.eval()
 
-        path = './net_twice'
+        path = os.path.join(self.temp_dir.name, './net_twice')
 
         # NOTE: check input_specs_compatible
         new_specs = [self.x_spec, True, "bn", 10]
@@ -264,6 +271,7 @@ class TestNetWithNonTensorSpecWithPrune(unittest.TestCase):
         self.y_spec = paddle.static.InputSpec([16], name='y')
         self.x = paddle.randn([4, 16])
         self.y = paddle.randn([16])
+        self.temp_dir = tempfile.TemporaryDirectory()
 
     @classmethod
     def setUpClass(cls):
@@ -271,7 +279,7 @@ class TestNetWithNonTensorSpecWithPrune(unittest.TestCase):
 
     def test_non_tensor_with_prune(self):
         specs = [self.x_spec, self.y_spec, True]
-        path = './net_non_tensor_prune_'
+        path = os.path.join(self.temp_dir.name, './net_non_tensor_prune_')
 
         net = NetWithNonTensorSpecPrune(self.in_num, self.out_num)
         net.eval()

python/paddle/fluid/tests/unittests/test_layout_autotune.py

@@ -12,14 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import paddle
+import os
+import json
+import tempfile
 import unittest
+import warnings
 import numpy
+
+import paddle
 import paddle.nn.functional as F
-import tempfile
-import warnings
-import json
-import os
 from paddle.fluid.framework import _enable_legacy_dygraph
 
 _enable_legacy_dygraph()

python/paddle/fluid/tests/unittests/test_optimizer.py

@@ -14,6 +14,8 @@
 
 from __future__ import print_function
 
+import os
+import tempfile
 import unittest
 
 import paddle.fluid as fluid
@@ -29,8 +31,6 @@ import paddle
 from paddle.io import Dataset
 import numpy
 
-paddle.enable_static()
-
 
 class TestOptimizer(unittest.TestCase):
 
@@ -1279,6 +1279,12 @@ class TestMasterWeightSaveForFP16(unittest.TestCase):
     Master weights will be saved by optimizer::state_dict.
     '''
 
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
+
     def check_with_opt_state_dict(self, use_save_load=True):
         paddle.seed(100)
         numpy.random.seed(100)
@@ -1340,10 +1346,12 @@ class TestMasterWeightSaveForFP16(unittest.TestCase):
             optimizer.clear_grad(set_to_zero=False)
 
             if use_save_load and i == 5:
-                paddle.save(model.state_dict(), "model.pdparams")
-                paddle.save(optimizer.state_dict(), "opt.pdopt")
-                model.set_state_dict(paddle.load("model.pdparams"))
-                optimizer.set_state_dict(paddle.load("opt.pdopt"))
+                model_path = os.path.join(self.temp_dir.name, "model.pdparams")
+                optimizer_path = os.path.join(self.temp_dir.name, "opt.pdopt")
+                paddle.save(model.state_dict(), model_path)
+                paddle.save(optimizer.state_dict(), optimizer_path)
+                model.set_state_dict(paddle.load(model_path))
+                optimizer.set_state_dict(paddle.load(optimizer_path))
 
         return loss.numpy()
 
@@ -1359,4 +1367,5 @@ class TestMasterWeightSaveForFP16(unittest.TestCase):
 
 
 if __name__ == '__main__':
+    paddle.enable_static()
     unittest.main()

python/paddle/fluid/tests/unittests/test_traced_layer_err_msg.py

@@ -13,12 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import numpy as np
+import tempfile
+import unittest
+
 import paddle
 import paddle.fluid as fluid
-import unittest
 import paddle.nn as nn
-import os
 
 
 class SimpleFCLayer(nn.Layer):
@@ -54,6 +56,10 @@ class TestTracedLayerErrMsg(unittest.TestCase):
         self.fc_size = 2
         self.layer = self._train_simple_net()
         self.type_str = 'class'
+        self.temp_dir = tempfile.TemporaryDirectory()
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
 
     def test_trace_err(self):
         if fluid.framework.in_dygraph_mode():
@@ -122,7 +128,7 @@ class TestTracedLayerErrMsg(unittest.TestCase):
             dygraph_out, traced_layer = fluid.dygraph.TracedLayer.trace(
                 self.layer, [in_x])
 
-            path = './traced_layer_err_msg'
+            path = os.path.join(self.temp_dir.name, './traced_layer_err_msg')
             with self.assertRaises(TypeError) as e:
                 traced_layer.save_inference_model([0])
             self.assertEqual(
@@ -193,11 +199,15 @@ class TestTracedLayerSaveInferenceModel(unittest.TestCase):
     """test save_inference_model will automaticlly create non-exist dir"""
 
     def setUp(self):
-        self.save_path = "./nonexist_dir/fc"
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.save_path = os.path.join(self.temp_dir.name, "./nonexist_dir/fc")
         import shutil
         if os.path.exists(os.path.dirname(self.save_path)):
             shutil.rmtree(os.path.dirname(self.save_path))
 
+    def tearDown(self):
+        self.temp_dir.cleanup()
+
     def test_mkdir_when_input_path_non_exist(self):
         if fluid.framework.in_dygraph_mode():
             return