Cherry-pick PR#43237 from deveop (#43685)

* merge_release_and_dev

* merge_release_dev

* update

* Use tempfile to place the temporary files (#43237)

* tempfile_fix

* update

* fix_CI

* update_word2vec.inference.model

* remove_change_in_word2vec_book

* fix_word2vec_book

* rm_affine

* update

python/paddle/fluid/contrib/tests/test_image_classification_fp16.py

@@ -24,6 +24,7 @@ import unittest
 import os
 import copy
 import numpy as np
+import tempfile
 from paddle.static.amp import decorate
 
 paddle.enable_static()
@@ -268,18 +269,26 @@ def infer(use_cuda, save_dirname=None):
             clip_extra=True)
 
 
-def main(net_type, use_cuda, is_local=True):
-    if use_cuda and not fluid.core.is_compiled_with_cuda():
-        return
+class TestImageClassification(unittest.TestCase):
 
-    # Directory for saving the trained model
-    save_dirname = "image_classification_" + net_type + ".inference.model"
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
 
-    train(net_type, use_cuda, save_dirname, is_local)
-    #infer(use_cuda, save_dirname)
+    def tearDown(self):
+        self.temp_dir.cleanup()
 
+    def main(self, net_type, use_cuda, is_local=True):
+        if use_cuda and not fluid.core.is_compiled_with_cuda():
+            return
+
+        # Directory for saving the trained model
+        save_dirname = os.path.join(
+            self.temp_dir.name,
+            "image_classification_" + net_type + ".inference.model")
+
+        train(net_type, use_cuda, save_dirname, is_local)
+        #infer(use_cuda, save_dirname)
 
-class TestImageClassification(unittest.TestCase):
     def test_amp_lists(self):
         white_list = copy.copy(
             fluid.contrib.mixed_precision.fp16_lists.white_list)
@@ -408,11 +417,11 @@ class TestImageClassification(unittest.TestCase):
 
     def test_vgg_cuda(self):
         with self.scope_prog_guard():
-            main('vgg', use_cuda=True)
+            self.main('vgg', use_cuda=True)
 
     def test_resnet_cuda(self):
         with self.scope_prog_guard():
-            main('resnet', use_cuda=True)
+            self.main('resnet', use_cuda=True)
 
     @contextlib.contextmanager
     def scope_prog_guard(self):

python/paddle/fluid/tests/book/test_fit_a_line.py

@@ -25,6 +25,7 @@ import math
 import sys
 import os
 import struct
+import tempfile
 
 paddle.enable_static()
 
@@ -192,11 +193,13 @@ def main(use_cuda, is_local=True, use_bf16=False, pure_bf16=False):
     if use_bf16 and not fluid.core.is_compiled_with_mkldnn():
         return
 
+    temp_dir = tempfile.TemporaryDirectory()
     # Directory for saving the trained model
-    save_dirname = "fit_a_line.inference.model"
+    save_dirname = os.path.join(temp_dir.name, "fit_a_line.inference.model")
 
     train(use_cuda, save_dirname, is_local, use_bf16, pure_bf16)
     infer(use_cuda, save_dirname, use_bf16)
+    temp_dir.cleanup()
 
 
 class TestFitALineBase(unittest.TestCase):

python/paddle/fluid/tests/book/test_image_classification.py

@@ -22,6 +22,7 @@ import sys
 import numpy
 import unittest
 import os
+import tempfile
 import numpy as np
 
 paddle.enable_static()
@@ -240,10 +241,13 @@ def main(net_type, use_cuda, is_local=True):
         return
 
     # Directory for saving the trained model
-    save_dirname = "image_classification_" + net_type + ".inference.model"
+    temp_dir = tempfile.TemporaryDirectory()
+    save_dirname = os.path.join(
+        temp_dir.name, "image_classification_" + net_type + ".inference.model")
 
     train(net_type, use_cuda, save_dirname, is_local)
     infer(use_cuda, save_dirname)
+    temp_dir.cleanup()
 
 
 class TestImageClassification(unittest.TestCase):

python/paddle/fluid/tests/book/test_label_semantic_roles.py

@@ -20,6 +20,7 @@ import numpy as np
 import os
 import time
 import unittest
+import tempfile
 
 import paddle
 import paddle.dataset.conll05 as conll05
@@ -350,12 +351,16 @@ def main(use_cuda, is_local=True):
     if use_cuda and not fluid.core.is_compiled_with_cuda():
         return
 
+    temp_dir = tempfile.TemporaryDirectory()
     # Directory for saving the trained model
-    save_dirname = "label_semantic_roles.inference.model"
+    save_dirname = os.path.join(temp_dir.name,
+                                "label_semantic_roles.inference.model")
 
     train(use_cuda, save_dirname, is_local)
     infer(use_cuda, save_dirname)
 
+    temp_dir.cleanup()
+
 
 class TestLabelSemanticRoles(unittest.TestCase):
     def test_cuda(self):

python/paddle/fluid/tests/book/test_recommender_system.py

@@ -23,6 +23,7 @@ import paddle.fluid as fluid
 import paddle.fluid.framework as framework
 import paddle.fluid.layers as layers
 import paddle.fluid.nets as nets
+import tempfile
 from paddle.fluid.executor import Executor
 from paddle.fluid.optimizer import SGDOptimizer
 
@@ -320,10 +321,13 @@ def main(use_cuda):
         return
 
     # Directory for saving the inference model
-    save_dirname = "recommender_system.inference.model"
+    temp_dir = tempfile.TemporaryDirectory()
+    save_dirname = os.path.join(temp_dir.name,
+                                "recommender_system.inference.model")
 
     train(use_cuda, save_dirname)
     infer(use_cuda, save_dirname)
+    temp_dir.cleanup()
 
 
 if __name__ == '__main__':

python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py

@@ -23,7 +23,9 @@ import paddle.fluid.layers as layers
 import contextlib
 import math
 import sys
+import os
 import unittest
+import tempfile
 from paddle.fluid.executor import Executor
 import paddle
 
@@ -257,10 +259,13 @@ def main(use_cuda):
         return
 
     # Directory for saving the trained model
-    save_dirname = "rnn_encoder_decoder.inference.model"
+    temp_dir = tempfile.TemporaryDirectory()
+    save_dirname = os.path.join(temp_dir.name,
+                                "rnn_encoder_decoder.inference.model")
 
     train(use_cuda, save_dirname)
     infer(use_cuda, save_dirname)
+    temp_dir.cleanup()
 
 
 class TestRnnEncoderDecoder(unittest.TestCase):

python/paddle/fluid/tests/book/test_word2vec_book.py

@@ -22,6 +22,7 @@ import os
 import numpy as np
 import math
 import sys
+import tempfile
 
 paddle.enable_static()
 
@@ -238,7 +239,7 @@ def infer(target, save_dirname=None):
         infer_inputs = [to_infer_tensor(t) for t in infer_inputs]
 
         infer_config = fluid.core.NativeConfig()
-        infer_config.model_dir = 'word2vec.inference.model'
+        infer_config.model_dir = save_dirname
         if target == "cuda":
             infer_config.use_gpu = True
             infer_config.device = 0
@@ -264,8 +265,9 @@ def main(target, is_sparse, is_parallel, use_bf16, pure_bf16):
     if use_bf16 and not fluid.core.is_compiled_with_mkldnn():
         return
 
+    temp_dir = tempfile.TemporaryDirectory()
     if not is_parallel:
-        save_dirname = "word2vec.inference.model"
+        save_dirname = os.path.join(temp_dir.name, "word2vec.inference.model")
     else:
         save_dirname = None
 
@@ -282,6 +284,7 @@ def main(target, is_sparse, is_parallel, use_bf16, pure_bf16):
             use_bf16=use_bf16,
             pure_bf16=pure_bf16)
     infer(target, save_dirname)
+    temp_dir.cleanup()
 
 
 FULL_TEST = os.getenv('FULL_TEST',

python/paddle/fluid/tests/unittests/rnn/test_rnn_nets.py

@@ -16,8 +16,10 @@ import paddle
 paddle.set_default_dtype("float64")
 from paddle.fluid.layers import sequence_mask
 
+import os
 import numpy as np
 import unittest
+import tempfile
 
 from convert import convert_params_for_net
 from rnn_numpy import SimpleRNN, LSTM, GRU
@@ -318,9 +320,11 @@ def predict_test_util(place, mode, stop_gradient=True):
     rnn.train()
 
     rnn = paddle.jit.to_static(
-        rnn, [paddle.static.InputSpec(
-            shape=[None, None, 16], dtype=x.dtype)])
-    paddle.jit.save(rnn, "./inference/%s_infer" % mode)
+        rnn, [paddle.static.InputSpec(shape=[None, None, 16], dtype=x.dtype)])
+    temp_dir = tempfile.TemporaryDirectory()
+    save_dirname = os.path.join(temp_dir.name, "./inference/%s_infer" % mode)
+
+    paddle.jit.save(rnn, save_dirname)
 
     paddle.enable_static()
 
@@ -328,8 +332,7 @@ def predict_test_util(place, mode, stop_gradient=True):
     with paddle.static.scope_guard(new_scope):
         exe = paddle.static.Executor(place)
         [inference_program, feed_target_names,
-         fetch_targets] = paddle.static.load_inference_model(
-             "./inference/%s_infer" % mode, exe)
+         fetch_targets] = paddle.static.load_inference_model(save_dirname, exe)
         results = exe.run(inference_program,
                           feed={feed_target_names[0]: x.numpy()},
                           fetch_list=fetch_targets)
@@ -337,6 +340,8 @@ def predict_test_util(place, mode, stop_gradient=True):
             y.numpy(), results[0])  # eval results equal predict results
     paddle.disable_static()
 
+    temp_dir.cleanup()
+
 
 def load_tests(loader, tests, pattern):
     suite = unittest.TestSuite()

python/paddle/fluid/tests/unittests/test_dataset_dataloader.py

@@ -18,6 +18,7 @@ import numpy as np
 import six
 import os
 import unittest
+import tempfile
 from simple_nets import simple_fc_net_with_inputs
 
 BATCH_SIZE = 32
@@ -27,8 +28,6 @@ EPOCH_NUM = 4
 IMAGE_SHAPE = [2, 3]
 LABEL_SHAPE = [1]
 
-ALL_WRITTEN_FILES = set()
-
 
 def get_place_string(p):
     if isinstance(p, (fluid.CPUPlace or fluid.CUDAPlace)):
@@ -42,13 +41,7 @@ def get_place_string(p):
         return 'CUDAPlace()'
 
 
-def remove_all_written_files():
-    for filename in ALL_WRITTEN_FILES:
-        os.remove(filename)
-
-
 def write_reader_data_to_file(filename, reader):
-    ALL_WRITTEN_FILES.add(filename)
     with open(filename, 'w') as fid:
         for instance_list in reader():
             for i, instance in enumerate(instance_list):
@@ -77,10 +70,10 @@ class DatasetLoaderTestBase(unittest.TestCase):
     def setUp(self):
         self.dataset_name = "QueueDataset"
         self.drop_last = False
+        self.temp_dir = tempfile.TemporaryDirectory()
 
     def tearDown(self):
-        return
-        remove_all_written_files()
+        self.temp_dir.cleanup()
 
     def build_network(self):
         main_prog = fluid.Program()
@@ -123,7 +116,8 @@ class DatasetLoaderTestBase(unittest.TestCase):
             random_delta_batch_size = np.zeros(shape=[file_num])
 
         for i in six.moves.range(file_num):
-            filename = 'dataset_test_{}.txt'.format(i)
+            filename = os.path.join(self.temp_dir.name,
+                                    'dataset_test_{}.txt'.format(i))
             filelist.append(filename)
             write_reader_data_to_file(
                 filename,
@@ -207,18 +201,21 @@ class QueueDatasetTestWithoutDropLast(DatasetLoaderTestBase):
     def setUp(self):
         self.dataset_name = "QueueDataset"
         self.drop_last = True
+        self.temp_dir = tempfile.TemporaryDirectory()
 
 
 class InMemoryDatasetTestWithoutDropLast(DatasetLoaderTestBase):
     def setUp(self):
         self.dataset_name = "InMemoryDataset"
         self.drop_last = False
+        self.temp_dir = tempfile.TemporaryDirectory()
 
 
 class InMemoryDatasetTestWithDropLast(DatasetLoaderTestBase):
     def setUp(self):
         self.dataset_name = "InMemoryDataset"
         self.drop_last = True
+        self.temp_dir = tempfile.TemporaryDirectory()
 
 
 if __name__ == '__main__':

python/paddle/fluid/tests/unittests/test_ops_nms.py

@@ -12,10 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import unittest
 import numpy as np
 import paddle
 from test_nms_op import nms
+import tempfile
 
 
 def _find(condition):
@@ -78,6 +80,11 @@ class TestOpsNMS(unittest.TestCase):
         self.devices = ['cpu']
         if paddle.is_compiled_with_cuda():
             self.devices.append('gpu')
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.path = os.path.join(self.temp_dir.name, './net')
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
 
     def test_nms(self):
         for device in self.devices:
@@ -165,7 +172,6 @@ class TestOpsNMS(unittest.TestCase):
                                                 categories, 10)
                     return out
 
-                path = "./net"
                 boxes = np.random.rand(64, 4).astype('float32')
                 boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
                 boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
@@ -173,12 +179,14 @@ class TestOpsNMS(unittest.TestCase):
                 origin = fun(paddle.to_tensor(boxes))
                 paddle.jit.save(
                     fun,
-                    path,
+                    self.path,
                     input_spec=[
-                        paddle.static.InputSpec(
-                            shape=[None, 4], dtype='float32', name='x')
-                    ], )
-                load_func = paddle.jit.load(path)
+                        paddle.static.InputSpec(shape=[None, 4],
+                                                dtype='float32',
+                                                name='x')
+                    ],
+                )
+                load_func = paddle.jit.load(self.path)
                 res = load_func(paddle.to_tensor(boxes))
                 self.assertTrue(
                     np.array_equal(origin, res),

python/paddle/tests/test_hapi_amp.py

@@ -19,7 +19,7 @@ import os
 os.environ['FLAGS_cudnn_deterministic'] = '1'
 
 import unittest
-
+import tempfile
 import numpy as np
 
 import paddle
@@ -98,7 +98,9 @@ class TestHapiWithAmp(unittest.TestCase):
                   batch_size=64,
                   num_iters=2,
                   log_freq=1)
-        model.save('./lenet_amp')
+        temp_dir = tempfile.TemporaryDirectory()
+        lenet_amp_path = os.path.join(temp_dir.name, './lenet_amp')
+        model.save(lenet_amp_path)
 
         with paddle.fluid.unique_name.guard():
             paddle.seed(2021)
@@ -116,7 +118,8 @@ class TestHapiWithAmp(unittest.TestCase):
                model._scaler.state_dict()['incr_count']))
 
         # equal after load
-        new_model.load('./lenet_amp')
+        new_model.load(lenet_amp_path)
+        temp_dir.cleanup()
         self.assertEqual(new_model._scaler.state_dict()['incr_count'],
                          model._scaler.state_dict()['incr_count'])
         self.assertEqual(new_model._scaler.state_dict()['decr_count'],

python/paddle/tests/test_read_file.py

@@ -16,6 +16,7 @@ import os
 import cv2
 import shutil
 import unittest
+import tempfile
 import numpy as np
 
 import paddle
@@ -25,23 +26,25 @@ from paddle.vision.ops import read_file, decode_jpeg
 class TestReadFile(unittest.TestCase):
     def setUp(self):
         fake_img = (np.random.random((400, 300, 3)) * 255).astype('uint8')
-        cv2.imwrite('fake.jpg', fake_img)
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.img_path = os.path.join(self.temp_dir.name, 'fake.jpg')
+        cv2.imwrite(self.img_path, fake_img)
 
     def tearDown(self):
-        os.remove('fake.jpg')
+        self.temp_dir.cleanup()
 
     def read_file_decode_jpeg(self):
         if not paddle.is_compiled_with_cuda():
             return
 
-        img_bytes = read_file('fake.jpg')
+        img_bytes = read_file(self.img_path)
 
         img = decode_jpeg(img_bytes, mode='gray')
         img = decode_jpeg(img_bytes, mode='rgb')
 
         img = decode_jpeg(img_bytes)
 
-        img_cv2 = cv2.imread('fake.jpg')
+        img_cv2 = cv2.imread(self.img_path)
         if paddle.in_dynamic_mode():
             np.testing.assert_equal(img.shape, img_cv2.transpose(2, 0, 1).shape)
         else:

python/paddle/tests/test_transforms.py

@@ -641,7 +641,8 @@ class TestFunctional(unittest.TestCase):
         fake_img = Image.fromarray((np.random.random((32, 32, 3)) * 255).astype(
             'uint8'))
 
-        path = 'temp.jpg'
+        temp_dir = tempfile.TemporaryDirectory()
+        path = os.path.join(temp_dir.name, 'temp.jpg')
         fake_img.save(path)
 
         set_image_backend('pil')
@@ -654,7 +655,7 @@ class TestFunctional(unittest.TestCase):
 
         np_img = image_load(path)
 
-        os.remove(path)
+        temp_dir.cleanup()
 
     def test_rotate(self):
         np_img = (np.random.rand(28, 28, 3) * 255).astype('uint8')