add hdfs ls retry time and sleep time, fix save inference (#22433)

* add hdfs ls retry time and sleep time, fix save inference
* test=develop

python/paddle/fluid/incubate/fleet/utils/fleet_util.py

@@ -920,7 +920,7 @@ class FleetUtil(object):
                         feeded_var_names=feeded_var_names,
                         target_vars=target_vars,
                         executor=executor,
-                        main_program=program,
+                        main_program=program.clone(),
                         params_filename="params")
                 else:
                     fluid.io.save_inference_model(
@@ -928,7 +928,7 @@ class FleetUtil(object):
                         feeded_var_names=feeded_var_names,
                         target_vars=target_vars,
                         executor=executor,
-                        main_program=program)
+                        main_program=program.clone())
 
             configs = {
                 "fs.default.name": hadoop_fs_name,

python/paddle/fluid/incubate/fleet/utils/hdfs.py

@@ -22,7 +22,7 @@ from datetime import datetime
 import re
 import copy
 import errno
-
+import time
 import logging
 
 __all__ = ["HDFSClient"]
@@ -83,6 +83,7 @@ class HDFSClient(object):
         ret_code = 0
         ret_out = None
         ret_err = None
+        retry_sleep_second = 3
         whole_commands = " ".join(whole_commands)
         for x in range(retry_times + 1):
             proc = subprocess.Popen(
@@ -99,6 +100,7 @@ class HDFSClient(object):
 
             if ret_code == 0:
                 break
+            time.sleep(retry_sleep_second)
 
         return ret_code, ret_out, ret_err
 
@@ -329,7 +331,7 @@ class HDFSClient(object):
 
         ls_commands = ['-ls', hdfs_path]
         returncode, output, errors = self.__run_hdfs_cmd(
-            ls_commands, retry_times=1)
+            ls_commands, retry_times=10)
 
         if returncode:
             _logger.error("HDFS list path: {} failed".format(hdfs_path))