未验证 提交 6e4f39a0 编写于 作者: X xujiaqi01 提交者: GitHub

add hdfs ls retry time and sleep time, fix save inference (#22433)

* add hdfs ls retry time and sleep time, fix save inference
* test=develop
上级 d51ffe86
...@@ -920,7 +920,7 @@ class FleetUtil(object): ...@@ -920,7 +920,7 @@ class FleetUtil(object):
feeded_var_names=feeded_var_names, feeded_var_names=feeded_var_names,
target_vars=target_vars, target_vars=target_vars,
executor=executor, executor=executor,
main_program=program, main_program=program.clone(),
params_filename="params") params_filename="params")
else: else:
fluid.io.save_inference_model( fluid.io.save_inference_model(
...@@ -928,7 +928,7 @@ class FleetUtil(object): ...@@ -928,7 +928,7 @@ class FleetUtil(object):
feeded_var_names=feeded_var_names, feeded_var_names=feeded_var_names,
target_vars=target_vars, target_vars=target_vars,
executor=executor, executor=executor,
main_program=program) main_program=program.clone())
configs = { configs = {
"fs.default.name": hadoop_fs_name, "fs.default.name": hadoop_fs_name,
......
...@@ -22,7 +22,7 @@ from datetime import datetime ...@@ -22,7 +22,7 @@ from datetime import datetime
import re import re
import copy import copy
import errno import errno
import time
import logging import logging
__all__ = ["HDFSClient"] __all__ = ["HDFSClient"]
...@@ -83,6 +83,7 @@ class HDFSClient(object): ...@@ -83,6 +83,7 @@ class HDFSClient(object):
ret_code = 0 ret_code = 0
ret_out = None ret_out = None
ret_err = None ret_err = None
retry_sleep_second = 3
whole_commands = " ".join(whole_commands) whole_commands = " ".join(whole_commands)
for x in range(retry_times + 1): for x in range(retry_times + 1):
proc = subprocess.Popen( proc = subprocess.Popen(
...@@ -99,6 +100,7 @@ class HDFSClient(object): ...@@ -99,6 +100,7 @@ class HDFSClient(object):
if ret_code == 0: if ret_code == 0:
break break
time.sleep(retry_sleep_second)
return ret_code, ret_out, ret_err return ret_code, ret_out, ret_err
...@@ -329,7 +331,7 @@ class HDFSClient(object): ...@@ -329,7 +331,7 @@ class HDFSClient(object):
ls_commands = ['-ls', hdfs_path] ls_commands = ['-ls', hdfs_path]
returncode, output, errors = self.__run_hdfs_cmd( returncode, output, errors = self.__run_hdfs_cmd(
ls_commands, retry_times=1) ls_commands, retry_times=10)
if returncode: if returncode:
_logger.error("HDFS list path: {} failed".format(hdfs_path)) _logger.error("HDFS list path: {} failed".format(hdfs_path))
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册