提交 f06f86f9 编写于 作者: B barrierye

fix codestyle

上级 e82e6805
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" """
Usage: Usage:
Start monitor with one line command Start monitor with one line command
...@@ -10,6 +23,10 @@ import argparse ...@@ -10,6 +23,10 @@ import argparse
class Monitor(object): class Monitor(object):
'''
Monitor base class. It is used to monitor the remote model, pull and update the local model.
'''
def __init__(self, interval): def __init__(self, interval):
self._remote_path = None self._remote_path = None
self._remote_model_name = None self._remote_model_name = None
...@@ -20,20 +37,28 @@ class Monitor(object): ...@@ -20,20 +37,28 @@ class Monitor(object):
self._interval = interval self._interval = interval
self._remote_donefile_timestamp = None self._remote_donefile_timestamp = None
self._local_tmp_dir = None self._local_tmp_dir = None
def set_remote_path(self, remote_path): def set_remote_path(self, remote_path):
self._remote_path = remote_path self._remote_path = remote_path
def set_remote_model_name(self, model_name): def set_remote_model_name(self, model_name):
self._remote_model_name = model_name self._remote_model_name = model_name
def set_remote_donefile_name(self, donefile_name): def set_remote_donefile_name(self, donefile_name):
self._remote_donefile_name = donefile_name self._remote_donefile_name = donefile_name
def set_local_path(self, local_path): def set_local_path(self, local_path):
self._local_path = local_path self._local_path = local_path
def set_local_model_name(self, model_name): def set_local_model_name(self, model_name):
self._local_model_name = model_name self._local_model_name = model_name
def set_local_donefile_name(self, donefile_name): def set_local_donefile_name(self, donefile_name):
self._local_donefile_name = donefile_name self._local_donefile_name = donefile_name
def set_local_tmp_dir(self, tmp_dir): def set_local_tmp_dir(self, tmp_dir):
self._local_tmp_dir = tmp_dir self._local_tmp_dir = tmp_dir
def _check_params(self): def _check_params(self):
if self._remote_path is None: if self._remote_path is None:
raise Exception('remote_path not set.') raise Exception('remote_path not set.')
...@@ -49,7 +74,11 @@ class Monitor(object): ...@@ -49,7 +74,11 @@ class Monitor(object):
raise Exception('local_donefile_name not set.') raise Exception('local_donefile_name not set.')
if self._local_tmp_dir is None: if self._local_tmp_dir is None:
raise Exception('local_tmp_dir not set.') raise Exception('local_tmp_dir not set.')
def run(self): def run(self):
'''
Monitor the remote address model by polling and update the local model.
'''
self._check_params() self._check_params()
if not os.path.exists(self._local_tmp_dir): if not os.path.exists(self._local_tmp_dir):
os.makedirs(self._local_tmp_dir) os.makedirs(self._local_tmp_dir)
...@@ -69,12 +98,16 @@ class Monitor(object): ...@@ -69,12 +98,16 @@ class Monitor(object):
print('[INFO] no donefile.') print('[INFO] no donefile.')
print('[INFO] sleep {}s'.format(self._interval)) print('[INFO] sleep {}s'.format(self._interval))
time.sleep(self._interval) time.sleep(self._interval)
def _exist_remote_donefile(self): def _exist_remote_donefile(self):
raise Exception('This function must be inherited.') raise Exception('This function must be inherited.')
def _pull_remote_model(self): def _pull_remote_model(self):
raise Exception('This function must be inherited.') raise Exception('This function must be inherited.')
def _update_local_model(self): def _update_local_model(self):
raise Exception('This function must be inherited.') raise Exception('This function must be inherited.')
def _update_local_donefile(self): def _update_local_donefile(self):
raise Exception('This function must be inherited.') raise Exception('This function must be inherited.')
...@@ -87,32 +120,42 @@ class FTPMonitor(Monitor): ...@@ -87,32 +120,42 @@ class FTPMonitor(Monitor):
self._ftp_port = ftp_port self._ftp_port = ftp_port
self._ftp = ftplib.FTP() self._ftp = ftplib.FTP()
self._connect(ftp_ip, ftp_port, username, password) self._connect(ftp_ip, ftp_port, username, password)
def _connect(self, ftp_ip, ftp_port, username, password): def _connect(self, ftp_ip, ftp_port, username, password):
self._ftp.connect(ftp_ip, ftp_port) self._ftp.connect(ftp_ip, ftp_port)
self._ftp.login(username, password) self._ftp.login(username, password)
def _exist_remote_donefile(self): def _exist_remote_donefile(self):
try: try:
donefile_path = '{}/{}'.format(self._remote_path, self._remote_donefile_name) donefile_path = '{}/{}'.format(self._remote_path,
timestamp = self._ftp.voidcmd('MDTM {}'.format(donefile_path))[4:].strip() self._remote_donefile_name)
timestamp = self._ftp.voidcmd('MDTM {}'.format(donefile_path))[
4:].strip()
return [True, timestamp] return [True, timestamp]
except ftplib.error_perm: except ftplib.error_perm:
return [False, None] return [False, None]
def _pull_remote_model(self): def _pull_remote_model(self):
cmd = 'wget -nH -r -P {} ftp://{}:{}/{}/{} &> /dev/null'.format(self._local_tmp_dir, cmd = 'wget -nH -r -P {} ftp://{}:{}/{}/{} &> /dev/null'.format(
self._ftp_ip, self._ftp_port, self._remote_path, self._remote_model_name) self._local_tmp_dir, self._ftp_ip, self._ftp_port,
self._remote_path, self._remote_model_name)
if os.system(cmd) != 0: if os.system(cmd) != 0:
raise Exception('pull remote model failed.') raise Exception('pull remote model failed.')
def _update_local_model(self): def _update_local_model(self):
cmd = 'cp -r {}/{}/* {}/{}'.format(self._local_tmp_dir, self._remote_model_name, cmd = 'cp -r {}/{}/* {}/{}'.format(
self._local_path, self._local_model_name) self._local_tmp_dir, self._remote_model_name, self._local_path,
self._local_model_name)
if os.system(cmd) != 0: if os.system(cmd) != 0:
raise Exception('update local model failed.') raise Exception('update local model failed.')
def _update_local_donefile(self): def _update_local_donefile(self):
cmd = 'touch {}/{}/{}'.format(self._local_path, self._local_model_name, cmd = 'touch {}/{}/{}'.format(self._local_path, self._local_model_name,
self._local_donefile_name) self._local_donefile_name)
if os.system(cmd) != 0: if os.system(cmd) != 0:
raise Exception('update local donefile failed.') raise Exception('update local donefile failed.')
def parse_args(): def parse_args():
parser = argparse.ArgumentParser(description="Monitor") parser = argparse.ArgumentParser(description="Monitor")
parser.add_argument( parser.add_argument(
...@@ -120,24 +163,33 @@ def parse_args(): ...@@ -120,24 +163,33 @@ def parse_args():
parser.add_argument( parser.add_argument(
"--remote_path", type=str, required=True, help="Remote path") "--remote_path", type=str, required=True, help="Remote path")
parser.add_argument( parser.add_argument(
"--remote_model_name", type=str, required=True, "--remote_model_name",
type=str,
required=True,
help="Remote model name") help="Remote model name")
parser.add_argument( parser.add_argument(
"--remote_donefile_name", type=str, required=True, "--remote_donefile_name",
type=str,
required=True,
help="Remote donefile name") help="Remote donefile name")
parser.add_argument( parser.add_argument(
"--local_path", type=str, required=True, help="Local path") "--local_path", type=str, required=True, help="Local path")
parser.add_argument( parser.add_argument(
"--local_model_name", type=str, required=True, help="Local model name") "--local_model_name", type=str, required=True, help="Local model name")
parser.add_argument( parser.add_argument(
"--local_donefile_name", type=str, required=True, "--local_donefile_name",
type=str,
required=True,
help="Local donfile name(fluid_time_file in model file)") help="Local donfile name(fluid_time_file in model file)")
parser.add_argument("--local_tmp_dir", type=str, default='tmp', help="Local tmp dir") parser.add_argument(
parser.add_argument("--interval", type=int, default=10, help="Time interval") "--local_tmp_dir", type=str, default='tmp', help="Local tmp dir")
parser.add_argument(
"--interval", type=int, default=10, help="Time interval")
parser.add_argument("--ftp_ip", type=str, help="Ip the ftp") parser.add_argument("--ftp_ip", type=str, help="Ip the ftp")
parser.add_argument("--ftp_port", type=int, help="Port the ftp") parser.add_argument("--ftp_port", type=int, help="Port the ftp")
return parser.parse_args() return parser.parse_args()
def start_ftp_monitor(): def start_ftp_monitor():
args = parse_args() args = parse_args()
obj = FTPMonitor(args.ftp_ip, args.ftp_port, interval=args.interval) obj = FTPMonitor(args.ftp_ip, args.ftp_port, interval=args.interval)
...@@ -150,6 +202,7 @@ def start_ftp_monitor(): ...@@ -150,6 +202,7 @@ def start_ftp_monitor():
obj.set_local_tmp_dir(args.local_tmp_dir) obj.set_local_tmp_dir(args.local_tmp_dir)
obj.run() obj.run()
if __name__ == "__main__": if __name__ == "__main__":
args = parse_args() args = parse_args()
if args.type == 'ftp': if args.type == 'ftp':
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册