diff --git a/python/paddle_serving_server/monitor.py b/python/paddle_serving_server/monitor.py index 758722e6c7409b5bb92889554ef854dc45ade3b2..e34cf2476fcaa5febe8cd29849a0dcfc0e96265e 100644 --- a/python/paddle_serving_server/monitor.py +++ b/python/paddle_serving_server/monitor.py @@ -1,3 +1,16 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ Usage: Start monitor with one line command @@ -10,6 +23,10 @@ import argparse class Monitor(object): + ''' + Monitor base class. It is used to monitor the remote model, pull and update the local model. + ''' + def __init__(self, interval): self._remote_path = None self._remote_model_name = None @@ -20,20 +37,28 @@ class Monitor(object): self._interval = interval self._remote_donefile_timestamp = None self._local_tmp_dir = None + def set_remote_path(self, remote_path): self._remote_path = remote_path + def set_remote_model_name(self, model_name): self._remote_model_name = model_name + def set_remote_donefile_name(self, donefile_name): self._remote_donefile_name = donefile_name + def set_local_path(self, local_path): self._local_path = local_path + def set_local_model_name(self, model_name): self._local_model_name = model_name + def set_local_donefile_name(self, donefile_name): self._local_donefile_name = donefile_name + def set_local_tmp_dir(self, tmp_dir): self._local_tmp_dir = tmp_dir + def _check_params(self): if self._remote_path is None: raise Exception('remote_path not set.') @@ -49,7 +74,11 @@ class Monitor(object): raise Exception('local_donefile_name not set.') if self._local_tmp_dir is None: raise Exception('local_tmp_dir not set.') + def run(self): + ''' + Monitor the remote address model by polling and update the local model. + ''' self._check_params() if not os.path.exists(self._local_tmp_dir): os.makedirs(self._local_tmp_dir) @@ -69,12 +98,16 @@ class Monitor(object): print('[INFO] no donefile.') print('[INFO] sleep {}s'.format(self._interval)) time.sleep(self._interval) + def _exist_remote_donefile(self): raise Exception('This function must be inherited.') + def _pull_remote_model(self): raise Exception('This function must be inherited.') + def _update_local_model(self): raise Exception('This function must be inherited.') + def _update_local_donefile(self): raise Exception('This function must be inherited.') @@ -87,32 +120,42 @@ class FTPMonitor(Monitor): self._ftp_port = ftp_port self._ftp = ftplib.FTP() self._connect(ftp_ip, ftp_port, username, password) + def _connect(self, ftp_ip, ftp_port, username, password): self._ftp.connect(ftp_ip, ftp_port) self._ftp.login(username, password) + def _exist_remote_donefile(self): try: - donefile_path = '{}/{}'.format(self._remote_path, self._remote_donefile_name) - timestamp = self._ftp.voidcmd('MDTM {}'.format(donefile_path))[4:].strip() + donefile_path = '{}/{}'.format(self._remote_path, + self._remote_donefile_name) + timestamp = self._ftp.voidcmd('MDTM {}'.format(donefile_path))[ + 4:].strip() return [True, timestamp] except ftplib.error_perm: return [False, None] + def _pull_remote_model(self): - cmd = 'wget -nH -r -P {} ftp://{}:{}/{}/{} &> /dev/null'.format(self._local_tmp_dir, - self._ftp_ip, self._ftp_port, self._remote_path, self._remote_model_name) + cmd = 'wget -nH -r -P {} ftp://{}:{}/{}/{} &> /dev/null'.format( + self._local_tmp_dir, self._ftp_ip, self._ftp_port, + self._remote_path, self._remote_model_name) if os.system(cmd) != 0: raise Exception('pull remote model failed.') + def _update_local_model(self): - cmd = 'cp -r {}/{}/* {}/{}'.format(self._local_tmp_dir, self._remote_model_name, - self._local_path, self._local_model_name) + cmd = 'cp -r {}/{}/* {}/{}'.format( + self._local_tmp_dir, self._remote_model_name, self._local_path, + self._local_model_name) if os.system(cmd) != 0: raise Exception('update local model failed.') + def _update_local_donefile(self): cmd = 'touch {}/{}/{}'.format(self._local_path, self._local_model_name, - self._local_donefile_name) + self._local_donefile_name) if os.system(cmd) != 0: raise Exception('update local donefile failed.') + def parse_args(): parser = argparse.ArgumentParser(description="Monitor") parser.add_argument( @@ -120,24 +163,33 @@ def parse_args(): parser.add_argument( "--remote_path", type=str, required=True, help="Remote path") parser.add_argument( - "--remote_model_name", type=str, required=True, + "--remote_model_name", + type=str, + required=True, help="Remote model name") parser.add_argument( - "--remote_donefile_name", type=str, required=True, + "--remote_donefile_name", + type=str, + required=True, help="Remote donefile name") parser.add_argument( "--local_path", type=str, required=True, help="Local path") parser.add_argument( "--local_model_name", type=str, required=True, help="Local model name") parser.add_argument( - "--local_donefile_name", type=str, required=True, + "--local_donefile_name", + type=str, + required=True, help="Local donfile name(fluid_time_file in model file)") - parser.add_argument("--local_tmp_dir", type=str, default='tmp', help="Local tmp dir") - parser.add_argument("--interval", type=int, default=10, help="Time interval") + parser.add_argument( + "--local_tmp_dir", type=str, default='tmp', help="Local tmp dir") + parser.add_argument( + "--interval", type=int, default=10, help="Time interval") parser.add_argument("--ftp_ip", type=str, help="Ip the ftp") parser.add_argument("--ftp_port", type=int, help="Port the ftp") return parser.parse_args() + def start_ftp_monitor(): args = parse_args() obj = FTPMonitor(args.ftp_ip, args.ftp_port, interval=args.interval) @@ -150,6 +202,7 @@ def start_ftp_monitor(): obj.set_local_tmp_dir(args.local_tmp_dir) obj.run() + if __name__ == "__main__": args = parse_args() if args.type == 'ftp':