diff --git a/paddlehub/commands/download.py b/paddlehub/commands/download.py index c70123aef5a921cda0360c1fb339aa5c9f51fb0d..a7db88918fa1918fd4252b28586934264330c55e 100644 --- a/paddlehub/commands/download.py +++ b/paddlehub/commands/download.py @@ -55,16 +55,26 @@ class DownloadCommand(BaseCommand): self.args = self.parser.parse_args(argv[1:]) self.args.type = self.check_type(self.args.type) + extra = {"command": "download"} if self.args.type in ["Module", "Model"]: search_result = default_hub_server.get_resource_url( - mod_name, resource_type=self.args.type, version=mod_version) + mod_name, + resource_type=self.args.type, + version=mod_version, + extra=extra) else: search_result = default_hub_server.get_resource_url( - mod_name, resource_type="Module", version=mod_version) + mod_name, + resource_type="Module", + version=mod_version, + extra=extra) self.args.type = "Module" if search_result == {}: search_result = default_hub_server.get_resource_url( - mod_name, resource_type="Model", version=mod_version) + mod_name, + resource_type="Model", + version=mod_version, + extra=extra) self.args.type = "Model" url = search_result.get('url', None) except_md5_value = search_result.get('md5', None) diff --git a/paddlehub/commands/hub.py b/paddlehub/commands/hub.py index 5618a1991938c3ac43c25bcf05135c2531d0c74f..0a1a59441909a41a0928c2d333066bc17e7d2310 100644 --- a/paddlehub/commands/hub.py +++ b/paddlehub/commands/hub.py @@ -51,7 +51,6 @@ class HubCommand(BaseCommand): help.command.execute(argv) exit(1) return False - srv_utils.hub_stat(['hub'] + argv) command = BaseCommand.command_dict[sub_command] return command.execute(argv[1:]) diff --git a/paddlehub/commands/install.py b/paddlehub/commands/install.py index 73d8b2ab1e4f2e8fc3735ff7719ac12a0c39cadb..e9ba9ba46fca22106736699675ca9824a857916b 100644 --- a/paddlehub/commands/install.py +++ b/paddlehub/commands/install.py @@ -47,8 +47,9 @@ class InstallCommand(BaseCommand): "==")[1] module_name = module_name if "==" not in module_name else module_name.split( "==")[0] + extra = {"command": "install"} result, tips, module_dir = default_module_manager.install_module( - module_name=module_name, module_version=module_version) + module_name=module_name, module_version=module_version, extra=extra) print(tips) return True diff --git a/paddlehub/commands/run.py b/paddlehub/commands/run.py index 8778388f83376cf5dfe8d67538347907fe265119..33f377f22e9e875662d7b5e813b5fe06291f7f5d 100644 --- a/paddlehub/commands/run.py +++ b/paddlehub/commands/run.py @@ -62,8 +62,9 @@ class RunCommand(BaseCommand): module_dir = (module_name, None) else: print("Install Module %s" % module_name) + extra = {"command": "install"} result, tips, module_dir = default_module_manager.install_module( - module_name) + module_name, extra=extra) print(tips) if not result: return None diff --git a/paddlehub/commands/search.py b/paddlehub/commands/search.py index bb090ad6dc850314da4079f74dca8c3b5c5a5e0e..24699f52abcd8a873573d0df6c1efbeeba68a12d 100644 --- a/paddlehub/commands/search.py +++ b/paddlehub/commands/search.py @@ -43,8 +43,9 @@ class SearchCommand(BaseCommand): argv = ['.*'] resource_name = argv[0] + extra = {"command": "search"} resource_list = default_hub_server.search_resource( - resource_name, resource_type="Module") + resource_name, resource_type="Module", extra=extra) if utils.is_windows(): placeholders = [20, 8, 8, 20] else: diff --git a/paddlehub/common/hub_server.py b/paddlehub/common/hub_server.py index 95b255ffe3da1214e30664c2b086dad306d75af5..c47f775b57c6bb35a3a0ea6b80a7584ae470b69e 100644 --- a/paddlehub/common/hub_server.py +++ b/paddlehub/common/hub_server.py @@ -96,13 +96,17 @@ class HubServer(object): return False return True - def search_resource(self, resource_key, resource_type=None, update=False): + def search_resource(self, + resource_key, + resource_type=None, + update=False, + extra=None): try: payload = {'word': resource_key} if resource_type: payload['type'] = resource_type api_url = srv_utils.uri_path(self.get_server_url(), 'search') - r = srv_utils.hub_request(api_url, payload) + r = srv_utils.hub_request(api_url, payload, extra=extra) if r['status'] == 0 and len(r['data']) > 0: return [(item['name'], item['type'], item['version'], item['summary']) for item in r['data']] @@ -147,7 +151,8 @@ class HubServer(object): resource_name, resource_type=None, version=None, - update=False): + update=False, + extra=None): try: payload = {'word': resource_name} if resource_type: @@ -155,7 +160,7 @@ class HubServer(object): if version: payload['version'] = version api_url = srv_utils.uri_path(self.get_server_url(), 'search') - r = srv_utils.hub_request(api_url, payload) + r = srv_utils.hub_request(api_url, payload, extra) if r['status'] == 0 and len(r['data']) > 0: for item in r['data']: if resource_name.lower() == item['name'].lower(): @@ -200,19 +205,26 @@ class HubServer(object): return {} - def get_module_url(self, module_name, version=None, update=False): + def get_module_url(self, + module_name, + version=None, + update=False, + extra=None): return self.get_resource_url( resource_name=module_name, resource_type="Module", version=version, - update=update) + update=update, + extra=extra) - def get_model_url(self, module_name, version=None, update=False): + def get_model_url(self, module_name, version=None, update=False, + extra=None): return self.get_resource_url( resource_name=module_name, resource_type="Model", version=version, - update=update) + update=update, + extra=extra) def request(self): if not os.path.exists(hub.CACHE_HOME): diff --git a/paddlehub/common/server_config.py b/paddlehub/common/server_config.py index ae73fc7732aab787e2ef288c100d236316dc8b7e..9ccf4dc6de683880070e442ecbc580680b23c8cb 100644 --- a/paddlehub/common/server_config.py +++ b/paddlehub/common/server_config.py @@ -14,16 +14,9 @@ HUB_SERVERS = ["http://paddlepaddle.org.cn/paddlehub"] -STAT_SERVERS = [ - "http://paddlepaddle.org.cn/paddlehub/stat", - "http://paddlepaddle.org.cn/paddlehub/stat" -] - default_server_config = { "server_url": HUB_SERVERS, "resource_storage_server_url": "https://bj.bcebos.com/paddlehub-data/", "debug": False, "log_level": "DEBUG" } - -default_stat_config = {"server_list": STAT_SERVERS} diff --git a/paddlehub/common/srv_utils.py b/paddlehub/common/srv_utils.py index 9d43dc32333fd476ea5fb46499cfaeb268d11f2d..71b85ab4ff1f439ef4453b8c0d3524e1d8c9afac 100644 --- a/paddlehub/common/srv_utils.py +++ b/paddlehub/common/srv_utils.py @@ -17,38 +17,11 @@ import requests import time import paddle import socket +import json from random import randint, seed from paddlehub import version -from paddlehub.common.server_config import default_stat_config - - -def get_stat_server(): - seed(int(time.time())) - stat_env = os.environ.get('HUB_SERVER_STAT_SRV') - if stat_env: - server_list = stat_env.split(';') - else: - server_list = default_stat_config['server_list'] - return server_list[randint(0, len(server_list) - 1)] - - -def hub_stat(argv): - try: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.connect(('bj.bcebos.com', 80)) - ip_addr = s.getsockname()[0] - params = { - 'command': ' '.join(argv), - 'hub_version': version.hub_version, - 'paddle_version': paddle.__version__, - 'ip_addr': ip_addr - } - stat_api = get_stat_server() - r = requests.get(stat_api, params=params, timeout=0.5) - except: - pass def uri_path(server_url, api): @@ -63,8 +36,9 @@ def uri_path(server_url, api): return srv -def hub_request(api, params): +def hub_request(api, params, extra=None): params['hub_version'] = version.hub_version params['paddle_version'] = paddle.__version__ + params["extra"] = json.dumps(extra) r = requests.get(api, params) return r.json() diff --git a/paddlehub/module/manager.py b/paddlehub/module/manager.py index 1b1a8d629dc787e10433e96dfba4a171f7485542..821f32f753bc4837de3c4c352c140fd9a7e20db2 100644 --- a/paddlehub/module/manager.py +++ b/paddlehub/module/manager.py @@ -75,7 +75,11 @@ class LocalModuleManager(object): self.all_modules(update=update) return self.modules_dict.get(module_name, None) - def install_module(self, module_name, module_version=None, upgrade=False): + def install_module(self, + module_name, + module_version=None, + upgrade=False, + extra=None): self.all_modules(update=True) module_info = self.modules_dict.get(module_name, None) if module_info: @@ -84,13 +88,12 @@ class LocalModuleManager(object): module_dir = self.modules_dict[module_name][0] module_tag = module_name if not module_version else '%s-%s' % ( module_name, module_version) - srv_utils.hub_stat(['installed', module_tag]) tips = "Module %s already installed in %s" % (module_tag, module_dir) return True, tips, self.modules_dict[module_name] search_result = hub.default_hub_server.get_module_url( - module_name, version=module_version) + module_name, version=module_version, extra=extra) name = search_result.get('name', None) url = search_result.get('url', None) md5_value = search_result.get('md5', None) @@ -102,7 +105,6 @@ class LocalModuleManager(object): tips += " with version %s" % module_version module_tag = module_name if not module_version else '%s-%s' % ( module_name, module_version) - srv_utils.hub_stat(['install fail', module_tag]) return False, tips, None result, tips, module_zip_file = default_downloader.download_file( @@ -127,7 +129,6 @@ class LocalModuleManager(object): shutil.move(module_dir, save_path) module_dir = save_path tips = "Successfully installed %s" % module_name - srv_utils.hub_stat(['install', module_name, url]) if installed_module_version: tips += "-%s" % installed_module_version return True, tips, (module_dir, installed_module_version) @@ -143,7 +144,6 @@ class LocalModuleManager(object): 1]: tips = "%s-%s is not installed" % (module_name, module_version) return True, tips - srv_utils.hub_stat(['uninstall', module_name]) tips = "Successfully uninstalled %s" % module_name if module_version: tips += '-%s' % module_version diff --git a/paddlehub/module/module.py b/paddlehub/module/module.py index ca4ffe49f0b1332d9ff8266a52f1b8564d45dab8..aed27a1958bf228fe721f30f0f12f32ef08110de 100644 --- a/paddlehub/module/module.py +++ b/paddlehub/module/module.py @@ -150,8 +150,9 @@ class Module(object): if version: log_msg += "-%s" % version logger.info(log_msg) + extra = {"command": "install"} result, tips, module_dir = default_module_manager.install_module( - module_name=name, module_version=version) + module_name=name, module_version=version, extra=extra) if not result: logger.error(tips) exit(1)