From 9532fbdc57c0e600d3cd79e6800f3877d4e58cd9 Mon Sep 17 00:00:00 2001 From: Kaipeng Deng Date: Wed, 26 May 2021 10:50:07 +0800 Subject: [PATCH] update version and config download (#3153) * update version and config download --- .gitignore | 2 ++ ppdet/__init__.py | 10 ++++++ ppdet/utils/download.py | 68 +++++++++++++++++++---------------------- setup.py | 41 ++++++++++++++++++++++++- 4 files changed, 84 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index ad3454f03..36fea6908 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,5 @@ dataset/wider_face/WIDER_test dataset/wider_face/WIDER_train dataset/wider_face/WIDER_val dataset/wider_face/wider_face_split + +ppdet/version.py diff --git a/ppdet/__init__.py b/ppdet/__init__.py index 56b687dd9..6fcc982fb 100644 --- a/ppdet/__init__.py +++ b/ppdet/__init__.py @@ -14,3 +14,13 @@ from . import (core, data, engine, modeling, model_zoo, optimizer, metrics, utils, slim) + + +try: + from .version import full_version as __version__ + from .version import commit as __git_commit__ +except ImportError: + import sys + sys.stderr.write("Warning: import ppdet from source directory " \ + "without installing, run 'python setup.py install' to " \ + "install ppdet firstly\n") diff --git a/ppdet/utils/download.py b/ppdet/utils/download.py index daa2793b7..9c7d80216 100644 --- a/ppdet/utils/download.py +++ b/ppdet/utils/download.py @@ -118,44 +118,36 @@ def get_config_path(url): download it from url. """ url = parse_url(url) - path, _ = get_path(url, CONFIGS_HOME) - _download_config(path, url, CONFIGS_HOME) - - return path - - -def _download_config(cfg_path, cfg_url, cur_dir): - with open(cfg_path) as f: - cfg = yaml.load(f, Loader=yaml.Loader) + path = map_path(url, CONFIGS_HOME, path_depth=2) + if os.path.isfile(path): + return path - # download dependence base ymls - if BASE_KEY in cfg: - base_ymls = list(cfg[BASE_KEY]) - for base_yml in base_ymls: - if base_yml.startswith("~"): - base_yml = os.path.expanduser(base_yml) - relpath = osp.relpath(base_yml, cfg_path) - if not base_yml.startswith('/'): - relpath = base_yml - base_yml = os.path.join(os.path.dirname(cfg_path), base_yml) + # config file not found, try download + # 1. clear configs directory + if osp.isdir(CONFIGS_HOME): + shutil.rmtree(CONFIGS_HOME) - if osp.isfile(base_yml): - logger.debug("Found _BASE_ config: {}".format(base_yml)) - continue + # 2. get url + try: + from ppdet import __version__ as version + except ImportError: + version= None - # download to CONFIGS_HOME firstly - base_yml_url = osp.join(osp.split(cfg_url)[0], relpath) - path, _ = get_path(base_yml_url, CONFIGS_HOME) + cfg_url = "ppdet://configs/{}/configs.tar".format(version) \ + if version else "ppdet://configs/configs.tar" + cfg_url = parse_url(cfg_url) - # move from CONFIGS_HOME to dst_path to restore config directory structure - dst_path = osp.join(cur_dir, relpath) - dst_dir = osp.split(dst_path)[0] - if not osp.isdir(dst_dir): - os.makedirs(dst_dir) - shutil.move(path, dst_path) + # 3. download and decompress + cfg_fullname = _download(cfg_url, osp.dirname(CONFIGS_HOME)) + _decompress(cfg_fullname) - # perfrom download base yml recursively - _download_config(dst_path, base_yml_url, osp.split(dst_path)[0]) + # 4. check config file existing + if os.path.isfile(path): + return path + else: + logger.error("Get config {} failed after download, please contact us on " \ + "https://github.com/PaddlePaddle/PaddleDetection/issues".format(path)) + sys.exit(1) def get_dataset_path(path, annotation, image_dir): @@ -235,11 +227,15 @@ def create_voc_list(data_dir, devkit_subdir='VOCdevkit'): logger.debug("Create voc file list finished") -def map_path(url, root_dir): +def map_path(url, root_dir, path_depth=1): # parse path after download to decompress under root_dir - fname = osp.split(url)[-1] + assert path_depth > 0, "path_depth should be a positive integer" + dirname = url + for _ in range(path_depth): + dirname = osp.dirname(dirname) + fpath = osp.relpath(url, dirname) + zip_formats = ['.zip', '.tar', '.gz'] - fpath = fname for zip_format in zip_formats: fpath = fpath.replace(zip_format, '') return osp.join(root_dir, fpath) diff --git a/setup.py b/setup.py index b9af4bc7d..ad29338b0 100644 --- a/setup.py +++ b/setup.py @@ -16,9 +16,48 @@ import os import os.path as osp import glob import shutil +import subprocess from setuptools import find_packages, setup +# ============== version definition ============== + +PPDET_VERSION = "2.1.0" + +def parse_version(): + return PPDET_VERSION.replace('-', '') + + +def git_commit(): + try: + cmd = ['git', 'rev-parse', 'HEAD'] + git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE, + ).communicate()[0].strip() + git_commit = git_commit.decode() + except: + git_commit = 'Unknown' + + return str(git_commit) + + +def write_version_py(filename='ppdet/version.py'): + ver_str = """# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY +# +full_version = '%(version)s' +commit = '%(commit)s' +""" + + _git_commit = git_commit() + with open(filename, 'w') as f: + f.write(ver_str % { + 'version': PPDET_VERSION, + 'commit': _git_commit}) + +write_version_py() + +# ============== version definition ============== + + def readme(): with open('README.md', encoding='utf-8') as f: content = f.read() @@ -71,7 +110,7 @@ if __name__ == "__main__": packages=find_packages(exclude=("configs", "tools", "deploy")), package_data={'ppdet.model_zoo': package_model_zoo()}, author='PaddlePaddle', - version='2.1.0', + version=parse_version(), install_requires=parse_requirements('./requirements.txt'), description='Object detection and instance segmentation toolkit based on PaddlePaddle', long_description=readme(), -- GitLab