update version and config download (#3153)

* update version and config download

.gitignore

@@ -78,3 +78,5 @@ dataset/wider_face/WIDER_test
 dataset/wider_face/WIDER_train
 dataset/wider_face/WIDER_val
 dataset/wider_face/wider_face_split
+
+ppdet/version.py

ppdet/__init__.py

@@ -14,3 +14,13 @@
 
 from . import (core, data, engine, modeling, model_zoo, optimizer, metrics,
                utils, slim)
+
+
+try:
+    from .version import full_version as __version__
+    from .version import commit as __git_commit__
+except ImportError:
+    import sys
+    sys.stderr.write("Warning: import ppdet from source directory " \
+            "without installing, run 'python setup.py install' to " \
+            "install ppdet firstly\n")

ppdet/utils/download.py

@@ -118,44 +118,36 @@ def get_config_path(url):
     download it from url.
     """
     url = parse_url(url)
-    path, _ = get_path(url, CONFIGS_HOME)
-    _download_config(path, url, CONFIGS_HOME)
-
-    return path
-
-
-def _download_config(cfg_path, cfg_url, cur_dir):
-    with open(cfg_path) as f:
-        cfg = yaml.load(f, Loader=yaml.Loader)
+    path = map_path(url, CONFIGS_HOME, path_depth=2)
+    if os.path.isfile(path):
+        return path
 
-    # download dependence base ymls
-    if BASE_KEY in cfg:
-        base_ymls = list(cfg[BASE_KEY])
-        for base_yml in base_ymls:
-            if base_yml.startswith("~"):
-                base_yml = os.path.expanduser(base_yml)
-                relpath = osp.relpath(base_yml, cfg_path)
-            if not base_yml.startswith('/'):
-                relpath = base_yml
-                base_yml = os.path.join(os.path.dirname(cfg_path), base_yml)
+    # config file not found, try download
+    # 1. clear configs directory
+    if osp.isdir(CONFIGS_HOME):
+        shutil.rmtree(CONFIGS_HOME)
 
-            if osp.isfile(base_yml):
-                logger.debug("Found _BASE_ config: {}".format(base_yml))
-                continue
+    # 2. get url
+    try:
+        from ppdet import __version__ as version
+    except ImportError:
+        version= None
 
-            # download to CONFIGS_HOME firstly
-            base_yml_url = osp.join(osp.split(cfg_url)[0], relpath)
-            path, _ = get_path(base_yml_url, CONFIGS_HOME)
+    cfg_url = "ppdet://configs/{}/configs.tar".format(version) \
+                if version else "ppdet://configs/configs.tar"
+    cfg_url = parse_url(cfg_url)
 
-            # move from CONFIGS_HOME to dst_path to restore config directory structure
-            dst_path = osp.join(cur_dir, relpath)
-            dst_dir = osp.split(dst_path)[0]
-            if not osp.isdir(dst_dir):
-                os.makedirs(dst_dir)
-            shutil.move(path, dst_path)
+    # 3. download and decompress
+    cfg_fullname = _download(cfg_url, osp.dirname(CONFIGS_HOME))
+    _decompress(cfg_fullname)
 
-            # perfrom download base yml recursively
-            _download_config(dst_path, base_yml_url, osp.split(dst_path)[0])
+    # 4. check config file existing
+    if os.path.isfile(path):
+        return path
+    else:
+        logger.error("Get config {} failed after download, please contact us on " \
+            "https://github.com/PaddlePaddle/PaddleDetection/issues".format(path))
+        sys.exit(1)
 
 
 def get_dataset_path(path, annotation, image_dir):
@@ -235,11 +227,15 @@ def create_voc_list(data_dir, devkit_subdir='VOCdevkit'):
     logger.debug("Create voc file list finished")
 
 
-def map_path(url, root_dir):
+def map_path(url, root_dir, path_depth=1):
     # parse path after download to decompress under root_dir
-    fname = osp.split(url)[-1]
+    assert path_depth > 0, "path_depth should be a positive integer"
+    dirname = url
+    for _ in range(path_depth):
+        dirname = osp.dirname(dirname)
+    fpath = osp.relpath(url, dirname)
+
     zip_formats = ['.zip', '.tar', '.gz']
-    fpath = fname
     for zip_format in zip_formats:
         fpath = fpath.replace(zip_format, '')
     return osp.join(root_dir, fpath)

setup.py

@@ -16,9 +16,48 @@ import os
 import os.path as osp
 import glob
 import shutil
+import subprocess
 from setuptools import find_packages, setup
 
 
+# ==============  version definition  ==============
+
+PPDET_VERSION = "2.1.0"
+
+def parse_version():
+    return PPDET_VERSION.replace('-', '')
+
+
+def git_commit():
+    try:
+        cmd = ['git', 'rev-parse', 'HEAD']
+        git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE,
+            ).communicate()[0].strip()
+        git_commit = git_commit.decode()
+    except:
+        git_commit = 'Unknown'
+
+    return str(git_commit)
+
+
+def write_version_py(filename='ppdet/version.py'):
+    ver_str = """# THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY
+#
+full_version    = '%(version)s'
+commit          = '%(commit)s'
+"""
+
+    _git_commit = git_commit()
+    with open(filename, 'w') as f:
+        f.write(ver_str % {
+            'version': PPDET_VERSION,
+            'commit': _git_commit})
+
+write_version_py()
+
+# ==============  version definition  ==============
+
+
 def readme():
     with open('README.md', encoding='utf-8') as f:
         content = f.read()
@@ -71,7 +110,7 @@ if __name__ == "__main__":
         packages=find_packages(exclude=("configs", "tools", "deploy")),
         package_data={'ppdet.model_zoo': package_model_zoo()},
         author='PaddlePaddle',
-        version='2.1.0',
+        version=parse_version(),
         install_requires=parse_requirements('./requirements.txt'),
         description='Object detection and instance segmentation toolkit based on PaddlePaddle',
         long_description=readme(),