resource list add md5 column

paddlehub/commands/download.py

@@ -17,6 +17,7 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
+import os
 
 from paddlehub.common.logger import logger
 from paddlehub.common import utils
@@ -57,8 +58,10 @@ class DownloadCommand(BaseCommand):
             self.args.output_path = "."
         utils.check_path(self.args.output_path)
 
-        url = default_hub_server.get_model_url(
+        search_result = default_hub_server.get_model_url(
             model_name, version=model_version)
+        url = search_result.get('url', None)
+        except_md5_value = search_result.get('md5', None)
         if not url:
             tips = "can't found model %s" % model_name
             if model_version:
@@ -66,13 +69,34 @@ class DownloadCommand(BaseCommand):
             print(tips)
             return True
 
-        if self.args.uncompress:
-            result, tips, file = default_downloader.download_file_and_uncompress(
-                url=url, save_path=self.args.output_path, print_progress=True)
-        else:
+        need_to_download_file = True
+        file_name = os.path.basename(url)
+        file = os.path.join(self.args.output_path, file_name)
+        if os.path.exists(file):
+            print("File %s already existed\nWait to check the MD5 value" %
+                  file_name)
+            file_md5_value = utils.md5_of_file(file)
+            if except_md5_value == file_md5_value:
+                print("MD5 check pass.")
+                need_to_download_file = False
+            else:
+                print("MD5 check failed!\nDelete invalid file.")
+                os.remove(file)
+
+        if need_to_download_file:
             result, tips, file = default_downloader.download_file(
                 url=url, save_path=self.args.output_path, print_progress=True)
-        print(tips)
+            if not result:
+                print(tips)
+                return False
+
+        if self.args.uncompress:
+            result, tips, file = default_downloader.uncompress(
+                file=file,
+                dirname=self.args.output_path,
+                delete_file=False,
+                print_progress=True)
+            print(tips)
         return True
 
 

paddlehub/common/downloader.py

@@ -32,14 +32,18 @@ from paddlehub.common.logger import logger
 __all__ = ['Downloader']
 FLUSH_INTERVAL = 0.1
 
+lasttime = time.time()
 
-def md5file(fname):
-    hash_md5 = hashlib.md5()
-    f = open(fname, "rb")
-    for chunk in iter(lambda: f.read(4096), b""):
-        hash_md5.update(chunk)
-    f.close()
-    return hash_md5.hexdigest()
+
+def progress(str, end=False):
+    global lasttime
+    if end:
+        str += "\n"
+        lasttime = 0
+    if time.time() - lasttime >= FLUSH_INTERVAL:
+        sys.stdout.write("\r%s" % str)
+        lasttime = time.time()
+        sys.stdout.flush()
 
 
 class Downloader:
@@ -85,30 +89,37 @@ class Downloader:
                         f.write(data)
                         if print_progress:
                             done = int(50 * dl / total_length)
-                            if time.time() - starttime >= FLUSH_INTERVAL:
-                                sys.stdout.write(
-                                    "\r%s : [%-50s] %.2f%%" %
-                                    (save_name, '=' * done,
-                                     float(dl / total_length * 100)))
-                                starttime = time.time()
-                                sys.stdout.flush()
+                            progress("%s : [%-50s] %.2f%%" %
+                                     (save_name, '=' * done,
+                                      float(dl / total_length * 100)))
                 if print_progress:
-                    sys.stdout.write(
-                        "\r%s : [%-50s]%.2f%%\n" %
-                        (save_name, '=' * done, float(dl / total_length * 100)))
-                    sys.stdout.flush()
+                    progress(
+                        "%s : [%-50s] %.2f%%" % (save_name, '=' * 50, 100),
+                        end=True)
 
         tips = "file %s download completed!" % (file_name)
         return True, tips, file_name
 
-    def uncompress(self, file, dirname=None, delete_file=False):
+    def uncompress(self,
+                   file,
+                   dirname=None,
+                   delete_file=False,
+                   print_progress=False):
         dirname = os.path.dirname(file) if dirname is None else dirname
         with tarfile.open(file, "r:gz") as tar:
             file_names = tar.getnames()
+            size = len(file_names) - 1
             module_dir = os.path.join(dirname, file_names[0])
-            for file_name in file_names:
+            for index, file_name in enumerate(file_names):
+                if print_progress:
+                    done = int(50 * float(index) / size)
+                    progress("%s : [%-50s] %.2f%%" %
+                             (file, '=' * done, float(index / size * 100)))
                 tar.extract(file_name, dirname)
 
+            if print_progress:
+                progress(
+                    "%s : [%-50s] %.2f%%" % (file, '=' * 50, 100), end=True)
         if delete_file:
             os.remove(file)
 
@@ -131,7 +142,8 @@ class Downloader:
             replace=replace)
         if not result:
             return result, tips_1, file
-        result, tips_2, file = self.uncompress(file, delete_file=delete_file)
+        result, tips_2, file = self.uncompress(
+            file, delete_file=delete_file, print_progress=print_progress)
         if not result:
             return result, tips_2, file
         if save_name:

paddlehub/common/hub_server.py

@@ -106,7 +106,7 @@ class HubServer:
             self.request()
 
         if not self._load_resource_list_file_if_valid():
-            return None
+            return {}
 
         resource_index_list = [
             index
@@ -123,14 +123,17 @@ class HubServer:
         resource_version_list = sorted(resource_version_list)
         if not version:
             if not resource_version_list:
-                return None
+                return {}
             version = resource_version_list[-1]
 
         for index in resource_index_list:
             if self.resource_list_file['version'][index] == version:
-                return self.resource_list_file['url'][index]
+                return {
+                    'url': self.resource_list_file['url'][index],
+                    'md5': self.resource_list_file['md5'][index]
+                }
 
-        return None
+        return {}
 
     def get_module_url(self, module_name, version=None, update=False):
         return self.get_resource_url(

paddlehub/module/manager.py

@@ -64,8 +64,10 @@ class LocalModuleManager:
             tips = "Module %s already installed in %s" % (module_name,
                                                           module_dir)
             return True, tips, module_dir
-        url = hub.default_hub_server.get_module_url(
+        search_result = hub.default_hub_server.get_module_url(
             module_name, version=module_version)
+        url = search_result.get('url', None)
+        md5_value = search_result.get('md5', None)
         #TODO(wuzewu): add compatibility check
         if not url:
             tips = "Can't find module %s" % module_name