Merge pull request #165 from ShenYuhan/develop

add multiprocessing lock

RELEASE.md

-# `v1.2.0` 
+# `v1.2.0`
 
 * 新增**超参优化Auto Fine-tune**，实现给定超参搜索空间，PaddleHub自动给出较佳的超参组合
   * 支持两种优化策略：HAZero和PSHE2

paddlehub/common/hub_server.py

@@ -30,6 +30,7 @@ from paddlehub.common import utils, srv_utils
 from paddlehub.common.downloader import default_downloader
 from paddlehub.common.server_config import default_server_config
 from paddlehub.io.parser import yaml_parser
+from paddlehub.common.lock import lock
 import paddlehub as hub
 
 RESOURCE_LIST_FILE = "resource_list_file.yml"
@@ -44,15 +45,20 @@ class HubServer(object):
             utils.mkdir(hub.CONF_HOME)
         if not os.path.exists(config_file_path):
             with open(config_file_path, 'w+') as fp:
+                lock.flock(fp, lock.LOCK_EX)
                 fp.write(json.dumps(default_server_config))
+                lock.flock(fp, lock.LOCK_UN)
 
         with open(config_file_path) as fp:
             self.config = json.load(fp)
+        fp_lock = open(config_file_path)
+        lock.flock(fp_lock, lock.LOCK_EX)
 
         utils.check_url(self.config['server_url'])
         self.server_url = self.config['server_url']
         self.request()
         self._load_resource_list_file_if_valid()
+        lock.flock(fp_lock, lock.LOCK_UN)
 
     def get_server_url(self):
         random.seed(int(time.time()))

paddlehub/common/lock.py

+import fcntl
+import os
+
+
+class WinLock(object):
+    def flock(self, *args):
+        pass
+
+    def __init__(self):
+        self.LOCK_EX = ""
+        self.LOCK_UN = ""
+
+
+class Lock(object):
+    _owner = None
+
+    def __init__(self):
+        self.LOCK_EX = fcntl.LOCK_EX
+        self.LOCK_UN = fcntl.LOCK_UN
+        self.LOCK_TE = ""
+        if os.name == "posix":
+            self.lock = fcntl
+        else:
+            self.lock = WinLock()
+        _lock = self.lock
+
+    def get_lock(self):
+        return self.lock
+
+    def flock(self, fp, cmd):
+        if cmd == fcntl.LOCK_UN:
+            Lock._owner = None
+            self.lock.flock(fp, cmd)
+        elif cmd == fcntl.LOCK_EX:
+            if Lock._owner is None:
+                Lock._owner = os.getpid()
+                self.lock.flock(fp, cmd)
+            else:
+                if Lock._owner == os.getpid():
+                    pass
+                else:
+                    self.lock.flock(fp, cmd)
+
+
+lock = Lock()

paddlehub/module/module.py

@@ -29,8 +29,10 @@ import paddle.fluid as fluid
 from paddlehub.common import utils
 from paddlehub.common import paddle_helper
 from paddlehub.common.logger import logger
+from paddlehub.common.lock import lock
 from paddlehub.common.downloader import default_downloader
 from paddlehub.module import module_desc_pb2
+from paddlehub.common.dir import CONF_HOME
 from paddlehub.module import check_info_pb2
 from paddlehub.module.signature import Signature, create_signature
 from paddlehub.module.checker import ModuleChecker
@@ -117,10 +119,14 @@ class Module(object):
         self.cache_fetch_dict = None
         self.cache_program = None
 
+        fp_lock = open(os.path.join(CONF_HOME, 'config.json'))
+        lock.flock(fp_lock, lock.LOCK_EX)
         if name:
             self._init_with_name(name=name, version=version)
+            lock.flock(fp_lock, lock.LOCK_UN)
         elif module_dir:
             self._init_with_module_file(module_dir=module_dir[0])
+            lock.flock(fp_lock, lock.LOCK_UN)
         elif signatures:
             if processor:
                 if not issubclass(processor, BaseProcessor):
@@ -134,7 +140,9 @@ class Module(object):
             self.processor = processor
             self._generate_module_info(module_info)
             self._init_with_signature(signatures=signatures)
+            lock.flock(fp_lock, lock.LOCK_UN)
         else:
+            lock.flock(fp_lock, lock.LOCK_UN)
             raise ValueError("Module initialized parameter is empty")
 
     def _init_with_name(self, name, version=None):