add visualdl dev (#812)

docs/file_system/BOS.md

@@ -6,7 +6,7 @@
 ### VisualDL
 VisualDL在2.0.0b8版本之后增加了对BOS的支持，可使用下述命令安装
 ```shell
-pip install visualdl==2.0.0b8
+pip install visualdl
 ```
 
 ### BOS

visualdl/__init__.py

@@ -19,5 +19,8 @@ import os
 
 from visualdl.writer.writer import LogWriter  # noqa
 from visualdl.version import vdl_version as __version__
+from visualdl.utils.dir import init_vdl_config
+
+init_vdl_config()
 
 ROOT = os.path.dirname(__file__)

visualdl/io/bfile.py

@@ -202,17 +202,76 @@ class HDFileSystem(object):
         return (['hdfs://' + root, dirs, files] for root, dirs, files in walks)
 
 
-class BosFileSystem(object):
-    def __init__(self):
-        self.max_contents_count = 1
-        self.max_contents_time = 1
-        self.get_bos_config()
+def get_object_info(path):
+    path = path[6:]
+    index = path.index('/')
+    bucket_name = path[0:index]
+    object_key = path[index + 1:]
+    return bucket_name, object_key
+
+
+class BosConfigClient(object):
+    def __init__(self, bos_ak, bos_sk, bos_sts, bos_host="bj.bcebos.com"):
+        self.config = BceClientConfiguration(
+            credentials=BceCredentials(bos_ak, bos_sk),
+            endpoint=bos_host, security_token=bos_sts)
         self.bos_client = BosClient(self.config)
-        self.file_length_map = {}
 
-        self._file_contents_to_add = b''
-        self._file_contents_count = 0
-        self._start_append_time = time.time()
+    def exists(self, path):
+        bucket_name, object_key = get_object_info(path)
+        try:
+            self.bos_client.get_object_meta_data(bucket_name, object_key)
+            return True
+        except exception.BceError:
+            return False
+
+    def makedirs(self, path):
+        if not path.endswith('/'):
+            path += '/'
+        if self.exists(path):
+            return
+        bucket_name, object_key = get_object_info(path)
+        if not object_key.endswith('/'):
+            object_key += '/'
+        init_data = b''
+        self.bos_client.append_object(bucket_name=bucket_name,
+                                      key=object_key,
+                                      data=init_data,
+                                      content_md5=content_md5(init_data),
+                                      content_length=len(init_data))
+
+    @staticmethod
+    def join(path, *paths):
+        result = os.path.join(path, *paths)
+        result.replace('\\', '/')
+        return result
+
+    def upload_object_from_file(self, path, filename):
+        if not self.exists(path):
+            self.makedirs(path)
+        bucket_name, object_key = get_object_info(path)
+
+        object_key = self.join(object_key, filename)
+        # if not object_key.endswith('/'):
+        #     object_key += '/'
+        print('Uploading file `%s`' % filename)
+        self.bos_client.put_object_from_file(bucket=bucket_name,
+                                             key=object_key,
+                                             file_name=filename)
+
+
+class BosFileSystem(object):
+    def __init__(self, write_flag=True):
+        if write_flag:
+            self.max_contents_count = 1
+            self.max_contents_time = 1
+            self.get_bos_config()
+            self.bos_client = BosClient(self.config)
+            self.file_length_map = {}
+
+            self._file_contents_to_add = b''
+            self._file_contents_count = 0
+            self._start_append_time = time.time()
 
     def get_bos_config(self):
         bos_host = os.getenv("BOS_HOST")
@@ -231,24 +290,22 @@ class BosFileSystem(object):
             credentials=BceCredentials(access_key_id, secret_access_key),
             endpoint=bos_host, security_token=bos_sts)
 
+    def set_bos_config(self, bos_ak, bos_sk, bos_sts, bos_host="bj.bcebos.com"):
+        self.config = BceClientConfiguration(
+            credentials=BceCredentials(bos_ak, bos_sk),
+            endpoint=bos_host, security_token=bos_sts)
+        self.bos_client = BosClient(self.config)
+
     def isfile(self, filename):
         return exists(filename)
 
     def read_file(self, filename, binary=True):
-        bucket_name, object_key = BosFileSystem._get_object_info(filename)
+        bucket_name, object_key = get_object_info(filename)
         result = self.bos_client.get_object_as_string(bucket_name, object_key)
         return result
 
-    @staticmethod
-    def _get_object_info(path):
-        path = path[6:]
-        index = path.index('/')
-        bucket_name = path[0:index]
-        object_key = path[index + 1:]
-        return bucket_name, object_key
-
     def exists(self, path):
-        bucket_name, object_key = BosFileSystem._get_object_info(path)
+        bucket_name, object_key = get_object_info(path)
         try:
             self.bos_client.get_object_meta_data(bucket_name, object_key)
             return True
@@ -263,7 +320,7 @@ class BosFileSystem(object):
             path += '/'
         if self.exists(path):
             return
-        bucket_name, object_key = BosFileSystem._get_object_info(path)
+        bucket_name, object_key = get_object_info(path)
         if not object_key.endswith('/'):
             object_key += '/'
         init_data = b''
@@ -280,7 +337,7 @@ class BosFileSystem(object):
         return result
 
     def read(self, filename, binary_mode=False, size=0, continue_from=None):
-        bucket_name, object_key = BosFileSystem._get_object_info(filename)
+        bucket_name, object_key = get_object_info(filename)
         offset = 0
         if continue_from is not None:
             offset = continue_from.get("last_offset", 0)
@@ -311,7 +368,7 @@ class BosFileSystem(object):
         if not force and not self.ready_to_append():
             return
         file_content = self._file_contents_to_add
-        bucket_name, object_key = BosFileSystem._get_object_info(filename)
+        bucket_name, object_key = get_object_info(filename)
         if not self.exists(filename):
             init_data = b''
             self.bos_client.append_object(bucket_name=bucket_name,
@@ -394,7 +451,7 @@ class BosFileSystem(object):
                 else:
                     raise StopIteration
 
-        bucket_name, object_key = BosFileSystem._get_object_info(dir)
+        bucket_name, object_key = get_object_info(dir)
 
         if object_key in ['.', './']:
             prefix = None

visualdl/server/app.py

@@ -32,6 +32,7 @@ from flask_babel import Babel
 
 import visualdl.server
 from visualdl.server.api import create_api_call
+from visualdl.server.serve import upload_to_dev
 from visualdl.server.args import (ParseArgs, parse_args)
 from visualdl.server.log import info
 from visualdl.server.template import Template
@@ -167,7 +168,11 @@ def run(logdir=None, **options):
 
 def main():
     args = parse_args()
-    _run(args)
+    if args.get('dest') == 'service':
+        if args.get('behavior') == 'upload':
+            upload_to_dev(args.get('logdir'), args.get('model'))
+    else:
+        _run(args)
 
 
 if __name__ == '__main__':

visualdl/server/args.py

@@ -40,6 +40,8 @@ class DefaultArgs(object):
         self.model = args.get('model', '')
         self.product = args.get('product', default_product)
         self.telemetry = args.get('telemetry', True)
+        self.dest = args.get('dest', '')
+        self.behavior = args.get('behavior', '')
 
 
 def get_host(host=default_host, port=default_port):
@@ -99,6 +101,8 @@ class ParseArgs(object):
         self.model = args.model
         self.product = args.product
         self.telemetry = args.telemetry
+        self.dest = args.dest
+        self.behavior = args.behavior
 
 
 def parse_args():
@@ -190,6 +194,15 @@ def parse_args():
         default=True,
         help="disable telemetry"
     )
+    parser.add_argument(
+        'dest',
+        nargs='?',
+        help='set destination for log'
+    )
+    parser.add_argument(
+        "behavior",
+        nargs='?'
+    )
 
     args = parser.parse_args()
 

visualdl/server/serve.py

+#!/user/bin/env python
+
+# Copyright (c) 2020 VisualDL Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =======================================================================
+import requests
+import json
+import os
+
+from visualdl.io import bfile
+from visualdl.reader.reader import is_VDLRecord_file
+from visualdl.utils.dir import CONFIG_PATH
+from visualdl.server.log import logger
+
+
+def get_server_url():
+    with open(CONFIG_PATH, 'r') as fp:
+        server_url = json.load(fp)['server_url']
+    return server_url
+
+
+def apply_for_token():
+    url = get_server_url() + '/sts/'
+    res = requests.post(url=url).json()
+
+    return res
+
+
+def get_url(path='', model='', **kwargs):
+    server_url = get_server_url() + '/url/'
+    data = json.dumps({'path': path, 'model': model})
+    headers = {"Content-Type": "application/json"}
+    res = requests.post(url=server_url, headers=headers, data=data).json()
+    err_code = res.get('code')
+    msg = res.get('msg')
+
+    if '000000' == err_code:
+        url = msg.get('url')
+        return url
+    else:
+        logger.error(msg)
+        return
+
+
+def get_vdl_log_file(logdirs):
+    """Get logs.
+
+    Every dir(means `run` in vdl) has only one log(meads `actual log file`).
+
+    Returns:
+        walks: A dict like {"exp1": "vdlrecords.1587375595.log",
+                            "exp2": "vdlrecords.1587375685.log"}
+    """
+    walks = {}
+    for logdir in logdirs:
+        for root, dirs, files in bfile.walk(logdir):
+            walks.update({root: files})
+
+            walks_temp = {}
+            for run, tags in walks.items():
+                tags_temp = [tag for tag in tags if is_VDLRecord_file(tag)]
+                tags_temp.sort(reverse=True)
+                if len(tags_temp) > 0:
+                    walks_temp.update({run: tags_temp[0]})
+
+    return walks_temp
+
+
+def upload_to_dev(logdir=None, model=None):
+    if not logdir and not model:
+        logger.error("Must specify directory to upload via `--logdir` or specify model to upload via `--model`.")
+        return
+    walks = {}
+    if logdir:
+        walks = get_vdl_log_file(logdir)
+
+    res = apply_for_token()
+
+    err_code = res.get('code')
+    msg = res.get('msg')
+
+    if '000000' == err_code:
+        sts_ak = msg.get('sts_ak')
+        sts_sk = msg.get('sts_sk')
+        sts_token = msg.get('token')
+        bucket_id = msg.get('dir')
+    else:
+        logger.error(msg)
+        return
+
+    if not sts_ak or not sts_sk or not sts_token:
+        return
+    bos_fs = bfile.BosConfigClient(bos_ak=sts_ak,
+                                   bos_sk=sts_sk,
+                                   bos_sts=sts_token)
+
+    for key, value in walks.items():
+        filename = bos_fs.join(key, value)
+        bos_fs.upload_object_from_file(path=bucket_id, filename=filename)
+
+    if model:
+        if os.path.getsize(model) > 1024 * 1024 * 100:
+            logger.error('Size of model must less than 100M.')
+        else:
+            bos_fs.upload_object_from_file(path=bucket_id, filename=model)
+    url = get_url(path=bucket_id, model=model)
+
+    print("You can view the visualization results on page`%s`." % url)

visualdl/utils/dir.py

+# Copyright (c) 2020 VisualDL Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =======================================================================
+
+import os
+import json
+
+
+VDL_SERVER = "http://paddlepaddle.org.cn/visualdl"
+
+default_vdl_config = {
+    'server_url': VDL_SERVER
+}
+
+USER_HOME = os.path.expanduser('~')
+VDL_HOME = os.path.join(USER_HOME, '.visualdl')
+CONF_HOME = os.path.join(VDL_HOME, 'conf')
+CONFIG_PATH = os.path.join(CONF_HOME, 'config.json')
+
+
+def init_vdl_config():
+    if not os.path.exists(CONF_HOME):
+        os.makedirs(CONF_HOME)
+    if not os.path.exists(CONFIG_PATH) or 0 == os.path.getsize(CONFIG_PATH):
+        with open(CONFIG_PATH, 'w') as fp:
+            fp.write(json.dumps(default_vdl_config))

visualdl/utils/md5_util.py

+# Copyright (c) 2020 VisualDL Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =======================================================================
+
+import hashlib
+
+
+def md5(text):
+    if isinstance(text, str):
+        text = text.encode("utf8")
+    md5 = hashlib.md5()
+    md5.update(text)
+    return md5.hexdigest()