From 63086205d9809d372071f8a9e4796b3bd18a5e0d Mon Sep 17 00:00:00 2001 From: Jeff Wang Date: Tue, 13 Mar 2018 17:47:30 -0700 Subject: [PATCH] Docs update for release (#317) * Update the version number Update the doc to include Vue Update the doc to give more details on the sync_cycle Update the doc to include python 3 * Update the LogWriter constructor doc. * Update the package copy directory. * Update the version to 0.0.2 (implying beta now) * Update the vdl_create_scratch_log to python 3 as well. --- README.cn.md | 7 +++++-- README.md | 9 +++++---- VERSION_NUMBER | 2 +- demo/vdl_create_scratch_log | 29 +++++++++++++++++++++-------- docs/how_to_dev_frontend_en.md | 10 +++++----- docs/quick_start_cn.md | 7 ++++++- docs/quick_start_en.md | 8 ++++++-- setup.py | 2 +- visualdl/python/storage.py | 9 +++++++++ 9 files changed, 59 insertions(+), 24 deletions(-) diff --git a/README.cn.md b/README.cn.md index 265d8067..e9ab70f6 100644 --- a/README.cn.md +++ b/README.cn.md @@ -70,7 +70,10 @@ visualDL --logdir=scratch_log --port=8080 ## SDK VisualDL 同时提供了python SDK 和 C++ SDK 来实现不同方式的使用。 + ### Python SDK +VisualDL 现在支持 Python 2和 Python 3。 + 以最简单的Scalar组件为例，尝试创建一个scalar组件并插入多个时间步的数据： ```python @@ -78,7 +81,7 @@ import random from visualdl import LogWriter logdir = "./tmp" -logger = LogWriter(logdir, sync_cycle=10) +logger = LogWriter(logdir, sync_cycle=10000) # mark the components with 'train' label. with logger.mode("train"): @@ -102,7 +105,7 @@ namespace cp = visualdl::components; int main() { const std::string dir = "./tmp"; - vs::LogWriter logger(dir, 10); + vs::LogWriter logger(dir, 10000); logger.SetMode("train"); auto tablet = logger.AddTablet("scalars/scalar0"); diff --git a/README.md b/README.md index 70b6714d..031196fa 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ New features will be continuously added. At present, most DNN frameworks use Python as their primary language. VisualDL supports Python by nature. Users can get plentiful visualization results by simply add a few lines of Python code into their model before training. - Besides Python SDK, VisualDL was writen in C++ on the low level. It also provides C++ SDK that can be integrated into other platforms. @@ -80,6 +79,7 @@ VisualDL provides both Python SDK and C++ SDK in order to fit more use cases. ### Python SDK +VisualDL now supports both Python 2 and Python 3. Below is an example of creating a simple Scalar component and inserting data from different timestamps: ```python @@ -87,7 +87,7 @@ import random from visualdl import LogWriter logdir = "./tmp" -logger = LogWriter(logdir, sync_cycle=10) +logger = LogWriter(logdir, sync_cycle=10000) # mark the components with 'train' label. with logger.mode("train"): @@ -112,7 +112,7 @@ namespace cp = visualdl::components; int main() { const std::string dir = "./tmp"; - vs::LogWriter logger(dir, 10); + vs::LogWriter logger(dir, 10000); logger.SetMode("train"); auto tablet = logger.AddTablet("scalars/scalar0"); @@ -161,7 +161,8 @@ pip install --upgrade dist/visualdl-*.whl ### Run a demo from scratch ``` -vdl_scratch.py +# vdl_create_scratch_log is a helper commend that creates mock data. +vdl_create_scratch_log visualDL --logdir=scratch_log --port=8080 ``` that will start a server locally on port 8080, then diff --git a/VERSION_NUMBER b/VERSION_NUMBER index 3fc37286..4e379d2b 100644 --- a/VERSION_NUMBER +++ b/VERSION_NUMBER @@ -1 +1 @@ -0.0.1-alpha.2 +0.0.2 diff --git a/demo/vdl_create_scratch_log b/demo/vdl_create_scratch_log index df95b10a..72a7cc09 100644 --- a/demo/vdl_create_scratch_log +++ b/demo/vdl_create_scratch_log @@ -9,7 +9,7 @@ from visualdl.server.log import logger as log logdir = './scratch_log' -logw = LogWriter(logdir, sync_cycle=30) +logw = LogWriter(logdir, sync_cycle=2000) # create scalars in mode train and test. with logw.mode('train') as logger: @@ -50,13 +50,13 @@ with logw.mode("train") as logger: image0 = logger.image("scratch/random", 4) dog_jpg = Image.open(os.path.join(ROOT, 'python/dog.jpg')) - dog_jpg = dog_jpg.resize(np.array(dog_jpg.size) / 2) + dog_jpg = dog_jpg.resize(np.floor_divide(np.array(dog_jpg.size), 2)) shape = [dog_jpg.size[1], dog_jpg.size[0], 3] # add dog's image - for pass_ in xrange(4): + for pass_ in range(4): image.start_sampling() - for sample in xrange(10): + for sample in range(10): # randomly crop a dog's image. target_shape = [100, 100, 3] # width, height, channels(3 for RGB) left_x = random.randint(0, shape[1] - target_shape[1]) @@ -82,9 +82,9 @@ with logw.mode("train") as logger: image.finish_sampling() # add randomly generated image - for pass_ in xrange(4): + for pass_ in range(4): image0.start_sampling() - for sample in xrange(10): + for sample in range(10): shape = [40, 30, 3] data = np.random.random(shape).flatten() image0.add_sample(shape, list(data)) @@ -98,10 +98,23 @@ def download_graph_image(): For real cases, just refer to README. ''' - import urllib + + import sys + + if sys.version_info[0] == 3: + import urllib.request as ur + + else: + # Not Python 3 - today, it is most likely to be Python 2 + import urllib as ur + + import ssl + myssl = ssl.create_default_context() + myssl.check_hostname = False + myssl.verify_mode = ssl.CERT_NONE image_url = "https://github.com/PaddlePaddle/VisualDL/blob/develop/demo/mxnet/super_resolution_graph.png?raw=true" log.warning('download graph demo from {}'.format(image_url)) - graph_image = urllib.urlopen(image_url).read() + graph_image = ur.urlopen(image_url, context=myssl).read() with open(os.path.join(logdir, 'graph.jpg'), 'wb') as f: f.write(graph_image) log.warning('graph ready!') diff --git a/docs/how_to_dev_frontend_en.md b/docs/how_to_dev_frontend_en.md index f183c4f8..c0621bbc 100644 --- a/docs/how_to_dev_frontend_en.md +++ b/docs/how_to_dev_frontend_en.md @@ -31,7 +31,7 @@ The VisualDL Web app uses multiple frameworks to help manage the project. They a 1. webpack: To manage all assets 1. npm: To manage dependencies -1. San: Javascript Component framework +1. Vue: Javascript Component framework 1. ECharts: To pilot charts ## Webpack @@ -63,13 +63,13 @@ npm install This command will go through `package.json` and install the dependencies in the local node_modules folder. -## San +## Vue -San is a JavaScript component framework that helps the developer to implement web component in MVVM architecture pattern. +Vue is a JavaScript component framework that helps the developer to implement web component in MVVM architecture pattern. -San allows you to define a self-contained view model in a .san file and attach view model objects to DOM objects. +Vue allows you to define a self-contained view model in a .vue file and attach view model objects to DOM objects. -To learn more about [san](https://github.com/ecomfe/san) +To learn more about [Vue](https://vuejs.org/) ## ECharts diff --git a/docs/quick_start_cn.md b/docs/quick_start_cn.md index 0902b95d..1aa09807 100644 --- a/docs/quick_start_cn.md +++ b/docs/quick_start_cn.md @@ -20,11 +20,16 @@ VisualDL提供原生的Python和C++ SDK，可以支持多种深度学习平台 from VisualDL import LogWriter from random import random -logw = LogWriter("./random_log", sync_cycle=30) +logw = LogWriter("./random_log", sync_cycle=10000) ``` 其中， 第一个参数指定存储数据的目录；第二个参数 `sync_cycle` 指定多少次写操作执行一次内存到磁盘的数据持久化。 +### sync_cycle +写IO是一项繁重的工作。设置` sync_cycle `太低可能会减慢你的训练。 +我们建议将 `sync_cycle` 设置为约要捕捉的数据点的两倍。 + + 模型训练会有不同的模式，比如训练、验证、测试等，这些对应到 VisualDL中就是 `mode`，可以用如下方式指定一个训练模式 ```python diff --git a/docs/quick_start_en.md b/docs/quick_start_en.md index 5cf3ff05..2fa179cd 100644 --- a/docs/quick_start_en.md +++ b/docs/quick_start_en.md @@ -23,11 +23,15 @@ The first step of using VisualDL is to create a `LogWriter' that can store visua from VisualDL import LogWriter from random import random -logw = LogWriter("./random_log", sync_cycle=30) +logw = LogWriter("./random_log", sync_cycle=10000) ``` The first parameter points to a folder; the second parameter `sync_cycle` specifies out of how memory operations should be -store the data into hard drive. +store the data into hard drive. + +### sync_cycle +Writing is a heavy operation. Setting `sync_cycle` might slow down your training. +A good starting point is to set the `sync_cycle` to be at least twice the amount of data point your would like to capture. There are different modes for model training, such as training, validating and testing. All these correspond to `mode' in VisualDL. We can use the following pattern to specify mode: diff --git a/setup.py b/setup.py index 773885f0..434733ed 100644 --- a/setup.py +++ b/setup.py @@ -90,7 +90,7 @@ setup( install_requires=install_requires, package_data={ 'visualdl.server': - ['dist/*.js', 'dist/*.html', 'dist/fonts/*', 'dist/images/*'], + ['dist/*.js', 'dist/*.html', 'dist/fonts/*', 'dist/assets/*'], 'visualdl': ['core.so'], 'visualdl.python': ['core.so', 'dog.jpg'] }, diff --git a/visualdl/python/storage.py b/visualdl/python/storage.py index 4f719c6b..bf17fc1d 100644 --- a/visualdl/python/storage.py +++ b/visualdl/python/storage.py @@ -126,6 +126,15 @@ class LogWriter(object): """LogWriter is a Python wrapper to write data to log file with the data format defined in storage.proto. A User can get Scalar Reader/Image Reader/ Histogram Reader from this module and use them to write the data to log file. + + :param dir: The directory path to the saved log files. + :type dir: basestring + :param sync_cycle: Specify how often should the system store data into the file system. + Typically adding a record requires 6 operations. + System will save the data into the file system once operations count reaches sync_cycle. + :type sync_cycle: integer + :return: a new LogWriter instance + :rtype: LogWriter """ cur_mode = None -- GitLab