From c5b5dc0811c4313585b073bd6920c9218c29b957 Mon Sep 17 00:00:00 2001 From: Jeff Wang Date: Fri, 20 Apr 2018 16:37:14 -0700 Subject: [PATCH] Update VisualDL documentation structure on PPO (#416) * Complete re layout the documentation on PPO. * Add how to develop in cn. Also rearrange the demo location. * Update the doc to use CDN * Update installation guide. * Update the doc setting to use layout.html template * Update the documentations to fix some grammer error --- docs/api/index.rst | 8 ++ docs/api/initialize_logger.rst | 5 + docs/api/visualdl.rst | 14 -- docs/api/write_logs.rst | 4 + docs/backend_architecture.md | 50 ------- docs/demo | 1 - docs/{ => develop}/contribute_code_cn.md | 0 docs/{ => develop}/contribute_code_en.md | 2 +- docs/develop/how_to_dev_backend_cn.md | 124 ++++++++++++++++++ docs/{ => develop}/how_to_dev_backend_en.md | 12 +- docs/develop/how_to_dev_frontend_cn.md | 78 +++++++++++ docs/{ => develop}/how_to_dev_frontend_en.md | 2 +- docs/develop/index_cn.rst | 9 ++ docs/develop/index_en.rst | 9 ++ docs/getting_started/demo | 1 + .../{introduction => getting_started}/dog.png | Bin .../graph.png | Bin .../histogram.png | Bin .../image-gan.png | Bin docs/getting_started/index_cn.rst | 10 ++ docs/getting_started/index_en.rst | 10 ++ docs/getting_started/installation_cn.md | 103 +++++++++++++++ docs/getting_started/installation_en.md | 107 +++++++++++++++ .../introduction_cn.md} | 103 +++++++-------- .../introduction_en.md | 124 ++++++++---------- docs/{ => getting_started}/quick_start_cn.md | 0 docs/{ => getting_started}/quick_start_en.md | 4 +- .../scalar.png | Bin docs/index_cn.rst | 8 +- docs/index_en.rst | 9 +- docs/templates/conf.py.cn.in | 2 +- docs/templates/conf.py.en.in | 2 +- docs/templates/layout.html | 11 ++ 33 files changed, 601 insertions(+), 211 deletions(-) create mode 100644 docs/api/index.rst create mode 100644 docs/api/initialize_logger.rst delete mode 100644 docs/api/visualdl.rst create mode 100644 docs/api/write_logs.rst delete mode 100644 docs/backend_architecture.md delete mode 120000 docs/demo rename docs/{ => develop}/contribute_code_cn.md (100%) rename docs/{ => develop}/contribute_code_en.md (99%) create mode 100644 docs/develop/how_to_dev_backend_cn.md rename docs/{ => develop}/how_to_dev_backend_en.md (89%) create mode 100644 docs/develop/how_to_dev_frontend_cn.md rename docs/{ => develop}/how_to_dev_frontend_en.md (99%) create mode 100644 docs/develop/index_cn.rst create mode 100644 docs/develop/index_en.rst create mode 120000 docs/getting_started/demo rename docs/{introduction => getting_started}/dog.png (100%) rename docs/{introduction => getting_started}/graph.png (100%) rename docs/{introduction => getting_started}/histogram.png (100%) rename docs/{introduction => getting_started}/image-gan.png (100%) create mode 100644 docs/getting_started/index_cn.rst create mode 100644 docs/getting_started/index_en.rst create mode 100644 docs/getting_started/installation_cn.md create mode 100644 docs/getting_started/installation_en.md rename docs/{introduction/introduction.md => getting_started/introduction_cn.md} (57%) rename docs/{introduction => getting_started}/introduction_en.md (53%) rename docs/{ => getting_started}/quick_start_cn.md (100%) rename docs/{ => getting_started}/quick_start_en.md (98%) rename docs/{introduction => getting_started}/scalar.png (100%) create mode 100644 docs/templates/layout.html diff --git a/docs/api/index.rst b/docs/api/index.rst new file mode 100644 index 00000000..22723636 --- /dev/null +++ b/docs/api/index.rst @@ -0,0 +1,8 @@ +API +========== + +.. toctree:: + :maxdepth: 1 + + initialize_logger.rst + write_logs.rst \ No newline at end of file diff --git a/docs/api/initialize_logger.rst b/docs/api/initialize_logger.rst new file mode 100644 index 00000000..cb7e5a19 --- /dev/null +++ b/docs/api/initialize_logger.rst @@ -0,0 +1,5 @@ +Initialize Logger +----------------------- + +.. automodule:: visualdl + :members: LogWriter, LogReader \ No newline at end of file diff --git a/docs/api/visualdl.rst b/docs/api/visualdl.rst deleted file mode 100644 index bf3e8030..00000000 --- a/docs/api/visualdl.rst +++ /dev/null @@ -1,14 +0,0 @@ -========== -API -========== - -Python API ----------- - -.. automodule:: visualdl - :members: LogReader, LogWriter - -PyBind Wrapper Classes ------------------------ - -.. automodule:: visualdl.core \ No newline at end of file diff --git a/docs/api/write_logs.rst b/docs/api/write_logs.rst new file mode 100644 index 00000000..ae6730c0 --- /dev/null +++ b/docs/api/write_logs.rst @@ -0,0 +1,4 @@ +Write Logs +----------------------- + +.. automodule:: visualdl.core \ No newline at end of file diff --git a/docs/backend_architecture.md b/docs/backend_architecture.md deleted file mode 100644 index eaa3b0ae..00000000 --- a/docs/backend_architecture.md +++ /dev/null @@ -1,50 +0,0 @@ -# 后端架构设计 -总体上划分为3层，由上至下分别为： - -- Service Layer - - `Server` 提供服务 - - `Frontend API` 后端前端的交互接口 -- Logic Layer - - SDK, 负责为其他应用提供使用接口 - - python SDK，方便基于 python 的应用的读取 - - C SDK, 方便应用嵌入 - - Information Maintainer, 负责周期性将数据预读到内存中，方便Server直接读取 -- Storage Layer - - 负责Log 的结构化存储 - - Protobuf API, 底层的存储结构 - -具体架构图如下 - -

- -

- -以下展开一些基本的想法 -## Service Layer -### server -- 基于简便的server框架实现，提供两种服务 - - 作为main程序，支持一个前端网页 - - 提供一系列的HTTP接口，以JSON格式支持与前端的交互 -### Frontend API -- `Frontend API` 以单独模块的方式封装了与前端交互的接口逻辑，server依赖 `Frontend API` 与前端交互 - -## Logic Layer -### Information Maintainer (IM) -IM提供具体的数据处理以及计算，比如数据抽样，方差计算(histogram)等， -为了支持类似 embedding 等比较计算复杂的可视化，中间的计算结果以异步的方式更新。 - -IM 与 Server 的关系是 - -- IM 异步更新共享内存，Server 直接读取共享内存 -- IM需要提供接口方便Server读取数据 - -### SDK -- Python SDK 用来支持任何Python界面的应用。 - - 比如 Paddle, Tensorflow 等平台能够很方便地使用 Python SDK 初步支持可视化 -- C SDK 用来方便嵌入到 C/C++ 的应用中 - - 类似Paddle的平台可以通过嵌入 C 的方式实现原生可视化 - -## Storage Layer -### Protobuf API -- 决定了底层的数据格式，protobuf 提供了序列化的公共界面 -- SDK通过Protobuf API操作底层数据 diff --git a/docs/demo b/docs/demo deleted file mode 120000 index 337bec7b..00000000 --- a/docs/demo +++ /dev/null @@ -1 +0,0 @@ -../demo/ \ No newline at end of file diff --git a/docs/contribute_code_cn.md b/docs/develop/contribute_code_cn.md similarity index 100% rename from docs/contribute_code_cn.md rename to docs/develop/contribute_code_cn.md diff --git a/docs/contribute_code_en.md b/docs/develop/contribute_code_en.md similarity index 99% rename from docs/contribute_code_en.md rename to docs/develop/contribute_code_en.md index 1226a41b..b9812f40 100644 --- a/docs/contribute_code_en.md +++ b/docs/develop/contribute_code_en.md @@ -1,4 +1,4 @@ -# Contribute Code +# How To Contribute VisualDL is an open source project initiated by PaddlePaddle and ECharts. Our goal is to break through the deep learning black box and help the developers to see the training trend by providing visualization. diff --git a/docs/develop/how_to_dev_backend_cn.md b/docs/develop/how_to_dev_backend_cn.md new file mode 100644 index 00000000..bcdeb7e6 --- /dev/null +++ b/docs/develop/how_to_dev_backend_cn.md @@ -0,0 +1,124 @@ +# 后台指南 + +VisualDL has three components. +1. The Python/C++ SDK that logs the data during training. +1. The single page client side app that visualizes training data. +1. The server powered by Flask, reads the log data and delivers it to the client side app for displaying graphs (scalars/histograms) and embeddings. + +This doc will go over Backend / SDK structure with development guide. + +For front end architecture and development guide, looks [here](https://github.com/PaddlePaddle/VisualDL/blob/develop/docs/how_to_dev_frontend_en.md) + + +## Code structure + +All backend and sdk logic is under visualdl sub directory + +``` +├── build +├── build.sh +├── CMakeLists.txt +├── frontend +├── visualdl + ├── logic - API for C++ SDK, information maintainer + ├── python - API for Python SDK + ├── server - Flask server, provides front end APIs for read data + ├── storage - Data structure and protobuf + └── utils - Helper function and classes +``` + +## Development + +Any code changes in ```server``` folder, simply run + +``` +python visualdl/server/visualDL --logdir={LOG_DIR} --port=8080 +``` +to restart flask server + +Notice if you install visualDL from pip install wheel, make sure resetting PYTHONPATH to get the command work. + +Any code changes in ```logic```, ```python```, ```storage```, ```utils``` is modified, need to rebuild core.so or sdk to get effect. + +Either run VisualDL/build.sh that builds everything or a target like this +``` +mkdir {VisualDL_ROOT}/build +cd build +cmake .. +make {TARGET MODULE} +``` + + +## Tests + +Modification in SDK API or any major function should be adding test cases. + + +In {VisualDL_ROOT}/CMakeList.txt, we have a list of test files as target + +``` +add_executable(vl_test + ${PROJECT_SOURCE_DIR}/visualdl/test.cc + ${PROJECT_SOURCE_DIR}/visualdl/logic/sdk_test.cc + ${PROJECT_SOURCE_DIR}/visualdl/logic/histogram_test.cc + ${PROJECT_SOURCE_DIR}/visualdl/storage/storage_test.cc + ${PROJECT_SOURCE_DIR}/visualdl/storage/test_binary_record.cc + ${PROJECT_SOURCE_DIR}/visualdl/utils/test_concurrency.cc + ${PROJECT_SOURCE_DIR}/visualdl/utils/test_image.cc + ${PROJECT_SOURCE_DIR}/visualdl/utils/concurrency.h + ${PROJECT_SOURCE_DIR}/visualdl/utils/filesystem.h + ) +``` + +Add new test cases in corresponding files where code is modified or create new test files. You can make vl_test target and just run vl_test executable for all test cases. + +``` +mkdir {VisualDL_ROOT}/build +cd build +cmake .. +make vl_test +./vl_test +``` + +## Backend / SDK architecture + +### Server + +- Implement a server using a lightweight framework Flask, provides two services: + - Host a main application with support of a front end web app + - Provide a series of HTTP end points, using JSON for front end data communication +- ```visualDL``` : main app entry point + - defines server arguments and config + - provides API and router for front end +- ```lib.py``` : delegate wrapper for function calls + - read and cache log data + - convert data to JSON + - retry mechanism +- ```graph.py``` and ```onnx``` + - graph.py defines the logic to read and parse onnx model file + - create edges and nodes as JSON format for front end to read + - onnx folder contains onnx protobuf interface that ports from onnx repo + + +### Logic & SDK + +#### Python SDK +- ```pybind.cc``` : create python interface from C++ layer +- ```storage.py``` : ```LogReader``` and ```LogWriter``` APIs of Python SDK that maps to pybind + +#### C++ SDK +- ```sdk.h``` + - defines ```LogReader``` and ```LogWriter``` APIs of C++ SDK + - defines feature components such as ```Scalar```, ```Histogram```, ```Image``` and their corresponding component```Reader``` +- ```sdk.cc``` : implements ```Reader``` to use ```Storage``` layer for reading and writing + +#### Information Container +- Currently it just handles logic of sync cycle (when to write) + +#### Storage +- Defines log data format as following: + - ```Storage``` : main set of data with different modes such as "train" or "test", contain multiple ```Tablet``` + - ```Tablet``` : group a single type of data with ```tag```. One tablet will only represent one data type such as scalar, histogram, image and contains multiple ```Record```. For example one tablet represents a line in scalar chart. + - ```Record``` : represent any data structure for any component +- Handles the actual logic to serialize to disk and deserialize from disk +- ```storage.proto```: protobuf file that defines the exact interface to be read/write diff --git a/docs/how_to_dev_backend_en.md b/docs/develop/how_to_dev_backend_en.md similarity index 89% rename from docs/how_to_dev_backend_en.md rename to docs/develop/how_to_dev_backend_en.md index 4ddb673c..e0382e69 100644 --- a/docs/how_to_dev_backend_en.md +++ b/docs/develop/how_to_dev_backend_en.md @@ -1,9 +1,9 @@ -# VisualDL Backend / SDK structure and Develop Guide +# Backend Guide VisualDL has three components. 1. The Python/C++ SDK that logs the data during training. -1. The single page client side app that visualized training data. -1. The server (powered on Flask) that reads the logs data and delivers it to the client side app for displaying graphs (scalars/histograms) and embeddings. +1. The single page client side app that visualizes training data. +1. The server powered by Flask, reads the log data and delivers it to the client side app for displaying graphs (scalars/histograms) and embeddings. This doc will go over Backend / SDK structure with development guide. @@ -51,7 +51,7 @@ make {TARGET MODULE} ## Tests -Modification in SDK API and any major function changes should be adding test cases. +Modification in SDK API or any major function should be adding test cases. In {VisualDL_ROOT}/CMakeList.txt, we have a list of test files as target @@ -91,7 +91,7 @@ make vl_test - defines server arguments and config - provides API and router for front end - ```lib.py``` : delegate wrapper for function calls - - read and caches log data + - read and cache log data - convert data to JSON - retry mechanism - ```graph.py``` and ```onnx``` @@ -120,5 +120,5 @@ make vl_test - ```Storage``` : main set of data with different modes such as "train" or "test", contain multiple ```Tablet``` - ```Tablet``` : group a single type of data with ```tag```. One tablet will only represent one data type such as scalar, histogram, image and contains multiple ```Record```. For example one tablet represents a line in scalar chart. - ```Record``` : represent any data structure for any component -- Handles the actual logic to serialized to disk and deserialized from disk +- Handles the actual logic to serialize to disk and deserialize from disk - ```storage.proto```: protobuf file that defines the exact interface to be read/write diff --git a/docs/develop/how_to_dev_frontend_cn.md b/docs/develop/how_to_dev_frontend_cn.md new file mode 100644 index 00000000..c2fd3770 --- /dev/null +++ b/docs/develop/how_to_dev_frontend_cn.md @@ -0,0 +1,78 @@ +# 前端指南 + +VisualDL has three components. +1. The Python/C++ SDK that logs the data during training. +1. The single page client side app that visualized training data. +1. The server (powered on Flask) that reads the logs data and delivers it to the client side app for displaying graphs (scalars/histograms) and embeddings. + +This article will go over the basic web-app architecture and development guide. + +## TL;DR +If you are not interested in the details of mumbo jumbo and want to start development as soon as possible, please run the following commands. + +To clone the repo and to prepare the environment +```bash +git clone git@github.com:PaddlePaddle/VisualDL.git +cd VisualDL +# Install all the requirements, we recommend you to setup a virtual environment to keep it clean. +pip install -r requirements.txt +# Set up the node dependencies and the VisualDL SDK +./scripts/setup_dev_env.sh +``` +To run the dev server and start coding +```bash +./scripts/start_dev_server.sh --logdir= +``` +The script will run a dev server at `http://0.0.0.0:8040`. Open a browser and navigate to `http://0.0.0.0:8040/` to see it + +## Web App Architecture + +The VisualDL Web app uses multiple frameworks to help manage the project. They are + +1. webpack: To manage all assets +1. npm: To manage dependencies +1. Vue: Javascript Component framework +1. ECharts: To pilot charts + +## Webpack +webpack is a module bundler. Its main purpose is to bundle JavaScript files for usage in a browser. + +For a long time, web developers need to worry about backward compatibility or how to support different versions of browsers. +webpack can help to transpile the Javascript files to the version where a browser can take. The developers don't need to implement version specific code anymore. +It can also help to minify and uglify Javascript files. In a nutshell, webpack helps to manage all of your assets (.js, .css, .gif, .sass, etc) so you don't have to. + +To learn more about [webpack](https://webpack.js.org/). + +## npm package +npm is the package manager for JavaScript and is a great tool to manage multiple dependencies. To confirm that you have npm installed. Please run this command +``` bash +npm -v +``` +If your environment does not have npm already, please [download npm](https://www.npmjs.com/get-npm). + +The basic packaging information is stored in `package.json`. It specifies build scripts and dependencies for each environment. +npm reads `package.json` to build different package according to the packaging script. +If you are not familiar with npm, please refer to the [npm documentation](https://docs.npmjs.com/) for more detail. + +Install the necessary dependencies by running +```bash +# Navigate to the web-app folder +cd VisualDL/frontend +npm install +``` + +This command will go through `package.json` and install the dependencies in the local node_modules folder. + +## Vue + +Vue is a JavaScript component framework that helps the developer to implement web component in MVVM architecture pattern. + +Vue allows you to define a self-contained view model in a .vue file and attach view model objects to DOM objects. + +To learn more about [Vue](https://vuejs.org/) + +## ECharts + +We use ECharts javascript library to render our charts and graphs. ECharts is a leading open source charting library that supports numerous data visualization. + +To learn more about ECharts framework, please visit [ECharts](https://ecomfe.github.io/echarts-doc/public/en/index.html). diff --git a/docs/how_to_dev_frontend_en.md b/docs/develop/how_to_dev_frontend_en.md similarity index 99% rename from docs/how_to_dev_frontend_en.md rename to docs/develop/how_to_dev_frontend_en.md index c0621bbc..1e617080 100644 --- a/docs/how_to_dev_frontend_en.md +++ b/docs/develop/how_to_dev_frontend_en.md @@ -1,4 +1,4 @@ -# VisualDL Web app Develop +# Frontend Guide VisualDL has three components. 1. The Python/C++ SDK that logs the data during training. diff --git a/docs/develop/index_cn.rst b/docs/develop/index_cn.rst new file mode 100644 index 00000000..2d481f41 --- /dev/null +++ b/docs/develop/index_cn.rst @@ -0,0 +1,9 @@ +开发和贡献 +============ + +.. toctree:: + :maxdepth: 1 + + contribute_code_cn.md + how_to_dev_backend_cn.md + how_to_dev_frontend_cn.md diff --git a/docs/develop/index_en.rst b/docs/develop/index_en.rst new file mode 100644 index 00000000..acfc5d3c --- /dev/null +++ b/docs/develop/index_en.rst @@ -0,0 +1,9 @@ +Develop +============ + +.. toctree:: + :maxdepth: 1 + + contribute_code_en.md + how_to_dev_backend_en.md + how_to_dev_frontend_en.md diff --git a/docs/getting_started/demo b/docs/getting_started/demo new file mode 120000 index 00000000..a6dfc5f4 --- /dev/null +++ b/docs/getting_started/demo @@ -0,0 +1 @@ +../../demo/ \ No newline at end of file diff --git a/docs/introduction/dog.png b/docs/getting_started/dog.png similarity index 100% rename from docs/introduction/dog.png rename to docs/getting_started/dog.png diff --git a/docs/introduction/graph.png b/docs/getting_started/graph.png similarity index 100% rename from docs/introduction/graph.png rename to docs/getting_started/graph.png diff --git a/docs/introduction/histogram.png b/docs/getting_started/histogram.png similarity index 100% rename from docs/introduction/histogram.png rename to docs/getting_started/histogram.png diff --git a/docs/introduction/image-gan.png b/docs/getting_started/image-gan.png similarity index 100% rename from docs/introduction/image-gan.png rename to docs/getting_started/image-gan.png diff --git a/docs/getting_started/index_cn.rst b/docs/getting_started/index_cn.rst new file mode 100644 index 00000000..1597cddd --- /dev/null +++ b/docs/getting_started/index_cn.rst @@ -0,0 +1,10 @@ +新手入门 +=============== + +.. toctree:: + :maxdepth: 1 + + introduction_cn.md + installation_cn.md + quick_start_cn.md + demo/demo_cn.rst \ No newline at end of file diff --git a/docs/getting_started/index_en.rst b/docs/getting_started/index_en.rst new file mode 100644 index 00000000..45d1cfea --- /dev/null +++ b/docs/getting_started/index_en.rst @@ -0,0 +1,10 @@ +Getting Started +============ + +.. toctree:: + :maxdepth: 1 + + introduction_en.md + installation_en.md + quick_start_en.md + demo/demo_en.rst diff --git a/docs/getting_started/installation_cn.md b/docs/getting_started/installation_cn.md new file mode 100644 index 00000000..ab4c5d1b --- /dev/null +++ b/docs/getting_started/installation_cn.md @@ -0,0 +1,103 @@ +# 安装 + +### 使用PIP安装 +Visual DL提供独立的Python SDK，如果训练任务是基于Python的话，直接安装visualdl的whl包，import到自己项目中即可使用。 + +``` +# Install the VisualDL. Preferably under a virtual environment or anaconda. +pip install --upgrade visualdl + +# run a demo, vdl_create_scratch_log will create logs for testing. +vdl_create_scratch_log +visualDL --logdir=scratch_log --port=8080 + +# visit http://127.0.0.1:8080 +``` + +如果以上步骤出现问题，很可能是因为python或pip不同版本或不同位置所致，以下安装方法能解决。 + +## 使用 virtualenv 安装 + +[Virtualenv](https://virtualenv.pypa.io/en/stable/) 能创建独立Python环境，也能确保Python和pip的相对位置正确。 + +在macOS上，安装pip和virtualenv如下： +``` +sudo easy_install pip +pip install --upgrade virtualenv +``` + +在Linux上，安装pip和virtualenv如下: +``` +sudo apt-get install python3-pip python3-dev python-virtualenv +``` + +然后创建一个虚拟环境： +``` +virtualenv ~/vdl # for Python2.7 +virtualenv -p python3 ~/vdl for Python 3.x +``` + +```~/vdl``` 是你的Virtualenv目录, 你也可以选择任一目录。 + +激活虚拟环境如下： +``` +source ~/vdl/bin/activate +``` + +现在再安装 VisualDL 和运行范例： + +``` +pip install --upgrade visualdl + +# 运行一个例子，vdl_create_scratch_log 将创建测试日志 +vdl_create_scratch_log +visualDL --logdir=scratch_log --port=8080 + +# 访问 http://127.0.0.1:8080 +``` + +如果在虚拟环境下仍然遇到安装问题，请尝试以下方法。 + + +## 使用 Anaconda 安装 + +Anaconda是一个用于科学计算的Python发行版，提供了包管理与环境管理的功能，可以很方便地解决多版本python并存、切换以及各种第三方包安装问题。 + +请根据[Anaconda下载网站](https://www.anaconda.com/download) 的指示去下载和安装Anaconda. +下载Python 3.6版本的command-Line installer. + +创建conda环境名字为```vdl```或任何名字: +``` +conda create -n vdl pip python=2.7 # or python=3.3, etc. +``` + +激活conda环境如下: +``` +source activate vdl +``` + +现在再安装 VisualDL 和运行范例： + +``` +pip install --upgrade visualdl + +# 运行一个例子，vdl_create_scratch_log 将创建测试日志 +vdl_create_scratch_log +visualDL --logdir=scratch_log --port=8080 + +# 访问 http://127.0.0.1:8080 +``` + +如果仍然遇到安装问题，请尝试以下用源代码安装方法。 + +## 使用代码安装 +``` +#建議是在虚拟环境或anaconda下。 +git clone https://github.com/PaddlePaddle/VisualDL.git +cd VisualDL + +python setup.py bdist_wheel +pip install --upgrade dist/visualdl-*.whl +``` + +如果打包和安装遇到其他问题，不安装只想运行Visual DL可以看[这里](https://github.com/PaddlePaddle/VisualDL/blob/develop/docs/how_to_dev_frontend_en.md) diff --git a/docs/getting_started/installation_en.md b/docs/getting_started/installation_en.md new file mode 100644 index 00000000..21852027 --- /dev/null +++ b/docs/getting_started/installation_en.md @@ -0,0 +1,107 @@ +# Installation + +### How to install from pypi + +``` +# Install the VisualDL. Preferably under a virtual environment or anaconda. +pip install --upgrade visualdl + +# run a demo, vdl_create_scratch_log will create logs for testing. +vdl_create_scratch_log +visualDL --logdir=scratch_log --port=8080 + +# visit http://127.0.0.1:8080 +``` + +If you run into issues in above steps, it could be error caused by environmental issues by different python or pip versions. +Following installation methods might fix the issues. + +## Install with Virtualenv + +[Virtualenv](https://virtualenv.pypa.io/en/stable/) creates isolated Python environment that prevents interfering +by other Python programs on the same machine and make sure Python and pip are located properly. + +On macOS, install pip and virtualenv by: +``` +sudo easy_install pip +pip install --upgrade virtualenv +``` + +On Linux, install pip and virtualenv by: +``` +sudo apt-get install python3-pip python3-dev python-virtualenv +``` + +Then create a Virtualenv environment by one of following command: +``` +virtualenv ~/vdl # for Python2.7 +virtualenv -p python3 ~/vdl for Python 3.x +``` + +```~/vdl``` will be your Virtualenv directory, you may choose to install anywhere. + +Activate your Virtualenv environment by: +``` +source ~/vdl/bin/activate +``` + +Now you should be able to install VisualDL and run our demo: + +``` +pip install --upgrade visualdl + +# run a demo, vdl_create_scratch_log will create logs for testing. +vdl_create_scratch_log +visualDL --logdir=scratch_log --port=8080 + +# visit http://127.0.0.1:8080 +``` + +If you still have issues installing VisualDL from Virtualenv, try following installation method. + + +## Install with Anaconda + +Anaconda is a python distribution, with installation and package management tools. Also it is an environment manager, +which provides the facility to create different python environments, each with their own settings. + +Follow the instructions on the [Anaconda download site](https://www.anaconda.com/download) to download and install Anaconda. +Download Python 3.6 version command-Line installer. + +Create a conda environment named ```vdl``` or anything you want by: +``` +conda create -n vdl pip python=2.7 # or python=3.3, etc. +``` + +Activate the conda environment by: +``` +source activate vdl +``` + +Now you should be able to install VisualDL and run our demo: + +``` +pip install --upgrade visualdl + +# run a demo, vdl_create_scratch_log will create logs for testing. +vdl_create_scratch_log +visualDL --logdir=scratch_log --port=8080 + +# visit http://127.0.0.1:8080 +``` + +If you still have issues installing VisualDL, try installing from sources as in following section. + + +## Install from source +``` +#Preferably under a virtualenv or anaconda. +git clone https://github.com/PaddlePaddle/VisualDL.git +cd VisualDL + +python setup.py bdist_wheel +pip install --upgrade dist/visualdl-*.whl +``` + +If there are still issues regarding the ```pip install```, you can still start Visual DL by starting the dev server +[here](https://github.com/PaddlePaddle/VisualDL/blob/develop/docs/how_to_dev_frontend_en.md) diff --git a/docs/introduction/introduction.md b/docs/getting_started/introduction_cn.md similarity index 57% rename from docs/introduction/introduction.md rename to docs/getting_started/introduction_cn.md index abb838a0..f28fceb5 100644 --- a/docs/introduction/introduction.md +++ b/docs/getting_started/introduction_cn.md @@ -1,71 +1,35 @@ -## Visual DL -`Visual DL`：让你的深度学习任务变得生动形象，可视化起来。 - -目前大部分深度学习框架都是基于Python语言，训练过程的状态通过日志的方式记录下来，类似下面这种： - -```shell -loss:[2.532566] acc:[0.0859375] pass_acc:[0.0859375] -loss:[2.6916795] acc:[0.09375] pass_acc:[0.08984375] -loss:[2.7038598] acc:[0.1171875] pass_acc:[0.09895834] -loss:[2.6405232] acc:[0.125] pass_acc:[0.10546875] -``` - -这种方式可以短期看一下训练状态，但是很难从全局把握训练过程的一些变化趋势，提取的信息有限。 - -Visual DL 让你方便的将训练过程可视化起来，形成类似下面的图片：

- +

-上述只是Visual DL诸多功能中的一项，Visual DL有以下几方面的优点。 +# 介绍 +VisualDL是一个面向深度学习任务设计的可视化工具，包含了scalar、参数分布、模型结构、图像可视化等功能，项目正处于高速迭代中，新的组件会不断加入。 -### 功能全 -1. Scalar. 支持Scalar打点数据展示，如上图所示： - - 可以将训练过程中的loss，准确率等信息以折现的形式体现出来，方便看出整体趋势。 - - 可以在同一个图片中画出多条曲线，方便对比分析。 -2. Image. 支持图片展示。 - - 输入图片展示，方便查看数据样本的质量 - - 支持卷积层的结果展示，方便查看卷积之后的效果。 -

- -

- - 支持图像生成任务生成图片的预览。 -

- -

+目前大多数DNN平台均使用Python作为配置语言，VisualDL原生支持python的使用， +通过在模型的Python配置中添加几行，便可以为训练过程提供丰富的可视化支持。 -3. Histogram. 参数分布展示，方便查看参数矩阵中数值的分布曲线，以及随着训练的进行，参数数值分布的变化趋势。 - - 帮助用户理解训练过程，随着训练的进行，参数从一种分布变化到另外一种分布，背后对应的原因。 - - 方便用户判断训练是否正常，例如参数迅速变小到0或者迅速变大，说明有梯度消失或者梯度爆炸的问题。 -

- -

+除了Python SDK之外，VisualDL底层采用C++编写，其暴露的C++ SDK也可以集成到其他平台中， +实现原生的性能和定制效果。 -4. Graph. 方便查看深度神经网络的模型结构。 - - Graph支持直接对[ONNX](http://onnx.ai/)的模型进行预览，因为MXNet，Caffe2，Pytorch和CNTK都支持转成ONNX的模型，也就间接支持了这些框架的模型可视化功能。 - - 便于排查网络配置的错误 - - 帮助理解网络结构 +## 我为什么要用 VisualDL？ +Visual DL有以下几方面的优点。 +### 可视化 +目前大部分深度学习框架都是基于Python语言，训练过程的状态通过日志的方式记录下来， +这种方式可以短期看一下训练状态，但是很难从全局把握训练过程的一些变化趋势，提取的信息有限。 + +Visual DL 让你方便的将训练过程可视化起来，形成类似下面的图片：

- +

+ ### 易集成 Visual DL提供独立的Python SDK，如果训练任务是基于Python的话，直接安装visualdl的whl包，import到自己项目中即可使用。 - -a. 安装visualdl包。 - -```shell -pip install visualdl -``` -b. 在自己的Python代码中加入visualdl记日志的逻辑。 - -```python -import visualdl -``` +详情请参阅安装。 ### 易使用 -a. 在自己的Python代码中加入visualdl记日志的逻辑。 +- 在自己的Python代码中加入visualdl记日志的逻辑。 ```python import visualdl @@ -79,11 +43,40 @@ for step in steps: loss_writer.add_record(step, loss) ``` -b. 启动visualdl service即可通过浏览器查看日志的可视化结果。 +- 启动visualdl service即可通过浏览器查看日志的可视化结果。 ```shell visualDL --logdir ./log --port 8080 ``` +### 功能全 +1. Scalar. 支持Scalar打点数据展示，如上图所示： + - 可以将训练过程中的loss，准确率等信息以折现的形式体现出来，方便看出整体趋势。 + - 可以在同一个图片中画出多条曲线，方便对比分析。 +2. Image. 支持图片展示。 + - 输入图片展示，方便查看数据样本的质量 + - 支持卷积层的结果展示，方便查看卷积之后的效果。 +

+ +

+ - 支持图像生成任务生成图片的预览。 +

+ +

+ +3. Histogram. 参数分布展示，方便查看参数矩阵中数值的分布曲线，以及随着训练的进行，参数数值分布的变化趋势。 + - 帮助用户理解训练过程，随着训练的进行，参数从一种分布变化到另外一种分布，背后对应的原因。 + - 方便用户判断训练是否正常，例如参数迅速变小到0或者迅速变大，说明有梯度消失或者梯度爆炸的问题。 +

+ +

+ +4. Graph. 方便查看深度神经网络的模型结构。 + - Graph支持直接对[ONNX](http://onnx.ai/)的模型进行预览，因为MXNet，Caffe2，Pytorch和CNTK都支持转成ONNX的模型，也就间接支持了这些框架的模型可视化功能。 + - 便于排查网络配置的错误 + - 帮助理解网络结构 +

+ +

### 完全开放 Visual DL作为一个深度学习任务可视化工具，同时支持所有的深度学习框架。SDK层面可以方便的集成到Python或者C++项目中，Graph通过支持ONNX支持了PaddlePaddle、MxNet、PyTorch和Caffe2等流行的深度学习框架。 diff --git a/docs/introduction/introduction_en.md b/docs/getting_started/introduction_en.md similarity index 53% rename from docs/introduction/introduction_en.md rename to docs/getting_started/introduction_en.md index f9af1a43..6b1fbb51 100644 --- a/docs/introduction/introduction_en.md +++ b/docs/getting_started/introduction_en.md @@ -1,26 +1,55 @@ -## Visual DL -`Visual DL`: makes your deep learning jobs more alive via visualization. - -At present, most deep learning frameworks are using Python. The status of training process is recorded -by logs. A sample log is as follow: +

+ +

+# Introduction +VisualDL is a deep learning visualization tool that can help design deep learning jobs. +It includes features such as scalar, parameter distribution, model structure and image visualization. +Currently it is being developed at a high pace. +New features will be continuously added. -```shell -loss:[2.532566] acc:[0.0859375] pass_acc:[0.0859375] -loss:[2.6916795] acc:[0.09375] pass_acc:[0.08984375] -loss:[2.7038598] acc:[0.1171875] pass_acc:[0.09895834] -loss:[2.6405232] acc:[0.125] pass_acc:[0.10546875] -``` +At present, most DNN frameworks use Python as their primary language. VisualDL supports Python by nature. +Users can get plentiful visualization results by simply add a few lines of Python code into their model before training. -This method can show short-term training status. But it can hardly show long-term trends. -Visual DL can help you visualize the whole training process and construct plots as below: +Besides Python SDK, VisualDL was writen in C++ on the low level. It also provides C++ SDK that +can be integrated into other platforms. +## Why should I use VisualDL? +Visual DL has the following advantages: +### Visualization +At present, most deep learning frameworks are using Python. The status of training process is recorded by logs. +This method can show short-term training status. But it can hardly show long-term trends. Visual DL can help you visualize the whole training process and construct plots as below:

- +

-The above is just one of Visual DL's many features. Visual DL has the following advantages: +### Easy to Integrate + +Visual DL provides independent Python SDK. If the training task is based on Python, user can simply +use Visual DL by installing the Visual DL wheel package and importing it into her/his own project. +Please refer to Installation for more details. + +### Easy to Use +- Add log collecting logic to your own Python code. + +```python +import visualdl + +log_writer = LogWriter("./log", sync_cycle=300) +with log_writer.mode('train') as logger: + loss_writer = logger.scalar("loss") + +for step in steps: + loss = train_process() // Insert your training code here + loss_writer.add_record(step, loss) +``` + +- Launch Visual DL service and you can see the visualization results. + +```shell +visualDL --logdir ./log --port 8080 +``` ### Comprehensive Usability @@ -31,14 +60,13 @@ The above is just one of Visual DL's many features. Visual DL has the following 2. Image: support image display. - display input and intermediate images, easy to check data sample - support display for convolutional layer, easy to see results after each layer - -

- -

+

+ +

- support image-generating tasks to preview generated image -

- -

+

+ +

3. Histogram: display of parameter distribution, easy to check distribution curves in each tensor, show the trend of parameter distribution. @@ -46,58 +74,18 @@ show the trend of parameter distribution. - help users understand the training process and the underneath reason for the change from one parameter distribution to another - help users judge if the training is on the track. For example, if parameter change rate becomes close to 0 or grows rapidly, then exploding and vanishing gradients might happen -

- -

+

+ +

4. Graph: visualize the model structure of deep learning networks. - Graph supports the preview of [ONNX](http://onnx.ai/) model. Since models of MXNet, Caffe2, PyTorch and CNTK can be converted to ONNX models easily, Visual DL can also support these models indirectly - easy to see wrong configuration of a network - help understand network structure - - -

- -

- -### Easy to Integrate - -Visual DL provides independent Python SDK. If the training task is based on Python, user can simply -use Visual DL by installing the Visual DL wheel package and importing it into her/his own project. - - -a. Install Visual DL package. - -```shell -pip install visualdl -``` -b. In your own Python code, add logic for Visual DL to collect logs. - -```python -import visualdl -``` - -### Easy to Use -a. Add log collecting logic to your own Python code. - -```python -import visualdl - -log_writer = LogWriter("./log", sync_cycle=30) -with log_writer.mode('train') as logger: - loss_writer = logger.scalar("loss") - -for step in steps: - loss = train_process() - loss_writer.add_record(step, loss) -``` - -b. Launch Visual DL service and you can see the visualization results. - -```shell -visualDL --logdir ./log --port 8080 -``` +

+ +

### Purely Open Source As a deep learning visualization tool, Visual DL support most deep learning frameworks. On the SDK perspective, diff --git a/docs/quick_start_cn.md b/docs/getting_started/quick_start_cn.md similarity index 100% rename from docs/quick_start_cn.md rename to docs/getting_started/quick_start_cn.md diff --git a/docs/quick_start_en.md b/docs/getting_started/quick_start_en.md similarity index 98% rename from docs/quick_start_en.md rename to docs/getting_started/quick_start_en.md index 1fc6a0da..ddf542fa 100644 --- a/docs/quick_start_en.md +++ b/docs/getting_started/quick_start_en.md @@ -1,5 +1,5 @@ -Quick start -=========== +Setting up First Visualization +============================== VisualDL is a deep learning visualization tool. It can be used to visualize intermediate and final results for training. Currently, VisualDL supports visualization features as follows: diff --git a/docs/introduction/scalar.png b/docs/getting_started/scalar.png similarity index 100% rename from docs/introduction/scalar.png rename to docs/getting_started/scalar.png diff --git a/docs/index_cn.rst b/docs/index_cn.rst index 9c135217..47ec79d2 100644 --- a/docs/index_cn.rst +++ b/docs/index_cn.rst @@ -4,8 +4,6 @@ VisualDL .. toctree:: :maxdepth: 1 - 首页 - quick_start_cn.md - contribute_code_cn.md - demo/demo_cn.rst - api/visualdl.rst \ No newline at end of file + getting_started/index_cn.rst + api/index.rst + develop/index_cn.rst \ No newline at end of file diff --git a/docs/index_en.rst b/docs/index_en.rst index 307b3128..f2e3eb03 100644 --- a/docs/index_en.rst +++ b/docs/index_en.rst @@ -4,9 +4,6 @@ VisualDL .. toctree:: :maxdepth: 1 - Home - quick_start_en.md - contribute_code_en.md - how_to_dev_frontend_en.md - demo/demo_en.rst - api/visualdl.rst \ No newline at end of file + getting_started/index_en.rst + api/index.rst + develop/index_en.rst diff --git a/docs/templates/conf.py.cn.in b/docs/templates/conf.py.cn.in index 059c4c26..c4328e1c 100644 --- a/docs/templates/conf.py.cn.in +++ b/docs/templates/conf.py.cn.in @@ -22,7 +22,7 @@ AutoStructify = transform.AutoStructify # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#templates_path = ["@VISUALDL_SOURCE_DIR@/doc_theme/templates"] +templates_path = ["@VISUALDL_SOURCE_DIR@/docs/templates"] # -- General configuration ------------------------------------------------ diff --git a/docs/templates/conf.py.en.in b/docs/templates/conf.py.en.in index 6c23b516..2a4f3cf7 100644 --- a/docs/templates/conf.py.en.in +++ b/docs/templates/conf.py.en.in @@ -22,7 +22,7 @@ AutoStructify = transform.AutoStructify # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#templates_path = ["@VISUALDL_SOURCE_DIR@/doc_theme/templates"] +templates_path = ["@VISUALDL_SOURCE_DIR@/docs/templates"] # -- General configuration ------------------------------------------------ diff --git a/docs/templates/layout.html b/docs/templates/layout.html new file mode 100644 index 00000000..410a6c18 --- /dev/null +++ b/docs/templates/layout.html @@ -0,0 +1,11 @@ +{# layout.html #} +{# Import the theme's layout. #} +{% extends "!layout.html" %} + +{# SIDE NAV, TOGGLES ON MOBILE #} +{% block menu %} + +{% endblock %} -- GitLab