feature/pip build (#64)

.travis.yml

@@ -3,6 +3,7 @@ cache:
   - pip
   - ccache
   - yarn
+  - npm
 sudo: required
 dist: trusty
 os:

README.md

 # VisualDL
 
 
-### How to use
-#### Step 1: build frontend
-```shell
-cd frontend
-npm install
-npm run build
+### How to install
 ```
-
-this step will generate a dist directory under frontend
-
-### Step 2: copy frontend/dist to server/visualdl/frontend/dist
-```shell
-mkdir -p server/visualdl/frontend/dist
-cp -r frontend/dist server/visualdl/frontend/dist
-```
-
-#### Step 3: build and install Python package
-```shell
-cd server/
-sh build.sh
-cd dist
-sudo pip install --upgrade visualdl-0.0.1-py2-none-any.whl
+python setup.py bdist_wheel
+pip install --upgrade dist/visualdl-0.0.1-py2-none-any.whl
 ```
 
 
 ### Step 3: run
 ```
-# cd to visualdl install dir
-cd /usr/local/lib/python2.7/site-packages/visualdl/
-python visual_dl.py --port=8888
+visualDL --logdir=<some log> --port=8888
 ```

VERSION_NUMBER

+0.0.1
\ No newline at end of file

build.sh

+#!/bin/bash
+set -ex
+
+TOP_DIR=$(pwd)
+FRONTEND_DIR=$TOP_DIR/frontend
+BACKEND_DIR=$TOP_DIR/visualdl
+BUILD_DIR=$TOP_DIR/build
+
+mkdir -p $BUILD_DIR
+
+build_frontend() {
+    cd $FRONTEND_DIR
+    if [ ! -d "dist" ]; then
+      npm install
+      npm run build
+    fi
+}
+
+build_frontend_fake() {
+    cd $FRONTEND_DIR
+    mkdir -p dist
+}
+
+build_backend() {
+    cd $BUILD_DIR
+    cmake ..
+    make -j2
+}
+
+build_onnx_graph() {
+    # TODO(ChunweiYan) check protoc version here
+    cd $TOP_DIR/visualdl/server/onnx
+    protoc onnx.proto --python_out .
+}
+
+package() {
+    cp -rf $FRONTEND_DIR/dist $TOP_DIR/visualdl/server/
+    cp $BUILD_DIR/visualdl/logic/core.so $TOP_DIR/visualdl
+    cp $BUILD_DIR/visualdl/logic/core.so $TOP_DIR/visualdl/python/
+}
+
+ARG=$1
+echo "ARG: " $ARG
+
+
+if [ $ARG = "travis-CI" ]; then
+    build_frontend_fake
+else
+    build_frontend
+fi
+
+build_backend
+build_onnx_graph
+package

server/visualdl/mock.sh

-#!/bin/bash
-set -ex
-
-export PYTHONPATH="/home/superjom/project/VisualDL/build/visualdl/logic:/home/superjom/project/VisualDL/visualdl/python"
-
-python lib_test.py -v

setup.py

+from __future__ import absolute_import
+
+import os
+import sys
+from distutils.spawn import find_executable
+from distutils import sysconfig, dep_util, log
+import setuptools.command.build_py
+import setuptools
+from setuptools import setup, find_packages
+import subprocess
+
+TOP_DIR = os.path.realpath(os.path.dirname(__file__))
+PYTHON_SDK_DIR = os.path.join(TOP_DIR, 'visualdl/python')
+BUILD_DIR = os.path.join(TOP_DIR, 'build')
+MODE = os.environ.get('VS_BUILD_MODE', 'RELEASE')
+
+
+def read(name):
+    return open(os.path.join(TOP_DIR, name)).read()
+
+
+def readlines(name):
+    return read(name).split('\n')
+
+
+VERSION_NUMBER = read('VERSION_NUMBER')
+LICENSE = readlines('LICENSE')[0].strip()
+
+install_requires = ['Flask', 'numpy', 'Pillow', 'protobuf']
+execute_requires = ['npm', 'node', 'protoc', 'bash']
+
+
+def die(msg):
+    log.error(msg)
+    sys.exit(1)
+
+
+def CHECK(cond, msg):
+    if not cond:
+        die(msg)
+
+
+for exe in execute_requires:
+    CHECK(find_executable(exe), "{} should be installed.".format(exe))
+
+
+class BaseCommand(setuptools.Command):
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+
+class build_py(setuptools.command.build_py.build_py):
+    def run(self):
+        cmd = ['bash', 'build.sh']
+        if MODE == "travis-CI":
+            cmd.append('travis-CI')
+        subprocess.check_call(cmd)
+        return setuptools.command.build_py.build_py.run(self)
+
+
+cmdclass = {
+    'build_py': build_py,
+}
+
+packages = [
+    'visualdl',
+    'visualdl.python',
+    'visualdl.server',
+    'visualdl.server.mock',
+    'visualdl.server.onnx',
+]
+
+setup(
+    name="visualdl",
+    version=VERSION_NUMBER,
+    author="PaddlePaddle and Echarts team.",
+    description="Visualize Deep Learning.",
+    license=LICENSE,
+    keywords="visualization deeplearning",
+    long_description=read('README.md'),
+    install_requires=install_requires,
+    package_data={'visualdl.server': ['dist/*', 'dist/fonts/*'],
+                  'visualdl':['core.so'],
+                  'visualdl.python':['core.so']},
+    packages=packages,
+    scripts=['visualdl/server/visualDL'],
+    cmdclass=cmdclass)

tests.sh

@@ -2,15 +2,39 @@
 set -ex
 
 mode=$1
-readonly cur=$(pwd)
-readonly core_path=$cur/build/visualdl/logic
-readonly python_path=$cur/visualdl/python
+readonly TOP_DIR=$(pwd)
+readonly core_path=$TOP_DIR/build/visualdl/logic
+readonly python_path=$TOP_DIR/visualdl/python
 readonly max_file_size=1000000 # 1MB
 
 export PYTHONPATH="${core_path}:${python_path}"
 
+# install the visualdl wheel first
+package() {
+    # some bug with frontend build
+    # a environment variable to skip frontend build
+    export VS_BUILD_MODE="travis-CI"
+
+    cd $TOP_DIR/visualdl/server
+    # manully install protobuf3
+    curl -OL https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip
+    unzip protoc-3.1.0-linux-x86_64.zip -d protoc3
+    export PATH="$PATH:$(pwd)/protoc3/bin"
+    chmod +x protoc3/bin/*
+
+    sudo pip install numpy
+    sudo pip install Flask
+    sudo pip install Pillow
+    sudo pip install protobuf
+
+    #sudo apt-get install protobuf-compiler
+    cd $TOP_DIR
+    python setup.py bdist_wheel
+    sudo pip install dist/visualdl-0.0.1-py2-none-any.whl
+}
+
 backend_test() {
-    cd $cur
+    cd $TOP_DIR
     sudo pip install numpy
     sudo pip install Pillow
     mkdir -p build
@@ -21,7 +45,7 @@ backend_test() {
 }
 
 frontend_test() {
-    cd $cur
+    cd $TOP_DIR
     cd frontend
     npm install
     npm run build
@@ -31,29 +55,20 @@ server_test() {
     sudo pip install google
     sudo pip install protobuf==3.1.0
 
-    cd $cur/server
-    curl -OL https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip
-    unzip protoc-3.1.0-linux-x86_64.zip -d protoc3
-    export PATH=$PATH:protoc3/bin
-    sudo chmod +x protoc3/bin/protoc
-    sudo chown `whoami` protoc3/bin/protoc
-
-    bash build.sh
-
-    cd visualdl
+    cd $TOP_DIR/visualdl/server
     bash graph_test.sh
 
-    cd $cur/server/visualdl
+    cd $TOP_DIR/visualdl/server
     python lib_test.py
 }
 
 # check the size of files in the repo.
 # reject PR that has some big data included.
 bigfile_reject() {
-    cd $cur
+    cd $TOP_DIR
     # it failed to exclude .git, remove it first.
     rm -rf .git
-    local largest_file=$(find . -path .git -prune -o -printf '%s %p\n' | sort -nr | head -n1)
+    local largest_file="$(find . -path .git -prune -o -printf '%s %p\n' | sort -nr | head -n1)"
     local size=$(echo $largest_file | awk '{print $1}')
     if [ "$size" -ge "$max_file_size" ]; then
         echo $largest_file
@@ -68,10 +83,16 @@ echo "mode" $mode
 if [ $mode = "backend" ]; then
     backend_test
 elif [ $mode = "all" ]; then
+    # bigfile_reject should be tested first, or some files downloaded may fail this test.
     bigfile_reject
+    package
     frontend_test
     backend_test
     server_test
+elif [ $mode = "local" ]; then
+    #frontend_test
+    backend_test
+    server_test
 else
     frontend_test
 fi

visualdl/__init__.py

+from __future__ import absolute_import
+
+from .python.storage import *

visualdl/python/__init__.py

-from storage import *

visualdl/python/storage.py

-__all__ = [
-    'StorageReader',
-    'StorageWriter',
-]
-import core
+from __future__ import absolute_import
+
+from visualdl import core
 
 dtypes = ("float", "double", "int32", "int64")
 

visualdl/python/test_storage.py

@@ -5,13 +5,16 @@ import unittest
 import numpy as np
 from PIL import Image
 
-import storage
+import sys, pprint
+pprint.pprint(sys.path)
+
+from visualdl import LogWriter, LogReader
 
 
 class StorageTest(unittest.TestCase):
     def setUp(self):
         self.dir = "./tmp/storage_test"
-        self.writer = storage.LogWriter(
+        self.writer = LogWriter(
             self.dir, sync_cycle=1).as_mode("train")
 
     def test_scalar(self):
@@ -22,7 +25,7 @@ class StorageTest(unittest.TestCase):
             scalar.add_record(i, float(i))
 
         print 'test read'
-        self.reader = storage.LogReader(self.dir)
+        self.reader = LogReader(self.dir)
         with self.reader.mode("train") as reader:
             scalar = reader.scalar("model/scalar/min")
             self.assertEqual(scalar.caption(), "train")
@@ -50,7 +53,7 @@ class StorageTest(unittest.TestCase):
                     image_writer.set_sample(index, shape, list(data))
             image_writer.finish_sampling()
 
-        self.reader = storage.LogReader(self.dir)
+        self.reader = LogReader(self.dir)
         with self.reader.mode("train") as reader:
             image_reader = reader.image(tag)
             self.assertEqual(image_reader.caption(), tag)
@@ -77,7 +80,7 @@ class StorageTest(unittest.TestCase):
         shape = [image.size[1], image.size[0], 3]
         origin_data = np.array(image.getdata()).flatten()
 
-        self.reader = storage.LogReader(self.dir)
+        self.reader = LogReader(self.dir)
         with self.reader.mode("train") as reader:
 
             image_writer.start_sampling()
@@ -110,14 +113,14 @@ class StorageTest(unittest.TestCase):
             for i in range(10):
                 scalar.add_record(i, float(i))
 
-        self.reader = storage.LogReader(self.dir)
+        self.reader = LogReader(self.dir)
         with self.reader.mode("train") as reader:
             scalar = reader.scalar("model/scalar/average")
             self.assertEqual(scalar.caption(), "train")
 
     def test_modes(self):
         dir = "./tmp/storagetest0"
-        store = storage.LogWriter(
+        store = LogWriter(
             self.dir, sync_cycle=1)
 
         scalars = []

visualdl/server/mock/data.py

+def sequence_data():
+    return [[1465980660.726887, 1, 0.07000000029802322],
+            [1465980660.767164, 2, 0.18000000715255737],
+            [1465980660.799817, 3, 0.3199999928474426],
+            [1465980660.831853, 4, 0.33000001311302185],
+            [1465980660.86555, 5, 0.3400000035762787],
+            [1465980660.898716, 6, 0.6000000238418579],
+            [1465980660.930804, 7, 0.6299999952316284],
+            [1465980660.963156, 8, 0.6299999952316284],
+            [1465980660.995727, 9, 0.6299999952316284],
+            [1465980661.603699, 11, 0.75],
+            [1465980661.641232, 12, 0.7900000214576721],
+            [1465980661.674311, 13, 0.7099999785423279],
+            [1465980661.704281, 14, 0.7099999785423279],
+            [1465980661.737259, 15, 0.7200000286102295],
+            [1465980661.768047, 16, 0.75],
+            [1465980661.801236, 17, 0.8600000143051147],
+            [1465980661.832749, 18, 0.7799999713897705],
+            [1465980661.862822, 19, 0.8399999737739563],
+            [1465980662.481258, 21, 0.699999988079071],
+            [1465980662.521448, 22, 0.6700000166893005],
+            [1465980662.557197, 23, 0.7699999809265137],
+            [1465980662.593898, 24, 0.7900000214576721],
+            [1465980662.629991, 25, 0.7200000286102295],
+            [1465980662.671198, 26, 0.8100000023841858],
+            [1465980662.711186, 27, 0.7599999904632568],
+            [1465980662.750267, 28, 0.7799999713897705],
+            [1465980662.791909, 29, 0.8299999833106995],
+            [1465980663.47027, 31, 0.8100000023841858],
+            [1465980663.538732, 32, 0.8500000238418579],
+            [1465980663.57077, 33, 0.8600000143051147],
+            [1465980663.60126, 34, 0.8199999928474426],
+            [1465980663.631059, 35, 0.7900000214576721],
+            [1465980663.665972, 36, 0.7799999713897705],
+            [1465980663.697275, 37, 0.9100000262260437],
+            [1465980663.726395, 38, 0.8700000047683716],
+            [1465980663.760169, 39, 0.9200000166893005],
+            [1465980664.45205, 41, 0.8299999833106995],
+            [1465980664.484207, 42, 0.7599999904632568],
+            [1465980664.515375, 43, 0.7699999809265137],
+            [1465980664.547608, 44, 0.8299999833106995],
+            [1465980664.580122, 45, 0.949999988079071],
+            [1465980664.611019, 46, 0.8999999761581421],
+            [1465980664.642956, 47, 0.8700000047683716],
+            [1465980664.674636, 48, 0.8500000238418579],
+            [1465980664.705622, 49, 0.8899999856948853],
+            [1465980665.379549, 51, 0.8399999737739563],
+            [1465980665.422869, 52, 0.8500000238418579],
+            [1465980665.466136, 53, 0.8199999928474426],
+            [1465980665.508361, 54, 0.9300000071525574],
+            [1465980665.544331, 55, 0.9399999976158142],
+            [1465980665.589887, 56, 0.8700000047683716],
+            [1465980665.633466, 57, 0.9300000071525574],
+            [1465980665.674978, 58, 0.7799999713897705],
+            [1465980665.716878, 59, 0.9300000071525574],
+            [1465980666.653456, 61, 0.8799999952316284],
+            [1465980666.697294, 62, 0.9300000071525574],
+            [1465980666.742066, 63, 0.8700000047683716],
+            [1465980666.780127, 64, 0.8299999833106995],
+            [1465980666.818287, 65, 0.9200000166893005],
+            [1465980666.855386, 66, 0.9399999976158142],
+            [1465980666.897352, 67, 0.9300000071525574],
+            [1465980666.931322, 68, 0.8899999856948853],
+            [1465980666.96562, 69, 0.8600000143051147],
+            [1465980667.619625, 71, 0.8700000047683716],
+            [1465980667.655166, 72, 0.9200000166893005], [
+                1465980667.687101, 73, 0.8199999928474426
+            ], [1465980667.720176, 74, 0.8100000023841858],
+            [1465980667.751985, 75, 0.8500000238418579], [
+                1465980667.785244, 76, 0.8600000143051147
+            ], [1465980667.820445, 77, 0.9200000166893005], [
+                1465980667.857163, 78, 0.8899999856948853
+            ], [1465980667.891868, 79, 0.8999999761581421], [
+                1465980668.56409, 81, 0.8500000238418579
+            ], [1465980668.599529, 82, 0.8299999833106995], [
+                1465980668.630751, 83, 0.8500000238418579
+            ], [1465980668.665135, 84, 0.8199999928474426], [
+                1465980668.697928, 85, 0.8199999928474426
+            ], [1465980668.730525, 86, 0.8799999952316284], [
+                1465980668.769772, 87, 0.9200000166893005
+            ], [1465980668.803344, 88, 0.8299999833106995], [
+                1465980668.834414, 89, 0.800000011920929
+            ], [1465980669.814826, 91, 0.8600000143051147], [
+                1465980669.851511, 92, 0.8899999856948853
+            ], [1465980669.891407, 93, 0.8799999952316284], [
+                1465980669.927507, 94, 0.9399999976158142
+            ], [1465980669.968384, 95, 0.9300000071525574], [
+                1465980670.007071, 96, 0.8500000238418579
+            ], [1465980670.044314, 97, 0.8500000238418579], [
+                1465980670.083472, 98, 0.9100000262260437
+            ], [1465980670.214597, 99, 0.8600000143051147], [
+                1465980670.934513, 101, 0.8799999952316284
+            ], [1465980670.971317, 102, 0.8700000047683716], [
+                1465980671.003626, 103, 0.8600000143051147
+            ], [1465980671.037037, 104, 0.8399999737739563], [
+                1465980671.070037, 105, 0.9200000166893005
+            ], [1465980671.104992, 106, 0.8600000143051147], [
+                1465980671.137882, 107, 0.8100000023841858
+            ], [1465980671.173917, 108, 0.7400000095367432], [
+                1465980671.205898, 109, 0.8799999952316284
+            ], [1465980671.833723, 111, 0.9100000262260437]]
+
+
+def graph_data():
+        return """{
+    "title": {
+        "text": "MLP"
+    },
+    "tooltip": {},
+    "animationDurationUpdate": 1500,
+    "animationEasingUpdate": "quinticInOut",
+    "series": [
+        {
+            "type": "graph",
+            "layout": "none",
+            "symbolSize": 50,
+            "roam": true,
+            "label": {
+                "normal": {
+                    "show": true
+                }
+            },
+            "edgeSymbol": [
+                "circle",
+                "arrow"
+            ],
+            "edgeSymbolSize": [
+                4,
+                10
+            ],
+            "edgeLabel": {
+                "normal": {
+                    "textStyle": {
+                        "fontSize": 20
+                    }
+                }
+            },
+            "data": [
+                {
+                    "name": "X",
+                    "x": 450,
+                    "y": 600
+                },
+                {
+                    "name": "W1",
+                    "x": 750,
+                    "y": 600
+                },
+                {
+                    "name": "B1",
+                    "x": 650,
+                    "y": 600
+                },
+                {
+                    "name": "W2",
+                    "x": 650,
+                    "y": 800
+                },
+                {
+                    "name": "B2",
+                    "x": 750,
+                    "y": 800
+                },
+                {
+                    "name": "FC1",
+                    "x": 550,
+                    "y": 700
+                },
+                {
+                    "name": "RELU",
+                    "x": 550,
+                    "y": 800
+                },
+                {
+                    "name": "FC2",
+                    "x": 550,
+                    "y": 900
+                },
+                {
+                    "name": "Y",
+                    "x": 550,
+                    "y": 1000
+                }
+            ],
+            "links": [
+                {
+                    "source": "X",
+                    "target": "FC1"
+                },
+                {
+                    "source": "W1",
+                    "target": "FC1"
+                },
+                {
+                    "source": "B1",
+                    "target": "FC1"
+                },
+                {
+                    "source": "FC1",
+                    "target": "RELU"
+                },
+                {
+                    "source": "RELU",
+                    "target": "FC2"
+                },
+                {
+                    "source": "W2",
+                    "target": "FC2"
+                },
+                {
+                    "source": "B2",
+                    "target": "FC2"
+                },
+                {
+                    "source": "FC2",
+                    "target": "Y"
+                }
+            ],
+            "lineStyle": {
+                "normal": {
+                    "opacity": 0.9,
+                    "width": 2,
+                    "curveness": 0
+                }
+            }
+        }
+    ]
+}"""
\ No newline at end of file

server/visualdl/visual_dl.py → visualdl/server/visualDL

+#! /user/bin/env python
+
 import json
 import os
 import re
@@ -7,12 +9,14 @@ from optparse import OptionParser
 from flask import (Flask, Response, redirect, request, send_file,
                    send_from_directory)
 
-import graph
-import lib
-import storage
-import visualdl.mock.data as mock_data
-import visualdl.mock.tags as mock_tags
-from visualdl.log import logger
+import visualdl.server
+from visualdl.server import graph, lib
+from visualdl.server import log as logger
+from visualdl.server.mock import data as mock_data
+from visualdl.server.mock import data as mock_tags
+from visualdl.python.storage import (LogWriter, LogReader)
+
+
 
 app = Flask(__name__, static_url_path="")
 # set static expires in a short time to reduce browser's memory usage.
@@ -50,7 +54,7 @@ server_path = os.path.abspath(os.path.dirname(sys.argv[0]))
 static_file_path = "./frontend/dist/"
 mock_data_path = "./mock_data/"
 
-log_reader = storage.LogReader(options.logdir)
+log_reader = LogReader(options.logdir)
 
 
 # return data
@@ -178,4 +182,8 @@ def graph():
 
 if __name__ == '__main__':
     logger.info(" port=" + str(options.port))
+    if not options.logdir:
+        logger.error("should pass in logdir option")
+        sys.exit(-1)
+
     app.run(debug=False, host=options.host, port=options.port)