Merge branch 'feature/add_image_component-add_server_test' into feature/add_image_component

.travis.yml

 language: cpp
 cache:
-  directories:
-    - $HOME/.ccache
-    - $HOME/.cache/pip
+  - pip
+  - ccache
+  - yarn
 sudo: required
 dist: trusty
 os:
@@ -19,8 +19,11 @@ addons:
       - python-wheel
       - clang-format-3.8
       - ccache
+      - npm
+      - nodejs
+
 script:
-  /bin/bash ./tests.sh
+  /bin/bash ./tests.sh all
 
 notifications:
   email:

server/visualdl/lib.py

@@ -9,17 +9,22 @@ from PIL import Image
 import storage
 
 
+def get_modes(storage):
+    return storage.modes()
+
+
 def get_scalar_tags(storage, mode):
     result = {}
-    print 'modes', storage.modes()
     for mode in storage.modes():
-        result[mode] = {}
         reader = storage.as_mode(mode)
-        for tag in reader.tags('scalar'):
-            result[mode][tag] = {
-                'displayName': reader.scalar(tag).caption(),
-                'description': "",
-            }
+        tags = reader.tags('scalar')
+        if tags:
+            result[mode] = {}
+            for tag in tags:
+                result[mode][tag] = {
+                    'displayName': reader.scalar(tag).caption(),
+                    'description': "",
+                }
     return result
 
 
@@ -35,27 +40,20 @@ def get_scalar(storage, mode, tag):
     return result
 
 
-def get_image_tags(storage, mode):
+def get_image_tags(storage):
     result = {}
 
-    print 'modes', storage.modes()
     for mode in storage.modes():
         reader = storage.as_mode(mode)
-        print 'tags', reader.tags('image')
-        result[mode] = {}
-        for tag in reader.tags('image'):
-            image = reader.image(tag)
-            if image.num_samples() <= 1:
-                result[mode][tag] = {
-                    'displayName': image.caption(),
-                    'description': "",
-                    'samples': 1,
-                }
-            else:
-                for i in xrange(image.num_samples()):
-                    caption = tag + '/%d' % i
-                    result[mode][caption] = {
-                        'displayName': caption,
+        tags = reader.tags('image')
+        if tags:
+            result[mode] = {}
+            for tag in tags:
+                image = reader.image(tag)
+                for i in xrange(max(1, image.num_samples())):
+                    tag = image.caption() if image.num_samples() <= 1 else '%s/%d'%(tag, i)
+                    result[mode][tag] = {
+                        'displayName': tag,
                         'description': "",
                         'samples': 1,
                     }
@@ -75,7 +73,6 @@ def get_image_tag_steps(storage, mode, tag):
     image = reader.image(tag)
     res = []
 
-
     for step_index in range(image.num_records()):
         record = image.record(step_index, sample_index)
         shape = record.shape()
@@ -116,7 +113,7 @@ def get_invididual_image(storage, mode, tag, step_index):
 
 if __name__ == '__main__':
     reader = storage.StorageReader('./tmp/mock')
-    tags = get_image_tags(reader, 'train')
+    tags = get_image_tags(reader)
 
     tags = get_image_tag_steps(reader, 'train', 'layer1/layer2/image0/0')
     pprint.pprint(tags)

server/visualdl/storage_mock.py

@@ -4,46 +4,45 @@ import unittest
 
 import numpy as np
 
-import storage
 
-dir = "./tmp/mock"
-writer = storage.StorageWriter(dir, sync_cycle=20)
-
-train_writer = writer.as_mode("train")
-test_writer = writer.as_mode("test")
-
-train_scalar = train_writer.scalar("model/scalar/min")
-test_scalar = test_writer.scalar("model/scalar/min")
-
-train_scalar1 = train_writer.scalar("model/scalar/max")
-test_scalar1 = test_writer.scalar("model/scalar/max")
-
-for i in range(100):
-    train_scalar.add_record(i, random.random())
-    train_scalar1.add_record(i, random.random())
-    if i % 10 == 0:
-        test_scalar.add_record(i, random.random())
-        test_scalar1.add_record(i, random.random())
-
-def add_image(mode):
+def add_scalar(writer, mode, tag, num_steps, skip):
+    my_writer = writer.as_mode(mode)
+    scalar = my_writer.scalar(tag)
+    for i in range(num_steps):
+        if i % skip == 0:
+            scalar.add_record(i, random.random())
+
+
+def add_image(writer,
+              mode,
+              tag,
+              num_samples,
+              num_passes,
+              step_cycle,
+              shape=[50, 50, 3]):
     writer_ = writer.as_mode(mode)
-    tag = "layer1/layer2/image0"
-    # TODO check step_cycle
-    num_samples = 10
-    num_passes = 20
-    image_writer = writer_.image(tag, num_samples, 1)
-    shape = [50, 50, 3]
+    image_writer = writer_.image(tag, num_samples, step_cycle)
 
     for pass_ in xrange(num_passes):
         image_writer.start_sampling()
-        for ins in xrange(2*num_samples):
+        for ins in xrange(2 * num_samples):
             print '.',
-            index =  image_writer.is_sample_taken()
+            index = image_writer.is_sample_taken()
             if index != -1:
                 data = np.random.random(shape) * 256
                 data = np.ndarray.flatten(data)
                 image_writer.set_sample(index, shape, list(data))
         image_writer.finish_sampling()
 
-add_image("train")
-add_image("test")
+
+if __name__ == '__main__':
+    add_scalar("train", "layer/scalar0/min", 1000, 1)
+    add_scalar("test", "layer/scalar0/min", 1000, 10)
+    add_scalar("valid", "layer/scalar0/min", 1000, 10)
+
+    add_scalar("train", "layer/scalar0/max", 1000, 1)
+    add_scalar("test", "layer/scalar0/max", 1000, 10)
+    add_scalar("valid", "layer/scalar0/max", 1000, 10)
+
+    add_image("train", "layer/image0", 7, 10, 1)
+    add_image("test", "layer/image0", 7, 10, 3)

server/visualdl/test.sh

-#!/bin/bash
-set -ex
-
-export PYTHONPATH="$(pwd)/..:/home/superjom/project/VisualDL/build/visualdl/logic:/home/superjom/project/VisualDL/visualdl/python"
-
-python lib.py

server/visualdl/visual_dl.py

@@ -89,7 +89,7 @@ def logdir():
 @app.route('/data/runs')
 def runs():
     modes = storage.modes()
-    result = gen_result(0, "", ["train", "test"])
+    result = gen_result(0, "", lib.modes())
     return Response(json.dumps(result), mimetype='application/json')
 
 
@@ -101,16 +101,16 @@ def scalar_tags():
         result = mock_tags.data()
     else:
         result = lib.get_scalar_tags(storage, mode)
-    print 'tags', result
+    print 'scalar tags (mode: %s)' % mode, result
     result = gen_result(0, "", result)
     return Response(json.dumps(result), mimetype='application/json')
 
 
 @app.route("/data/plugin/images/tags")
 def image_tags():
-    mode = request.args.get('mode')
-    result = lib.get_image_tags(storage, mode)
-    print 'tags', result
+    mode = request.args.get('run')
+    result = lib.get_image_tags(storage)
+    print 'image tags (mode: %s)'%mode, result
     result = gen_result(0, "", result)
     return Response(json.dumps(result), mimetype='application/json')
 
@@ -132,6 +132,7 @@ def scalars():
 @app.route('/data/plugin/images/images')
 def images():
     mode = request.args.get('run')
+    # TODO(ChunweiYan) update this when frontend fix the field name
     #tag = request.args.get('tag')
     tag = request.args.get('displayName')
 

tests.sh

 #!/bin/bash
 set -ex
 
-sudo pip install numpy
-#sudo apt-get install --only-upgrade cmake -y
-mkdir -p build
-cd build
-cmake ..
-make
-make test
-
-#if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then bash ./travis/run_on_pull_requests; fi
-#if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then bash ./travis/run_on_non_pull_requests; fi
+mode=$1
+readonly cur=$(pwd)
+readonly core_path=$cur/build/visualdl/logic
+readonly python_path=$cur/visualdl/python
+
+export PYTHONPATH="${core_path}:${python_path}"
+
+backend_test() {
+    cd $cur
+    sudo pip install numpy
+    mkdir -p build
+    cd build
+    cmake ..
+    make
+    make test
+}
+
+frontend_test() {
+    cd $cur
+    cd frontend
+    npm install
+    npm run build
+}
+
+server_test() {
+    cd $cur/server/visualdl
+    python lib_test.py
+}
+
+echo "mode" $mode
+
+if [ $mode = "backend" ]; then
+    backend_test
+elif [ $mode = "all" ]; then
+    frontend_test
+    backend_test
+    server_test
+else
+    frontend_test
+fi