Merge branch 'develop' of github.com:PaddlePaddle/VisualDL into feature/support_pytorch_demo

.gitignore

@@ -106,3 +106,7 @@ ENV/
 node_modules
 /.vscode
 package-lock.json
+
+# PyCharm IDE
+.idea/
+

CMakeLists.txt

@@ -46,7 +46,7 @@ add_executable(vl_test
         ${PROJECT_SOURCE_DIR}/visualdl/utils/concurrency.h
         ${PROJECT_SOURCE_DIR}/visualdl/utils/filesystem.h
         )
-target_link_libraries(vl_test sdk storage entry tablet im gtest glog protobuf gflags pthread)
+target_link_libraries(vl_test sdk storage entry tablet im gtest glog protobuf gflags pthread eigen3)
 
 enable_testing ()
 

server/build.sh

+#!/bin/bash
+
 script=$(readlink -f "$0")
 script_path=$(dirname "$script")
 
 pushd $script_path
-  protoc visualdl/onnx/onnx.proto --python_out .
+  protoc3/bin/protoc visualdl/onnx/onnx.proto --python_out .
   python setup.py bdist_wheel
 popd

server/visualdl/graph.py

@@ -31,6 +31,8 @@ def reorganize_inout(json_obj, key):
 
 
 def add_edges(json_obj):
+    # TODO(daming-lu): should try to de-duplicate node's out-edge
+    # Currently it is counted twice: 1 as out-edge, 1 as in-edge
     json_obj['edges'] = []
     label_incrementer = 0
 

server/visualdl/graph_test.py

+import unittest
+import graph
+import json
+
+
+class GraphTest(unittest.TestCase):
+    def setUp(self):
+        self.mock_dir = "./mock"
+
+    def test_graph_edges_squeezenet(self):
+        json_str = graph.load_model(self.mock_dir + '/squeezenet_model.pb')
+        json_obj = json.loads(json_str)
+
+        # 126 edges + 66 nodes (out-edge of each node is counted twice)
+        self.assertEqual(len(json_obj['edges']), 126 + 66)
+
+        # label_0: (in-edge)
+        # {u'source': u'data_0', u'target': u'node_0', u'label': u'label_0'}
+        self.assertEqual(json_obj['edges'][0]['source'], 'data_0')
+        self.assertEqual(json_obj['edges'][0]['target'], 'node_0')
+        self.assertEqual(json_obj['edges'][0]['label'], 'label_0')
+
+        # label_50: (in-edge)
+        # {u'source': u'fire3/concat_1', u'target': u'node_17', u'label': u'label_50'}
+        self.assertEqual(json_obj['edges'][50]['source'], 'fire3/concat_1')
+        self.assertEqual(json_obj['edges'][50]['target'], 'node_17')
+        self.assertEqual(json_obj['edges'][50]['label'], 'label_50')
+
+        # label_100: (in-edge)
+        # {u'source': u'fire6/squeeze1x1_1', u'target': u'node_34', u'label': u'label_100'}
+        self.assertEqual(json_obj['edges'][100]['source'], 'fire6/squeeze1x1_1')
+        self.assertEqual(json_obj['edges'][100]['target'], 'node_34')
+        self.assertEqual(json_obj['edges'][100]['label'], 'label_100')
+
+        # label_111: (out-edge)
+        # {u'source': u'node_37', u'target': u'fire6/expand3x3_1', u'label': u'label_111'}
+        self.assertEqual(json_obj['edges'][111]['source'], 'node_37')
+        self.assertEqual(json_obj['edges'][111]['target'], 'fire6/expand3x3_1')
+        self.assertEqual(json_obj['edges'][111]['label'], 'label_111')
+
+    def test_graph_edges_inception_v1(self):
+        json_str = graph.load_model(self.mock_dir + '/inception_v1_model.pb')
+        json_obj = json.loads(json_str)
+
+        # 286 edges + 143 nodes (out-edge of each node is counted twice)
+        self.assertEqual(len(json_obj['edges']), 286 + 143)
+
+        # label_0: (in-edge)
+        # {u'source': u'data_0', u'target': u'node_0', u'label': u'label_0'}
+        self.assertEqual(json_obj['edges'][0]['source'], 'data_0')
+        self.assertEqual(json_obj['edges'][0]['target'], 'node_0')
+        self.assertEqual(json_obj['edges'][0]['label'], 'label_0')
+
+        # label_50: (in-edge)
+        # {u'source': u'inception_3a/5x5_reduce_2', u'target': u'node_18', u'label': u'label_50'}
+        self.assertEqual(json_obj['edges'][50]['source'], 'inception_3a/5x5_reduce_2')
+        self.assertEqual(json_obj['edges'][50]['target'], 'node_18')
+        self.assertEqual(json_obj['edges'][50]['label'], 'label_50')
+
+        # label_100: (out-edge)
+        # {u'source': u'node_34', u'target': u'inception_3b/pool_1', u'label': u'label_100'}
+        self.assertEqual(json_obj['edges'][100]['source'], 'node_34')
+        self.assertEqual(json_obj['edges'][100]['target'], 'inception_3b/pool_1')
+        self.assertEqual(json_obj['edges'][100]['label'], 'label_100')
+
+        # label_420: (out-edge)
+        # {u'source': u'node_139', u'target': u'pool5/7x7_s1_2', u'label': u'label_420'}
+        self.assertEqual(json_obj['edges'][420]['source'], 'node_139')
+        self.assertEqual(json_obj['edges'][420]['target'], 'pool5/7x7_s1_2')
+        self.assertEqual(json_obj['edges'][420]['label'], 'label_420')
+
+
+if __name__ == '__main__':
+    unittest.main()

server/visualdl/graph_test.sh

+#!/bin/bash
+set -ex
+
+cd mock
+bash download_mock_models.sh
+
+cd ..
+
+python graph_test.py
+
+rm ./mock/*.pb
+

server/visualdl/mock/download_mock_models.sh

 
 # Download inception_v1 model
 curl -LOk http://visualdl.bj.bcebos.com/inception_v1.tar.gz
-
+#curl -LOk https://www.dropbox.com/s/twbfdqgvowzy762/inception_v1.tar.gz?dl=0
 tar -xvzf inception_v1.tar.gz
 cp inception_v1/model.pb inception_v1_model.pb
 
@@ -11,7 +11,7 @@ rm inception_v1.tar.gz
 
 # Download squeezenet model
 curl -LOk http://visualdl.bj.bcebos.com/squeezenet.tar.gz
-
+#curl -LOk https://www.dropbox.com/s/fip3jzxsjf2g6zc/squeezenet.tar.gz?dl=0
 tar -xvzf squeezenet.tar.gz
 cp squeezenet/model.pb squeezenet_model.pb
 

server/visualdl/visual_dl.py

@@ -50,7 +50,7 @@ server_path = os.path.abspath(os.path.dirname(sys.argv[0]))
 static_file_path = "./frontend/dist/"
 mock_data_path = "./mock_data/"
 
-storage = storage.LogReader(options.logdir)
+log_reader = storage.LogReader(options.logdir)
 
 
 # return data
@@ -87,7 +87,7 @@ def logdir():
 
 @app.route('/data/runs')
 def runs():
-    result = gen_result(0, "", lib.get_modes(storage))
+    result = gen_result(0, "", lib.get_modes(log_reader))
     return Response(json.dumps(result), mimetype='application/json')
 
 
@@ -98,7 +98,7 @@ def scalar_tags():
     if is_debug:
         result = mock_tags.data()
     else:
-        result = lib.get_scalar_tags(storage, mode)
+        result = lib.get_scalar_tags(log_reader, mode)
     print 'scalar tags (mode: %s)' % mode, result
     result = gen_result(0, "", result)
     return Response(json.dumps(result), mimetype='application/json')
@@ -107,7 +107,7 @@ def scalar_tags():
 @app.route("/data/plugin/images/tags")
 def image_tags():
     mode = request.args.get('run')
-    result = lib.get_image_tags(storage)
+    result = lib.get_image_tags(log_reader)
     print 'image tags (mode: %s)'%mode, result
     result = gen_result(0, "", result)
     return Response(json.dumps(result), mimetype='application/json')
@@ -121,7 +121,7 @@ def scalars():
     if is_debug:
         result = mock_data.sequence_data()
     else:
-        result = lib.get_scalar(storage, run, tag)
+        result = lib.get_scalar(log_reader, run, tag)
 
     result = gen_result(0, "", result)
     return Response(json.dumps(result), mimetype='application/json')
@@ -132,7 +132,7 @@ def images():
     mode = request.args.get('run')
     tag = request.args.get('tag')
 
-    result = lib.get_image_tag_steps(storage, mode, tag)
+    result = lib.get_image_tag_steps(log_reader, mode, tag)
     result = gen_result(0, "", result)
 
     return Response(json.dumps(result), mimetype='application/json')
@@ -145,7 +145,7 @@ def individual_image():
     step_index = int(request.args.get('index'))  # index of step
     offset = 0
 
-    imagefile = lib.get_invididual_image(storage, mode, tag, step_index)
+    imagefile = lib.get_invididual_image(log_reader, mode, tag, step_index)
     response = send_file(
         imagefile, as_attachment=True, attachment_filename='img.png')
     return response

tests.sh

@@ -27,8 +27,21 @@ frontend_test() {
 }
 
 server_test() {
+    sudo pip install google
+    sudo pip install protobuf==3.1.0
+
     cd $cur/server
+    curl -OL https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip
+    unzip protoc-3.1.0-linux-x86_64.zip -d protoc3
+    export PATH=$PATH:protoc3/bin
+    sudo chmod +x protoc3/bin/protoc
+    sudo chown `whoami` protoc3/bin/protoc
+
     bash build.sh
+
+    cd visualdl
+    bash graph_test.sh
+
     cd $cur/server/visualdl
     python lib_test.py
 }

visualdl/logic/im.cc

@@ -36,6 +36,7 @@ template class SimpleWriteSyncGuard<Entry<float>>;
 template class SimpleWriteSyncGuard<Entry<double>>;
 template class SimpleWriteSyncGuard<Entry<bool>>;
 template class SimpleWriteSyncGuard<Entry<long>>;
+template class SimpleWriteSyncGuard<Entry<long long>>;
 template class SimpleWriteSyncGuard<Entry<std::string>>;
 template class SimpleWriteSyncGuard<Entry<std::vector<byte_t>>>;
 template class SimpleWriteSyncGuard<Entry<int>>;

visualdl/logic/sdk.cc

@@ -10,7 +10,7 @@ template <typename T>
 std::vector<T> ScalarReader<T>::records() const {
   std::vector<T> res;
   for (int i = 0; i < reader_.total_records(); i++) {
-    res.push_back(reader_.record(i).data<T>(0).Get());
+    res.push_back(reader_.record(i).template data<T>(0).Get());
   }
   return res;
 }

visualdl/logic/sdk.h

@@ -132,7 +132,7 @@ struct Scalar {
     record.SetId(id);
     time_t time = std::time(nullptr);
     record.SetTimeStamp(time);
-    auto entry = record.AddData<T>();
+    auto entry = record.template AddData<T>();
     entry.Set(value);
   }
 

visualdl/logic/sdk_test.cc

@@ -34,7 +34,7 @@ TEST(Scalar, write) {
   ASSERT_TRUE(!reader.storage().modes().empty());
 
   // check tags
-  ASSERT_EQ(reader.all_tags().size(), 1);
+  ASSERT_EQ(reader_.all_tags().size(), 1);
   auto tags = reader.tags("scalar");
   ASSERT_EQ(tags.size(), 2);
   ASSERT_EQ(tags.front(), "scalar0");

visualdl/utils/image.h

@@ -67,9 +67,6 @@ static void NormalizeImage(Uint8Image* image,
     scale = (image_max < kZeroThreshold ? 0.0f : 255.0f) / image_max;
     offset = 0.0f;
   }
-
-  // LOG(INFO) << "scale " << scale;
-
   // Transform image, turning nonfinite values to bad_color
   for (int i = 0; i < depth; i++) {
     auto tmp = scale * values.row(i).array() + offset;