Merge branch 'feature/add_image_component-debug_image' into feature/add_image_component

server/visualdl/lib.py

@@ -51,9 +51,9 @@ def get_image_tags(storage):
             for tag in tags:
                 image = reader.image(tag)
                 for i in xrange(max(1, image.num_samples())):
-                    tag = image.caption() if image.num_samples() <= 1 else '%s/%d'%(tag, i)
-                    result[mode][tag] = {
-                        'displayName': tag,
+                    caption = tag if image.num_samples() <= 1 else '%s/%d'%(tag, i)
+                    result[mode][caption] = {
+                        'displayName': caption,
                         'description': "",
                         'samples': 1,
                     }
@@ -61,6 +61,7 @@ def get_image_tags(storage):
 
 
 def get_image_tag_steps(storage, mode, tag):
+    print 'image_tag_steps,mode,tag:', mode, tag
     # remove suffix '/x'
     res = re.search(r".*/([0-9]+$)", tag)
     sample_index = 0
@@ -76,6 +77,7 @@ def get_image_tag_steps(storage, mode, tag):
     for step_index in range(image.num_records()):
         record = image.record(step_index, sample_index)
         shape = record.shape()
+        assert shape, "%s,%s" % (mode, tag)
         query = urllib.urlencode({
             'sample': 0,
             'index': step_index,

server/visualdl/lib_test.py

+import lib
+import unittest
+import storage
+import pprint
+from storage_mock import add_scalar, add_image
+
+
+class LibTest(unittest.TestCase):
+    def setUp(self):
+        dir = "./tmp/mock"
+        writer = storage.StorageWriter(dir, sync_cycle=20)
+
+        add_scalar(writer, "train", "layer/scalar0/min", 1000, 1)
+        add_scalar(writer, "test", "layer/scalar0/min", 1000, 10)
+        add_scalar(writer, "valid", "layer/scalar0/min", 1000, 10)
+
+        add_scalar(writer, "train", "layer/scalar0/max", 1000, 1)
+        add_scalar(writer, "test", "layer/scalar0/max", 1000, 10)
+        add_scalar(writer, "valid", "layer/scalar0/max", 1000, 10)
+
+        add_image(writer, "train", "layer/image0", 7, 10, 1)
+        add_image(writer, "test", "layer/image0", 7, 10, 3)
+
+        self.reader = storage.StorageReader(dir)
+
+    def test_modes(self):
+        modes = lib.get_modes(self.reader)
+        self.assertEqual(sorted(modes), sorted(["train", "test", "valid"]))
+
+    def test_scalar(self):
+
+        for mode in "train test valid".split():
+            tags = lib.get_scalar_tags(self.reader, mode)
+            print 'scalar tags:'
+            pprint.pprint(tags)
+            self.assertEqual(len(tags), 3)
+            self.assertEqual(sorted(tags.keys()), sorted("train test valid".split()))
+
+    def test_image(self):
+        tags = lib.get_image_tags(self.reader)
+        self.assertEqual(len(tags), 2)
+
+        tags = lib.get_image_tag_steps(self.reader, 'train', 'layer/image0/0')
+        pprint.pprint(tags)
+
+        image = lib.get_invididual_image(self.reader, "train", 'layer/image0/0', 2)
+        print image
+
+
+if __name__ == '__main__':
+    unittest.main()

server/visualdl/mock.sh

@@ -3,4 +3,4 @@ set -ex
 
 export PYTHONPATH="/home/superjom/project/VisualDL/build/visualdl/logic:/home/superjom/project/VisualDL/visualdl/python"
 
-python storage_mock.py
+python lib_test.py

server/visualdl/storage_mock.py

@@ -26,11 +26,12 @@ def add_image(writer,
     for pass_ in xrange(num_passes):
         image_writer.start_sampling()
         for ins in xrange(2 * num_samples):
-            print '.',
             index = image_writer.is_sample_taken()
             if index != -1:
                 data = np.random.random(shape) * 256
                 data = np.ndarray.flatten(data)
+                assert shape
+                assert len(data) > 0
                 image_writer.set_sample(index, shape, list(data))
         image_writer.finish_sampling()
 

server/visualdl/visual_dl.py

@@ -89,7 +89,7 @@ def logdir():
 @app.route('/data/runs')
 def runs():
     modes = storage.modes()
-    result = gen_result(0, "", lib.modes())
+    result = gen_result(0, "", lib.get_modes())
     return Response(json.dumps(result), mimetype='application/json')
 
 

tests.sh

@@ -26,6 +26,8 @@ frontend_test() {
 }
 
 server_test() {
+    cd $cur/server
+    bash build.sh
     cd $cur/server/visualdl
     python lib_test.py
 }

visualdl/logic/sdk.cc

@@ -48,9 +48,8 @@ template class ScalarReader<float>;
 template class ScalarReader<double>;
 
 void Image::StartSampling() {
-  // TODO(ChunweiYan) big bug here, every step will be stored in protobuf
-  // and that might result in explosion in some scenerios, Just sampling
-  // some steps should be better.
+  if (!ToSampleThisStep()) return;
+
   step_ = writer_.AddRecord();
   step_.SetId(step_id_);
 
@@ -103,6 +102,7 @@ void Image::SetSample(int index,
   // production
   int size = std::accumulate(
       shape.begin(), shape.end(), 1., [](float a, float b) { return a * b; });
+  CHECK_GT(size, 0);
   CHECK_EQ(size, data.size()) << "image's shape not match data";
   CHECK_LT(index, num_samples_);
   CHECK_LE(index, num_records_);