add try call to enhance robust of server (#133)

visualdl/server/graph.py

-import os
 import json
+import os
 
 from google.protobuf.json_format import MessageToJson
+from PIL import Image
 
-import onnx
 import graphviz_graph as gg
-from PIL import Image
+import onnx
 
 
 def debug_print(json_obj):

visualdl/server/graph_test.py

+import json
 import unittest
+
 import graph
-import json
 
 
 class GraphTest(unittest.TestCase):

visualdl/server/graphviz_graph.py

-import subprocess
-import tempfile
-import sys
+import os
 import random
 import subprocess
-import os
+import sys
+import tempfile
 
 
 def crepr(v):

visualdl/server/lib.py

 import pprint
-import random
 import re
+import time
 import urllib
 from tempfile import NamedTemporaryFile
 
@@ -133,7 +133,7 @@ def get_invididual_image(storage, mode, tag, step_index, max_size=80):
         shape = record.shape()
 
         if shape[2] == 1:
-          shape = [shape[0], shape[1]]
+            shape = [shape[0], shape[1]]
         data = np.array(record.data(), dtype='uint8').reshape(shape)
         tempfile = NamedTemporaryFile(mode='w+b', suffix='.png')
         with Image.fromarray(data) as im:
@@ -174,12 +174,13 @@ def get_histogram(storage, mode, tag):
         return res
 
 
-if __name__ == '__main__':
-    reader = storage.LogReader('./tmp/mock')
-    tags = get_image_tags(reader)
-
-    tags = get_image_tag_steps(reader, 'train', 'layer1/layer2/image0/0')
-    pprint.pprint(tags)
-
-    image = get_invididual_image(reader, "train", 'layer1/layer2/image0/0', 2)
-    print image
+def retry(ntimes, function, time2sleep, *args, **kwargs):
+    '''
+    try to execute `function` `ntimes`, if exception catched, the thread will
+    sleep `time2sleep` seconds.
+    '''
+    for i in xrange(ntimes):
+        try:
+            return function(*args, **kwargs)
+        except:
+            time.sleep(time2sleep)

visualdl/server/lib_test.py

 import pprint
 import unittest
 
-import lib
 from visualdl import LogReader, LogWriter
 
-import pprint
-from storage_mock import add_scalar, add_image, add_histogram
+import lib
+from storage_mock import add_histogram, add_image, add_scalar
 
+_retry_counter = 0
 
 class LibTest(unittest.TestCase):
     def setUp(self):
         dir = "./tmp/mock"
-        writer = LogWriter(dir, sync_cycle=10)
+        writer = LogWriter(dir, sync_cycle=30)
 
         add_scalar(writer, "train", "layer/scalar0/min", 1000, 1)
         add_scalar(writer, "test", "layer/scalar0/min", 1000, 10)
@@ -32,6 +32,21 @@ class LibTest(unittest.TestCase):
 
         self.reader = LogReader(dir)
 
+    def test_retry(self):
+        ntimes = 7
+        time2sleep = 1
+
+        def func():
+            global _retry_counter
+            if _retry_counter < 5:
+                _retry_counter += 1
+                raise
+            return _retry_counter
+
+        lib.retry(ntimes, func, time2sleep)
+
+        self.assertEqual(_retry_counter, 5)
+
     def test_modes(self):
         modes = lib.get_modes(self.reader)
         self.assertEqual(

visualdl/server/mock/data.py

@@ -225,4 +225,4 @@ def graph_data():
             }
         }
     ]
-}"""
\ No newline at end of file
+}"""

visualdl/server/visualDL

@@ -2,6 +2,8 @@
 
 import json
 import os
+import exceptions
+import time
 import sys
 from optparse import OptionParser
 
@@ -21,9 +23,20 @@ app = Flask(__name__, static_url_path="")
 # set static expires in a short time to reduce browser's memory usage.
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 30
 
+error_retry_times = 3
+error_sleep_time = 2  # seconds
+
 SERVER_DIR = os.path.join(visualdl.ROOT, 'server')
 
 
+def try_call(function, *args, **kwargs):
+    res = lib.retry(error_retry_times, function, error_sleep_time, *args,
+                    **kwargs)
+    if not res:
+        raise exceptions.IOError("server IO error, will retry latter.")
+    return res
+
+
 def option_parser():
     """
     :return:
@@ -64,6 +77,7 @@ log_reader = LogReader(options.logdir)
 
 graph_image_path = None
 
+
 # return data
 # status, msg, data
 def gen_result(status, msg, data):
@@ -89,11 +103,12 @@ def serve_static(filename):
     return send_from_directory(
         os.path.join(server_path, static_file_path), filename)
 
+
 @app.route('/graphs/image')
 def serve_graph():
-    print 'send file', graph_image_path
     return send_file(graph_image_path)
 
+
 @app.route('/data/logdir')
 def logdir():
     result = gen_result(0, "", {"logdir": options.logdir})
@@ -110,10 +125,7 @@ def runs():
 def scalar_tags():
     mode = request.args.get('mode')
     is_debug = bool(request.args.get('debug'))
-    if is_debug:
-        result = mock_tags.data()
-    else:
-        result = lib.get_scalar_tags(log_reader)
+    result = try_call(lib.get_scalar_tags, log_reader)
     result = gen_result(0, "", result)
     return Response(json.dumps(result), mimetype='application/json')
 
@@ -121,7 +133,7 @@ def scalar_tags():
 @app.route("/data/plugin/images/tags")
 def image_tags():
     mode = request.args.get('run')
-    result = lib.get_image_tags(log_reader)
+    result = try_call(lib.get_image_tags, log_reader)
     result = gen_result(0, "", result)
     return Response(json.dumps(result), mimetype='application/json')
 
@@ -129,7 +141,8 @@ def image_tags():
 @app.route("/data/plugin/histograms/tags")
 def histogram_tags():
     mode = request.args.get('run')
-    result = lib.get_histogram_tags(log_reader)
+    # hack to avlid IO conflicts
+    result = try_call(lib.get_histogram_tags, log_reader)
     result = gen_result(0, "", result)
     return Response(json.dumps(result), mimetype='application/json')
 
@@ -138,12 +151,7 @@ def histogram_tags():
 def scalars():
     run = request.args.get('run')
     tag = request.args.get('tag')
-    is_debug = bool(request.args.get('debug'))
-    if is_debug:
-        result = mock_data.sequence_data()
-    else:
-        result = lib.get_scalar(log_reader, run, tag)
-
+    result = try_call(lib.get_scalar, log_reader, run, tag)
     result = gen_result(0, "", result)
     return Response(json.dumps(result), mimetype='application/json')
 
@@ -153,7 +161,7 @@ def images():
     mode = request.args.get('run')
     tag = request.args.get('tag')
 
-    result = lib.get_image_tag_steps(log_reader, mode, tag)
+    result = try_call(lib.get_image_tag_steps, log_reader, mode, tag)
     result = gen_result(0, "", result)
 
     return Response(json.dumps(result), mimetype='application/json')
@@ -166,7 +174,8 @@ def individual_image():
     step_index = int(request.args.get('index'))  # index of step
     offset = 0
 
-    imagefile = lib.get_invididual_image(log_reader, mode, tag, step_index)
+    imagefile = try_call(lib.get_invididual_image, log_reader, mode, tag,
+                         step_index)
     response = send_file(
         imagefile, as_attachment=True, attachment_filename='img.png')
     return response
@@ -176,7 +185,7 @@ def individual_image():
 def histogram():
     run = request.args.get('run')
     tag = request.args.get('tag')
-    result = lib.get_histogram(log_reader, run, tag)
+    result = try_call(lib.get_histogram, log_reader, run, tag)
     result = gen_result(0, "", result)
     return Response(json.dumps(result), mimetype='application/json')