Add Python 3 support (#286)

* Add Python 3 support

* Update protobuf to 3.5 and pybind to 2.2.2

* Update test scripts to use python -m

* Change zip/xrange/import usages

* Remove file generated by

* Graph viz set up (#282)

* Include the Graph requirement, GraphViz` in docs.

* Provide information to the ONNX format model export.

* Ask the user to download Graphviz via the website instead of brew.

* Make test and build scripts compatible with python 2 & 3

* A WITH_PYTHON3 option was added to cmake.

* setup.py will detect runing version python, and setup up WITH_PYTHON3 environemnt variable before calling build.sh

* build.sh and test.sh will handle WITH_PYTHON3

* Fix pybind warning

* Pass WITH_PYTHON3 correctly

* Fix coding style

* Add six for Python 2 and 3 Compatibility

* Fix coding style

.travis.yml

@@ -28,6 +28,9 @@ addons:
       - python-pip
       - python2.7-dev
       - python-wheel
+      - python3-pip
+      - python3-dev
+      - python3-wheel
       - clang-format-3.8
       - ccache
       - npm
@@ -42,7 +45,7 @@ before_install:
 
 script:
   - if [[ "$JOB" == "check_style" ]]; then ./scripts/check_style.sh; fi
-  - if [[ "$JOB" == "test" ]]; then ./scripts/tests.sh all; fi
+  - if [[ "$JOB" == "test" ]]; then ./scripts/tests.sh all; WITH_PYTHON3=ON ./scripts/tests.sh all; fi
   - |
     if [[ "$JOB" != "build_doc" ]]; then exit 0; fi;
     if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then exit 0; fi;

CMakeLists.txt

@@ -32,6 +32,7 @@ set(THIRD_PARTY_PATH "${CMAKE_BINARY_DIR}/third_party" CACHE STRING
 ################################ Configurations #######################################
 option(WITH_DOC         "Compile VisualDL with documentation"       OFF)
 option(WITH_TESTING     "Compile VisualDL with unit testing"        ON)
+option(WITH_PYTHON3     "Compile VisualDL with Python 3"            OFF)
 option(ON_RELEASE       "RELEASE mode" ON)
 
 

build.sh

@@ -36,7 +36,11 @@ build_frontend_fake() {
 
 build_backend() {
     cd $BUILD_DIR
-    cmake .. ${PYTHON_FLAGS}
+    if [[ $WITH_PYTHON3 ]]; then
+        cmake -DWITH_PYTHON3=ON .. ${PYTHON_FLAGS}
+    else
+        cmake .. ${PYTHON_FLAGS}
+    fi
     make -j2
 }
 

cmake/external/protobuf.cmake

@@ -185,7 +185,7 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
         UPDATE_COMMAND  ""
         DEPENDS         zlib
         GIT_REPOSITORY  "https://github.com/google/protobuf.git"
-        GIT_TAG         "9f75c5aa851cd877fb0d93ccc31b8567a6706546"
+        GIT_TAG         "2761122b810fe8861004ae785cc3ab39f384d342"
         CONFIGURE_COMMAND
         ${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/${TARGET_NAME}/cmake
             ${OPTIONAL_ARGS}
@@ -203,7 +203,7 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
     )
 ENDFUNCTION()
 
-SET(PROTOBUF_VERSION 3.1)
+SET(PROTOBUF_VERSION 3.5)
 IF(CMAKE_CROSSCOMPILING)
     build_protobuf(protobuf_host TRUE)
     LIST(APPEND external_project_dependencies protobuf_host)

cmake/external/pybind11.cmake

@@ -22,7 +22,7 @@ ExternalProject_Add(
         extern_pybind
         ${EXTERNAL_PROJECT_LOG_ARGS}
         GIT_REPOSITORY  "https://github.com/pybind/pybind11.git"
-        GIT_TAG         "v2.1.1"
+        GIT_TAG         "v2.2.2"
         PREFIX          ${PYBIND_SOURCE_DIR}
         UPDATE_COMMAND  ""
         CONFIGURE_COMMAND ""

cmake/external/python.cmake

@@ -14,8 +14,14 @@
 
 INCLUDE(python_module)
 
-FIND_PACKAGE(PythonInterp 2.7)
-FIND_PACKAGE(PythonLibs 2.7)
+IF(WITH_PYTHON3)
+    FIND_PACKAGE(PythonInterp 3)
+    FIND_PACKAGE(PythonLibs 3)
+ELSE()
+    FIND_PACKAGE(PythonInterp 2.7)
+    FIND_PACKAGE(PythonLibs 2.7)
+ENDIF(WITH_PYTHON3)
+
 # Fixme: Maybe find a static library. Get SHARED/STATIC by FIND_PACKAGE.
 ADD_LIBRARY(python SHARED IMPORTED GLOBAL)
 SET_PROPERTY(TARGET python PROPERTY IMPORTED_LOCATION ${PYTHON_LIBRARIES})

requirements.txt

@@ -8,3 +8,4 @@ Pillow==5.0.0
 pre-commit==1.5.1
 flask==0.12.2
 protobuf==3.5.1
+six==1.11.0

scripts/build.sh

@@ -39,7 +39,11 @@ build_frontend_fake() {
 
 build_backend() {
     cd $BUILD_DIR
-    cmake .. ${PYTHON_FLAGS}
+    if [[ $WITH_PYTHON3 ]]; then
+        cmake -DWITH_PYTHON3=ON .. ${PYTHON_FLAGS}
+    else
+        cmake .. ${PYTHON_FLAGS}
+    fi
     make -j2
 }
 

scripts/tests.sh

@@ -5,6 +5,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd $SCRIPT_DIR/..
 
 mode=$1
+
 readonly TOP_DIR=$(pwd)
 readonly core_path=$TOP_DIR/build/visualdl/logic
 readonly python_path=$TOP_DIR/visualdl/python
@@ -13,20 +14,28 @@ readonly max_file_size=1000000 # 1MB
 readonly version_number=`cat VERSION_NUMBER | sed 's/\([0-9]*.[0-9]*.[0-9]*\).*/\1/g'`
 
 sudo="sudo"
+pip="pip"
+python="python2"
+
+if [[ "$WITH_PYTHON3" == "ON" ]]; then
+    pip="pip3"
+    python="python3"
+    $sudo python3 -m pip install --upgrade pip
+fi
 
 if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo=""; fi
 
 if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
 		curl -O http://python-distribute.org/distribute_setup.py
-		python distribute_setup.py
+		$python distribute_setup.py
 		curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py
-		python get-pip.py
+		$python get-pip.py
 fi
 
-$sudo pip install numpy
-$sudo pip install Flask
-$sudo pip install Pillow
-$sudo pip install protobuf
+$sudo $pip install numpy
+$sudo $pip install Flask
+$sudo $pip install Pillow
+$sudo $pip install protobuf
 
 export PYTHONPATH="${core_path}:${python_path}"
 
@@ -38,22 +47,26 @@ package() {
 
     cd $TOP_DIR/visualdl/server
     # manully install protobuf3
-    curl -OL https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip
-    unzip protoc-3.1.0-linux-x86_64.zip -d protoc3
+    curl -OL https://github.com/google/protobuf/releases/download/v3.5.0/protoc-3.5.0-linux-x86_64.zip
+    unzip protoc-3.5.0-linux-x86_64.zip -d protoc3
     export PATH="$PATH:$(pwd)/protoc3/bin"
     chmod +x protoc3/bin/*
 
 
     cd $TOP_DIR
-    python setup.py bdist_wheel
-    $sudo pip install dist/visualdl-${version_number}*.whl
+    $python setup.py bdist_wheel
+    $sudo $pip install dist/visualdl-${version_number}*.whl
 }
 
 backend_test() {
     cd $TOP_DIR
     mkdir -p build
     cd build
-    cmake ..
+    if [[ "$WITH_PYTHON3" == "ON" ]]; then
+        cmake -DWITH_PYTHON3=ON ..
+    else
+        cmake ..
+    fi
     make
     make test
 }
@@ -66,14 +79,14 @@ frontend_test() {
 }
 
 server_test() {
-    $sudo pip install google
-    $sudo pip install protobuf==3.1.0
+    $sudo $pip install google
+    $sudo $pip install protobuf==3.5.1
 
     cd $TOP_DIR/visualdl/server
     bash graph_test.sh
 
-    cd $TOP_DIR/visualdl/server
-    python lib_test.py
+    cd $TOP_DIR/
+    $python -m visualdl.server.lib_test
 }
 
 # check the size of files in the repo.
@@ -92,11 +105,24 @@ bigfile_reject() {
     fi
 }
 
+clean_env() {
+    rm -rf $TOP_DIR/build
+    rm -rf $TOP_DIR/dist
+    rm -rf $TOP_DIR/frontend/node_modules
+    rm -rf $TOP_DIR/frontend/dist
+    rm -rf $TOP_DIR/visualdl/core.so
+    rm -rf $TOP_DIR/visualdl/python/core.so
+    rm -rf $TOP_DIR/visualdl/server/protoc3
+    rm -rf $TOP_DIR/visualdl/server/protoc*.zip
+}
+
 echo "mode" $mode
 
 if [ $mode = "backend" ]; then
     backend_test
 elif [ $mode = "all" ]; then
+    # Clean before test
+    clean_env
     # bigfile_reject should be tested first, or some files downloaded may fail this test.
     bigfile_reject
     package

setup.py

@@ -58,9 +58,12 @@ class BaseCommand(setuptools.Command):
 class build_py(setuptools.command.build_py.build_py):
     def run(self):
         cmd = ['bash', 'build.sh']
+        env = dict(os.environ)
         if MODE == "travis-CI":
             cmd.append('travis-CI')
-        subprocess.check_call(cmd)
+        if sys.version_info[0] >= 3:
+            env["WITH_PYTHON3"] = "ON"
+        subprocess.check_call(cmd, env=env)
         return setuptools.command.build_py.build_py.run(self)
 
 

tests.sh

@@ -35,8 +35,8 @@ package() {
 
     cd $TOP_DIR/visualdl/server
     # manully install protobuf3
-    curl -OL https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip
-    unzip protoc-3.1.0-linux-x86_64.zip -d protoc3
+    curl -OL https://github.com/google/protobuf/releases/download/v3.5.0/protoc-3.5.0-linux-x86_64.zip
+    unzip protoc-3.5.0-linux-x86_64.zip -d protoc3
     export PATH="$PATH:$(pwd)/protoc3/bin"
     chmod +x protoc3/bin/*
 
@@ -64,13 +64,13 @@ frontend_test() {
 
 server_test() {
     $sudo pip install google
-    $sudo pip install protobuf==3.1.0
+    $sudo pip install protobuf==3.5.1
 
     cd $TOP_DIR/visualdl/server
     bash graph_test.sh
 
-    cd $TOP_DIR/visualdl/server
-    python lib_test.py
+    cd $TOP_DIR/
+    python -m visualdl.server.lib_test
 }
 
 # check the size of files in the repo.

visualdl/logic/pybind.cc

@@ -28,14 +28,12 @@ namespace cp = visualdl::components;
   CODE(float);                   \
   CODE(double);
 
-PYBIND11_PLUGIN(core) {
-  py::module m("core", "C++ core of VisualDL");
+PYBIND11_MODULE(core, m) {
+  m.doc() = "C++ core of VisualDL";
 
   py::class_<vs::LogReader>(m, "LogReader")
-      .def("__init__",
-           [](vs::LogReader& instance, const std::string& dir) {
-             new (&instance) vs::LogReader(dir);
-           })
+      .def(py::init(
+          [](const std::string& dir) { return new vs::LogReader(dir); }))
       .def("as_mode", &vs::LogReader::AsMode)
       .def("set_mode", &vs::LogReader::SetMode)
       .def("modes", [](vs::LogReader& self) { return self.storage().modes(); })
@@ -70,10 +68,9 @@ PYBIND11_PLUGIN(core) {
 
   // clang-format on
   py::class_<vs::LogWriter>(m, "LogWriter")
-      .def("__init__",
-           [](vs::LogWriter& instance, const std::string& dir, int sync_cycle) {
-             new (&instance) vs::LogWriter(dir, sync_cycle);
-           })
+      .def(py::init([](const std::string& dir, int sync_cycle) {
+        return new vs::LogWriter(dir, sync_cycle);
+      }))
       .def("set_mode", &vs::LogWriter::SetMode)
       .def("as_mode", &vs::LogWriter::AsMode)
 // clang-format off

visualdl/python/test_storage.py

+from __future__ import print_function
 import pprint
 import sys
 import unittest
@@ -15,13 +16,13 @@ class StorageTest(unittest.TestCase):
         self.writer = LogWriter(self.dir, sync_cycle=1).as_mode("train")
 
     def test_scalar(self):
-        print 'test write'
+        print('test write')
         scalar = self.writer.scalar("model/scalar/min")
         # scalar.set_caption("model/scalar/min")
         for i in range(10):
             scalar.add_record(i, float(i))
 
-        print 'test read'
+        print('test read')
         self.reader = LogReader(self.dir)
         with self.reader.mode("train") as reader:
             scalar = reader.scalar("model/scalar/min")
@@ -31,8 +32,8 @@ class StorageTest(unittest.TestCase):
             self.assertTrue(
                 np.equal(records, [float(i) for i in range(10 - 1)]).all())
             self.assertTrue(np.equal(ids, [float(i) for i in range(10)]).all())
-            print 'records', records
-            print 'ids', ids
+            print('records', records)
+            print('ids', ids)
 
     def test_image(self):
         tag = "layer1/layer2/image0"
@@ -41,9 +42,9 @@ class StorageTest(unittest.TestCase):
         num_samples = 100
         shape = [10, 10, 3]
 
-        for pass_ in xrange(num_passes):
+        for pass_ in range(num_passes):
             image_writer.start_sampling()
-            for ins in xrange(num_samples):
+            for ins in range(num_samples):
                 data = np.random.random(shape) * 256
                 data = np.ndarray.flatten(data)
                 image_writer.add_sample(shape, list(data))
@@ -68,7 +69,7 @@ class StorageTest(unittest.TestCase):
         '''
         check whether the storage will keep image data consistent
         '''
-        print 'check image'
+        print('check image')
         tag = "layer1/check/image1"
         image_writer = self.writer.image(tag, 10)
 
@@ -91,8 +92,8 @@ class StorageTest(unittest.TestCase):
 
             PIL_image_shape = (shape[0] * shape[1], shape[2])
             data = np.array(data, dtype='uint8').reshape(PIL_image_shape)
-            print 'origin', origin_data.flatten()
-            print 'data', data.flatten()
+            print('origin', origin_data.flatten())
+            print('data', data.flatten())
             image = Image.fromarray(data.reshape(shape))
             # manully check the image and found that nothing wrong with the image storage.
             # image.show()

visualdl/server/__init__.py

-import log
+from __future__ import absolute_import
+from . import log
 
 __all__ = ['log']

visualdl/server/graph.py

+from __future__ import absolute_import
 import json
 import os
 
 from google.protobuf.json_format import MessageToJson
 from PIL import Image
 
-import graphviz_graph as gg
-import onnx
+from . import graphviz_graph as gg
+from . import onnx
 
 
 def debug_print(json_obj):

visualdl/server/graph_test.py

+from __future__ import absolute_import
 import unittest
 
-import graph
+from . import graph
 
 
 class GraphTest(unittest.TestCase):
     def setUp(self):
-        self.mock_dir = "./mock"
+        self.mock_dir = "./visualdl/server/mock"
 
     def test_graph_edges_squeezenet(self):
         json_obj = graph.to_IR_json(self.mock_dir + '/squeezenet_model.pb')

visualdl/server/graph_test.sh

@@ -4,8 +4,12 @@ set -ex
 cd mock
 bash download_mock_models.sh
 
-cd ..
+cd ../../../
 
-python graph_test.py
+if [[ "$WITH_PYTHON3" == "ON" ]]; then
+    python3 -m visualdl.server.graph_test
+else
+    python2 -m visualdl.server.graph_test
+fi
 
-rm ./mock/*.pb
+rm visualdl/server/mock/*.pb

visualdl/server/graphviz_graph.py

+from __future__ import print_function
 import random
 import subprocess
+import six
 
 
 def crepr(v):
-    if type(v) is str or type(v) is unicode:
+    if type(v) is six.text_type:
         return '"%s"' % v
     return str(v)
 
@@ -195,5 +197,5 @@ if __name__ == '__main__':
     add_edge(n0, n2)
     add_edge(n1, n2)
 
-    print g_graph.code()
+    print(g_graph.code())
     g_graph.display('./1.dot')

visualdl/server/lib.py

+from __future__ import absolute_import
 import re
 import sys
 import time
-import urllib
 from tempfile import NamedTemporaryFile
-
 import numpy as np
 from PIL import Image
+from .log import logger
 
-from log import logger
+try:
+    from urllib.parse import urlencode
+except Exception:
+    from urllib import urlencode
 
 
 def get_modes(storage):
@@ -43,7 +46,7 @@ def get_scalar(storage, mode, tag, num_records=300):
         ids = scalar.ids()
         timestamps = scalar.timestamps()
 
-        data = zip(timestamps, ids, records)
+        data = list(zip(timestamps, ids, records))
         data_size = len(data)
 
         if data_size <= num_records:
@@ -78,7 +81,7 @@ def get_image_tags(storage):
                 result[mode] = {}
                 for tag in tags:
                     image = reader.image(tag)
-                    for i in xrange(max(1, image.num_samples())):
+                    for i in range(max(1, image.num_samples())):
                         caption = tag if image.num_samples(
                         ) <= 1 else '%s/%d' % (tag, i)
                         result[mode][caption] = {
@@ -109,7 +112,7 @@ def get_image_tag_steps(storage, mode, tag):
         if not shape:
             continue
         try:
-            query = urllib.urlencode({
+            query = urlencode({
                 'sample': 0,
                 'index': step_index,
                 'tag': origin_tag,
@@ -161,7 +164,7 @@ def get_histogram(storage, mode, tag, num_samples=100):
         histogram = reader.histogram(tag)
         res = []
 
-        for i in xrange(histogram.num_records()):
+        for i in range(histogram.num_records()):
             try:
                 # some bug with protobuf, some times may overflow
                 record = histogram.record(i)
@@ -175,7 +178,7 @@ def get_histogram(storage, mode, tag, num_samples=100):
             py_record.append([])
 
             data = py_record[-1]
-            for j in xrange(record.num_instances()):
+            for j in range(record.num_instances()):
                 instance = record.instance(j)
                 data.append(
                     [instance.left(), instance.right(), instance.frequency()])
@@ -202,7 +205,7 @@ def retry(ntimes, function, time2sleep, *args, **kwargs):
     try to execute `function` `ntimes`, if exception catched, the thread will
     sleep `time2sleep` seconds.
     '''
-    for i in xrange(ntimes):
+    for i in range(ntimes):
         try:
             return function(*args, **kwargs)
         except Exception:

visualdl/server/lib_test.py

+from __future__ import print_function
 import pprint
 import unittest
 
 from visualdl import LogReader, LogWriter
 
-import lib
-from storage_mock import add_histogram, add_image, add_scalar
+from . import lib
+from .storage_mock import add_histogram, add_image, add_scalar
 
 _retry_counter = 0
 
@@ -55,7 +56,7 @@ class LibTest(unittest.TestCase):
 
     def test_scalar(self):
         tags = lib.get_scalar_tags(self.reader)
-        print 'scalar tags:'
+        print('scalar tags:')
         pprint.pprint(tags)
         self.assertEqual(len(tags), 3)
         self.assertEqual(
@@ -70,7 +71,7 @@ class LibTest(unittest.TestCase):
 
         image = lib.get_invididual_image(self.reader, "train",
                                          'layer/image0/0', 2)
-        print image
+        print(image)
 
     def test_histogram(self):
         tags = lib.get_histogram_tags(self.reader)

visualdl/server/storage_mock.py

@@ -21,9 +21,9 @@ def add_image(writer,
     with writer.mode(mode) as writer_:
         image_writer = writer_.image(tag, num_samples, step_cycle)
 
-        for pass_ in xrange(num_passes):
+        for pass_ in range(num_passes):
             image_writer.start_sampling()
-            for ins in xrange(2 * num_samples):
+            for ins in range(2 * num_samples):
                 data = np.random.random(shape) * 256
                 data = np.ndarray.flatten(data)
                 image_writer.add_sample(shape, list(data))

visualdl/server/visualDL

@@ -2,7 +2,6 @@
 
 import json
 import os
-import exceptions
 import time
 import sys
 from argparse import ArgumentParser
@@ -20,6 +19,11 @@ from visualdl.server.mock import data as mock_tags
 from visualdl.python.cache import MemCache
 from visualdl.python.storage import (LogWriter, LogReader)
 
+try:
+    import exceptions
+except:
+    pass
+    
 app = Flask(__name__, static_url_path="")
 # set static expires in a short time to reduce browser's memory usage.
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 30

visualdl/storage/CMakeLists.txt

@@ -14,3 +14,5 @@ add_dependencies(entry storage_proto im)
 add_dependencies(record storage_proto entry)
 add_dependencies(tablet storage_proto)
 add_dependencies(storage storage_proto record binary_record tablet entry)
+
+target_link_libraries(storage storage_proto record binary_record tablet entry)

visualdl/test.py

@@ -24,7 +24,7 @@ class StorageTester(unittest.TestCase):
         self.assertEqual(self.storage.tablets_size(), 11)
 
     def test_timestamp(self):
-        print self.storage.timestamp()
+        print(self.storage.timestamp())
 
     def test_dir(self):
         dir = "./1.txt"
@@ -32,7 +32,7 @@ class StorageTester(unittest.TestCase):
         self.assertEqual(dir, self.storage.dir())
 
     def test_human_readable_buffer(self):
-        print self.storage.human_readable_buffer()
+        print(self.storage.human_readable_buffer())
 
 
 class TabletTester(unittest.TestCase):
@@ -41,7 +41,7 @@ class TabletTester(unittest.TestCase):
         self.tablet = im.add_tablet("tag101", 20)
 
     def test_human_readable_buffer(self):
-        print self.tablet.human_readable_buffer()
+        print(self.tablet.human_readable_buffer())
 
     def test_scalar(self):
         scalar = self.tablet.as_float_scalar()

visualdl/utils/logging.h

@@ -16,6 +16,7 @@ limitations under the License. */
 #define VISUALDL_UTILS_LOGGING_H
 
 #include <csignal>
+#include <functional>
 #include <iostream>
 #include <sstream>
 #include <stdexcept>