Merge pull request #34 from jacquesqiao/add-graph-data-format

support onnx as graph data format and add related API

README.md

@@ -20,7 +20,7 @@ cp -r frontend/dist server/visualdl/frontend/dist
 #### Step 3: build and install Python package
 ```shell
 cd server/
-python setup.py bdist_wheel
+sh build.sh
 cd dist
 sudo pip install --upgrade visualdl-0.0.1-py2-none-any.whl
 ```

server/build.sh

+script=$(readlink -f "$0")
+script_path=$(dirname "$script")
+
+pushd $script_path
+  protoc visualdl/onnx/onnx.proto --python_out .
+  python setup.py bdist_wheel
+popd

server/setup.py

 from setuptools import setup
 
-packages = ['visualdl', 'visualdl.frontend.dist', 'visualdl.mock']
+packages = ['visualdl',
+            'visualdl.onnx',
+            'visualdl.mock',
+            'visualdl.frontend.dist']
 
 setup(
     name="visualdl",

server/visualdl/graph.py

+import onnx
+import json
+from google.protobuf.json_format import MessageToJson
+
+
+def reorganize_inout(json_obj, key):
+    """
+    :param json_obj: the model's json obj
+    :param key: "input or output"
+    :return:
+    """
+    for index in range(len(json_obj[key])):
+        var = json_obj[key][index]
+        var_new = dict()
+
+        # set name
+        var_new['name'] = var['name']
+
+        tensor_type = var['type']['tensorType']
+
+        # set data_type
+        var_new['data_type'] = tensor_type['elemType']
+
+        # set shape
+        shape = [dim['dimValue'] for dim in tensor_type['shape']['dim']]
+        var_new['shape'] = shape
+
+        json_obj[key][index] = var_new
+
+
+def load_model(model_pb_path):
+    model = onnx.load(model_pb_path)
+    graph = model.graph
+    del graph.initializer[:]
+
+    # to json string
+    json_str = MessageToJson(model.graph)
+    json_obj = json.loads(json_str)
+    reorganize_inout(json_obj, 'input')
+    reorganize_inout(json_obj, 'output')
+    return json.dumps(json_obj, sort_keys=True, indent=4, separators=(',', ': '))
+
+
+if __name__ == '__main__':
+    import os
+    import sys
+    current_path = os.path.abspath(os.path.dirname(sys.argv[0]))
+    json_str = load_model(current_path + "/mock/inception_v1.pb")
+    print(json_str)

server/visualdl/onnx/READEME.md

+## Note
+the onnx.proto is port from onnx commit c3137751923f576d14b4212ea444d1562b127a66

server/visualdl/onnx/__init__.py

+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import google.protobuf.message
+
+from .onnx_pb2 import ModelProto
+
+
+def load(obj):
+    '''
+    Loads a binary protobuf that stores onnx model
+
+    @params
+    Takes a file-like object (has "read" function)
+    or a string containing a file name
+    @return ONNX ModelProto object
+    '''
+    if hasattr(obj, 'read') and callable(obj.read):
+        s = obj.read()
+    else:
+        with open(obj, 'rb') as f:
+            s = f.read()
+    return load_from_string(s)
+
+
+def load_from_string(s):
+    '''
+    Loads a binary string that stores onnx model
+
+    @params
+    Takes a string object containing protobuf
+    @return ONNX ModelProto object
+    '''
+    model = ModelProto()
+    decoded = model.ParseFromString(s)
+    # in python implementation ParseFromString returns None
+    if decoded is not None and decoded != len(s):
+        raise google.protobuf.message.DecodeError(
+            "Protobuf decoding consumed too few bytes: {} out of {}".format(
+                decoded, len(s)))
+    return model

server/visualdl/onnx/onnx.proto

+//
+// WARNING: This file is automatically generated!  Please edit onnx.in.proto.
+//
+
+
+// Copyright (c) Facebook Inc. and Microsoft Corporation.
+// Licensed under the MIT license.
+
+syntax = "proto2";
+
+package onnx;
+
+// Note [Release]
+// We are still in the very early stage of defining ONNX. The current
+// version of ONNX is a starting point. While we are actively working
+// towards a complete spec, we would like to get the community involved
+// by sharing our working version of ONNX.
+
+// Note [Protobuf compatibility]
+// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+// Based on experience working with downstream vendors, we generally can't
+// assume recent versions of protobufs. This means that we do not use any
+// protobuf features that are only available in proto3.
+//
+// Here are the most notable contortions we have to carry out to work around
+// these limitations:
+//
+//   - No 'map' (added protobuf 3.0). We instead represent mappings as lists
+//     of key-value pairs, where order does not matter and duplicates
+//     are not allowed.
+
+// Note [Namespaces]
+// ~~~~~~~~~~~~~~~~~
+// ONNX gives explicit names to graphs, intermediate values and
+// serialized tensors.  To make it easier to generate names, we organize
+// these into separate namespaces (so, e.g., a graph can have the same
+// name as a serialized tensor.)  The namespaces are as follows:
+//
+// - Node: These names identify specific nodes in the graph (but not, necessarily
+//   any particular input or output of the node.
+// - Graph: These names identify graphs in the protobuf.
+// - Attribute: These names identify attribute names for extra attributes that
+//   are passed to operators.
+// - Operator: These names identify particular operators.
+// - Value: These names identify intermediate values (typically tensors) flowing through
+//   the computation of a graph.
+// - Shape: These names represent parameters for unknown shape dimensions.
+//
+// We specify the namespace of a name in ONNX as comments in the form
+// of "namespace {Node,Graph,Operator,Attribute,Value,Shape}". Framework is responsible
+// for supporting the namespaces.
+//
+// Naming things is hard. Every element with a name has an optional doc_string associated
+// with it, providing a human-readable description in text markdown.
+
+// To be compatible with both proto2 and proto3, we will use a version number
+// that is not defined by the default value but an explicit enum number.
+enum Version {
+  // proto3 requires the first enum value to be zero.
+  // We add this just to appease the compiler.
+  _START_VERSION = 0;
+  // The version field is always serialized and we will use it to store the
+  // version that the  graph is generated from. This helps us set up version
+  // control. We should use version as
+  //     xx(major) - xx(minor) - xxxx(bugfix)
+  // and we are starting with 0x00000001 (0.0.1), which was the
+  //  version we published on Oct 10, 2017.
+  IR_VERSION_2017_10_10 = 0x00000001;
+
+  // IR_VERSION 0.0.2 published on Oct 30, 2017
+  // - Added type discriminator to AttributeProto to support proto3 users
+  IR_VERSION_2017_10_30 = 0x00000002;
+
+  // IR VERSION 0.0.3 published on Nov 3, 2017
+  // - For operator versioning:
+  //    - Added new message OperatorSetIdProto
+  //    - Added opset_import in ModelProto
+  // - For vendor extensions, added domain in NodeProto
+  IR_VERSION = 0x00000003;
+}
+
+// A named attribute containing either singular float, integer, string
+// and tensor values, or repeated float, integer, string and tensor values.
+// An AttributeProto MUST contain the name field, and *only one* of the
+// following content fields, effectively enforcing a C/C++ union equivalent.
+message AttributeProto {
+
+  // Note: this enum is structurally identical to the OpSchema::AttrType
+  // enum defined in schema.h.  If you rev one, you likely need to rev the other.
+  enum AttributeType {
+    UNDEFINED = 0;
+    FLOAT = 1;
+    INT = 2;
+    STRING = 3;
+    TENSOR = 4;
+    GRAPH = 5;
+
+    FLOATS = 6;
+    INTS = 7;
+    STRINGS = 8;
+    TENSORS = 9;
+    GRAPHS = 10;
+  }
+
+  // The name field MUST be present for this version of the IR.
+  optional string name = 1;           // namespace Attribute
+
+  // A human-readable documentation for this attribute. Markdown is allowed.
+  optional string doc_string = 13;
+
+  // The type field MUST be present for this version of the IR.
+  // For 0.0.1 versions of the IR, this field was not defined, and
+  // implementations needed to use has_field hueristics to determine
+  // which value field was in use.  For IR_VERSION 0.0.2 or later, this
+  // field MUST be set and match the f|i|s|t|... field in use.  This
+  // change was made to accomodate proto3 implementations.
+  optional AttributeType type = 20;   // discriminator that indicates which field below is in use
+
+  // Exactly ONE of the following fields must be present for this version of the IR
+  optional float f = 2;               // float
+  optional int64 i = 3;               // int
+  optional bytes s = 4;               // UTF-8 string
+  optional TensorProto t = 5;         // tensor value
+  optional GraphProto g = 6;          // graph
+  // Do not use field below, it's deprecated.
+  // optional ValueProto v = 12;         // value - subsumes everything but graph
+
+  repeated float floats = 7;          // list of floats
+  repeated int64 ints = 8;            // list of ints
+  repeated bytes strings = 9;         // list of UTF-8 strings
+  repeated TensorProto tensors = 10;  // list of tensors
+  repeated GraphProto graphs = 11;    // list of graph
+}
+
+// Defines information on value, including the name, the type, and
+// the shape of the value.
+message ValueInfoProto {
+  // This field MUST be present in this version of the IR.
+  optional string name = 1;     // namespace Value
+  // This field MUST be present in this version of the IR.
+  optional TypeProto type = 2;
+  // A human-readable documentation for this value. Markdown is allowed.
+  optional string doc_string = 3;
+}
+
+// NodeProto stores a node that is similar to the notion of "layer"
+// or "operator" in many deep learning frameworks. For example, it can be a
+// node of type "Conv" that takes in an image, a filter tensor and a bias
+// tensor, and produces the convolved output.
+message NodeProto {
+  repeated string input = 1;    // namespace Value
+  repeated string output = 2;   // namespace Value
+
+  // An optional identifier for this node in a graph.
+  // This field MAY be absent in ths version of the IR.
+  optional string name = 3;     // namespace Node
+
+  // The symbolic identifier of the Operator to execute.
+  optional string op_type = 4;  // namespace Operator
+  // The domain of the OperatorSet that specifies the operator named by op_type.
+  optional string domain = 7;   // namespace Domain
+
+  // Additional named attributes.
+  // NOTE: Simply using ValueProto.NameValuePairProto is the most general
+  //       solution.  I kept AttributeProto to minimize churn on CI results.
+  repeated AttributeProto attribute = 5;
+
+  // A human-readable documentation for this node. Markdown is allowed.
+  optional string doc_string = 6;
+}
+
+// ModelProto is a top-level file/container format for bundling a ML model.
+// The semantics of the model are described by the GraphProto that represents
+// a parameterized computation graph against a set of named operators that are
+// defined independently from the graph.
+message ModelProto {
+  // The version of the IR this model targets. See Version enum above.
+  // This field MUST be present.
+  optional int64 ir_version = 1;
+
+  // The OperatorSets this model relies on.
+  // All ModelProtos MUST have at least one entry that
+  // specifies which version of the ONNX OperatorSet is
+  // being imported.
+  //
+  // All nodes in the ModelProto's graph will bind against the operator
+  // with the same-domain/same-op_type operator with the HIGHEST version
+  // in the referenced operator sets.
+  repeated OperatorSetIdProto opset_import = 8;
+
+  // The name of the framework or tool used to generate this model.
+  // This field SHOULD be present to indicate which implementation/tool/framework
+  // emitted the model.
+  optional string producer_name = 2;
+
+  // The version of the framework or tool used to generate this model.
+  // This field SHOULD be present to indicate which implementation/tool/framework
+  // emitted the model.
+  optional string producer_version = 3;
+
+  // Domain name of the model.
+  // We use reverse domain names as name space indicators. For example:
+  // `com.facebook.fair` or `com.microsoft.cognitiveservices`
+  //
+  // Together with `model_version` and GraphProto.name, this forms the unique identity of
+  // the graph.
+  optional string domain = 4;
+
+  // The version of the graph encoded. See Version enum below.
+  optional int64 model_version = 5;
+
+  // A human-readable documentation for this model. Markdown is allowed.
+  optional string doc_string = 6;
+
+  // The parameterized graph that is evaluated to execute the model.
+  optional GraphProto graph = 7;
+
+  // Named metadata values; keys should be distinct.
+  repeated StringStringEntryProto metadata_props = 14;
+};
+
+// StringStringEntryProto follows the pattern for cross-proto-version maps.
+// See https://developers.google.com/protocol-buffers/docs/proto3#maps
+message StringStringEntryProto {
+  optional string key = 1;
+  optional string value= 2;
+};
+
+// GraphProto defines a parameterized series of nodes to form a directed acyclic graph.
+// This is the equivalent of the "network" and "graph" in many deep learning
+// frameworks.
+message GraphProto {
+  // The nodes in the graph.
+  repeated NodeProto node = 1;
+
+  // The name of the graph.
+  optional string name = 2;   // namespace Graph
+
+  // A list of named tensor values (constants), used to specify default
+  // values for some of the inputs of the graph.
+  // Each TensorProto entry must have a distinct name (within the list) that
+  // also appears in the input list.
+  // In an evaluation, the default value specified here is used if and only if
+  // user specifies no value for the corresponding input parameter.
+  // May be used to pass serialized parameters for networks.
+  repeated TensorProto initializer = 5;
+
+  // A human-readable documentation for this graph. Markdown is allowed.
+  optional string doc_string = 10;
+
+  // The inputs and outputs of the graph.
+  repeated ValueInfoProto input = 11;
+  repeated ValueInfoProto output = 12;
+
+  // Information for the values in the graph. The ValueInfoProto.name's
+  // must be distinct. It is optional for a value to appear in value_info list.
+  repeated ValueInfoProto value_info = 13;
+
+  // DO NOT USE the following fields, they were deprecated before
+  // repeated string input = 3;
+  // repeated string output = 4;
+  // optional int64 ir_version = 6;
+  // optional int64 producer_version = 7;
+  // optional string producer_tag = 8;
+  // optional string domain = 9;
+}
+
+// A message defined to store a tensor in its serialized format.
+message TensorProto {
+  enum DataType {
+    UNDEFINED = 0;
+    // Basic types.
+    FLOAT = 1;   // float
+    UINT8 = 2;   // uint8_t
+    INT8 = 3;    // int8_t
+    UINT16 = 4;  // uint16_t
+    INT16 = 5;   // int16_t
+    INT32 = 6;   // int32_t
+    INT64 = 7;   // int64_t
+    STRING = 8;  // string
+    BOOL = 9;    // bool
+
+    // Advanced types
+    FLOAT16 = 10;
+    DOUBLE = 11;
+    UINT32 = 12;
+    UINT64 = 13;
+    COMPLEX64 = 14;     // complex with float32 real and imaginary components
+    COMPLEX128 = 15;    // complex with float64 real and imaginary components
+    // Future extensions go here.
+  }
+
+  // The shape of the tensor.
+  repeated int64 dims = 1;
+
+  // The data type of the tensor.
+  optional DataType data_type = 2;
+
+  // For very large tensors, we may want to store them in chunks, in which
+  // case the following fields will specify the segment that is stored in
+  // the current TensorProto.
+  message Segment {
+    optional int64 begin = 1;
+    optional int64 end = 2;
+  }
+  optional Segment segment = 3;
+
+  // Tensor content must be in the row major order.
+  //
+  // Depending on the data_type field, exactly one of the fields below with
+  // name ending in _data is used to store the elements of the tensor.
+
+  // For float and complex64 values
+  // Complex64 tensors are encoded as a single array of floats,
+  // with the real components appearing in odd numbered positions,
+  // and the corresponding imaginary component apparing in the
+  // subsequent even numbered position. (e.g., [1.0 + 2.0i, 3.0 + 4.0i]
+  // is encoded as [1.0, 2.0 ,3.0 ,4.0]
+  // When this field is present, the data_type field MUST be FLOAT or COMPLEX64.
+  repeated float float_data = 4 [packed = true];
+
+  // For int32, uint8, int8, uint16, int16, bool, and float16 values
+  // float16 values must be bit-wise converted to an uint16_t prior
+  // to writing to the buffer.
+  // When this field is present, the data_type field MUST be
+  // INT32, INT16, INT8, UINT16, INT8, BOOL, or FLOAT32
+  repeated int32 int32_data = 5 [packed = true];
+
+  // For strings.
+  // Each element of string_data is a UTF-8 encoded Unicode
+  // string. No trailing null, no leading BOM. The protobuf "string"
+  // scalar type is not used to match ML community conventions.
+  // When this field is present, the data_type field MUST be STRING
+  repeated bytes string_data = 6;
+
+  // For int64.
+  // When this field is present, the data_type field MUST be INT64
+  repeated int64 int64_data = 7 [packed = true];
+
+  // Optionally, a name for the tensor.
+  optional string name = 8; // namespace Value
+
+  // A human-readable documentation for this tensor. Markdown is allowed.
+  optional string doc_string = 12;
+
+  // Serializations can either use one of the fields above, or use this
+  // raw bytes field. The only exception is the string case, where one is
+  // required to store the content in the repeated bytes string_data field.
+  //
+  // When this raw_data field is used to store tensor value, elements MUST
+  // be stored in as fixed-width, little-endian order.
+  // Floating-point data types MUST be stored in IEEE 754 format.
+  // Complex64 elements must be written as two consecutive FLOAT values, real component first.
+  // Complex128 elements must be written as two consecutive DOUBLE values, real component first.
+  // Boolean type MUST be written one byte per tensor element (00000001 for true, 00000000 for false).
+  //
+  // Note: the advantage of specific field rather than the raw_data field is
+  // that in some cases (e.g. int data), protobuf does a better packing via
+  // variable length storage, and may lead to smaller binary footprint.
+  // When this field is present, the data_type field MUST NOT be STRING or UNDEFINED
+  optional bytes raw_data = 9;
+
+  // For double
+  // Complex64 tensors are encoded as a single array of doubles,
+  // with the real components appearing in odd numbered positions,
+  // and the corresponding imaginary component apparing in the
+  // subsequent even numbered position. (e.g., [1.0 + 2.0i, 3.0 + 4.0i]
+  // is encoded as [1.0, 2.0 ,3.0 ,4.0]
+  // When this field is present, the data_type field MUST be DOUBLE or COMPLEX128
+  repeated double double_data = 10 [packed = true];
+
+  // For uint64 and uint32 values
+  // When this field is present, the data_type field MUST be
+  // UINT32 or UINT64
+  repeated uint64 uint64_data = 11 [packed = true];
+}
+
+// Defines a tensor shape. A dimension can be either an integer value
+// or a symbolic variable. A symbolic variable represents an unknown
+// dimension.
+message TensorShapeProto {
+  message Dimension {
+    oneof value {
+      int64 dim_value = 1;
+      string dim_param = 2;   // namespace Shape
+    };
+  };
+  repeated Dimension dim = 1;
+}
+
+// Define the types.
+message TypeProto {
+
+  message Tensor {
+    // This field MUST NOT have the value of UNDEFINED
+    // This field MUST be present for this version of the IR.
+    optional TensorProto.DataType elem_type = 1;
+    optional TensorShapeProto shape = 2;
+  }
+
+
+  oneof value {
+    // The type of a tensor.
+    Tensor tensor_type = 1;
+
+  }
+}
+
+// OperatorSets are uniquely identified by a (domain, opset_version) pair.
+message OperatorSetIdProto {
+  // The domain of the operator set being identified.
+  // The empty string ("") or absence of this field implies the operator
+  // set that is defined as part of the ONNX specification.
+  // This field MUST be present in this version of the IR when referring to any other operator set.
+  optional string domain = 1;
+
+  // The version of the operator set being identified.
+  // This field MUST be present in this version of the IR.
+  optional int64 version = 2;
+}
\ No newline at end of file

server/visualdl/visual_dl.py

@@ -14,6 +14,7 @@ from visualdl.log import logger
 import visualdl.mock.data as mock_data
 import visualdl.mock.tags as mock_tags
 import storage
+import graph
 
 app = Flask(__name__, static_url_path="")
 
@@ -135,6 +136,12 @@ def scalars():
     return Response(json.dumps(result), mimetype='application/json')
 
 
+@app.route('/data/plugin/graphs/graph')
+def graph():
+    model_json = graph.load_model("")
+    return Response(model_json, mimetype='application/json')
+
+
 if __name__ == '__main__':
     logger.info(" port=" + str(options.port))
     app.run(debug=False, host=options.host, port=options.port)