Skip to content

V
VisualDL

PaddlePaddle / VisualDL
大约 1 年 前同步成功

通知 86
Star 4655
Fork 641
V
VisualDL

体验新版 GitCode，发现更多精彩内容 >>

提交 a7d15082 编写于 作者: Q Qiao Longfei 提交者： GitHub
浏览文件
操作

Merge pull request #65 from jacquesqiao/auto-placement 

auto graph placement
上级 5bd5a3bc a997e591
隐藏空白更改
内联 并排
Showing with 396 addition and 245 deletion
server/visualdl/mock/data.py 已删除 100644 → 0
浏览文件 @ 5bd5a3bc
def sequence_data():
return [[1465980660.726887, 1, 0.07000000029802322],
[1465980660.767164, 2, 0.18000000715255737],
[1465980660.799817, 3, 0.3199999928474426],
[1465980660.831853, 4, 0.33000001311302185],
[1465980660.86555, 5, 0.3400000035762787],
[1465980660.898716, 6, 0.6000000238418579],
[1465980660.930804, 7, 0.6299999952316284],
[1465980660.963156, 8, 0.6299999952316284],
[1465980660.995727, 9, 0.6299999952316284],
[1465980661.603699, 11, 0.75],
[1465980661.641232, 12, 0.7900000214576721],
[1465980661.674311, 13, 0.7099999785423279],
[1465980661.704281, 14, 0.7099999785423279],
[1465980661.737259, 15, 0.7200000286102295],
[1465980661.768047, 16, 0.75],
[1465980661.801236, 17, 0.8600000143051147],
[1465980661.832749, 18, 0.7799999713897705],
[1465980661.862822, 19, 0.8399999737739563],
[1465980662.481258, 21, 0.699999988079071],
[1465980662.521448, 22, 0.6700000166893005],
[1465980662.557197, 23, 0.7699999809265137],
[1465980662.593898, 24, 0.7900000214576721],
[1465980662.629991, 25, 0.7200000286102295],
[1465980662.671198, 26, 0.8100000023841858],
[1465980662.711186, 27, 0.7599999904632568],
[1465980662.750267, 28, 0.7799999713897705],
[1465980662.791909, 29, 0.8299999833106995],
[1465980663.47027, 31, 0.8100000023841858],
[1465980663.538732, 32, 0.8500000238418579],
[1465980663.57077, 33, 0.8600000143051147],
[1465980663.60126, 34, 0.8199999928474426],
[1465980663.631059, 35, 0.7900000214576721],
[1465980663.665972, 36, 0.7799999713897705],
[1465980663.697275, 37, 0.9100000262260437],
[1465980663.726395, 38, 0.8700000047683716],
[1465980663.760169, 39, 0.9200000166893005],
[1465980664.45205, 41, 0.8299999833106995],
[1465980664.484207, 42, 0.7599999904632568],
[1465980664.515375, 43, 0.7699999809265137],
[1465980664.547608, 44, 0.8299999833106995],
[1465980664.580122, 45, 0.949999988079071],
[1465980664.611019, 46, 0.8999999761581421],
[1465980664.642956, 47, 0.8700000047683716],
[1465980664.674636, 48, 0.8500000238418579],
[1465980664.705622, 49, 0.8899999856948853],
[1465980665.379549, 51, 0.8399999737739563],
[1465980665.422869, 52, 0.8500000238418579],
[1465980665.466136, 53, 0.8199999928474426],
[1465980665.508361, 54, 0.9300000071525574],
[1465980665.544331, 55, 0.9399999976158142],
[1465980665.589887, 56, 0.8700000047683716],
[1465980665.633466, 57, 0.9300000071525574],
[1465980665.674978, 58, 0.7799999713897705],
[1465980665.716878, 59, 0.9300000071525574],
[1465980666.653456, 61, 0.8799999952316284],
[1465980666.697294, 62, 0.9300000071525574],
[1465980666.742066, 63, 0.8700000047683716],
[1465980666.780127, 64, 0.8299999833106995],
[1465980666.818287, 65, 0.9200000166893005],
[1465980666.855386, 66, 0.9399999976158142],
[1465980666.897352, 67, 0.9300000071525574],
[1465980666.931322, 68, 0.8899999856948853],
[1465980666.96562, 69, 0.8600000143051147],
[1465980667.619625, 71, 0.8700000047683716],
[1465980667.655166, 72, 0.9200000166893005], [
1465980667.687101, 73, 0.8199999928474426
], [1465980667.720176, 74, 0.8100000023841858],
[1465980667.751985, 75, 0.8500000238418579], [
1465980667.785244, 76, 0.8600000143051147
], [1465980667.820445, 77, 0.9200000166893005], [
1465980667.857163, 78, 0.8899999856948853
], [1465980667.891868, 79, 0.8999999761581421], [
1465980668.56409, 81, 0.8500000238418579
], [1465980668.599529, 82, 0.8299999833106995], [
1465980668.630751, 83, 0.8500000238418579
], [1465980668.665135, 84, 0.8199999928474426], [
1465980668.697928, 85, 0.8199999928474426
], [1465980668.730525, 86, 0.8799999952316284], [
1465980668.769772, 87, 0.9200000166893005
], [1465980668.803344, 88, 0.8299999833106995], [
1465980668.834414, 89, 0.800000011920929
], [1465980669.814826, 91, 0.8600000143051147], [
1465980669.851511, 92, 0.8899999856948853
], [1465980669.891407, 93, 0.8799999952316284], [
1465980669.927507, 94, 0.9399999976158142
], [1465980669.968384, 95, 0.9300000071525574], [
1465980670.007071, 96, 0.8500000238418579
], [1465980670.044314, 97, 0.8500000238418579], [
1465980670.083472, 98, 0.9100000262260437
], [1465980670.214597, 99, 0.8600000143051147], [
1465980670.934513, 101, 0.8799999952316284
], [1465980670.971317, 102, 0.8700000047683716], [
1465980671.003626, 103, 0.8600000143051147
], [1465980671.037037, 104, 0.8399999737739563], [
1465980671.070037, 105, 0.9200000166893005
], [1465980671.104992, 106, 0.8600000143051147], [
1465980671.137882, 107, 0.8100000023841858
], [1465980671.173917, 108, 0.7400000095367432], [
1465980671.205898, 109, 0.8799999952316284
], [1465980671.833723, 111, 0.9100000262260437]]
def graph_data():
return """{
"title": {
"text": "MLP"
},
"tooltip": {},
"animationDurationUpdate": 1500,
"animationEasingUpdate": "quinticInOut",
"series": [
{
"type": "graph",
"layout": "none",
"symbolSize": 50,
"roam": true,
"label": {
"normal": {
"show": true
}
},
"edgeSymbol": [
"circle",
"arrow"
],
"edgeSymbolSize": [
4,
10
],
"edgeLabel": {
"normal": {
"textStyle": {
"fontSize": 20
}
}
},
"data": [
{
"name": "X",
"x": 450,
"y": 600
},
{
"name": "W1",
"x": 750,
"y": 600
},
{
"name": "B1",
"x": 650,
"y": 600
},
{
"name": "W2",
"x": 650,
"y": 800
},
{
"name": "B2",
"x": 750,
"y": 800
},
{
"name": "FC1",
"x": 550,
"y": 700
},
{
"name": "RELU",
"x": 550,
"y": 800
},
{
"name": "FC2",
"x": 550,
"y": 900
},
{
"name": "Y",
"x": 550,
"y": 1000
}
],
"links": [
{
"source": "X",
"target": "FC1"
},
{
"source": "W1",
"target": "FC1"
},
{
"source": "B1",
"target": "FC1"
},
{
"source": "FC1",
"target": "RELU"
},
{
"source": "RELU",
"target": "FC2"
},
{
"source": "W2",
"target": "FC2"
},
{
"source": "B2",
"target": "FC2"
},
{
"source": "FC2",
"target": "Y"
}
],
"lineStyle": {
"normal": {
"opacity": 0.9,
"width": 2,
"curveness": 0
}
}
}
]
}"""
\ No newline at end of file
visualdl/server/graph.py
浏览文件 @ a7d15082
......@@ -5,6 +5,10 @@ from google.protobuf.json_format import MessageToJson
import onnx
def debug_print(json_obj):
print(json.dumps(json_obj, sort_keys=True, indent=4, separators=(',', ': ')))
def reorganize_inout(json_obj, key):
"""
:param json_obj: the model's json obj
......@@ -30,6 +34,265 @@ def reorganize_inout(json_obj, key):
json_obj[key][index] = var_new
def rename_model(model_json):
def rename_edge(model_json, old_name, new_name):
for node in model_json['node']:
inputs = node['input']
for idx in range(len(inputs)):
if inputs[idx] == old_name:
inputs[idx] = new_name
outputs = node['output']
for idx in range(len(outputs)):
if outputs[idx] == old_name:
outputs[idx] = new_name
def rename_variables(model, variables):
for variable in variables:
old_name = variable['name']
new_shape = [int(dim) for dim in variable['shape']]
new_name = old_name + '\ndata_type=' + str(variable['data_type']) \
+ '\nshape=' + str(new_shape)
variable['name'] = new_name
rename_edge(model, old_name, new_name)
rename_variables(model_json, model_json['input'])
rename_variables(model_json, model_json['output'])
# rename
all_nodes = model_json['node']
for idx in range(len(all_nodes)):
name = ""
if "name" in all_nodes[idx]:
name = all_nodes[idx]['name']
op_type = all_nodes[idx]['opType']
new_name = str(idx) + '\n' + str(op_type)
if name != "":
new_name = new_name + "\n" + name
all_nodes[idx]['name'] = new_name
def get_links(model_json):
links = []
for input in model_json['input']:
name = input['name']
for node in model_json['node']:
if name in node['input']:
links.append({'source': name,
"target": node['name']})
for source_node in model_json['node']:
for output in source_node['output']:
for target_node in model_json['node']:
if output in target_node['input']:
links.append({'source': source_node['name'],
'target': target_node['name']})
return links
def get_node_links(model_json):
"""
:return:
{
"0": {
"input": [],
"output": [
1
]
},
"1": {
"input": [
0
],
"output": [
2
]
}
}
"""
node_links = dict()
nodes = model_json['node']
# init all nodes
for idx in range(len(nodes)):
node_links[idx] = {'input': list(), 'output': list()}
for src_idx in range(len(nodes)):
for out_name in nodes[src_idx]['output']:
for dst_idx in range(len(nodes)):
if out_name in nodes[dst_idx]['input']:
node_links[src_idx]['output'].append(dst_idx)
node_links[dst_idx]['input'].append(src_idx)
return node_links
def add_level_to_node_links(node_links):
"""
:return:
{
"0": {
"input": [],
"output": [
1
],
"level": 1
},
"1": {
"input": [
0
],
"output": [
2
],
"level": 2
}
}
"""
# init level
for key in node_links:
node_links[key]['level'] = None
for idx in range(len(node_links)):
# the start up op's level is 1
if len(node_links[idx]['input']) == 0:
node_links[idx]['level'] = 1
else:
cur_level = node_links[idx]['level']
for in_idx in node_links[idx]['input']:
in_level = node_links[in_idx]['level']
assert in_level is not None
if cur_level is None or in_level >= cur_level:
node_links[idx]['level'] = in_level + 1
def get_level_to_all(node_links, model_json):
"""
level_to_nodes {level -> [node_1, node_2]}
output:
{
"35": {
"inputs": [
38,
39
],
"nodes": [
46
],
"outputs": []
}, {}
"""
level_to_nodes = dict()
for idx in node_links:
level = node_links[idx]['level']
if level not in level_to_nodes:
level_to_nodes[level] = list()
level_to_nodes[level].append(idx)
# debug_print(level_to_nodes)
"""
input_to_level {idx -> level}
level_to_inputs {level -> [input1, input2]}
"""
nodes = model_json['node']
input_to_level = dict()
inputs = model_json['input']
for in_idx in range(len(inputs)):
in_name = inputs[in_idx]['name']
for node_idx in range(len(nodes)):
if in_name in nodes[node_idx]['input']:
node_level = node_links[node_idx]['level']
in_level = node_level - 1
if in_idx not in input_to_level:
input_to_level[in_idx] = in_level
elif input_to_level[in_idx] > in_level:
input_to_level[in_idx] = in_level
level_to_inputs = dict()
for in_idx in input_to_level:
level = input_to_level[in_idx]
if level not in level_to_inputs:
level_to_inputs[level] = list()
level_to_inputs[level].append(in_idx)
# debug_print(level_to_inputs)
# get output level
output_to_level = dict()
outputs = model_json['output']
for out_idx in range(len(outputs)):
out_name = outputs[out_idx]['name']
for node_idx in range(len(nodes)):
if out_name in nodes[node_idx]['output']:
node_level = node_links[node_idx]['level']
out_level = node_level + 1
if out_level not in output_to_level:
output_to_level[out_idx] = out_level
else:
raise Exception("output " + out_name + "have multiple source")
level_to_outputs = dict()
for out_idx in output_to_level:
level = output_to_level[out_idx]
if level not in level_to_outputs:
level_to_outputs[level] = list()
level_to_outputs[level].append(out_idx)
level_to_all = dict()
def init_level(level):
if level not in level_to_all:
level_to_all[level] = {'nodes': list(), 'inputs': list(), 'outputs': list()}
# merge all levels
for level in level_to_nodes:
init_level(level)
level_to_all[level]['nodes'] = level_to_nodes[level]
for level in level_to_inputs:
init_level(level)
level_to_all[level]['inputs'] = level_to_inputs[level]
for level in level_to_outputs:
init_level(level)
level_to_all[level]['outputs'] = level_to_outputs[level]
# debug_print(level_to_all)
return level_to_all
def level_to_coordinate(level_to_all):
default_x = 100
x_step = 100
default_y = 10
y_step = 100
node_to_coordinate = dict()
input_to_coordinate = dict()
output_to_coordinate = dict()
def get_coordinate(x_idx, y_idx):
x = default_x + x_idx * x_step
y = default_y + y_idx * y_step
return {"x": int(x), "y": int(y)}
for level in level_to_all:
nodes = level_to_all[level]['nodes']
inputs = level_to_all[level]['inputs']
outputs = level_to_all[level]['outputs']
x_idx = 0
for node_idx in nodes:
node_to_coordinate[node_idx] = get_coordinate(x_idx, level)
x_idx += 1
for in_idx in inputs:
input_to_coordinate[in_idx] = get_coordinate(x_idx, level)
x_idx += 1
for out_idx in outputs:
output_to_coordinate[out_idx] = get_coordinate(x_idx, level)
x_idx += 1
return node_to_coordinate, input_to_coordinate, output_to_coordinate
def add_edges(json_obj):
# TODO(daming-lu): should try to de-duplicate node's out-edge
# Currently it is counted twice: 1 as out-edge, 1 as in-edge
......@@ -55,20 +318,136 @@ def add_edges(json_obj):
'label': 'label_' + str(label_incrementer)
})
label_incrementer += 1
return json_obj
def load_model(model_pb_path):
def transform_for_echars(model_json):
opItemStyle = {
"normal": {
"color": '#d95f02'
}
}
paraterItemStyle = {
"normal": {
"color": '#1b9e77'
}
};
paraSymbolSize = [12, 6]
paraSymbol = 'rect'
opSymbolSize = [5, 5]
option = {
"title": {
"text": 'Default Graph Name'
},
"tooltip": {
"show": False
},
"animationDurationUpdate": 1500,
"animationEasingUpdate": 'quinticInOut',
"series": [
{
"type": "graph",
"layout": "none",
"symbolSize": 8,
"roam": True,
"label": {
"normal": {
"show": True,
"color": 'black'
}
},
"edgeSymbol": ['none', 'arrow'],
"edgeSymbolSize": [0, 10],
"edgeLabel": {
"normal": {
"textStyle": {
"fontSize": 20
}
}
},
"lineStyle": {
"normal": {
"opacity": 0.9,
"width": 2,
"curveness": 0
}
},
"data": [],
"links": []
}
]
}
option['title']['text'] = model_json['name']
rename_model(model_json)
node_links = get_node_links(model_json)
add_level_to_node_links(node_links)
level_to_all = get_level_to_all(node_links, model_json)
node_to_coordinate, input_to_coordinate, output_to_coordinate = level_to_coordinate(level_to_all)
inputs = model_json['input']
nodes = model_json['node']
outputs = model_json['output']
echars_data = list()
for in_idx in range(len(inputs)):
input = inputs[in_idx]
data = dict()
data['name'] = input['name']
data['x'] = input_to_coordinate[in_idx]['x']
data['y'] = input_to_coordinate[in_idx]['y']
data['symbol'] = paraSymbol
data['itemStyle'] = paraterItemStyle
data['symbolSize'] = paraSymbolSize
echars_data.append(data)
for node_idx in range(len(nodes)):
node = nodes[node_idx]
data = dict()
data['name'] = node['name']
data['x'] = node_to_coordinate[node_idx]['x']
data['y'] = node_to_coordinate[node_idx]['y']
data['itemStyle'] = opItemStyle
data['symbolSize'] = opSymbolSize
echars_data.append(data)
for out_idx in range(len(outputs)):
output = outputs[out_idx]
data = dict()
data['name'] = output['name']
data['x'] = output_to_coordinate[out_idx]['x']
data['y'] = output_to_coordinate[out_idx]['y']
data['symbol'] = paraSymbol
data['itemStyle'] = paraterItemStyle
data['symbolSize'] = paraSymbolSize
echars_data.append(data)
option['series'][0]['data'] = echars_data
option['series'][0]['links'] = get_links(model_json)
return option
def to_IR_json(model_pb_path):
model = onnx.load(model_pb_path)
graph = model.graph
del graph.initializer[:]
# to json string
json_str = MessageToJson(model.graph)
json_obj = json.loads(json_str)
reorganize_inout(json_obj, 'input')
reorganize_inout(json_obj, 'output')
add_edges(json_obj)
return json.dumps(json_obj, sort_keys=True, indent=4, separators=(',', ': '))
model_json = json.loads(json_str)
reorganize_inout(model_json, 'input')
reorganize_inout(model_json, 'output')
return model_json
def load_model(model_pb_path):
model_json = to_IR_json(model_pb_path)
options = transform_for_echars(model_json)
return options
if __name__ == '__main__':
......
visualdl/server/graph_test.py
浏览文件 @ a7d15082
......@@ -8,8 +8,8 @@ class GraphTest(unittest.TestCase):
self.mock_dir = "./mock"
def test_graph_edges_squeezenet(self):
json_str = graph.load_model(self.mock_dir + '/squeezenet_model.pb')
json_obj = json.loads(json_str)
json_obj = graph.to_IR_json(self.mock_dir + '/squeezenet_model.pb')
json_obj = graph.add_edges(json_obj)
# 126 edges + 66 nodes (out-edge of each node is counted twice)
self.assertEqual(len(json_obj['edges']), 126 + 66)
......@@ -39,8 +39,8 @@ class GraphTest(unittest.TestCase):
self.assertEqual(json_obj['edges'][111]['label'], 'label_111')
def test_graph_edges_inception_v1(self):
json_str = graph.load_model(self.mock_dir + '/inception_v1_model.pb')
json_obj = json.loads(json_str)
json_obj = graph.to_IR_json(self.mock_dir + '/inception_v1_model.pb')
json_obj = graph.add_edges(json_obj)
# 286 edges + 143 nodes (out-edge of each node is counted twice)
self.assertEqual(len(json_obj['edges']), 286 + 143)
......
visualdl/server/visualDL
浏览文件 @ a7d15082
#! /user/bin/env python
#!/user/bin/env python
import json
import os
import re
import sys
from optparse import OptionParser
......@@ -11,7 +10,8 @@ from flask import (Flask, Response, redirect, request, send_file,
import visualdl
import visualdl.server
from visualdl.server import graph, lib
import visualdl.server.graph as vdl_graph
from visualdl.server import lib
from visualdl.server.log import logger
from visualdl.server.mock import data as mock_data
from visualdl.server.mock import data as mock_tags
......@@ -170,12 +170,12 @@ def histogram():
return Response(json.dumps(result), mimetype='application/json')
@app.route('/data/plugin/graphs/graph')
@app.route('/data/plugin/graphs/graphs')
def graph():
# run = request.args.get('run')
# model_json = graph.load_model("")
model_json_str = mock_data.graph_data()
model_json = json.loads(model_json_str)
# model_json_str = mock_data.graph_data()
# model_json = json.loads(model_json_str)
model_json = vdl_graph.load_model(options.logdir + "/model.pb")
result = gen_result(0, "", model_json)
return Response(json.dumps(result), mimetype='application/json')
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或