Skip to content

X
X2Paddle

PaddlePaddle / X2Paddle
大约 2 年 前同步成功

通知 329
Star 698
Fork 167
X
X2Paddle
提交 cf689f17 编写于 作者: J jiangjiajun
浏览文件
操作

add tensor infer function

上级 b5fe13be
隐藏空白更改
内联 并排
Showing with 1186 addition and 4001122 deletion
tensorflow2fluid/demo/diff_test/diff.py 已删除 100644 → 0
浏览文件 @ b5fe13be
import sys
import math
val1 = map(float, open(sys.argv[1]).read().strip().split('\n'))
val2 = map(float, open(sys.argv[2]).read().strip().split('\n'))
if len(val1) != len(val2):
raise Exception("Not Same Length")
max_diff = 0
avg_diff = 0
for i in range(len(val1)):
diff = math.fabs(val1[i] - val2[i])
if diff > max_diff:
max_diff = diff
avg_diff += diff
avg_diff /= len(val1)
print("max_diff: {}\tavg_diff: {}".format(max_diff, avg_diff))
tensorflow2fluid/demo/diff_test/resnet_v1_101_infer.py 已删除 100644 → 0
浏览文件 @ b5fe13be
# coding:utf-8
import sys
sys.path.append("..")
from paddle_resnet_v1_101.mymodel import KitModel
import paddle.fluid as fluid
import numpy
use_cuda = True
def model_initialize():
# 构建模型结构，并初始化参数
result = KitModel()
if use_cuda:
exe = fluid.Executor(fluid.CUDAPlace(0))
else:
exe = fluid.Executor(fluid.CPUPlace())
exe.run(fluid.default_startup_program())
# 根据save_var.list列表，加载模型参数
var_list = list()
global_block = fluid.default_main_program().global_block()
with open('../paddle_resnet_v1_101/save_var.list') as f:
for line in f:
try:
# 过滤部分不需要加载的参数（OP配置参数）
var = global_block.var(line.strip())
var_list.append(var)
except:
pass
fluid.io.load_vars(exe, '../paddle_resnet_v1_101', vars=var_list)
prog = fluid.default_main_program()
return exe, prog, result
def test_case(exe, prog, result):
# 测试随机数据输入
numpy.random.seed(13)
img_data = numpy.random.rand(1000, 224, 224, 3)
# tf中输入为NHWC，PaddlePaddle则为NCHW，需transpose
img_data = numpy.transpose(img_data, (0, 3, 1, 2))
# input_0为输入数据的张量名，张量名和数据类型须与my_model.py中定义一致
for i in range(0, 50):
r, = exe.run(
fluid.default_main_program(),
feed={
'input_0':
numpy.array(img_data[i * 20:i * 20 + 20], dtype='float32')
},
fetch_list=[result])
r = r.flatten()
files = open('fluid_resnet_v1_101.result', 'a+')
for i in range(0, r.shape[0]):
files.write(str(r[i]) + '\n')
files.close()
# 调用save_inference_model可将模型结构（当前以代码形式保存）和参数均序列化保存
# 保存后的模型可使用load_inference_model加载
# http://www.paddlepaddle.org/documentation/docs/zh/1.2/api_cn/api_guides/low_level/inference.html#api-guide-inference
# fluid.io.save_inference_model("./paddle_model", ["input_0"], [result], exe)
if __name__ == "__main__":
exe, prog, result = model_initialize()
test_case(exe, prog, result)
tensorflow2fluid/demo/diff_test/resnet_v1_50_infer.py 已删除 100644 → 0
浏览文件 @ b5fe13be
# coding:utf-8
import sys
sys.path.append("..")
from paddle_resnet_v1_50.mymodel import KitModel
import paddle.fluid as fluid
import numpy
use_cuda = True
def model_initialize():
# 构建模型结构，并初始化参数
result = KitModel()
if use_cuda:
exe = fluid.Executor(fluid.CUDAPlace(0))
else:
exe = fluid.Executor(fluid.CPUPlace())
exe.run(fluid.default_startup_program())
# 根据save_var.list列表，加载模型参数
var_list = list()
global_block = fluid.default_main_program().global_block()
with open('../paddle_resnet_v1_50/save_var.list') as f:
for line in f:
try:
# 过滤部分不需要加载的参数（OP配置参数）
var = global_block.var(line.strip())
var_list.append(var)
except:
pass
fluid.io.load_vars(exe, '../paddle_resnet_v1_50', vars=var_list)
prog = fluid.default_main_program()
return exe, prog, result
def test_case(exe, prog, result):
# 测试随机数据输入
numpy.random.seed(13)
img_data = numpy.random.rand(1000, 224, 224, 3)
# tf中输入为NHWC，PaddlePaddle则为NCHW，需transpose
img_data = numpy.transpose(img_data, (0, 3, 1, 2))
# input_0为输入数据的张量名，张量名和数据类型须与my_model.py中定义一致
for i in range(0, 50):
r, = exe.run(
fluid.default_main_program(),
feed={
'input_0':
numpy.array(img_data[i * 20:i * 20 + 20], dtype='float32')
},
fetch_list=[result])
r = r.flatten()
files = open('fluid_resnet_v1_50.result', 'a+')
for i in range(0, r.shape[0]):
files.write(str(r[i]) + '\n')
files.close()
# 调用save_inference_model可将模型结构（当前以代码形式保存）和参数均序列化保存
# 保存后的模型可使用load_inference_model加载
# http://www.paddlepaddle.org/documentation/docs/zh/1.2/api_cn/api_guides/low_level/inference.html#api-guide-inference
# fluid.io.save_inference_model("./paddle_model", ["input_0"], [result], exe)
if __name__ == "__main__":
exe, prog, result = model_initialize()
test_case(exe, prog, result)
tensorflow2fluid/demo/diff_test/run.sh 已删除 100644 → 0
浏览文件 @ b5fe13be
rm -rf fluid_vgg_19.result
python vgg_19_infer.py
echo "paddle fluid vgg_19 model"
python diff.py fluid_vgg_19.result tf_vgg_19.result
rm -rf fluid_vgg_16.result
python vgg_16_infer.py
echo "paddle fluid vgg_16 model"
python diff.py fluid_vgg_16.result tf_vgg_16.result
rm -rf fluid_resnet_v1_50.result
python resnet_v1_50_infer.py
echo "paddle fluid resnet_v1_50 model"
python diff.py fluid_resnet_v1_50.result tf_resnet_v1_50.result
rm -rf fluid_resnet_v1_101.result
python resnet_v1_101_infer.py
echo "paddle fluid resnet_v1_101 model"
python diff.py fluid_resnet_v1_101.result tf_resnet_v1_101.result
tensorflow2fluid/demo/diff_test/tf_resnet_v1_101.result 已删除 100644 → 0
浏览文件 @ b5fe13be
因为 它太大了无法显示 source diff 。你可以改为 查看blob。
tensorflow2fluid/demo/diff_test/tf_resnet_v1_50.result 已删除 100644 → 0
浏览文件 @ b5fe13be
因为 它太大了无法显示 source diff 。你可以改为 查看blob。
tensorflow2fluid/demo/diff_test/tf_vgg_16.result 已删除 100644 → 0
浏览文件 @ b5fe13be
因为 它太大了无法显示 source diff 。你可以改为 查看blob。
tensorflow2fluid/demo/diff_test/tf_vgg_19.result 已删除 100644 → 0
浏览文件 @ b5fe13be
因为 它太大了无法显示 source diff 。你可以改为 查看blob。
tensorflow2fluid/demo/diff_test/vgg_16_infer.py 已删除 100644 → 0
浏览文件 @ b5fe13be
# coding:utf-8
import sys
sys.path.append("..")
from paddle_vgg_16.mymodel import KitModel
import paddle.fluid as fluid
import numpy
use_cuda = True
def model_initialize():
# 构建模型结构，并初始化参数
result = KitModel()
if use_cuda:
exe = fluid.Executor(fluid.CUDAPlace(0))
else:
exe = fluid.Executor(fluid.CPUPlace())
exe.run(fluid.default_startup_program())
# 根据save_var.list列表，加载模型参数
var_list = list()
global_block = fluid.default_main_program().global_block()
with open('../paddle_vgg_16/save_var.list') as f:
for line in f:
try:
# 过滤部分不需要加载的参数（OP配置参数）
var = global_block.var(line.strip())
var_list.append(var)
except:
pass
fluid.io.load_vars(exe, '../paddle_vgg_16', vars=var_list)
prog = fluid.default_main_program()
return exe, prog, result
def test_case(exe, prog, result):
# 测试随机数据输入
numpy.random.seed(13)
img_data = numpy.random.rand(1000, 224, 224, 3)
# tf中输入为NHWC，PaddlePaddle则为NCHW，需transpose
img_data = numpy.transpose(img_data, (0, 3, 1, 2))
# input_0为输入数据的张量名，张量名和数据类型须与my_model.py中定义一致
for i in range(0, 50):
r, = exe.run(
fluid.default_main_program(),
feed={
'input_0':
numpy.array(img_data[i * 20:i * 20 + 20], dtype='float32')
},
fetch_list=[result])
r = r.flatten()
files = open('fluid_vgg_16.result', 'a+')
for i in range(0, r.shape[0]):
files.write(str(r[i]) + '\n')
files.close()
# 调用save_inference_model可将模型结构（当前以代码形式保存）和参数均序列化保存
# 保存后的模型可使用load_inference_model加载
# http://www.paddlepaddle.org/documentation/docs/zh/1.2/api_cn/api_guides/low_level/inference.html#api-guide-inference
# fluid.io.save_inference_model("./paddle_model", ["input_0"], [result], exe)
if __name__ == "__main__":
exe, prog, result = model_initialize()
test_case(exe, prog, result)
tensorflow2fluid/demo/diff_test/vgg_19_infer.py 已删除 100644 → 0
浏览文件 @ b5fe13be
# coding:utf-8
import sys
sys.path.append("..")
from paddle_vgg_19.mymodel import KitModel
import paddle.fluid as fluid
import numpy
use_cuda = True
def model_initialize():
# 构建模型结构，并初始化参数
result = KitModel()
if use_cuda:
exe = fluid.Executor(fluid.CUDAPlace(0))
else:
exe = fluid.Executor(fluid.CPUPlace())
exe.run(fluid.default_startup_program())
# 根据save_var.list列表，加载模型参数
var_list = list()
global_block = fluid.default_main_program().global_block()
with open('../paddle_vgg_19/save_var.list') as f:
for line in f:
try:
# 过滤部分不需要加载的参数（OP配置参数）
var = global_block.var(line.strip())
var_list.append(var)
except:
pass
fluid.io.load_vars(exe, '../paddle_vgg_19', vars=var_list)
prog = fluid.default_main_program()
return exe, prog, result
def test_case(exe, prog, result):
# 测试随机数据输入
numpy.random.seed(13)
img_data = numpy.random.rand(1000, 224, 224, 3)
# tf中输入为NHWC，PaddlePaddle则为NCHW，需transpose
img_data = numpy.transpose(img_data, (0, 3, 1, 2))
# input_0为输入数据的张量名，张量名和数据类型须与my_model.py中定义一致
for i in range(0, 50):
r, = exe.run(
fluid.default_main_program(),
feed={
'input_0':
numpy.array(img_data[i * 20:i * 20 + 20], dtype='float32')
},
fetch_list=[result])
r = r.flatten()
files = open('fluid_vgg_19.result', 'a+')
for i in range(0, r.shape[0]):
files.write(str(r[i]) + '\n')
files.close()
# 调用save_inference_model可将模型结构（当前以代码形式保存）和参数均序列化保存
# 保存后的模型可使用load_inference_model加载
# http://www.paddlepaddle.org/documentation/docs/zh/1.2/api_cn/api_guides/low_level/inference.html#api-guide-inference
# fluid.io.save_inference_model("./paddle_model", ["input_0"], [result], exe)
if __name__ == "__main__":
exe, prog, result = model_initialize()
test_case(exe, prog, result)
tensorflow2fluid/demo/export_to_checkpoint.py 已删除 100644 → 0
浏览文件 @ b5fe13be
from tensorflow.contrib.slim.nets import inception
from tensorflow.contrib.slim.nets import vgg as vgg
from tensorflow.contrib.slim.nets import resnet_v1 as resnet_v1
import tensorflow.contrib.slim as slim
import tensorflow as tf
from six import text_type as _text_type
def inception_v3(ckpt_file):
def get_tuned_variables():
CHECKPOINT_EXCLUDE_SCOPES = 'InceptionV3/Logits,InceptionV3/AuxLogits'
exclusions = [
scope.strip() for scope in CHECKPOINT_EXCLUDE_SCOPES.split(',')
]
variables_to_restore = []
for var in slim.get_model_variables():
excluded = False
for exclusion in exclusions:
if var.op.name.startswith(exclusion):
excluded = True
break
if not excluded:
variables_to_restore.append(var)
return variables_to_restore
img_size = inception.inception_v3.default_image_size
img = tf.placeholder(
tf.float32, shape=[None, img_size, img_size, 3], name='inputs')
with slim.arg_scope(inception.inception_v3_arg_scope()):
logits, _ = inception.inception_v3(
img, num_classes=1000, is_training=False)
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
load_model = tf.contrib.slim.assign_from_checkpoint_fn(
ckpt_file, get_tuned_variables(), ignore_missing_vars=True)
load_model(sess)
return sess
def resnet_v1_50(ckpt_file):
img_size = resnet_v1.resnet_v1.default_image_size
img = tf.placeholder(
tf.float32, shape=[None, img_size, img_size, 3], name='inputs')
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
net, endpoint = resnet_v1.resnet_v1_50(
img, num_classes=1000, is_training=False)
sess = tf.Session()
load_model = tf.contrib.slim.assign_from_checkpoint_fn(
ckpt_file, tf.contrib.slim.get_model_variables("resnet_v1_50"))
load_model(sess)
return sess
def resnet_v1_101(ckpt_file):
img_size = resnet_v1.resnet_v1.default_image_size
img = tf.placeholder(
tf.float32, shape=[None, img_size, img_size, 3], name='inputs')
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
net, endpoint = resnet_v1.resnet_v1_101(
img, num_classes=1000, is_training=False)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
load_model = tf.contrib.slim.assign_from_checkpoint_fn(
ckpt_file, tf.contrib.slim.get_model_variables("resnet_v1_101"))
load_model(sess)
return sess
def vgg_16(ckpt_file):
img_size = vgg.vgg_16.default_image_size
inputs = tf.placeholder(
tf.float32, shape=[None, img_size, img_size, 3], name="inputs")
logits, endpoint = vgg.vgg_16(inputs, num_classes=1000, is_training=False)
sess = tf.Session()
load_model = tf.contrib.slim.assign_from_checkpoint_fn(
ckpt_file, tf.contrib.slim.get_model_variables("vgg_16"))
load_model(sess)
return sess
def vgg_19(ckpt_file):
img_size = vgg.vgg_19.default_image_size
inputs = tf.placeholder(
tf.float32, shape=[None, img_size, img_size, 3], name="inputs")
logits, endpoint = vgg.vgg_19(inputs, num_classes=1000, is_training=False)
sess = tf.Session()
load_model = tf.contrib.slim.assign_from_checkpoint_fn(
ckpt_file, tf.contrib.slim.get_model_variables("vgg_19"))
load_model(sess)
return sess
def save_checkpoint(sess, save_dir):
saver = tf.train.Saver()
saver.save(sess, save_dir + "/model")
def get_parser():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
"--model",
"-m",
type=_text_type,
default=None,
help="inception_v3/resnet_v1_50/resnet_v1_101/vgg_16/vgg_19")
parser.add_argument(
"--ckpt_file",
"-c",
type=_text_type,
default=None,
help="parameters ckpt file")
parser.add_argument(
"--save_dir", "-s", type=_text_type, default=None, help="model path")
return parser
if __name__ == "__main__":
parser = get_parser()
args = parser.parse_args()
sess = None
if args.model is None or args.save_dir is None or args.ckpt_file is None:
raise Exception("--model, --ckpt_file and --save_dir are needed")
if args.model == "inception_v3":
sess = inception_v3(args.ckpt_file)
elif args.model == "resnet_v1_50":
sess = resnet_v1_50(args.ckpt_file)
elif args.model == "resnet_v1_101":
sess = resnet_v1_101(args.ckpt_file)
elif args.model == "vgg_16":
sess = vgg_16(args.ckpt_file)
elif args.model == "vgg_19":
sess = vgg_19(args.ckpt_file)
else:
raise Exception(
"Only support inception_v3/resnet_v1_50/resnet_v1_101/vgg_16/vgg_19"
)
save_checkpoint(sess, args.save_dir)
tensorflow2fluid/demo/resnet_v1_101.sh 已删除 100644 → 0
浏览文件 @ b5fe13be
export CUDA_VISIBLE_DEVICES=-1
wget http://download.tensorflow.org/models/resnet_v1_101_2016_08_28.tar.gz
tar xzvf resnet_v1_101_2016_08_28.tar.gz
python export_to_checkpoint.py --model resnet_v1_101 --ckpt_file resnet_v1_101.ckpt --save_dir resnet_v1_101_checkpoint
rm resnet_v1_101_2016_08_28.tar.gz resnet_v1_101.ckpt
tf2fluid --meta_file resnet_v1_101_checkpoint/model.meta \
--ckpt_dir resnet_v1_101_checkpoint \
--in_nodes inputs \
--input_shape None,224,224,3 \
--output_nodes resnet_v1_101/predictions/Softmax \
--save_dir paddle_resnet_v1_101
tensorflow2fluid/demo/resnet_v1_50.sh 已删除 100644 → 0
浏览文件 @ b5fe13be
export CUDA_VISIBLE_DEVICES=-1
wget http://download.tensorflow.org/models/resnet_v1_50_2016_08_28.tar.gz
tar xzvf resnet_v1_50_2016_08_28.tar.gz
python export_to_checkpoint.py --model resnet_v1_50 --ckpt_file resnet_v1_50.ckpt --save_dir resnet_v1_50_checkpoint
rm resnet_v1_50_2016_08_28.tar.gz resnet_v1_50.ckpt
tf2fluid --meta_file resnet_v1_50_checkpoint/model.meta \
--ckpt_dir resnet_v1_50_checkpoint \
--in_nodes inputs \
--input_shape None,224,224,3 \
--output_nodes resnet_v1_50/predictions/Softmax \
--save_dir paddle_resnet_v1_50
tensorflow2fluid/demo/vgg_16.sh 已删除 100644 → 0
浏览文件 @ b5fe13be
export CUDA_VISIBLE_DEVICES=-1
wget http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz
tar xzvf vgg_16_2016_08_28.tar.gz
python export_to_checkpoint.py --model vgg_16 --ckpt_file vgg_16.ckpt --save_dir vgg_16_checkpoint
rm vgg_16_2016_08_28.tar.gz vgg_16.ckpt
tf2fluid --meta_file vgg_16_checkpoint/model.meta \
--ckpt_dir vgg_16_checkpoint \
--in_nodes inputs \
--input_shape None,224,224,3 \
--output_nodes vgg_16/fc8/squeezed \
--save_dir paddle_vgg_16
tensorflow2fluid/demo/vgg_19.sh 已删除 100644 → 0
浏览文件 @ b5fe13be
export CUDA_VISIBLE_DEVICES=-1
wget http://download.tensorflow.org/models/vgg_19_2016_08_28.tar.gz
tar xzvf vgg_19_2016_08_28.tar.gz
python export_to_checkpoint.py --model vgg_19 --ckpt_file vgg_19.ckpt --save_dir vgg_19_checkpoint
rm vgg_19_2016_08_28.tar.gz vgg_19.ckpt
tf2fluid --meta_file vgg_19_checkpoint/model.meta \
--ckpt_dir vgg_19_checkpoint \
--in_nodes inputs \
--input_shape None,224,224,3 \
--output_nodes vgg_19/fc8/squeezed \
--save_dir paddle_vgg_19
tensorflow2fluid/setup.py 已删除 100644 → 0
浏览文件 @ b5fe13be
from __future__ import absolute_import
from setuptools import setup, find_packages
from io import open
setup(
name='tensorflow2fluid',
version='0.0.1',
description='Translate TensorFlow Model to PaddlePaddle',
url='http://paddlepaddle.org',
author = 'PaddlePaddle Development Group',
author_email='paddle-dev@baidu.com',
license='Apache 2',
packages=find_packages(),
install_requires=[],
entry_points={
'console_scripts': [
'tf2fluid = src.convert:_main',
],
},
)
tensorflow2fluid/src/convert.py
浏览文件 @ cf689f17
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from paddle_emitter import PaddleEmitter
from tensorflow_parser import TensorflowCkptParser
from tensorflow_parser import TensorflowPbParser
from six import text_type as _text_type
from utils import *
import argparse
import sys
import logging
import os
logging.basicConfig(level=logging.DEBUG)
def _get_parser():
parser = argparse.ArgumentParser()
parser.add_argument("--meta_file", "-m", type=_text_type, default=None, help="meta file path for checkpoint format")
parser.add_argument("--ckpt_dir", "-c", type=_text_type, default=None, help="checkpoint directory")
parser.add_argument("--pb_file", "-p", type=_text_type, default=None, help="pb model file path")
parser.add_argument("--in_nodes", "-i", type=_text_type, nargs="+", default=None, help="input nodes name")
parser.add_argument("--input_shape", "-is", type=_text_type, nargs="+", default=None, help="input tensor shape")
parser.add_argument("--output_nodes", "-o", type=_text_type, nargs="+", default=None, help="output nodes name")
parser.add_argument("--save_dir", "-s", type=_text_type, default=None, help="path to save transformed paddle model")
parser.add_argument("--version", "-v", action="version", version="tensorflow2fluid version=0.0.1 Release @2019.01.28")
parser.add_argument("--meta_file", "-m", type=_text_type,
default=None, help="meta file path for checkpoint format")
parser.add_argument("--ckpt_dir", "-c", type=_text_type,
default=None, help="checkpoint directory")
parser.add_argument("--pb_file", "-p", type=_text_type,
default=None, help="pb model file path")
parser.add_argument("--in_nodes", "-i", type=_text_type, nargs="+",
default=None, help="input nodes name")
parser.add_argument("--input_shape", "-is", type=_text_type, nargs="+",
default=None, help="input tensor shape")
parser.add_argument("--output_nodes", "-o", type=_text_type, nargs="+",
default=None, help="output nodes name")
parser.add_argument("--save_dir", "-s", type=_text_type,
default=None, help="path to save transformed paddle model")
parser.add_argument("--input_format", "-sf", type=_text_type,
default=None, help="input data format(NHWC/NCHW or OTHER)")
parser.add_argument("--use_cuda", "-u", type=_text_type,
default="True", help="True for use gpu")
return parser
def _convert(args):
if args.meta_file is None and args.pb_file is None:
raise Exception("Need to define --meta_file or --pb_file")
if args.input_format is None:
raise Exception("Input format need to be defined(NHWC, NCHW or OTHER)")
assert args.use_cuda == "True" or args.use_cuda == "False"
if args.use_cuda == "False":
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
if args.input_format == "NHWC":
input_format = NHWC
elif args.input_format == "NCHW":
input_format = NCHW
elif args.input_format == "OTHER":
input_format = OTHER
else:
raise Exception("Can not identify input format(NHWC/NCHW/OTHER)")
assert args.in_nodes is not None
assert args.output_nodes is not None
assert args.input_shape is not None
assert args.save_dir is not None
if os.path.exists(args.save_dir):
sys.stderr.write("save_dir already exists, change to a new path\n")
return
os.makedirs(args.save_dir)
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
input_shape = list()
for shape_str in args.input_shape:
......@@ -43,19 +81,21 @@ def _convert(args):
input_shape.append(items)
sys.stderr.write("\nLoading tensorflow model......\n")
logging.info("Loading tensorflow model...")
if args.meta_file is not None:
parser = TensorflowCkptParser(args.meta_file, args.ckpt_dir, args.output_nodes, input_shape, args.in_nodes)
parser = TensorflowCkptParser(args.meta_file, args.ckpt_dir,
args.output_nodes, input_shape, args.in_nodes, input_format)
else:
parser = TensorflowPbParser(args.pb_file, args.output_nodes, input_shape, args.in_nodes)
sys.stderr.write("Tensorflow model loaded!\n")
parser = TensorflowPbParser(args.pb_file, args.output_nodes,
input_shape, args.in_nodes, input_format)
logging.info("Tensorflow model loaded!")
emitter = PaddleEmitter(parser, args.save_dir)
emitter.run()
open(args.save_dir+"/__init__.py", "w").close()
def _main():
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
parser = _get_parser()
args = parser.parse_args()
_convert(args)
......
tensorflow2fluid/src/graph.py
浏览文件 @ cf689f17
......@@ -12,16 +12,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from name_generator import NameGenerator
from utils import *
import collections
class GraphNode(object):
def __init__(self, layer):
def __init__(self, layer, layer_name=None):
self.inputs = list()
self.outputs = list()
self.layer = layer
self.ref_name = None
self.output_name = None
if layer_name is not None:
self.layer_name = layer_name
else:
self.layer_name = layer.name
def __hash__(self):
return hash(self.layer.name)
......@@ -34,14 +39,14 @@ class GraphNode(object):
class Graph(object):
def __init__(self, model):
self.node_map = dict()
self.node_map = collections.OrderedDict()
self.input_nodes = list()
self.output_nodes = list()
self.topological_sort = list()
self.model = model
self.name_generator = NameGenerator()
def build(self):
def build(self, input_format):
self._make_input_nodes()
self._make_output_nodes()
self._get_topological_sort()
......@@ -78,9 +83,26 @@ class Graph(object):
for node_name in self.topological_sort:
node = self.node_map[node_name]
ref_name = self.name_generator.get_name(node)
if node.layer_type == 'split' or node.layer_type == 'splitv':
index = '0'
if len(node_name.split(':')) == 2:
index = node_name.split(':')[-1]
ref_name += '[{}]'.format(index)
self.node_map[node.layer.name].ref_name = ref_name
self.node_map[node.layer.name].output_name = ref_name.split('[')[0]
for node_name, node in self.node_map.items():
ref_name = self.name_generator.get_name(node)
if node.layer_type == 'split' or node.layer_type == 'splitv':
index = '0'
if len(node_name.split(':')) == 2:
index = node_name.split(':')[-1]
ref_name += '[{}]'.format(index)
self.node_map[node_name].ref_name = ref_name
self.node_map[node_name].output_name = ref_name.split('[')[0]
def get_node(self, name):
if name not in self.node_map:
raise Exception("Graph doesn't have node [%s]." % name)
......@@ -88,9 +110,8 @@ class Graph(object):
return self.node_map[name]
def _make_connection(self, src, dst):
if src.layer_name == dst.layer_name or src.layer_name not in self.node_map or dst.layer_name not in self.node_map:
if src.layer_name == dst.layer_name or src.layer_name not in \
self.node_map or dst.layer_name not in self.node_map:
raise Exception('Warning: Node not exist or there is a self-loop')
if src not in self.node_map[dst.layer_name].inputs:
self.node_map[dst.layer_name].inputs.append(src)
if dst not in self.node_map[src.layer_name].outputs:
self.node_map[src.layer_name].outputs.append(dst)
self.node_map[dst.layer_name].inputs.append(src)
self.node_map[src.layer_name].outputs.append(dst)
tensorflow2fluid/src/paddle_emitter.py
浏览文件 @ cf689f17
......@@ -14,43 +14,34 @@
from tensorflow.core.framework import attr_value_pb2
from tensorflow.python.framework import tensor_util
from utils import *
from functools import *
from six import string_types as _string_types
import framework_pb2 as framework
import logging
import math
import struct
import numpy
import logging
import sys
logging.basicConfig(level=logging.DEBUG)
class PaddleEmitter(object):
skip_op = set(['variablev2', 'identity'])
skip_op = set()
dtype_map = {1: "float32", 3: "int32", 9: "int64"}
def __init__(self, parser, save_dir):
self.graph = parser.tf_graph
self.weights = parser.weights
self.infer = parser.infer
self.inputs_sample_data = dict()
self.save_dir = save_dir
self.body_code = ""
self.tab = " " * 4
@staticmethod
def tensor_shape_to_list(shapes):
if isinstance(shapes, attr_value_pb2.AttrValue):
return [dim.size for dim in shapes.shape.dim]
else:
ret = []
for shape in shapes:
this_one = [dim.size for dim in shape.dim]
ret.append(this_one)
return ret
@staticmethod
def compute_padding_size(in_size, filter_size, stride):
new_size = int(math.ceil(in_size * 1.0 / stride))
pad_size = (new_size - 1) * stride + filter_size - in_size
return pad_size
pad_0 = int(pad_size / 2)
pad_1 = pad_size - pad_0
return [pad_0, pad_1]
def check_op(self, node_name_list):
uncovered_ops = set()
......@@ -58,29 +49,88 @@ class PaddleEmitter(object):
node = self.graph.get_node(name)
if len(node.inputs) == 0 and len(node.outputs) == 0:
continue
if node.layer_type in self.skip_op:
continue
if not hasattr(self, "emit_" + node.layer_type):
uncovered_ops.add(node.layer_type)
if len(uncovered_ops) > 0:
sys.stderr.write("Still {} tensorflow OP are not supported\n".format(len(uncovered_ops)))
logging.error("{} OP are not supported".format(len(uncovered_ops)))
for op in uncovered_ops:
sys.stderr.write("Unsupported OP: {}\n".format(op))
sys.exit(0)
logging.error("Unsupported OP: {}".format(op))
return False
return True
# trick method to solve NHWC problem
def get_axis(self, node1, node2):
shape1 = self.tensor_shape_to_list(node1.get_attr('_output_shapes'))[0]
shape2 = self.tensor_shape_to_list(node2.get_attr('_output_shapes'))[0]
if len(shape1) == 4 and len(
shape2) == 1 and node1.data_format == "NHWC":
shape1 = node1.shape_dim_size
shape2 = node2.shape_dim_size
if shape1 == 4 and shape2 == 1 and node1.data_format == NHWC:
axis = 1
elif len(shape2) == 4 and len(
shape1) == 1 and node2.data_format == "NHWC":
elif shape2 == 4 and shape1 == 1 and node2.data_format == NHWC:
axis = 1
else:
axis = -1
return axis
def elementwise(self, node, op):
data1 = node.inputs[0]
data2 = node.inputs[1]
axis = self.get_axis(data1, data2)
shape1 = self.infer.get_tensor_shape(data1.layer)
shape2 = self.infer.get_tensor_shape(data2.layer)
op = "elementwise_" + op
if shape2.shape[0] == shape1.shape[0]:
if (shape1 == shape2).all():
param_attr = {
'x':data1.ref_name,
'y':data2.ref_name,
}
node.code.add_layer(op, None, node.output_name, param_attr)
return
index1_not_one = list(numpy.argwhere(shape1 != 1).flatten())
index1_one = list(numpy.argwhere(shape1 == 1).flatten())
perm1 = range(shape1.shape[0])
perm2 = range(shape1.shape[0])
if len(index1_one) != 0:
perm1 = index1_not_one + index1_one
index2_not_one = list(numpy.argwhere(shape2 != 1).flatten())
index2_one = list(numpy.argwhere(shape2 == 1).flatten())
if len(index2_one) != 0:
perm2 = index2_not_one + index2_one
perm = list(numpy.array(perm1)[numpy.array(perm2)])
if perm != range(shape1.shape[0]):
param_attr = {"perm":perm}
node.code.add_layer("transpose", data1.ref_name,
"temp1", param_attr)
node.code.add_layer("transpose", data2.ref_name,
"temp2", param_attr)
if len(index2_one) > len(index1_one):
param_attr = {"x":"temp1", "y":"temp2"}
else:
param_attr = {"x":"temp2", "y":"temp1"}
node.code.add_layer(op, None, node.output_name, param_attr)
perm = sorted(range(len(perm)), key=lambda k:perm[k])
param_attr = {"perm":perm}
node.code.add_layer("transpose", node.output_name,
node.output_name, param_attr)
else:
if len(index2_one) > len(index1_one):
param_attr = {"x":data1.ref_name, "y":data2.ref_name}
else:
param_attr = {"x":data2.ref_name, "y":data1.ref_name}
node.code.add_layer(op, None, node.output_name, param_attr)
else:
param_attr = {"x":data1.ref_name, "y":data2.ref_name, "axis":axis}
if shape2.shape[0] > shape1.shape[0]:
param_attr = {
"x":data2.ref_name,
"y":data1.ref_name,
"axis":axis
}
node.code.add_layer(op, None, node.output_name, param_attr)
def export_weights(self, weight, paddle_var_name, dir):
self.save_var_set.add(paddle_var_name)
numpy_dtype_map = {
......@@ -115,10 +165,9 @@ class PaddleEmitter(object):
filew.write(tensor_desc.SerializeToString())
tensor_size = reduce(lambda x, y: x * y, shape)
weight = weight.flatten()
tensor_stream = ""
for i in range(0, tensor_size):
tensor_stream += struct.pack(struct_write_format[str(weight.dtype)], weight[i])
filew.write(tensor_stream)
filew.write(struct.pack(struct_write_format[str(weight.dtype)],
weight[i]))
filew.close()
@property
......@@ -126,6 +175,7 @@ class PaddleEmitter(object):
code = list()
code.append("import paddle.fluid.layers as layers")
code.append("import paddle.fluid as fluid")
code.append("import numpy")
code.append("")
code.append("def KitModel():")
return code
......@@ -133,6 +183,8 @@ class PaddleEmitter(object):
def add_codes(self, indent, codes):
if isinstance(codes, _string_types):
codes = codes.strip().split("\n")
if not isinstance(codes, list):
raise Exception("Unexpected error!")
for code in codes:
self.body_code += (self.tab * indent) + code + "\n"
......@@ -142,22 +194,29 @@ class PaddleEmitter(object):
self.save_var_set = set()
self.check_op(self.graph.topological_sort)
# filter branch nodes, like 'split:1'
translate_nodes = []
for node in self.graph.topological_sort:
if node.count(':') == 0:
translate_nodes.append(node)
# check if exists unsupported OPs in model
if not self.check_op(translate_nodes):
return
# ref_name.txt record relationship between
# paddle value name and tensorflow value name
ref_name_recorder = open(self.save_dir + "/ref_name.txt", 'w')
total_nodes_num = len(self.graph.topological_sort)
total_nodes_num = len(translate_nodes)
translated_nodes_count = 1
sys.stderr.write("\nModel Translating......\n")
sys.stderr.write("Start to translate all the nodes(Total_num:{})\n".
format(total_nodes_num))
for node in self.graph.topological_sort:
sys.stderr.write(
"\rTranslated nodes num: {}, Current node: {}".format(
translated_nodes_count, node))
sys.stderr.flush()
translated_nodes_count += 1
for node in translate_nodes:
logging.info("TotalNum:{},TraslatedNum:{},CurrentNode:{}".format(
total_nodes_num, translated_nodes_count, node))
current_node = self.graph.get_node(node)
ref_name_recorder.write("{}\t{}\n".format(
current_node.layer_name, current_node.output_name))
translated_nodes_count += 1
# skip isolated nodes
if len(current_node.inputs) == 0 and len(
......@@ -165,279 +224,257 @@ class PaddleEmitter(object):
continue
op = current_node.layer_type
if op in self.skip_op:
continue
if hasattr(self, "emit_" + op):
func = getattr(self, "emit_" + op)
codes = func(current_node)
if not isinstance(codes, list):
codes = [codes]
self.graph.get_node(node).codes = codes
func(current_node)
else:
raise Exception("Unknow node op: {}".format(op))
ref_name_recorder.close()
for node in self.graph.topological_sort:
codes = self.graph.get_node(node).codes
# merge all the generated python codes
for node in translate_nodes:
print(self.graph.get_node(node).output_name, node)
codes = self.graph.get_node(node).code.gen_codes()
self.add_codes(1, codes)
# add return value codes
outs = []
for node in self.graph.output_nodes:
outs.append(self.graph.get_node(node).output_name)
self.add_codes(1, "# {} : {}".format(
self.graph.get_node(node).output_name,
self.graph.get_node(node).layer_name))
self.add_codes(1, "return {}".format(", ".join(outs)))
# write python code to file "my_model.py"
filew = open(self.save_dir + "/mymodel.py", 'w')
filew.write(self.body_code)
filew.close()
# file "save_var.list" records name of dumped variables
filew = open(self.save_dir + "/save_var.list", 'w')
for var in self.save_var_set:
filew.write(var + '\n')
filew.close()
sys.stderr.write("Model translated!\n\n")
sys.stderr.flush()
logging.info("Model translated!")
return self.body_code
def emit_placeholder(self, node):
shape = self.tensor_shape_to_list(node.get_attr('_output_shapes'))[0]
shape = list(self.infer.get_tensor_shape(node.layer))
self.inputs_sample_data[node.layer_name] = []
if shape[0] < 0 or shape[0] is None:
self.batch_node = node
for i in range(1, 4):
sample_data = numpy.random.random_sample([i] + shape[1:])
self.inputs_sample_data[node.layer_name].append(sample_data)
else:
for i in range(1, 4):
sample_data = numpy.random.random_sample(shape)
self.inputs_sample_data[node.layer_name].append(sample_data)
if node.data_format == "NHWC" and len(shape) == 4:
if node.data_format == NHWC and len(shape) == 4:
shape = [shape[0], shape[3], shape[1], shape[2]]
dtype = node.get_attr("dtype")
if dtype in self.dtype_map:
dtype = self.dtype_map[dtype]
else:
raise Exception("Unknow dtype: {}".format(dtype))
code = list()
code.append("# placehoder[{}]:\t{}".format(node.output_name,
node.layer_name))
code.append(
"{} = layers.data(name=\'{}\', shape={}, dtype=\'{}\')".format(
node.output_name, node.ref_name, shape, dtype))
return code
param_attr = {
"name":"\'{}\'".format(node.ref_name),
"shape":shape,
"dtype":"\'{}\'".format(node.dtype),
"append_batch_size":False
}
node.code.add_str("# placeholder[{}]:\t{}".format(node.output_name,
node.layer_name))
node.code.add_layer("data", None, node.output_name, param_attr)
def emit_const(self, node):
try:
float_val = node.layer.attr['value'].tensor.float_val
if len(float_val) == 1:
code = list()
code.append("# {} {}".format(node.layer_name, node.ref_name))
code.append(
"{} = layers.fill_constant(shape=[1], value={}, dtype=\'float32\')"
.format(node.output_name, float_val[0]))
return code
int_val = node.layer.attr['value'].tensor.int_val
if len(int_val) == 1:
code = list()
code.append("# {} {}".format(node.layer_name, node.ref_name))
code.append(
"{} = layers.fill_constant(shape=[1], value={}, dtype=\'int32\')"
.format(node.output_name, int_val[0]))
return code
node.layer.attr['value'].tensor.tensor_content
weight = tensor_util.MakeNdarray(node.layer.attr['value'].tensor)
if len(weight) == 0:
return []
shape = list(weight.shape)
dtype = node.get_attr('dtype')
if dtype in self.dtype_map:
dtype = self.dtype_map[dtype]
else:
raise Exception("Unknow dtype[{}] of node[{}]".format(
dtype, node.layer_name))
code = list()
code.append("# {} {}".format(node.layer_name, node.ref_name))
if dtype.startswith('int'):
code.append(
"{} = layers.create_parameter({}, \'{}\', \'{}\', default_initializer=fluid.initializer.Constant(0))"
.format(node.output_name, shape, dtype, node.ref_name))
else:
code.append(
"{} = layers.create_parameter({}, \'{}\', \'{}\')".format(
node.output_name, shape, dtype, node.ref_name))
self.export_weights(weight, node.ref_name, self.save_dir)
value = self.infer.get_const_tensor_value(node.layer)
shape = list(value.shape)
return code
try:
dtype = node.dtype
except:
return []
node.code.add_str("#{} {} {}".format(node.layer_name,
node.ref_name, value.shape))
if len(shape) == 0 or (len(shape) ==1 and shape[0] < 2):
param_attr = {
"shape":[1],
"value":value,
"dtype":"\'{}\'".format(dtype),
}
node.code.add_layer("fill_constant", None,
node.output_name, param_attr)
else:
param_attr = {
"shape":shape,
"name":"\'{}\'".format(node.ref_name),
"dtype":"\'{}\'".format(dtype)
}
if node.dtype.startswith('int'):
param_attr["default_initializer"] = \
"fluid.initializer.Constant(0)"
node.code.add_layer("create_parameter", None,
node.output_name, param_attr)
self.export_weights(value, node.ref_name, self.save_dir)
def emit_conv2d(self, node):
data = node.inputs[0]
kernel = node.inputs[1]
if len(kernel.outputs) == 1:
kernel.codes = []
kernel.code.clear()
padding_mode = node.get_attr("padding")
strides = node.get_attr("strides")[1:3]
k_shape = self.tensor_shape_to_list(
kernel.get_attr("_output_shapes"))[0]
input_shape = self.tensor_shape_to_list(
data.get_attr("_output_shapes"))[0]
strides = node.get_attr("strides")[2:4]
k_shape = list(self.infer.get_tensor_shape(kernel.layer))
input_shape = list(self.infer.get_tensor_shape(data.layer))
input_h, input_w = input_shape[2:4]
kernel_num, channel, k_h, k_w = k_shape
if node.data_format == "NHWC":
k_h, k_w, channel, kernel_num = k_shape
k_h, k_w, channel, kernel_num = k_shape
if node.data_format == NHWC:
input_h, input_w = input_shape[1:3]
strides = node.get_attr("strides")[1:3]
if k_h < strides[0] or k_w < strides[1]:
raise Exception(
"Unexpected situation with kernel's height/width less than the corresponding stride"
)
raise Exception("Unexpected situation with kernel's height/width " \
"less than the corresponding stride")
if kernel.layer_name in self.weights:
if node.data_format == "NHWC":
weight = self.weights[kernel.layer_name]
self.weights[kernel.layer_name] = numpy.transpose(
weight, (3, 2, 0, 1))
weight = self.weights[kernel.layer_name]
self.weights[kernel.layer_name] = numpy.transpose(
weight, (3, 2, 0, 1))
self.export_weights(self.weights[kernel.layer_name],
kernel.ref_name, self.save_dir)
code = list()
padding = [0, 0]
if padding_mode == "SAME":
total_pad_h = self.compute_padding_size(input_h, k_h, strides[0])
total_pad_w = self.compute_padding_size(input_w, k_w, strides[1])
if total_pad_h % 2 == 0 and total_pad_w % 2 == 0:
padding = map(int, [total_pad_h / 2, total_pad_w / 2])
code.append(
"{} = layers.conv2d({}, {}, {}, padding={}, stride={}, param_attr=\'{}\', bias_attr=False)"
.format(node.output_name, data.ref_name, kernel_num,
[k_h, k_w], padding, strides, kernel.ref_name))
conv2d_param = {
"num_filters":kernel_num,
"filter_size":[k_h, k_w],
"stride":strides,
"param_attr":"\'{}\'".format(kernel.ref_name),
"bias_attr":False
}
if padding_mode == SAME:
pad_h = self.compute_padding_size(input_h, k_h, strides[0])
pad_w = self.compute_padding_size(input_w, k_w, strides[1])
if len(set(pad_h)) == 1 and len(set(pad_w)) == 1:
conv2d_param["padding"] = [pad_h[0], pad_w[0]]
node.code.add_layer("conv2d", data.ref_name,
node.output_name, conv2d_param)
else:
padding = [0] * 4
padding[0] = total_pad_h / 2
padding[1] = total_pad_h - padding[0]
padding[2] = total_pad_w / 2
padding[3] = total_pad_w - padding[2]
code.append("{} = layers.pad2d({}, {})".format(
node.output_name, data.ref_name, padding))
code.append(
"{} = layers.conv2d({}, {}, {}, stride={}, param_attr=\'{}\', bias_attr=False)"
.format(node.output_name, node.ref_name, kernel_num,
[k_h, k_w], strides, kernel.ref_name))
pad_param = {"paddings":pad_h + pad_w}
node.code.add_layer("pad2d", data.ref_name,
node.output_name, pad_param)
node.code.add_layer("conv2d", node.output_name,
node.output_name, conv2d_param)
else:
code.append(
"{} = layers.conv2d({}, {}, {}, stride={}, param_attr=\'{}\', bias_attr=False)"
.format(node.output_name, data.ref_name, kernel_num,
[k_h, k_w], strides, kernel.ref_name))
return code
node.code.add_layer("conv2d", data.ref_name,
node.output_name, conv2d_param)
def emit_variablev2(self, node):
shape = self.tensor_shape_to_list(node.get_attr("_output_shapes"))[0]
dtype = node.get_attr("dtype")
if dtype in self.dtype_map:
dtype = self.dtype_map[dtype]
else:
raise Exception("Unknow dtype[{}] of node[{}]".format(
dtype, node.layer_name))
shape = list(self.infer.get_tensor_shape(node.layer))
code = list()
code.append("# variable[{}]:\t{}".format(node.output_name,
node.layer_name))
if dtype.startswith('int'):
code.append(
"{} = layers.create_parameter(name=\'{}\', shape={}, dtype=\'{}\', default_initializer=fluid.initializer.Constant(0))"
.format(node.output_name, node.ref_name, shape, dtype))
else:
code.append(
"{} = layers.create_parameter(name=\'{}\', shape={}, dtype=\'{}\')"
.format(node.output_name, node.ref_name, shape, dtype))
return code
node.code.add_str("# variable[{}]:\t{}".format(node.output_name,
node.layer_name))
if node.layer_name in self.weights:
self.export_weights(self.weights[node.layer_name],
node.ref_name, self.save_dir)
param_attr = {
"name":"\'{}\'".format(node.ref_name),
"shape":shape,
"dtype":"\'{}\'".format(node.dtype)
}
if node.dtype.startswith('int'):
param_attr["default_initializer"] = "fluid.initializer.Constant(0)"
node.code.add_layer("create_parameter", None,
node.output_name, param_attr)
def emit_biasadd(self, node):
data = node.inputs[0]
bias = node.inputs[1]
axis = self.get_axis(data, bias)
if bias.layer_name in self.weights:
self.export_weights(self.weights[bias.layer_name], bias.ref_name,
self.save_dir)
code = list()
code = code + self.emit_variablev2(bias)
code.append("{} = layers.elementwise_add({}, {}, axis={})".format(
node.output_name, data.ref_name, bias.ref_name, axis))
return code
self.export_weights(self.weights[bias.layer_name],
bias.ref_name, self.save_dir)
self.emit_variablev2(bias)
param_attr = {
"x":data.ref_name,
"y":bias.ref_name,
"axis":1
}
node.code.add_layer("elementwise_add", None,
node.output_name, param_attr)
def emit_relu(self, node):
data = node.inputs[0]
code = "{} = layers.relu({})".format(node.output_name, data.ref_name)
return code
node.code.add_layer("relu", data.ref_name, node.output_name)
def emit_maxpool(self, node):
data = node.inputs[0]
padding_mode = node.get_attr("padding")
strides = node.get_attr("strides")[1:3]
pool_size = node.get_attr("ksize")[1:3]
if padding_mode == "SAME":
pad_right = (pool_size[0] - 1) / 2
pad_bottom = (pool_size[1] - 1) / 2
padding = [0, pad_right * 2, 0, pad_bottom * 2]
code = [
"pad_net = layers.pad2d({}, paddings={})".format(
data.ref_name, padding)
]
code.append("{} = layers.pool2d(pad_net, {}, \'max\', {})".format(
node.output_name, pool_size, strides))
return code
input_shape = list(self.infer.get_tensor_shape(node.layer))
input_h, input_w = input_shape[2:4]
strides = node.get_attr("strides")[2:4]
pool_size = node.get_attr("ksize")[2:4]
if node.data_format == NHWC:
input_h, input_w = input_shape[1:3]
strides = node.get_attr("strides")[1:3]
pool_size = node.get_attr("ksize")[1:3]
pool_param = {
"pool_size":pool_size,
"pool_type":"\'max\'",
"pool_stride":strides
}
if padding_mode == SAME:
pad_h = self.compute_padding_size(input_h, pool_size[0], strides[0])
pad_w = self.compute_padding_size(input_w, pool_size[1], strides[1])
pad_right = pad_w[0] + pad_w[1]
pad_bottom = pad_h[0] + pad_h[1]
padding = [0, pad_right, 0, pad_bottom]
pad_param = {"paddings":padding}
node.code.add_layer("pad2d", data.ref_name,
node.output_name, pad_param)
node.code.add_layer("pool2d", node.output_name,
node.output_name, pool_param)
else:
code = "{} = layers.pool2d({}, {}, \'max\', {})".format(
node.output_name, data.ref_name, pool_size, strides)
return code
node.code.add_layer("pool2d", data.ref_name,
node.output_name, pool_param)
def emit_squeeze(self, node):
data = node.inputs[0]
axis = node.get_attr("squeeze_dims")
input_shape = self.tensor_shape_to_list(
data.get_attr("_output_shapes"))[0]
if node.data_format == "NHWC" and len(input_shape) == 4:
input_shape_len = data.shape_dim_size
if node.data_format == NHWC and input_shape_len == 4:
for i in range(0, len(axis)):
if axis[i] == 1:
axis[i] = 2
elif axis[i] == 2:
axis[i] = 3
elif axis[i] == 3:
axis[i] = 1
code = "{} = layers.squeeze({}, {})".format(node.output_name,
data.ref_name, axis)
return code
if axis[i] > 0:
axis[i] = (axis[i] + 1) % 4 + int((axis[i] + 1) / 4)
param_attr = {"axes":axis}
node.code.add_layer("squeeze", data.ref_name,
node.output_name, param_attr)
def emit_add(self, node):
data1 = node.inputs[0]
data2 = node.inputs[1]
axis = self.get_axis(data1, data2)
code = "{} = layers.elementwise_add({}, {}, axis={})".format(node.output_name, data1.ref_name, data2.ref_name, axis)
return code
return self.elementwise(node, "add")
def emit_mean(self, node):
data = node.inputs[0]
reduce_idx = node.inputs[1]
reduce_idx.codes = []
idxs = tensor_util.MakeNdarray(reduce_idx.layer.attr['value'].tensor)
shape = idxs.shape
assert len(shape) == 1
data_shape = self.tensor_shape_to_list(
data.get_attr('_output_shapes'))[0]
reduce_idx.code.clear()
idxs = list(self.infer.get_const_tensor_value(
reduce_idx.layer).flatten())
data_shape_len = data.shape_dim_size
keep_dims = node.layer.attr['keep_dims'].b
if node.data_format == "NHWC" and len(data_shape) == 4:
for i in range(0, shape[0]):
if idxs[i] == 1:
idxs[i] = 2
elif idxs[i] == 2:
idxs[i] = 3
elif idxs[i] == 3:
idxs[i] = 1
code = "{} = layers.reduce_mean({}, {}, keep_dim={})".format(
node.output_name, data.ref_name, list(idxs), keep_dims)
return code
if node.data_format == NHWC and data_shape_len == 4:
for i in range(len(idxs)):
if idxs[i] > 0:
idxs[i] = (idxs[i] + 1) % 4 + int((idxs[i] + 1) / 4)
param_attr = {"dim":list(idxs), "keep_dim":keep_dims}
node.code.add_layer("reduce_mean", data.ref_name,
node.output_name, param_attr)
def emit_fusedbatchnorm(self, node):
data = node.inputs[0]
......@@ -446,13 +483,13 @@ class PaddleEmitter(object):
moving_mean = node.inputs[3]
moving_variance = node.inputs[4]
if len(gamma.outputs) == 1:
gamma.codes = []
gamma.code.clear()
if len(beta.outputs) == 1:
beta.codes = []
beta.code.clear()
if len(moving_mean.outputs) == 1:
moving_mean.codes = []
moving_mean.code.clear()
if len(moving_variance.outputs) == 1:
moving_variance.codes = []
moving_variance.code.clear()
epsilon = round(node.get_attr('epsilon'), 6)
is_training = node.get_attr('is_training')
......@@ -470,315 +507,531 @@ class PaddleEmitter(object):
self.export_weights(self.weights[moving_variance.layer_name],
moving_variance.ref_name, self.save_dir)
code = "{} = layers.batch_norm({}, epsilon={}, param_attr=\'{}\', bias_attr=\'{}\', moving_mean_name=\'{}\', moving_variance_name=\'{}\', is_test={})".format(
node.output_name, data.ref_name, epsilon, gamma.ref_name,
beta.ref_name, moving_mean.ref_name, moving_variance.ref_name,
not is_training)
return code
param_attr = {
"epsilon":epsilon,
"param_attr":"\'{}\'".format(gamma.ref_name),
"bias_attr":"\'{}\'".format(beta.ref_name),
"moving_mean_name":"\'{}\'".format(moving_mean.ref_name),
"moving_variance_name":"\'{}\'".format(moving_variance.ref_name),
"is_test": not is_training
}
node.code.add_layer("batch_norm", data.ref_name,
node.output_name, param_attr)
def emit_concatv2(self, node):
input_shape = self.tensor_shape_to_list(
node.inputs[0].get_attr('_output_shapes'))[0]
input_shape_len = node.inputs[0].shape_dim_size
axis = node.inputs[-1]
axis.codes = []
axis = axis.layer.attr['value'].tensor.int_val[0]
if node.data_format == "NHWC" and len(input_shape) == 4:
if axis == 1:
axis = 2
elif axis == 2:
axis = 3
elif axis == 3:
axis = 1
axis.code.clear()
axis = self.infer.get_const_tensor_value(axis.layer)
if axis < 0:
axis = input_shape_len + axis
if node.data_format == NHWC and input_shape_len == 4:
if axis >0:
axis = (axis + 1) % 4 + int((axis + 1) / 4)
num_tensor = len(node.inputs) - 1
code = "{} = layers.concat([{}], {})".format(
node.output_name, ", ".join(
[input.ref_name for input in node.inputs[:num_tensor]]), axis)
return code
input_list = [input.ref_name for input in node.inputs[:num_tensor]]
input_list = "[{}]".format(", ".join(input_list))
param_attr = {"axis":axis}
node.code.add_layer("concat", input_list, node.output_name, param_attr)
def emit_avgpool(self, node):
data = node.inputs[0]
padding_mode = node.get_attr("padding")
strides = node.get_attr("strides")[1:3]
pool_size = node.get_attr("ksize")[1:3]
padding = [0, 0]
if padding_mode == "SAME":
pad_h = (pool_size[0] - 1) / 2
pad_w = (pool_size[1] - 1) / 2
padding = [pad_h, pad_w]
code = "{} = layers.pool2d({}, {}, \'avg\', {}, {})".format(
node.output_name, data.ref_name, pool_size, strides, padding)
return code
input_shape = list(self.infer.get_tensor_shape(data.layer))
strides = node.get_attr("strides")[2:4]
pool_size = node.get_attr("ksize")[2:4]
input_h, input_w = input_shape[2:4]
if node.data_format == NHWC:
strides = node.get_attr("strides")[1:3]
pool_size = node.get_attr("ksize")[1:3]
input_h, input_w = input_shape[1:3]
param_attr = {
"pool_size":pool_size,
"pool_stride":strides,
"pool_type":"\'avg\'"
}
if padding_mode == SAME:
pad_h = self.compute_padding_size(input_h, pool_size[0], strides[0])
pad_w = self.compute_padding_size(input_w, pool_size[1], strides[0])
if len(set(pad_h)) == 1 and len(set(pad_w)) == 1:
padding = [pad_h[0], pad_w[0]]
param_attr["pool_padding"] = padding
else:
pad_param = {"paddings":pad_h + pad_w}
node.code.add_layer("pad2d", data.ref_name,
node.output_name, pad_param)
node.code.add_layer("pool2d", node.output_name,
node.output_name, param_attr)
node.code.add_layer("pool2d", data.ref_name, node.output_name, param_attr)
def emit_rsqrt(self, node):
data = node.inputs[0]
code = list()
code.append("sqrt_res = layers.sqrt({})".format(data.ref_name))
code.append("{} = layers.pow(sqrt_res, -1.0)".format(node.output_name))
return code
pow_param = {"factor":-1.0}
node.code.add_layer("sqrt", data.ref_name, node.output_name)
node.code.add_layer("pow", node.output_name, node.output_name, pow_param)
def emit_mul(self, node):
data1 = node.inputs[0]
data2 = node.inputs[1]
axis = self.get_axis(data1, data2)
code = "{} = layers.elementwise_mul({}, {}, axis={})".format(
node.output_name, data1.ref_name, data2.ref_name, axis)
shape1 = self.tensor_shape_to_list(data1.get_attr('_output_shapes'))[0]
shape2 = self.tensor_shape_to_list(data2.get_attr('_output_shapes'))[0]
if len(shape2) > len(shape1):
code = "{} = layers.elementwise_mul({}, {}, axis={})".format(
node.output_name, data2.ref_name, data1.ref_name, axis)
return code
return self.elementwise(node, "mul")
def emit_sub(self, node):
data1 = node.inputs[0]
data2 = node.inputs[1]
axis = self.get_axis(data1, data2)
code = "{} = layers.elementwise_sub({}, {}, axis={})".format(
node.output_name, data1.ref_name, data2.ref_name, axis)
return code
data1_shape = list(self.infer.get_tensor_shape(data1.layer))
data2_shape = list(self.infer.get_tensor_shape(data2.layer))
param_attr = {"x":data1.ref_name, "y":data2.ref_name, "axis":axis}
if len(data1_shape) == 4 and len(data2_shape) == 4 \
and node.data_format == NHWC:
if data1_shape[-1] != data2_shape[-1]:
node.code.add_layer("transpose", data1.ref_name,
"temp1", {"perm":[0, 2, 3, 1]})
node.code.add_layer("transpose", data2.ref_name,
"temp2", {"perm":[0, 2, 3, 1]})
param_attr = {"x":"temp1", "y":"temp2", "axis":-1}
node.code.add_layer("elementwise_sub", None,
node.output_name, param_attr)
node.code.add_layer("transpose", node.output_name,
node.output_name, {"perm":[0, 3, 1, 2]})
else:
node.code.add_layer("elementwise_sub", None,
node.output_name, param_attr)
def emit_shape(self, node):
data = node.inputs[0]
code = "{} = layers.shape({})".format(node.output_name, data.ref_name)
return code
input_shape_len = data.shape_dim_size
if input_shape_len == 4 and node.data_format == NHWC:
param = {"perm":[0, 2, 3, 1]}
node.code.add_layer("transpose", data.ref_name,
node.output_name, param)
node.code.add_layer("shape", node.output_name, node.output_name)
else:
node.code.add_layer("shape", data.ref_name, node.output_name)
param = {"dtype":"\'int32\'"}
node.code.add_layer("cast", node.output_name, node.output_name, param)
def emit_pad(self, node):
data = node.inputs[0]
padding = node.inputs[1]
padding.codes = []
padding.code.clear()
padding = padding.layer.attr['value'].tensor
padding = tensor_util.MakeNdarray(padding)
if node.data_format == "NHWC" and padding.shape[0] == 4:
padding = tensor_util.MakeNdarray(padding).astype('int32')
if node.data_format == NHWC and padding.shape[0] == 4:
padding = padding[[0, 3, 1, 2]]
code = "{} = layers.pad({}, {})".format(node.output_name,
data.ref_name,
list(padding.flatten()))
return code
param_attr = {"paddings":list(padding.flatten())}
node.code.add_layer("pad", data.ref_name, node.output_name, param_attr)
def emit_stridedslice(self, node):
data = node.inputs[0]
begin = node.inputs[1]
end = node.inputs[2]
strides = node.inputs[3]
begin.codes = []
end.codes = []
strides.codes = []
begin = list(tensor_util.MakeNdarray(begin.layer.attr['value'].tensor))
end = list(tensor_util.MakeNdarray(end.layer.attr['value'].tensor))
strides = list(
tensor_util.MakeNdarray(strides.layer.attr['value'].tensor))
begin.code.clear()
end.code.clear()
strides.code.clear()
begin = list(self.infer.get_const_tensor_value(begin.layer).flatten())
end = list(self.infer.get_const_tensor_value(end.layer).flatten())
strides = list(self.infer.get_const_tensor_value(
strides.layer).flatten())
for i in range(len(strides)):
assert strides[i] == 1
code = "{} = layers.slice({}, axes={}, starts={}, ends={})".format(
node.output_name, data.ref_name, [i for i in range(len(begin))],
begin, end)
return code
param_attr = {
"axes":range(len(begin)),
"starts":begin,
"ends":end
}
node.code.add_layer("slice", data.ref_name,
node.output_name, param_attr)
def emit_resizenearestneighbor(self, node):
data = node.inputs[0]
output_shape = node.inputs[1]
resize_shape = node.inputs[1]
resize_shape.code.clear()
align_corners = node.get_attr('align_corners')
if output_shape.layer_type == "const":
output_shape.codes = []
output_shape = tensor_util.MakeNdarray(
output_shape.layer.attr['value'].tensor)
code = "{} = layers.resize_nearest({}, {}, align_corners={}, align_mode=1)".format(
node.output_name, data.ref_name, list(output_shape),
align_corners)
else:
code = "{} = layers.resize_nearest({}, {}, align_corners={}, align_mode=1)".format(
node.output_name, data.ref_name, output_shape.ref_name,
align_corners)
logging.warn(
"\tNotice there's RESIZE_NEAREST in translated code, and the code list below:"
)
logging.warn("\t\t{}".format(code))
logging.warn(
"\tPaddle doesn't support tensor type for output_shape now")
logging.warn(
"\tYou need to change \'{}\'(in tf model: \'{}\') to a list with constant value, e.g. [28, 28]. IMPORTANT!!!\n"
.format(output_shape.ref_name, output_shape.layer_name))
return code
resize_shape = list(self.infer.get_shape_tensor(resize_shape.layer))
param_attr = {
"align_corners":align_corners,
"out_shape":resize_shape
}
node.code.add_layer("resize_nearest", data.ref_name,
node.output_name, param_attr)
def emit_maximum(self, node):
data1 = node.inputs[0]
data2 = node.inputs[1]
axis = self.get_axis(data1, data2)
code = "{} = layers.elementwise_max({}, {}, axis={})".format(
node.output_name, data1.ref_name, data2.ref_name, axis)
return code
return self.elementwise(node, "max")
def emit_minimum(self, node):
data1 = node.inputs[0]
data2 = node.inputs[1]
axis = self.get_axis(data1, data2)
code = "{} = layers.elementwise_min({}, {}, axis={})".format(
node.output_name, data1.ref_name, data2.ref_name, axis)
return code
return self.elementwise(node, "min")
def emit_sigmoid(self, node):
data = node.inputs[0]
code = "{} = layers.sigmoid({})".format(node.output_name,
data.ref_name)
return code
node.code.add_layer("sigmoid", data.ref_name, node.output_name)
def emit_pack(self, node):
ref_name = [input.ref_name for input in node.inputs]
code = "{} = layers.stack([{}])".format(node.output_name,
", ".join(ref_name))
return code
inputs = [input.ref_name for input in node.inputs]
inputs = "[{}]".format(", ".join(inputs))
node.code.add_layer("stack", inputs, node.output_name)
def emit_reshape(self, node):
data = node.inputs[0]
shape = node.inputs[1]
if shape.layer_type == "const":
shape = shape.layer.attr['value'].tensor
shape = list(tensor_util.MakeNdarray(shape))
code = "{} = layers.reshape({}, {})".format(
node.output_name, data.ref_name, shape)
input_shape_len = data.shape_dim_size
output_shape = list(self.infer.get_tensor_shape(node.layer))
shape = self.infer.get_shape_tensor(shape.layer, output_shape)
reshape_param = {"shape":list(shape)}
if node.data_format == NHWC and input_shape_len == 4:
param_attr = {"perm":[0, 2, 3, 1]}
node.code.add_layer("transpose", data.ref_name,
node.output_name, param_attr)
node.code.add_layer("reshape", node.output_name,
node.output_name, reshape_param)
if len(shape) == 4:
param_attr = {"perm":[0, 3, 1, 2]}
node.code.add_layer("transpose", node.output_name,
node.output_name, param_attr)
else:
code = "{} = layers.reshape({}, {})".format(
node.output_name, data.ref_name, shape.ref_name)
logging.warn(
"\tNotice there's RESHAPE in translated code, and the code list below:"
)
logging.warn("\t\t{}".format(code))
logging.warn(
"\tPaddle doesn't support tensor type for output_shape now")
logging.warn(
"\tYou need to change \'{}\'(in tf model: \'{}\') to a list with constant value, e.g. [28, 28]. IMPORTANT!!!\n"
.format(shape.ref_name, shape.layer_name))
return code
node.code.add_layer("reshape", data.ref_name,
node.output_name, reshape_param)
def emit_conv2dbackpropinput(self, node):
output_shape = node.inputs[0]
kernel = node.inputs[1]
data = node.inputs[2]
if len(kernel.outputs) == 1:
kernel.codes = []
kernel.code.clear()
output_shape.code.clear()
padding_mode = node.get_attr("padding")
strides = node.get_attr("strides")[1:3]
k_shape = self.tensor_shape_to_list(
kernel.get_attr("_output_shapes"))[0]
channel, k_num, k_h, k_w = k_shape
if node.data_format == "NHWC":
k_h, k_w, k_num, channel = k_shape
strides = node.get_attr("strides")[2:4]
k_shape = self.infer.get_tensor_shape(kernel.layer)
k_h, k_w, k_num, channel = k_shape
if node.data_format == NHWC:
strides = node.get_attr("strides")[1:3]
if strides[0] > k_h or strides[1] > k_w:
raise Exception(
"Paddle cannot process the situation now[kernel's height/width less than the corresponding stride]"
)
raise Exception("Paddle cannot process the situation now[kernel's" \
" height/width less than the corresponding stride]")
padding = [0, 0]
if padding_mode == "SAME":
padding = map(int, [(k_h - strides[0]) / 2,
(k_w - strides[1]) / 2])
if padding_mode == SAME:
padding = [int(val) for val in [(k_h - strides[0]) / 2, \
(k_w - strides[1]) / 2]]
if kernel.layer_name in self.weights:
if node.data_format == "NHWC":
weight = self.weights[kernel.layer_name]
self.weights[kernel.layer_name] = numpy.transpose(
weight, (3, 2, 0, 1))
self.export_weights(self.weights[kernel.layer_name],
kernel.ref_name, self.save_dir)
code = []
if output_shape.layer_type == "const":
output_shape.codes = []
output_shape = tensor_util.MakeNdarray(
output_shape.layer.attr['value'].tensor)
if node.data_format == "NHWC" and output_shape[0] == 4:
output_shape = output_shape[[0, 3, 1, 2]]
code.append(
"{} = layers.conv2d_transpose({}, {}, None, {}, {}, {}, param_attr=\'{}\', bias_attr=False)"
.format(node.output_name, data.ref_name, k_num, [k_h, k_w],
padding, strides, kernel.ref_name))
if padding_mode == "SAME":
code.append("{} = layers.crop({}, shape={})".format(
node.output_name, node.output_name, list(output_shape)))
else:
code.append(
"{} = layers.conv2d_transpose({}, {}, None, {}, 0, {}, param_attr=\'{}\', bias_attr=False)"
.format(node.output_name, data.ref_name, k_num, [k_h, k_w],
strides, kernel.ref_name))
if padding_mode == "SAME":
code.append("{} = layers.crop({}, shape={})".format(
node.output_name, node.output_name, output_shape.ref_name))
weight = self.weights[kernel.layer_name]
self.weights[kernel.layer_name] = numpy.transpose(
weight, (3, 2, 0, 1))
self.export_weights(self.weights[kernel.layer_name],
kernel.ref_name, self.save_dir)
return code
output_shape = list(self.infer.get_shape_tensor(output_shape.layer))
if node.data_format == NHWC and len(output_shape) == 4:
output_shape = [output_shape[0], output_shape[3],
output_shape[1], output_shape[2]]
param_attr = {
"num_filters":k_num,
"filter_size":[k_h, k_w],
"padding":padding,
"stride":strides,
"param_attr":"\'{}\'".format(kernel.ref_name),
"bias_attr":False
}
node.code.add_layer("conv2d_transpose", data.ref_name,
node.output_name, param_attr)
if padding_mode == SAME:
param_attr = {"shape":list(output_shape)}
node.code.add_layer("crop", node.output_name,
node.output_name, param_attr)
def emit_depthwiseconv2dnative(self, node):
data = node.inputs[0]
kernel = node.inputs[1]
if len(kernel.outputs) == 1:
kernel.codes = []
kernel.code.clear()
padding_mode = node.get_attr("padding")
strides = node.get_attr("strides")[1:3]
k_shape = self.tensor_shape_to_list(
kernel.get_attr("_output_shapes"))[0]
input_shape = self.tensor_shape_to_list(
data.get_attr("_output_shapes"))[0]
strides = node.get_attr("strides")[2:4]
k_shape = self.infer.get_tensor_shape(kernel.layer)
input_shape = self.infer.get_tensor_shape(data.layer)
input_h, input_w = input_shape[2:4]
in_channels, channel_multiplier, k_h, k_w = k_shape
if node.data_format == "NHWC":
k_h, k_w, in_channels, channel_multiplier = k_shape
k_h, k_w, in_channels, channel_multiplier = k_shape
if node.data_format == NHWC:
strides = node.get_attr("strides")[1:3]
input_h, input_w = input_shape[1:3]
if k_h < strides[0] or k_w < strides[1]:
raise Exception(
"Unexpected situation with kernel's height/width less than the corresponding stride"
)
groups = channel_multiplier * in_channels
if kernel.layer_name in self.weights:
if node.data_format == "NHWC":
weight = self.weights[kernel.layer_name]
self.weights[kernel.layer_name] = numpy.transpose(
weight, (2, 3, 0, 1))
weight = self.weights[kernel.layer_name]
self.weights[kernel.layer_name] = numpy.transpose(
weight, (2, 3, 0, 1))
self.export_weights(self.weights[kernel.layer_name],
kernel.ref_name, self.save_dir)
code = list()
padding = [0, 0]
if padding_mode == "SAME":
total_pad_h = self.compute_padding_size(input_h, k_h, strides[0])
total_pad_w = self.compute_padding_size(input_w, k_w, strides[1])
if total_pad_h % 2 == 0 and total_pad_w % 2 == 0:
padding = map(int, [total_pad_h / 2, total_pad_w / 2])
code.append(
"{} = layers.conv2d({}, {}, {}, padding={}, stride={}, param_attr=\'{}\', bias_attr=False, groups={})"
.format(node.output_name, data.ref_name, in_channels,
[k_h, k_w], padding, strides, kernel.ref_name,
groups))
conv_param = {
"num_filters":in_channels,
"filter_size":[k_h, k_w],
"stride":strides,
"groups":groups,
"param_attr":"\'{}\'".format(kernel.ref_name),
"bias_attr":False
}
if padding_mode == SAME:
pad_h = self.compute_padding_size(input_h, k_h, strides[0])
pad_w = self.compute_padding_size(input_w, k_w, strides[1])
if len(set(pad_h)) == 1 and len(set(pad_w)) == 1:
padding = [pad_h[0], pad_w[0]]
conv_param["padding"] = padding
node.code.add_layer("conv2d", data.ref_name,
node.output_name, conv_param)
else:
padding = [0] * 4
padding[0] = total_pad_h / 2
padding[1] = total_pad_h - padding[0]
padding[2] = total_pad_w / 2
padding[3] = total_pad_w - padding[2]
code.append("{} = layers.pad2d({}, {})".format(
node.output_name, data.ref_name, padding))
code.append(
"{} = layers.conv2d({}, {}, {}, stride={}, param_attr=\'{}\', bias_attr=False, groups={})"
.format(node.output_name, node.ref_name, in_channels,
[k_h, k_w], strides, kernel.ref_name, groups))
pad_param = {"paddings":pad_h + pad_w}
node.code.add_layer("pad2d", data.ref_name,
node.output_name, pad_param)
node.code.add_layer("conv2d", node.output_name,
node.output_name, conv_param)
else:
code.append(
"{} = layers.conv2d({}, {}, {}, stride={}, param_attr=\'{}\', bias_attr=False, groups={})"
.format(node.output_name, data.ref_name, in_channels,
[k_h, k_w], strides, kernel.ref_name, groups))
return code
node.code.add_layer("conv2d", data.ref_name,
node.output_name, conv_param)
def emit_softmax(self, node):
data = node.inputs[0]
code = "{} = layers.softmax({})".format(node.output_name,
data.ref_name)
return code
node.code.add_layer("softmax", data.ref_name, node.output_name)
def emit_matmul(self, node):
data0 = node.inputs[0]
data1 = node.inputs[1]
transpose_a = node.get_attr('transpose_a')
transpose_b = node.get_attr('transpose_b')
param_attr = {
"x":data0.ref_name,
"y":data1.ref_name,
"transpose_x":transpose_a,
"transpose_y":transpose_b
}
node.code.add_layer("matmul", None, node.output_name, param_attr)
def emit_transpose(self, node):
data = node.inputs[0]
perm = node.inputs[1]
perm.code.clear()
perm = list(self.infer.get_shape_tensor(perm.layer))
if node.data_format == NHWC and len(perm) == 4:
if perm == [0, 3, 1, 2]:
self.graph.set_data_format(node, NCHW)
node.code.add_str("{} = {}".format(node.output_name,
data.ref_name))
else:
raise Exception("Unexpected situation in OP transpose")
elif node.data_format == NCHW and len(perm) == 4:
if perm == [0, 2, 3, 1]:
self.graph.set_data_format(node, NHWC)
node.code.add_str("{} = {}".format(node.output_name,
data.ref_name))
else:
raise Exception("Unexpected situation in OP transpose")
else:
param_attr = {"perm":perm}
node.code.add_layer("transpose", data.ref_name,
node.output_name, param_attr)
def emit_randomuniform(self, node):
shape = node.inputs[0]
shape = self.infer.get_shape_tensor(shape.layer)
if node.data_format == NHWC and len(shape) == 4:
shape = shape[[0, 3, 1, 2]]
batch_index = list(numpy.argwhere(shape < 0).flatten())
shape = list(shape)
param_attr = {
"shape":shape,
"dtype":"\'float32\'",
"min":0.00001,
"max":0.99999
}
if len(batch_index) > 1:
raise Exception("More than one dimension value less than zero")
if len(batch_index) == 0:
node.code.add_layer("uniform_random", None,
node.output_name, param_attr)
else:
param_attr["input_dim_idx"] = batch_index[0]
node.code.add_layer("uniform_random_batch_size_like",
self.batch_node.ref_name, node.output_name, param_attr)
def emit_floor(self, node):
data = node.inputs[0]
node.code.add_layer("floor", data.ref_name, node.output_name)
def emit_exp(self, node):
data = node.inputs[0]
node.code.add_layer("exp", data.ref_name, node.output_name)
def emit_floordiv(self, node):
data1 = node.inputs[0]
data2 = node.inputs[1]
self.emit_div(node)
param = {"dtype":"\'float32\'"}
node.code.add_layer("cast", node.output_name, node.output_name, param)
node.code.add_layer("floor", node.output_name, node.output_name)
def emit_div(self, node):
data1 = node.inputs[0]
data2 = node.inputs[1]
axis = self.get_axis(data1, data2)
data1_shape = self.infer.get_tensor_shape(data1.layer)
data2_shape = self.infer.get_tensor_shape(data2.layer)
div_param = {
"x":data1.ref_name,
"y":data2.ref_name,
"axis":axis
}
if len(data1_shape) == 4 and len(data2_shape) == 4 \
and node.data_format == NHWC:
if data1_shape[-1] != data2_shape[-1]:
perm = {"perm":[0, 2, 3, 1]}
node.code.add_layer("transpose", data1.ref_name, "temp1", perm)
node.code.add_layer("transpose", data2.ref_name, "temp2", perm)
div_param["x"] = "temp1"
div_param["y"] = "temp2"
div_param["axis"] = -1
node.code.add_layer("elementwise_div", None,
node.output_name, div_param)
def emit_realdiv(self, node):
return self.emit_div(node)
def emit_slice(self, node):
data = node.inputs[0]
begin = node.inputs[1]
size = node.inputs[2]
begin.code.clear()
size.code.clear()
begin = list(self.infer.get_shape_tensor(begin.layer))
size = list(self.infer.get_shape_tensor(size.layer))
input_shape = self.infer.get_tensor_shape(data.layer)
if len(numpy.argwhere(input_shape < 0).flatten()) > 1:
input_shape = list(self.infer.get_tensor_shape(data.layer))
assert len(begin) == len(input_shape) and len(size) == len(input_shape)
if node.data_format == NHWC and len(input_shape) == 4:
begin = [begin[0], begin[3], begin[1], begin[2]]
size = [size[0], size[3], size[1], size[2]]
input_shape = [input_shape[0], input_shape[3],
input_shape[1], input_shape[2]]
for i in range(len(size)):
if size[i] < 0:
size[i] = input_shape[i] - begin[i]
param_attr = {"shape":size, "offsets":begin}
node.code.add_layer("crop", data.ref_name, node.output_name, param_attr)
def emit_sum(self, node):
data = node.inputs[0]
reduce_idx = node.inputs[1]
reduce_idx.code.clear()
idxs = tensor_util.MakeNdarray(reduce_idx.layer.attr['value'].tensor
).astype('int32').flatten()
data_shape_len = data.shape_dim_size
keep_dims = node.layer.attr['keep_dims'].b
if node.data_format == NHWC and data_shape_len == 4:
for i in range(idxs.shape[0]):
if idxs[i] > 0:
idxs[i] = (idxs[i] + 1) % 4 + int((idxs[i] + 1) / 4)
param = {"dim":list(idxs), "keep_dim":keep_dims}
node.code.add_layer("reduce_sum", data.ref_name, node.output_name, param)
def emit_max(self, node):
data = node.inputs[0]
reduce_idx = node.inputs[1]
reduce_idx.code.clear()
idxs = tensor_util.MakeNdarray(reduce_idx.layer.attr['value'].tensor
).astype('int32').flatten()
data_shape_len = data.shape_dim_size
keep_dims = node.layer.attr['keep_dims'].b
if node.data_format == NHWC and data_shape_len == 4:
for i in range(idxs.shape[0]):
if idxs[i] > 0:
idxs[i] = (idxs[i] + 1) % 4 + int((idxs[i] + 1) / 4)
param = {"dim":list(idxs), "keep_dim":keep_dims}
node.code.add_layer("reduce_max", data.ref_name,
node.output_name, param)
def emit_fill(self, node):
shape = node.inputs[0]
shape.code.clear()
value = node.inputs[1]
value.code.clear()
shape = list(self.infer.get_shape_tensor(shape.layer))
value = list(self.infer.get_const_tensor_value(value.layer).flatten())
assert len(value) == 1
value = value[0]
if node.data_format == NHWC and len(shape) == 4:
shape = [shape[0], shape[3], shape[1], shape[2]]
param = {
"shape":shape,
"dtype":"\'{}\'".format(value.dtype),
"value":value
}
if shape[0] < 0:
node.code.add_layer("fill_constant_batch_size_like",
self.batch_node.ref_name, node.output_name, param)
else:
node.code.add_layer("fill_constant", None, node.output_name, param)
def emit_range(self, node):
start = node.inputs[0]
end = node.inputs[1]
delta = node.inputs[2]
start.code.clear()
end.code.clear()
delta.code.clear()
start = self.infer.get_const_tensor_value(start.layer)
end = self.infer.get_const_tensor_value(end.layer)
delta = self.infer.get_const_tensor_value(delta.layer)
np_code = "np_array = numpy.arange({}, {}, {}).astype(\'{}\')".format(
start, end, delta, delta.dtype)
node.code.add_str(np_code)
node.code.add_layer("assign", "np_array", node.output_name)
def emit_tile(self, node):
data = node.inputs[0]
expand_times = node.inputs[1]
expand_times.code.clear()
expand_times = list(self.infer.get_const_tensor_value(
expand_times.layer))
param = {"expand_times":expand_times}
node.code.add_layer("expand", data.ref_name, node.output_name, param)
def emit_splitv(self, node):
data = node.inputs[0]
num_sections = node.inputs[1]
num_sections.code.clear()
split_dim = node.inputs[2]
split_dim.code.clear()
num_sections = self.infer.get_const_tensor_value(num_sections.layer)
split_dim = self.infer.get_const_tensor_value(split_dim.layer)
input_shape = self.infer.get_tensor_shape(data.layer)
if split_dim < 0:
split_dim += len(input_shape)
index = numpy.argwhere(num_sections < 0).flatten()
if index.shape[0] > 1:
raise Exception("More than one dimension less than 0")
if index.shape[0] == 1:
num_sections[index[0]] = input_shape[split_dim] - numpy.sum(num_sections) + num_sections[index[0]]
param = {
"num_or_sections":list(num_sections),
"dim":split_dim
}
node.code.add_layer("split", data.ref_name, node.output_name, param)
tensorflow2fluid/src/tensorflow_graph.py
浏览文件 @ cf689f17
......@@ -14,21 +14,35 @@
from graph import GraphNode, Graph
from tensorflow.core.framework import attr_value_pb2
from utils import *
class TensorflowGraphNode(GraphNode):
def __init__(self, layer):
super(TensorflowGraphNode, self).__init__(layer)
dtype_map = {1: "float32", 3: "int32", 9: "int64"}
def __init__(self, layer, input_format, layer_name=None):
super(TensorflowGraphNode, self).__init__(layer, layer_name)
self.codes = list()
self.data_format = 'NCHW'
self.code = FluidCode()
self.ref_as_const = 0
self.data_format = input_format
@property
def layer_type(self):
return self.layer.op.lower()
@property
def layer_name(self):
return self.layer.name
def shape_dim_size(self):
shape = self.layer.attr['_output_shapes']
return len(shape.list.shape[0].dim)
@property
def dtype(self):
dtype = self.get_attr("dtype")
if dtype in self.dtype_map:
dtype = self.dtype_map[dtype]
else:
raise Exception("Unknow dtype: {}".format(dtype))
return dtype
def get_attr(self, name, default_value=None):
if name in self.layer.attr:
......@@ -36,72 +50,89 @@ class TensorflowGraphNode(GraphNode):
field = attr.WhichOneof('value')
val = getattr(attr, field) if field else default_value
if isinstance(val, attr_value_pb2.AttrValue.ListValue):
return list(val.ListFields()[0][1])
result = list(val.ListFields()[0][1])
for i in range(len(result)):
if isinstance(result[i], int):
result[i] = int(result[i])
try:
if isinstance(result[i], long):
result[i] = int(result[i])
except:
pass
return result
else:
return val.decode('utf-8') if isinstance(val, bytes) else val
return val if isinstance(val, bytes) else val
else:
return default_value
def clear_code(self):
self.code.clear()
class TensorflowGraph(Graph):
useless_type = [
'identity',
'placeholderwithdefault',
'switch',
'merge'
]
def __init__(self, tf_graph):
super(TensorflowGraph, self).__init__(tf_graph)
self.tf_graph = tf_graph
def build(self):
def build(self, input_format):
skip_node = set(['const'])
for i, layer in enumerate(self.tf_graph.node):
self.node_map[layer.name] = TensorflowGraphNode(layer)
self.node_map[layer.name] = TensorflowGraphNode(layer, input_format)
for i, layer in enumerate(self.tf_graph.node):
if layer.op.lower() in skip_node:
continue
for pred in layer.input:
if pred not in self.node_map and pred.split(
':')[0] in self.node_map:
node = self.node_map[pred.split(':')[0]]
if node.layer_type == "switch":
self._make_connection(node, self.node_map[layer.name])
pred_node = self.node_map[pred.split(':')[0]]
if pred_node.layer_type == "switch":
self._make_connection(pred_node,
self.node_map[layer.name])
elif pred_node.layer_type == "split" or \
pred_node.layer_type == "splitv":
self.node_map[pred] = TensorflowGraphNode(
pred_node.layer, input_format, pred)
self._make_connection(self.node_map[pred],
self.node_map[layer.name])
self._make_connection(pred_node, self.node_map[pred])
else:
raise Exception("Need to fix here")
raise Exception("Unsupported situation(name:[{}], \
OP[{}])".format(node.layer_name, node.layer_type))
elif pred in self.node_map:
self._make_connection(self.node_map[pred],
self.node_map[layer.name])
self._make_connection(self.node_map[pred],
self.node_map[layer.name])
else:
raise Exception("input: {} not in node_map".format(pred))
super(TensorflowGraph, self).build()
super(TensorflowGraph, self).build(input_format)
self._remove_useless_nodes()
self._check_dataformat()
self._process_useless_nodes()
self._check_dataformat(input_format)
def _check_dataformat(self):
ss = list()
for i in range(0, len(self.topological_sort)):
def _check_dataformat(self, input_format):
for i in range(len(self.topological_sort)):
current_node = self.node_map[self.topological_sort[i]]
if 'data_format' in current_node.layer.attr:
if 'data_format'.encode() in current_node.layer.attr:
s = current_node.layer.attr['data_format'].s
if s != 'NHWC' and s != 'NCHW':
if s != NHWC and s != NCHW:
raise Exception('Unkown dataformat {}'.format(s))
ss.append(s)
self.set_data_format(current_node, s)
if len(set(ss)) > 1:
raise Exception("Two type of dataformat exist in this model")
if len(set(ss)) == 0:
return
for k, v in self.node_map.items():
self.node_map[k].data_format = ss[0]
def _remove_useless_nodes(self):
useless_type = set(
['identity', 'placeholderwithdefault', 'switch', 'merge'])
def _process_useless_nodes(self):
remove_index = list()
for i in range(0, len(self.topological_sort)):
for i in range(len(self.topological_sort)):
name = self.topological_sort[i]
current_node = self.node_map[name]
if current_node.layer_type in useless_type:
if current_node.layer_type in self.useless_type:
input = current_node.inputs[0]
for node in current_node.outputs:
for k in range(0, len(node.inputs)):
......@@ -118,5 +149,15 @@ class TensorflowGraph(Graph):
remove_index.append(i)
remove_index.sort(reverse=True)
for i in range(0, len(remove_index)):
for i in range(len(remove_index)):
del self.topological_sort[remove_index[i]]
def set_data_format(self, node, data_format):
assert data_format == 'NHWC'.encode() or data_format == 'NCHW'.encode()
if node.data_format == data_format:
return
node.data_format = data_format
if len(node.outputs) == 0:
return
for output in node.outputs:
self.set_data_format(output, data_format)
tensorflow2fluid/src/tensorflow_parser.py
浏览文件 @ cf689f17
......@@ -12,11 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import tensorflow
import tensorflow as tf
from tensorflow_graph import TensorflowGraph
from tensorflow.python.framework import tensor_util
from tensorflow.python.tools import strip_unused_lib
from tensorflow.python.framework import dtypes
import numpy
class TensorflowCkptParser(object):
def __init__(self,
......@@ -24,24 +25,25 @@ class TensorflowCkptParser(object):
checkpoint_file,
dest_nodes,
input_shape=None,
in_nodes=None):
in_nodes=None,
input_format="NCHW".encode()):
graph_def = None
self.weights = None
with tensorflow.Session() as sess:
if meta_file is None:
raise Exception("meta_file must be provided")
new_saver = tensorflow.train.import_meta_graph(meta_file)
if checkpoint_file is not None:
self.weights = dict()
new_saver.restore(
sess, tensorflow.train.latest_checkpoint(checkpoint_file))
for var in tensorflow.global_variables():
value = var.eval(sess)
self.weights[var.name.split(':')[0]] = value
graph_def, ver = tensorflow.get_default_graph()._as_graph_def(
add_shapes=True)
# self.sess = sess
sess = tf.Session()
if meta_file is None:
raise Exception("meta_file must be provided")
new_saver = tf.train.import_meta_graph(meta_file)
if checkpoint_file is not None:
self.weights = dict()
new_saver.restore(
sess, tf.train.latest_checkpoint(checkpoint_file))
for var in tf.global_variables():
value = var.eval(sess)
self.weights[var.name.split(':')[0]] = value
self.infer = ModelInfer(sess)
graph_def, ver = tf.get_default_graph()._as_graph_def(
add_shapes=True)
if in_nodes is not None and input_shape is not None:
graph_def = strip_unused_lib.strip_unused(
......@@ -50,23 +52,34 @@ class TensorflowCkptParser(object):
output_node_names=dest_nodes,
placeholder_type_enum=dtypes.float32.as_datatype_enum)
for node in graph_def.node:
if node.name in in_nodes:
index = in_nodes.index(node.name)
shape = [tf.Dimension(x) for x in input_shape[index]]
shape_proto = tf.TensorShape(shape).as_proto()
node.attr['_output_shapes'].list.shape.pop()
node.attr['_output_shapes'].list.shape.extend([shape_proto])
self.infer.gen_sample_data(node.name, input_shape[index])
self.tf_graph = TensorflowGraph(graph_def)
else:
raise Exception('in_nodes and output_nodes need be provided')
self.tf_graph.build()
self.tf_graph.build(input_format)
class TensorflowPbParser(object):
def __init__(self, pb_file, dest_nodes, input_shape=None, in_nodes=None):
with open(pb_file) as f:
def __init__(self, pb_file, dest_nodes, input_shape=None,
in_nodes=None, input_format="NCHW".encode()):
with open(pb_file, 'rb') as f:
serialized = f.read()
tensorflow.reset_default_graph()
original_graph_def = tensorflow.GraphDef()
tf.reset_default_graph()
original_graph_def = tf.GraphDef()
original_graph_def.ParseFromString(serialized)
# tensorflow.import_graph_def(origin_graph_def, name="")
# self.sess = tensorflow.Session(graph=tf.get_default_graph())
# self.sess.run(tensorflow.global_variables_initializer())
sess = tf.Session(graph=tf.get_default_graph())
sess.run(tf.global_variables_initializer())
self.infer = ModelInfer(sess)
original_graph_def = strip_unused_lib.strip_unused(
input_graph_def=original_graph_def,
......@@ -74,7 +87,7 @@ class TensorflowPbParser(object):
output_node_names=dest_nodes,
placeholder_type_enum=dtypes.float32.as_datatype_enum)
graph_def = tensorflow.GraphDef()
graph_def = tf.GraphDef()
graph_def.ParseFromString(original_graph_def.SerializeToString())
in_type_list = dict()
for node in graph_def.node:
......@@ -89,23 +102,23 @@ class TensorflowPbParser(object):
for i in range(len(input_shape)):
if in_type_list[in_nodes[i]] == 1 or in_type_list[
in_nodes[i]] == 0:
dtype = tensorflow.float32
x = tensorflow.placeholder(dtype, shape=input_shape[i])
dtype = tf.float32
x = tf.placeholder(dtype, shape=input_shape[i])
elif in_type_list[in_nodes[i]] == 3:
dtype = tensorflow.int32
x = tensorflow.placehoder(dtype, shape=input_shape[i])
dtype = tf.int32
x = tf.placehoder(dtype, shape=input_shape[i])
else:
raise Exception(
"Unexpected dtype for input, only support float32 and int32 now"
)
raise Exception("Unexpected dtype for input, only support " \
"float32 and int32 now")
input_map[in_nodes[i] + ":0"] = x
self.infer.gen_sample_data(x.name, input_shape[i])
tensorflow.import_graph_def(graph_def, name="", input_map=input_map)
graph_def = tensorflow.get_default_graph()._as_graph_def(
tf.import_graph_def(graph_def, name="", input_map=input_map)
graph_def = tf.get_default_graph()._as_graph_def(
add_shapes=True)[0]
node = graph_def.node[0]
self.tf_graph = TensorflowGraph(graph_def)
self.tf_graph.build()
self.tf_graph.build(input_format)
self.weights = dict()
for node in graph_def.node:
......@@ -116,3 +129,136 @@ class TensorflowPbParser(object):
self.weights[node.name] = weight
except:
continue
class ModelInfer(object):
""" Trick method for tensorflow2fluid
There are some Operators in PaddlePaddle not support
tensor as parameter, like reshape/transpose, Because these
parameters should be fixed in PaddlePaddle. So we
provide 'ModelInfer' here to solove this problem.
"""
def __init__(self, sess):
self.sess = sess
self.inputs_sample_data = dict()
def gen_sample_data(self, tensor_name, shape):
self.inputs_sample_data[tensor_name] = list()
if shape[0] is None or shape[0] < 0:
for i in range(1, 4):
data = numpy.random.random_sample([i] + shape[1:])
self.inputs_sample_data[tensor_name].append(data)
else:
for i in range(1, 4):
data = numpy.random.random_sample(shape)
self.inputs_sample_data[tensor_name].append(data)
def get_shape_tensor(self, layer, output_shape=None):
""" return value of shape parameter
return value of shape parameter which are tensor type
in tensorflow model
"""
tensor_name = layer.name
if len(tensor_name.split(':')) < 2:
tensor_name = tensor_name + ':0'
output_tensor = self.sess.graph.get_tensor_by_name(tensor_name)
tensor_values = []
for i in range(0, 3):
inputs_tensors = dict()
for name, values in self.inputs_sample_data.items():
if len(name.split(':')) < 2:
name = name + ':0'
tensor = self.sess.graph.get_tensor_by_name(name)
inputs_tensors[tensor] = values[i]
r, = self.sess.run([output_tensor], inputs_tensors)
tensor_values.append(r.flatten())
compare01 = (tensor_values[0] == tensor_values[1])
compare12 = (tensor_values[1] == tensor_values[2])
if compare01.all() and compare12.all():
return tensor_values[0]
if (compare01 == compare12).all():
index = numpy.argwhere(compare01==False).flatten()
if index.shape[0] != 1:
raise Exception("There's not only one unstable dimension")
tensor_values[0][index[0]] = -1
index = numpy.argwhere(tensor_values[0] < 0).flatten()
if index.shape[0] > 2:
raise Exception("There's more than two values less than zero")
if index.shape[0] == 2:
if output_shape is None:
raise Exception("Need output_shape parameter, " \
"get_shape_tensor(tensor_name, output_shape)")
tensor_values[0][index[1]] = output_shape[index[1]]
return tensor_values[0]
else:
raise Exception("Can not infer a stable shape tensor value")
def get_tensor_shape(self, layer):
shape = layer.attr['_output_shapes'].list.shape[0]
shape = numpy.array([dim.size for dim in shape.dim])
if numpy.argwhere(shape<0).shape[0] <= 1:
return shape
tensor_name = layer.name
if len(tensor_name.split(':')) < 2:
tensor_name = tensor_name + ':0'
output_tensor = self.sess.graph.get_tensor_by_name(tensor_name)
shapes = []
for i in range(0, 3):
inputs_tensors = dict()
for name, values in self.inputs_sample_data.items():
if len(name.split(':')) < 2:
name = name + ':0'
tensor = self.sess.graph.get_tensor_by_name(name)
inputs_tensors[tensor] = values[i]
r, = self.sess.run([output_tensor], inputs_tensors)
shapes.append(numpy.array(r.shape))
compare01 = (shapes[0] == shapes[1])
compare12 = (shapes[1] == shapes[2])
if compare01.all() and compare12.all():
return shapes[0]
if (compare01 == compare12).all():
index = numpy.argwhere(compare01==False).flatten()
if index.shape[0] != 1:
raise Exception("There's not only one unstable dimension")
if index[0] != 0:
raise Exception("Batch size not in the first dimension")
shapes[0][0] = -1
return shapes[0]
else:
raise Exception("Can not infer a stable tensor shape, failed!")
def get_const_tensor_value(self, layer):
tensor_name = layer.name
if len(tensor_name.split(':')) < 2:
tensor_name = tensor_name + ':0'
output_tensor = self.sess.graph.get_tensor_by_name(tensor_name)
result = []
for i in range(0, 3):
inputs_tensors = dict()
for name, values in self.inputs_sample_data.items():
if len(name.split(':')) < 2:
name = name + ':0'
tensor = self.sess.graph.get_tensor_by_name(name)
inputs_tensors[tensor] = values[i]
r, = self.sess.run([output_tensor], inputs_tensors)
result.append(r)
compare01 = (result[0] == result[1])
compare12 = (result[1] == result[2])
if compare01.all() and compare12.all():
return result[0]
else:
raise Exception("Can not infer a stable constant tensor value")
tensorflow2fluid/src/transformer.py 已删除 100644 → 0
浏览文件 @ b5fe13be
from paddle_emitter import PaddleEmitter
from tensorflow_parser import TensorflowCkptParser
from tensorflow_parser import TensorflowPbParser
class Transformer(object):
def __init__(self, meta_file, ckpt_file, out_nodes, in_shape, in_nodes, save_dir):
self.parser = TensorflowCkptParser(meta_file, ckpt_file, out_nodes,
in_shape, in_nodes)
self.emitter = PaddleEmitter(self.parser, save_dir)
def transform_code(self):
codes = self.emitter.run()
def run(self):
self.transform_code()
class PbTransformer(object):
def __init__(self, pb_file, out_nodes, in_shape, in_nodes, save_dir):
self.parser = TensorflowPbParser(pb_file, out_nodes, in_shape, in_nodes)
self.emitter = PaddleEmitter(self.parser, save_dir)
node = self.parser.tf_graph.tf_graph.node[0]
def transform_code(self):
codes = self.emitter.run()
def run(self):
self.transform_code()
tensorflow2fluid/src/utils.py 0 → 100644
浏览文件 @ cf689f17
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
NHWC = 'NHWC'.encode()
NCHW = 'NCHW'.encode()
OTHER = 'OTHER'.encode()
SAME = 'SAME'.encode()
VALID = 'VALID'.encode()
class NameGenerator(object):
def __init__(self):
self.param_index = 0
self.input_index = 0
self.net_index = 0
self.const_index = 0
self.names = dict()
def get_name(self, node):
ref_name = None
op_name = node.layer_type
if node.layer.name in self.names:
return self.names[node.layer.name]
if op_name == "variablev2":
ref_name = "param_" + str(self.param_index)
self.param_index += 1
elif op_name == "placeholder":
ref_name = "input_" + str(self.input_index)
self.input_index += 1
elif op_name == "const":
ref_name = "const_" + str(self.const_index)
self.const_index += 1
elif op_name.lower() == "identity":
ref_name = self.names[node.layer.input[0]]
else:
ref_name = "net_" + str(self.net_index)
self.net_index += 1
self.names[node.layer.name] = ref_name
return ref_name
class LayerCode(object):
def __init__(self):
self.op = None
self.param_attr = dict()
self.input = None
self.output = None
self.str_code = None
def get_str_code(self):
if self.str_code is not None:
return self.str_code
layer_code0 = ""
if self.output is not None:
layer_code0 = layer_code0 + self.output + " = "
layer_code0 += "layers."
layer_code1 = self.op + "("
if self.input is not None:
layer_code1 = layer_code1 + self.input + ", "
layer_code2 = ""
for k, v in self.param_attr.items():
layer_code2 = layer_code2 + k + "=" + "{}".format(v) + ", "
layer_code2 = layer_code2.strip(", ")
layer_code = (layer_code0 + layer_code1 + layer_code2).strip(", ") + ")"
return layer_code
class FluidCode(object):
def __init__(self):
self.codes = list()
def add_layer(self, op, input, output, param_attr=None):
if param_attr is None:
param_attr = dict()
layer_code = LayerCode()
layer_code.op = op
layer_code.input = input
layer_code.output = output
layer_code.param_attr = param_attr
self.codes.append(layer_code)
def add_str(self, str_code):
layer_code = LayerCode()
layer_code.str_code = str_code
self.codes.append(layer_code)
def clear(self):
self.codes = list()
def gen_codes(self):
res = list()
if len(self.codes) == 0:
return []
for code in self.codes:
if isinstance(code, LayerCode):
res.append(code.get_str_code())
else:
raise Exception("Unexcept situation!")
return res
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或