convert.py 9.7 KB
Newer Older
1
# Copyright (c) 2020  PaddlePaddle Authors. All Rights Reserved.
J
jiangjiajun 已提交
2 3 4 5 6 7 8 9 10 11 12 13
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
S
SunAhong1993 已提交
14

15 16
from six import text_type as _text_type
import argparse
J
jiangjiajun 已提交
17
import sys
18

J
jiangjiajun 已提交
19

20 21
def arg_parser():
    parser = argparse.ArgumentParser()
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45
    parser.add_argument(
        "--model",
        "-m",
        type=_text_type,
        default=None,
        help="define model file path for tensorflow or onnx")
    parser.add_argument(
        "--prototxt",
        "-p",
        type=_text_type,
        default=None,
        help="prototxt file of caffe model")
    parser.add_argument(
        "--weight",
        "-w",
        type=_text_type,
        default=None,
        help="weight file of caffe model")
    parser.add_argument(
        "--save_dir",
        "-s",
        type=_text_type,
        default=None,
        help="path to save translated model")
J
upgrade  
jiangjiajun 已提交
46 47 48 49 50
    parser.add_argument(
        "--framework",
        "-f",
        type=_text_type,
        default=None,
51 52
        help="define which deeplearning framework(tensorflow/caffe/onnx/paddle2onnx)"
    )
S
SunAhong1993 已提交
53 54 55 56 57
    parser.add_argument(
        "--caffe_proto",
        "-c",
        type=_text_type,
        default=None,
J
upgrade  
jiangjiajun 已提交
58 59
        help="optional: the .py file compiled by caffe proto file of caffe model"
    )
60 61 62 63 64 65
    parser.add_argument(
        "--version",
        "-v",
        action="store_true",
        default=False,
        help="get version of x2paddle")
66 67 68 69 70 71
    parser.add_argument(
        "--without_data_format_optimization",
        "-wo",
        action="store_true",
        default=False,
        help="tf model conversion without data format optimization")
72 73 74 75 76 77 78 79 80 81 82 83
    parser.add_argument(
        "--define_input_shape",
        "-d",
        action="store_true",
        default=False,
        help="define input shape for tf model")
    parser.add_argument(
        "--params_merge",
        "-pm",
        action="store_true",
        default=False,
        help="define whether merge the params")
J
jiangjiajun 已提交
84

85
    return parser
J
jiangjiajun 已提交
86

87

88 89 90
def tf2paddle(model_path,
              save_dir,
              without_data_format_optimization=False,
M
mamingjie-China 已提交
91
              define_input_shape=False,
M
mamingjie-China 已提交
92
              params_merge=False):
J
jiangjiajun 已提交
93 94
    # check tensorflow installation and version
    try:
95 96
        import os
        os.environ["TF_CPP_MIN_LOG_LEVEL"] = '3'
J
jiangjiajun 已提交
97 98 99 100
        import tensorflow as tf
        version = tf.__version__
        if version >= '2.0.0' or version < '1.0.0':
            print(
J
jiangjiajun@baidu.com 已提交
101
                "[ERROR] 1.0.0<=tensorflow<2.0.0 is required, and v1.14.0 is recommended"
J
jiangjiajun 已提交
102 103 104
            )
            return
    except:
J
jiangjiajun@baidu.com 已提交
105 106 107
        print(
            "[ERROR] Tensorflow is not installed, use \"pip install tensorflow\"."
        )
J
jiangjiajun 已提交
108 109
        return

J
jiangjiajun 已提交
110
    from x2paddle.decoder.tf_decoder import TFDecoder
J
jiangjiajun 已提交
111
    from x2paddle.op_mapper.tf_op_mapper import TFOpMapper
112
    from x2paddle.op_mapper.tf_op_mapper_nhwc import TFOpMapperNHWC
J
jiangjiajun 已提交
113
    from x2paddle.optimizer.tf_optimizer import TFOptimizer
J
jiangjiajun 已提交
114 115

    print("Now translating model from tensorflow to paddle.")
116 117 118 119 120 121 122
    model = TFDecoder(model_path, define_input_shape=define_input_shape)
    if not without_data_format_optimization:
        mapper = TFOpMapper(model)
        optimizer = TFOptimizer(mapper)
        # neccesary optimization
        optimizer.delete_redundance_code()
        # optimizer below is experimental
J
jiangjiajun 已提交
123
        optimizer.optimize_elementwise_op()
124 125
        optimizer.merge_activation()
        optimizer.merge_bias()
J
jiangjiajun 已提交
126 127 128 129
        optimizer.optimize_sub_graph()

#        optimizer.merge_batch_norm()
#        optimizer.merge_prelu()
130 131 132 133
    else:
        mapper = TFOpMapperNHWC(model)
        optimizer = TFOptimizer(mapper)
        optimizer.delete_redundance_code()
J
jiangjiajun 已提交
134
        optimizer.strip_graph()
135 136
        optimizer.merge_activation()
        optimizer.merge_bias()
J
jiangjiajun 已提交
137
        optimizer.make_nchw_input_output()
138
        optimizer.remove_transpose()
M
mamingjie-China 已提交
139
    mapper.save_inference_model(save_dir, params_merge)
140 141


M
mamingjie-China 已提交
142
def caffe2paddle(proto, weight, save_dir, caffe_proto, params_merge=False):
J
jiangjiajun 已提交
143 144
    from x2paddle.decoder.caffe_decoder import CaffeDecoder
    from x2paddle.op_mapper.caffe_op_mapper import CaffeOpMapper
145
    from x2paddle.optimizer.caffe_optimizer import CaffeOptimizer
S
SunAhong1993 已提交
146
    import google.protobuf as gpb
S
SunAhong1993 已提交
147 148 149 150 151
    ver_part = gpb.__version__.split('.')
    version_satisfy = False
    if (int(ver_part[0]) == 3 and int(ver_part[1]) >= 6) \
        or (int(ver_part[0]) > 3):
        version_satisfy = True
J
jiangjiajun@baidu.com 已提交
152
    assert version_satisfy, '[ERROR] google.protobuf >= 3.6.0 is required'
J
jiangjiajun 已提交
153
    print("Now translating model from caffe to paddle.")
S
SunAhong1993 已提交
154
    model = CaffeDecoder(proto, weight, caffe_proto)
J
jiangjiajun 已提交
155
    mapper = CaffeOpMapper(model)
156 157 158
    optimizer = CaffeOptimizer(mapper)
    optimizer.merge_bn_scale()
    optimizer.merge_op_activation()
M
mamingjie-China 已提交
159
    mapper.save_inference_model(save_dir, params_merge)
160 161


M
mamingjie-China 已提交
162
def onnx2paddle(model_path, save_dir, params_merge=False):
C
update  
channingss 已提交
163 164 165 166
    # check onnx installation and version
    try:
        import onnx
        version = onnx.version.version
167
        if version != '1.6.0':
J
jiangjiajun@baidu.com 已提交
168
            print("[ERROR] onnx==1.6.0 is required")
C
update  
channingss 已提交
169 170
            return
    except:
J
jiangjiajun@baidu.com 已提交
171
        print("[ERROR] onnx is not installed, use \"pip install onnx==1.6.0\".")
C
update  
channingss 已提交
172
        return
C
channingss 已提交
173
    print("Now translating model from onnx to paddle.")
C
update  
channingss 已提交
174

C
Channingss 已提交
175
    from x2paddle.op_mapper.onnx_op_mapper import ONNXOpMapper
C
update  
channingss 已提交
176
    from x2paddle.decoder.onnx_decoder import ONNXDecoder
R
root 已提交
177 178
    from x2paddle.optimizer.onnx_optimizer import ONNXOptimizer
    model = ONNXDecoder(model_path)
C
Channingss 已提交
179
    mapper = ONNXOpMapper(model)
180
    print("Model optimizing ...")
C
update  
channingss 已提交
181
    optimizer = ONNXOptimizer(mapper)
182
    print("Model optimized.")
C
channingss 已提交
183

184
    print("Paddle model and code generating ...")
M
mamingjie-China 已提交
185
    mapper.save_inference_model(save_dir, params_merge)
186 187 188 189 190 191 192 193 194 195 196
    print("Paddle model and code generated.")


def paddle2onnx(model_path, save_dir):
    from x2paddle.decoder.paddle_decoder import PaddleDecoder
    from x2paddle.op_mapper.paddle_op_mapper import PaddleOpMapper
    model = PaddleDecoder(model_path, '__model__', '__params__')
    mapper = PaddleOpMapper()
    mapper.convert(model.program, save_dir)


C
Channingss 已提交
197 198 199 200 201 202
def paddle2onnx(model_path, save_dir):
    from x2paddle.decoder.paddle_decoder import PaddleDecoder
    from x2paddle.op_mapper.paddle_op_mapper import PaddleOpMapper
    model = PaddleDecoder(model_path, '__model__', '__params__')
    mapper = PaddleOpMapper()
    mapper.convert(model.program, save_dir)
C
update  
channingss 已提交
203 204


205 206 207 208 209 210
def paddle2onnx(model_path, save_dir):
    from x2paddle.decoder.paddle_decoder import PaddleDecoder
    from x2paddle.op_mapper.paddle_op_mapper import PaddleOpMapper
    model = PaddleDecoder(model_path, '__model__', '__params__')
    mapper = PaddleOpMapper()
    mapper.convert(model.program, save_dir)
C
update  
channingss 已提交
211 212


213
def main():
J
jiangjiajun 已提交
214
    if len(sys.argv) < 2:
C
update  
channingss 已提交
215
        print("Use \"x2paddle -h\" to print the help information")
J
jiangjiajun 已提交
216 217
        print("For more information, please follow our github repo below:)")
        print("\nGithub: https://github.com/PaddlePaddle/X2Paddle.git\n")
J
jiangjiajun 已提交
218 219
        return

220 221 222
    parser = arg_parser()
    args = parser.parse_args()

J
jiangjiajun 已提交
223
    if args.version:
J
jiangjiajun 已提交
224
        import x2paddle
M
mamingjie-China 已提交
225
        print("x2paddle-{} with python>=3.5, paddlepaddle>=1.6.0\n".format(
J
jiangjiajun 已提交
226
            x2paddle.__version__))
J
jiangjiajun 已提交
227 228
        return

J
Jason 已提交
229
    assert args.framework is not None, "--framework is not defined(support tensorflow/caffe/onnx)"
230
    assert args.save_dir is not None, "--save_dir is not defined"
M
mamingjie-China 已提交
231

M
mamingjie-China 已提交
232 233 234
    try:
        import paddle
        v0, v1, v2 = paddle.__version__.split('.')
235 236 237 238
        print("paddle.__version__ = {}".format(paddle.__version__))
        if v0 == '0' and v1 == '0' and v2 == '0':
            print("[WARNING] You are use develop version of paddlepaddle")
        elif int(v0) != 1 or int(v1) < 6:
J
jiangjiajun@baidu.com 已提交
239
            print("[ERROR] paddlepaddle>=1.6.0 is required")
M
mamingjie-China 已提交
240 241
            return
    except:
J
jiangjiajun@baidu.com 已提交
242 243 244
        print(
            "[ERROR] paddlepaddle not installed, use \"pip install paddlepaddle\""
        )
245 246

    if args.framework == "tensorflow":
J
jiangjiajun 已提交
247
        assert args.model is not None, "--model should be defined while translating tensorflow model"
248 249
        without_data_format_optimization = False
        define_input_shape = False
M
mamingjie-China 已提交
250
        params_merge = False
251 252 253 254
        if args.without_data_format_optimization:
            without_data_format_optimization = True
        if args.define_input_shape:
            define_input_shape = True
M
mamingjie-China 已提交
255 256
        if args.params_merge:
            params_merge = True
257
        tf2paddle(args.model, args.save_dir, without_data_format_optimization,
M
mamingjie-China 已提交
258
                  define_input_shape, params_merge)
259 260

    elif args.framework == "caffe":
S
SunAhong1993 已提交
261
        assert args.prototxt is not None and args.weight is not None, "--prototxt and --weight should be defined while translating caffe model"
M
mamingjie-China 已提交
262 263 264
        params_merge = False
        if args.params_merge:
            params_merge = True
S
SunAhong1993 已提交
265
        caffe2paddle(args.prototxt, args.weight, args.save_dir,
M
mamingjie-China 已提交
266
                     args.caffe_proto, params_merge)
C
update  
channingss 已提交
267 268
    elif args.framework == "onnx":
        assert args.model is not None, "--model should be defined while translating onnx model"
M
mamingjie-China 已提交
269
        params_merge = False
270

M
mamingjie-China 已提交
271 272 273
        if args.params_merge:
            params_merge = True
        onnx2paddle(args.model, args.save_dir, params_merge)
274 275 276 277 278

    elif args.framework == "paddle2onnx":
        assert args.model is not None, "--model should be defined while translating paddle model to onnx"
        paddle2onnx(args.model, args.save_dir)

279
    else:
280 281
        raise Exception(
            "--framework only support tensorflow/caffe/onnx/paddle2onnx now")
282 283 284 285


if __name__ == "__main__":
    main()