From aa0ab8c992404fbf0dc67f16897c9095038ec59a Mon Sep 17 00:00:00 2001 From: whs Date: Tue, 21 Aug 2018 15:19:33 +0800 Subject: [PATCH] Make ocr and icnet support for python3 (#1178) * Make ocr and icnet support for python3 * Restore default arguments. --- fluid/icnet/cityscape.py | 7 +++-- fluid/icnet/eval.py | 7 ++--- fluid/icnet/icnet.py | 39 +++++++++++++----------- fluid/icnet/infer.py | 3 +- fluid/icnet/train.py | 15 +++++---- fluid/icnet/utils.py | 3 +- fluid/ocr_recognition/attention_model.py | 3 ++ fluid/ocr_recognition/crnn_ctc_model.py | 8 +++-- fluid/ocr_recognition/data_reader.py | 7 +++-- fluid/ocr_recognition/eval.py | 7 ++--- fluid/ocr_recognition/infer.py | 13 ++++---- fluid/ocr_recognition/train.py | 27 ++++++++-------- fluid/ocr_recognition/utility.py | 15 ++++----- 13 files changed, 89 insertions(+), 65 deletions(-) diff --git a/fluid/icnet/cityscape.py b/fluid/icnet/cityscape.py index 0b862d9a..c5c08afc 100644 --- a/fluid/icnet/cityscape.py +++ b/fluid/icnet/cityscape.py @@ -1,5 +1,8 @@ """Reader for Cityscape dataset. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import os import cv2 import numpy as np @@ -173,8 +176,8 @@ class DataGenerater: """ Scale label according to factor. """ - h = label.shape[0] / factor - w = label.shape[1] / factor + h = label.shape[0] // factor + w = label.shape[1] // factor return cv2.resize( label, (h, w), interpolation=cv2.INTER_NEAREST)[:, :, np.newaxis] diff --git a/fluid/icnet/eval.py b/fluid/icnet/eval.py index bdebe7ad..dc2f5743 100644 --- a/fluid/icnet/eval.py +++ b/fluid/icnet/eval.py @@ -64,7 +64,7 @@ def eval(args): exe.run(fluid.default_startup_program()) assert os.path.exists(args.model_path) fluid.io.load_params(exe, args.model_path) - print "loaded model from: %s" % args.model_path + print("loaded model from: %s" % args.model_path) sys.stdout.flush() fetch_vars = [iou, out_w, out_r] @@ -80,11 +80,10 @@ def eval(args): fetch_list=fetch_vars) out_wrong += result[1] out_right += result[2] - print "count: %s; current iou: %.3f;\r" % (count, result[0]), sys.stdout.flush() iou = cal_mean_iou(out_wrong, out_right) - print "\nmean iou: %.3f" % iou - print "kpis test_acc %f" % iou + print("\nmean iou: %.3f" % iou) + print("kpis test_acc %f" % iou) def main(): diff --git a/fluid/icnet/icnet.py b/fluid/icnet/icnet.py index afe3fa9d..d640621e 100644 --- a/fluid/icnet/icnet.py +++ b/fluid/icnet/icnet.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import paddle.fluid as fluid import numpy as np import sys @@ -20,8 +23,8 @@ def conv(input, if padding == "SAME": padding_h = max(k_h - s_h, 0) padding_w = max(k_w - s_w, 0) - padding_top = padding_h / 2 - padding_left = padding_w / 2 + padding_top = padding_h // 2 + padding_left = padding_w // 2 padding_bottom = padding_h - padding_top padding_right = padding_w - padding_left padding = [ @@ -57,8 +60,8 @@ def atrous_conv(input, if padding == "SAME": padding_h = max(k_h - s_h, 0) padding_w = max(k_w - s_w, 0) - padding_top = padding_h / 2 - padding_left = padding_w / 2 + padding_top = padding_h // 2 + padding_left = padding_w // 2 padding_bottom = padding_h - padding_top padding_right = padding_w - padding_left padding = [ @@ -141,15 +144,15 @@ def dilation_convs(input): def pyramis_pooling(input, input_shape): - shape = np.ceil(input_shape / 32).astype("int32") + shape = np.ceil(input_shape // 32).astype("int32") h, w = shape pool1 = avg_pool(input, h, w, h, w) pool1_interp = interp(pool1, shape) - pool2 = avg_pool(input, h / 2, w / 2, h / 2, w / 2) + pool2 = avg_pool(input, h // 2, w // 2, h // 2, w // 2) pool2_interp = interp(pool2, shape) - pool3 = avg_pool(input, h / 3, w / 3, h / 3, w / 3) + pool3 = avg_pool(input, h // 3, w // 3, h // 3, w // 3) pool3_interp = interp(pool3, shape) - pool4 = avg_pool(input, h / 4, w / 4, h / 4, w / 4) + pool4 = avg_pool(input, h // 4, w // 4, h // 4, w // 4) pool4_interp = interp(pool4, shape) conv5_3_sum = input + pool4_interp + pool3_interp + pool2_interp + pool1_interp return conv5_3_sum @@ -172,14 +175,14 @@ def shared_convs(image): def res_block(input, filter_num, padding=0, dilation=None, name=None): - tmp = conv(input, 1, 1, filter_num / 4, 1, 1, name=name + "_1_1_reduce") + tmp = conv(input, 1, 1, filter_num // 4, 1, 1, name=name + "_1_1_reduce") tmp = bn(tmp, relu=True) tmp = zero_padding(tmp, padding=padding) if dilation is None: - tmp = conv(tmp, 3, 3, filter_num / 4, 1, 1, name=name + "_3_3") + tmp = conv(tmp, 3, 3, filter_num // 4, 1, 1, name=name + "_3_3") else: tmp = atrous_conv( - tmp, 3, 3, filter_num / 4, dilation, name=name + "_3_3") + tmp, 3, 3, filter_num // 4, dilation, name=name + "_3_3") tmp = bn(tmp, relu=True) tmp = conv(tmp, 1, 1, filter_num, 1, 1, name=name + "_1_1_increase") tmp = bn(tmp, relu=False) @@ -195,7 +198,7 @@ def proj_block(input, filter_num, padding=0, dilation=None, stride=1, proj_bn = bn(proj, relu=False) tmp = conv( - input, 1, 1, filter_num / 4, stride, stride, name=name + "_1_1_reduce") + input, 1, 1, filter_num // 4, stride, stride, name=name + "_1_1_reduce") tmp = bn(tmp, relu=True) tmp = zero_padding(tmp, padding=padding) @@ -208,7 +211,7 @@ def proj_block(input, filter_num, padding=0, dilation=None, stride=1, tmp, 3, 3, - filter_num / 4, + filter_num // 4, 1, 1, padding=padding, @@ -218,7 +221,7 @@ def proj_block(input, filter_num, padding=0, dilation=None, stride=1, tmp, 3, 3, - filter_num / 4, + filter_num // 4, dilation, padding=padding, name=name + "_3_3") @@ -232,12 +235,12 @@ def proj_block(input, filter_num, padding=0, dilation=None, stride=1, def sub_net_4(input, input_shape): - tmp = interp(input, out_shape=np.ceil(input_shape / 32)) + tmp = interp(input, out_shape=np.ceil(input_shape // 32)) tmp = dilation_convs(tmp) tmp = pyramis_pooling(tmp, input_shape) tmp = conv(tmp, 1, 1, 256, 1, 1, name="conv5_4_k1") tmp = bn(tmp, relu=True) - tmp = interp(tmp, input_shape / 16) + tmp = interp(tmp, input_shape // 16) return tmp @@ -265,7 +268,7 @@ def CCF24(sub2_out, sub4_out, input_shape): tmp = bn(tmp, relu=False) tmp = tmp + sub2_out tmp = fluid.layers.relu(tmp) - tmp = interp(tmp, input_shape / 8) + tmp = interp(tmp, input_shape // 8) return tmp @@ -275,7 +278,7 @@ def CCF124(sub1_out, sub24_out, input_shape): tmp = bn(tmp, relu=False) tmp = tmp + sub1_out tmp = fluid.layers.relu(tmp) - tmp = interp(tmp, input_shape / 4) + tmp = interp(tmp, input_shape // 4) return tmp diff --git a/fluid/icnet/infer.py b/fluid/icnet/infer.py index 63fb3268..f93469f1 100644 --- a/fluid/icnet/infer.py +++ b/fluid/icnet/infer.py @@ -1,4 +1,5 @@ """Infer for ICNet model.""" +from __future__ import print_function import cityscape import argparse import functools @@ -101,7 +102,7 @@ def infer(args): exe.run(fluid.default_startup_program()) assert os.path.exists(args.model_path) fluid.io.load_params(exe, args.model_path) - print "loaded model from: %s" % args.model_path + print("loaded model from: %s" % args.model_path) sys.stdout.flush() if not os.path.isdir(args.out_path): diff --git a/fluid/icnet/train.py b/fluid/icnet/train.py index 9b0f4fd4..1059e353 100644 --- a/fluid/icnet/train.py +++ b/fluid/icnet/train.py @@ -1,4 +1,7 @@ """Trainer for ICNet model.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function from icnet import icnet import cityscape import argparse @@ -94,7 +97,7 @@ def train(args): exe.run(fluid.default_startup_program()) if args.init_model is not None: - print "load model from: %s" % args.init_model + print("load model from: %s" % args.init_model) sys.stdout.flush() fluid.io.load_params(exe, args.init_model) @@ -111,7 +114,7 @@ def train(args): for data in train_reader(): if iter_id > TOTAL_STEP: end_time = time.time() - print "kpis train_duration %f" % (end_time - start_time) + print("kpis train_duration %f" % (end_time - start_time)) return iter_id += 1 results = exe.run( @@ -123,10 +126,10 @@ def train(args): sub124_loss += results[3] # training log if iter_id % LOG_PERIOD == 0: - print "Iter[%d]; train loss: %.3f; sub4_loss: %.3f; sub24_loss: %.3f; sub124_loss: %.3f" % ( + print("Iter[%d]; train loss: %.3f; sub4_loss: %.3f; sub24_loss: %.3f; sub124_loss: %.3f" % ( iter_id, t_loss / LOG_PERIOD, sub4_loss / LOG_PERIOD, - sub24_loss / LOG_PERIOD, sub124_loss / LOG_PERIOD) - print "kpis train_cost %f" % (t_loss / LOG_PERIOD) + sub24_loss / LOG_PERIOD, sub124_loss / LOG_PERIOD)) + print("kpis train_cost %f" % (t_loss / LOG_PERIOD)) t_loss = 0. sub4_loss = 0. @@ -137,7 +140,7 @@ def train(args): if iter_id % CHECKPOINT_PERIOD == 0 and args.checkpoint_path is not None: dir_name = args.checkpoint_path + "/" + str(iter_id) fluid.io.save_persistables(exe, dirname=dir_name) - print "Saved checkpoint: %s" % (dir_name) + print("Saved checkpoint: %s" % (dir_name)) def main(): diff --git a/fluid/icnet/utils.py b/fluid/icnet/utils.py index 699841d6..7d58060e 100644 --- a/fluid/icnet/utils.py +++ b/fluid/icnet/utils.py @@ -19,6 +19,7 @@ from __future__ import print_function import distutils.util import numpy as np from paddle.fluid import core +import six def print_arguments(args): @@ -37,7 +38,7 @@ def print_arguments(args): :type args: argparse.Namespace """ print("----------- Configuration Arguments -----------") - for arg, value in sorted(vars(args).iteritems()): + for arg, value in sorted(six.iteritems(vars(args))): print("%s: %s" % (arg, value)) print("------------------------------------------------") diff --git a/fluid/ocr_recognition/attention_model.py b/fluid/ocr_recognition/attention_model.py index 0f24da99..363c0307 100755 --- a/fluid/ocr_recognition/attention_model.py +++ b/fluid/ocr_recognition/attention_model.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import paddle.fluid as fluid decoder_size = 128 diff --git a/fluid/ocr_recognition/crnn_ctc_model.py b/fluid/ocr_recognition/crnn_ctc_model.py index 82d5c9ff..aa46d4ff 100755 --- a/fluid/ocr_recognition/crnn_ctc_model.py +++ b/fluid/ocr_recognition/crnn_ctc_model.py @@ -1,7 +1,11 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import paddle.fluid as fluid from paddle.fluid.layers.learning_rate_scheduler import _decay_step_counter from paddle.fluid.initializer import init_on_cpu import math +import six def conv_bn_pool(input, @@ -15,7 +19,7 @@ def conv_bn_pool(input, pooling=True, use_cudnn=False): tmp = input - for i in xrange(group): + for i in six.moves.xrange(group): tmp = fluid.layers.conv2d( input=tmp, num_filters=out_ch[i], @@ -192,7 +196,7 @@ def ctc_train_net(args, data_shape, num_classes): inference_program = fluid.default_main_program().clone(for_test=True) if learning_rate_decay == "piecewise_decay": learning_rate = fluid.layers.piecewise_decay([ - args.total_step / 4, args.total_step / 2, args.total_step * 3 / 4 + args.total_step // 4, args.total_step // 2, args.total_step * 3 // 4 ], [LR, LR * 0.1, LR * 0.01, LR * 0.001]) else: learning_rate = LR diff --git a/fluid/ocr_recognition/data_reader.py b/fluid/ocr_recognition/data_reader.py index e537914e..1a1c5c87 100644 --- a/fluid/ocr_recognition/data_reader.py +++ b/fluid/ocr_recognition/data_reader.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import os import cv2 import tarfile @@ -67,11 +70,11 @@ class DataGenerator(object): batchsize ) + "; i++) print $(4*i+1)\" \"$(4*i+2)\" \"$(4*i+3)\" \"$(4*i+4);}}' > " + to_file os.system(cmd) - print "finish batch shuffle" + print("finish batch shuffle") img_label_lines = open(to_file, 'r').readlines() def reader(): - sizes = len(img_label_lines) / batchsize + sizes = len(img_label_lines) // batchsize if sizes == 0: raise ValueError('Batch size is bigger than the dataset size.') while True: diff --git a/fluid/ocr_recognition/eval.py b/fluid/ocr_recognition/eval.py index 35603757..1d553999 100644 --- a/fluid/ocr_recognition/eval.py +++ b/fluid/ocr_recognition/eval.py @@ -55,17 +55,16 @@ def evaluate(args): model_dir = os.path.dirname(args.model_path) model_file_name = os.path.basename(args.model_path) fluid.io.load_params(exe, dirname=model_dir, filename=model_file_name) - print "Init model from: %s." % args.model_path + print("Init model from: %s." % args.model_path) evaluator.reset(exe) count = 0 for data in test_reader(): count += 1 exe.run(fluid.default_main_program(), feed=get_feeder_data(data, place)) - print "Read %d samples;\r" % count, avg_distance, avg_seq_error = evaluator.eval(exe) - print "Read %d samples; avg_distance: %s; avg_seq_error: %s" % ( - count, avg_distance, avg_seq_error) + print("Read %d samples; avg_distance: %s; avg_seq_error: %s" % ( + count, avg_distance, avg_seq_error)) def main(): diff --git a/fluid/ocr_recognition/infer.py b/fluid/ocr_recognition/infer.py index ca1fd118..5ba77ced 100755 --- a/fluid/ocr_recognition/infer.py +++ b/fluid/ocr_recognition/infer.py @@ -1,3 +1,4 @@ +from __future__ import print_function import paddle.v2 as paddle import paddle.fluid as fluid from utility import add_arguments, print_arguments, to_lodtensor, get_ctc_feeder_data, get_attention_feeder_for_infer @@ -64,7 +65,7 @@ def inference(args): with open(args.dict) as dict_file: for i, word in enumerate(dict_file): dict_map[i] = word.strip() - print "Loaded dict from %s" % args.dict + print("Loaded dict from %s" % args.dict) # load init model model_dir = args.model_path @@ -73,7 +74,7 @@ def inference(args): model_dir = os.path.dirname(args.model_path) model_file_name = os.path.basename(args.model_path) fluid.io.load_params(exe, dirname=model_dir, filename=model_file_name) - print "Init model from: %s." % args.model_path + print("Init model from: %s." % args.model_path) batch_times = [] iters = 0 @@ -96,17 +97,17 @@ def inference(args): fps = args.batch_size / batch_time batch_times.append(batch_time) if dict_map is not None: - print "Iteration %d, latency: %.5f s, fps: %f, result: %s" % ( + print("Iteration %d, latency: %.5f s, fps: %f, result: %s" % ( iters, batch_time, fps, - [dict_map[index] for index in indexes], ) + [dict_map[index] for index in indexes], )) else: - print "Iteration %d, latency: %.5f s, fps: %f, result: %s" % ( + print("Iteration %d, latency: %.5f s, fps: %f, result: %s" % ( iters, batch_time, fps, - indexes, ) + indexes, )) iters += 1 diff --git a/fluid/ocr_recognition/train.py b/fluid/ocr_recognition/train.py index 12c89456..7954d23d 100755 --- a/fluid/ocr_recognition/train.py +++ b/fluid/ocr_recognition/train.py @@ -1,4 +1,7 @@ """Trainer for OCR CTC or attention model.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import paddle.fluid as fluid from utility import add_arguments, print_arguments, to_lodtensor, get_ctc_feeder_data, get_attention_feeder_data import paddle.fluid.profiler as profiler @@ -85,7 +88,7 @@ def train(args): model_dir = os.path.dirname(args.init_model) model_file_name = os.path.basename(args.init_model) fluid.io.load_params(exe, dirname=model_dir, filename=model_file_name) - print "Init model from: %s." % args.init_model + print("Init model from: %s." % args.init_model) train_exe = exe error_evaluator.reset(exe) @@ -112,18 +115,18 @@ def train(args): for data in test_reader(): exe.run(inference_program, feed=get_feeder_data(data, place)) _, test_seq_error = error_evaluator.eval(exe) - print "\nTime: %s; Iter[%d]; Test seq error: %s.\n" % ( - time.time(), iter_num, str(test_seq_error[0])) + print("\nTime: %s; Iter[%d]; Test seq error: %s.\n" % ( + time.time(), iter_num, str(test_seq_error[0]))) #Note: The following logs are special for CE monitoring. #Other situations do not need to care about these logs. - print "kpis test_acc %f" % (1 - test_seq_error[0]) + print("kpis test_acc %f" % (1 - test_seq_error[0])) def save_model(args, exe, iter_num): filename = "model_%05d" % iter_num fluid.io.save_params( exe, dirname=args.save_model_dir, filename=filename) - print "Saved model to: %s/%s." % (args.save_model_dir, filename) + print("Saved model to: %s/%s." % (args.save_model_dir, filename)) iter_num = 0 stop = False @@ -152,14 +155,14 @@ def train(args): iter_num += 1 # training log if iter_num % args.log_period == 0: - print "\nTime: %s; Iter[%d]; Avg loss: %.3f; Avg seq err: %.3f" % ( + print("\nTime: %s; Iter[%d]; Avg loss: %.3f; Avg seq err: %.3f" % ( time.time(), iter_num, total_loss / (args.log_period * args.batch_size), - total_seq_error / (args.log_period * args.batch_size)) - print "kpis train_cost %f" % (total_loss / (args.log_period * - args.batch_size)) - print "kpis train_acc %f" % ( - 1 - total_seq_error / (args.log_period * args.batch_size)) + total_seq_error / (args.log_period * args.batch_size))) + print("kpis train_cost %f" % (total_loss / (args.log_period * + args.batch_size))) + print("kpis train_acc %f" % ( + 1 - total_seq_error / (args.log_period * args.batch_size))) total_loss = 0.0 total_seq_error = 0.0 @@ -179,7 +182,7 @@ def train(args): else: save_model(args, exe, iter_num) end_time = time.time() - print "kpis train_duration %f" % (end_time - start_time) + print("kpis train_duration %f" % (end_time - start_time)) # Postprocess benchmark data latencies = batch_times[args.skip_batch_num:] latency_avg = np.average(latencies) diff --git a/fluid/ocr_recognition/utility.py b/fluid/ocr_recognition/utility.py index d401b225..fb8d066c 100755 --- a/fluid/ocr_recognition/utility.py +++ b/fluid/ocr_recognition/utility.py @@ -20,6 +20,7 @@ import distutils.util import numpy as np from paddle.fluid import core import paddle.fluid as fluid +import six def print_arguments(args): @@ -38,7 +39,7 @@ def print_arguments(args): :type args: argparse.Namespace """ print("----------- Configuration Arguments -----------") - for arg, value in sorted(vars(args).iteritems()): + for arg, value in sorted(six.iteritems(vars(args))): print("%s: %s" % (arg, value)) print("------------------------------------------------") @@ -82,9 +83,9 @@ def get_ctc_feeder_data(data, place, need_label=True): pixel_tensor = core.LoDTensor() pixel_data = None pixel_data = np.concatenate( - map(lambda x: x[0][np.newaxis, :], data), axis=0).astype("float32") + list(map(lambda x: x[0][np.newaxis, :], data)), axis=0).astype("float32") pixel_tensor.set(pixel_data, place) - label_tensor = to_lodtensor(map(lambda x: x[1], data), place) + label_tensor = to_lodtensor(list(map(lambda x: x[1], data)), place) if need_label: return {"pixel": pixel_tensor, "label": label_tensor} else: @@ -95,10 +96,10 @@ def get_attention_feeder_data(data, place, need_label=True): pixel_tensor = core.LoDTensor() pixel_data = None pixel_data = np.concatenate( - map(lambda x: x[0][np.newaxis, :], data), axis=0).astype("float32") + list(map(lambda x: x[0][np.newaxis, :], data)), axis=0).astype("float32") pixel_tensor.set(pixel_data, place) - label_in_tensor = to_lodtensor(map(lambda x: x[1], data), place) - label_out_tensor = to_lodtensor(map(lambda x: x[2], data), place) + label_in_tensor = to_lodtensor(list(map(lambda x: x[1], data)), place) + label_out_tensor = to_lodtensor(list(map(lambda x: x[2], data)), place) if need_label: return { "pixel": pixel_tensor, @@ -126,7 +127,7 @@ def get_attention_feeder_for_infer(data, place): pixel_tensor = core.LoDTensor() pixel_data = None pixel_data = np.concatenate( - map(lambda x: x[0][np.newaxis, :], data), axis=0).astype("float32") + list(map(lambda x: x[0][np.newaxis, :], data)), axis=0).astype("float32") pixel_tensor.set(pixel_data, place) return { "pixel": pixel_tensor, -- GitLab