提交 812de6e8 编写于 作者: M minqiyang

Port utils to Python3

上级 6988aea9
......@@ -37,9 +37,9 @@ if __name__ == '__main__':
assert isinstance(conf, TrainerConfig_pb2.TrainerConfig)
if whole_conf:
print conf
print(conf)
else:
if binary:
sys.stdout.write(conf.model_config.SerializeToString())
else:
print conf.model_config
print(conf.model_config)
......@@ -15,7 +15,8 @@
import os, sys
import numpy as np
from PIL import Image
from cStringIO import StringIO
import six
from six.moves import cStringIO as StringIO
import multiprocessing
import functools
import itertools
......@@ -187,7 +188,8 @@ class PILTransformer(ImageTransformer):
return self.transform(im)
def job(is_img_string, transformer, (data, label)):
def job(is_img_string, transformer, data_label_pack):
(data, label) = data_label_pack
if is_img_string:
return transformer.transform_from_string(data), label
else:
......@@ -208,7 +210,7 @@ class MultiProcessImageTransformer(object):
"""
Processing image with multi-process. If it is used in PyDataProvider,
the simple usage for CNN is as follows:
.. code-block:: python
def hool(settings, is_train, **kwargs):
......@@ -229,7 +231,7 @@ class MultiProcessImageTransformer(object):
@provider(init_hook=hook, pool_size=20480)
def process(settings, file_list):
with open(file_list, 'r') as fdata:
for line in fdata:
for line in fdata:
data_dic = np.load(line.strip()) # load the data batch pickled by Pickle.
data = data_dic['data']
labels = data_dic['label']
......@@ -249,10 +251,10 @@ class MultiProcessImageTransformer(object):
:type channel_swap: tuple or list
:param mean: the mean values of image, per-channel mean or element-wise mean.
:type mean: array, The dimension is 1 for per-channel mean.
The dimension is 3 for element-wise mean.
The dimension is 3 for element-wise mean.
:param is_train: training peroid or testing peroid.
:type is_train: bool.
:param is_color: the image is color or gray.
:param is_color: the image is color or gray.
:type is_color: bool.
:param is_img_string: The input can be the file name of image or image string.
:type is_img_string: bool.
......@@ -273,4 +275,4 @@ class MultiProcessImageTransformer(object):
def run(self, data, label):
fun = functools.partial(job, self.is_img_string, self.transformer)
return self.pool.imap_unordered(
fun, itertools.izip(data, label), chunksize=100 * self.procnum)
fun, six.moves.zip(data, label), chunksize=100 * self.procnum)
......@@ -14,7 +14,7 @@
import numpy as np
from PIL import Image
from cStringIO import StringIO
from six.moves import cStringIO as StringIO
def resize_image(img, target_size):
......@@ -34,7 +34,7 @@ def flip(im):
"""
Return the flipped image.
Flip an image along the horizontal direction.
im: input image, (H x W x K) ndarrays
im: input image, (H x W x K) ndarrays
"""
if len(im.shape) == 3:
return im[:, :, ::-1]
......@@ -132,7 +132,7 @@ def load_meta(meta_path, mean_img_size, crop_size, color=True):
def load_image(img_path, is_color=True):
"""
Load image and return.
Load image and return.
img_path: image path.
is_color: is color image or not.
"""
......@@ -205,7 +205,7 @@ class ImageTransformer:
def set_mean(self, mean):
if mean is not None:
# mean value, may be one value per channel
# mean value, may be one value per channel
if mean.ndim == 1:
mean = mean[:, np.newaxis, np.newaxis]
else:
......
......@@ -15,6 +15,9 @@
# Generate dot diagram file for the given paddle model config
# The generated file can be viewed using Graphviz (http://graphviz.org)
from __future__ import print_function
import six
import sys
import traceback
......@@ -61,9 +64,9 @@ def make_diagram_from_proto(model_config, dot_file):
name2id[mem.link_name])
return s
print >> f, 'digraph graphname {'
print >> f, 'node [width=0.375,height=0.25];'
for i in xrange(len(model_config.layers)):
print('digraph graphname {', file=f)
print('node [width=0.375,height=0.25];', file=f)
for i in six.moves.xrange(len(model_config.layers)):
l = model_config.layers[i]
name2id[l.name] = i
......@@ -71,12 +74,12 @@ def make_diagram_from_proto(model_config, dot_file):
for sub_model in model_config.sub_models:
if sub_model.name == 'root':
continue
print >> f, 'subgraph cluster_%s {' % i
print >> f, 'style=dashed;'
print('subgraph cluster_%s {' % i, file=f)
print('style=dashed;', file=f)
label = '%s ' % sub_model.name
if sub_model.reversed:
label += '<=='
print >> f, 'label = "%s";' % label
print('label = "%s";' % label, file=f)
i += 1
submodel_layers.add(sub_model.name)
for layer_name in sub_model.layer_names:
......@@ -84,37 +87,41 @@ def make_diagram_from_proto(model_config, dot_file):
lid = name2id[layer_name]
layer_config = model_config.layers[lid]
label = make_layer_label(layer_config)
print >> f, 'l%s [label="%s", shape=box];' % (lid, label)
print >> f, '}'
print('l%s [label="%s", shape=box];' % (lid, label), file=f)
print('}', file=f)
for i in xrange(len(model_config.layers)):
for i in six.moves.xrange(len(model_config.layers)):
l = model_config.layers[i]
if l.name not in submodel_layers:
label = make_layer_label(l)
print >> f, 'l%s [label="%s", shape=box];' % (i, label)
print('l%s [label="%s", shape=box];' % (i, label), file=f)
for sub_model in model_config.sub_models:
if sub_model.name == 'root':
continue
for link in sub_model.in_links:
print >> f, make_link(link)
print(make_link(link), file=f)
for link in sub_model.out_links:
print >> f, make_link(link)
print(make_link(link), file=f)
for mem in sub_model.memories:
print >> f, make_mem(mem)
print(make_mem(mem), file=f)
for i in xrange(len(model_config.layers)):
for i in six.moves.xrange(len(model_config.layers)):
for l in model_config.layers[i].inputs:
print >> f, 'l%s -> l%s [label="%s"];' % (
name2id[l.input_layer_name], i, l.input_parameter_name)
print(
'l%s -> l%s [label="%s"];' % (name2id[l.input_layer_name], i,
l.input_parameter_name),
file=f)
print >> f, '}'
print('}', file=f)
f.close()
def usage():
print >> sys.stderr, ("Usage: python show_model_diagram.py" +
" CONFIG_FILE DOT_FILE [config_str]")
print(
("Usage: python show_model_diagram.py" +
" CONFIG_FILE DOT_FILE [config_str]"),
file=sys.stderr)
exit(1)
......
......@@ -70,4 +70,4 @@ def merge_v2_model(net, param_file, output_file):
for pname in param_names:
params.serialize(pname, f)
print 'Generate %s success!' % (output_file)
print('Generate %s success!' % (output_file))
......@@ -44,6 +44,7 @@ To use this script to generate plot for AvgCost, error:
python plotcurve.py -i paddle.INFO -o figure.png AvgCost error
"""
import six
import sys
import matplotlib
# the following line is added immediately after import matplotlib
......@@ -91,7 +92,7 @@ def plot_paddle_curve(keys, inputfile, outputfile, format='png',
sys.stderr.write("No data to plot. Exiting!\n")
return
m = len(keys) + 1
for i in xrange(1, m):
for i in six.moves.xrange(1, m):
pyplot.plot(
x[:, 0],
x[:, i],
......
......@@ -13,6 +13,7 @@
# limitations under the License.
import numpy as np
import six
import os
from paddle.trainer.config_parser import *
from paddle.utils.preprocess_img import \
......@@ -112,7 +113,7 @@ def simple_conv_net(data_conf, is_color=False):
num_classes: num of classes.
is_color: whether the input images are color.
"""
for k, v in data_conf.iteritems():
for k, v in six.iteritems(data_conf):
globals()[k] = v
data_input, label_input, num_image_channels = \
image_data_layers(image_size, num_classes, is_color, is_predict)
......@@ -340,7 +341,7 @@ def small_vgg(data_conf, is_predict=False):
num_classes: num of classes.
is_color: whether the input images are color.
"""
for k, v in data_conf.iteritems():
for k, v in six.iteritems(data_conf):
globals()[k] = v
vgg_conv_net(image_size, num_classes,
num_layers=[2, 2, 3, 3],
......
......@@ -17,9 +17,9 @@ import os
import random
import numpy as np
import PIL.Image as Image
import StringIO
import preprocess_util
from image_util import crop_img
from six.moves import cStringIO as StringIO
from . import preprocess_util
from .image_util import crop_img
def resize_image(img, target_size):
......@@ -52,7 +52,7 @@ class DiskImage:
def read_image(self):
if self.img is None:
print "reading: " + self.path
print("reading: " + self.path)
image = resize_image(Image.open(self.path), self.target_size)
self.img = image
......@@ -69,7 +69,7 @@ class DiskImage:
convert the image into the paddle batch format.
"""
self.read_image()
output = StringIO.StringIO()
output = StringIO()
self.img.save(output, "jpeg")
contents = output.getvalue()
return contents
......@@ -127,7 +127,7 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater):
image_path = items[0]
label_name = items[1]
if not label_name in label_set:
label_set[label_name] = len(label_set.keys())
label_set[label_name] = len(list(label_set.keys()))
img = DiskImage(path=image_path, target_size=self.target_size)
label = preprocess_util.Lablel(
label=label_set[label_name], name=label_name)
......@@ -144,7 +144,7 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater):
return create_dataset_from_list(path)
label_set = preprocess_util.get_label_set_from_dir(path)
data = []
for l_name in label_set.keys():
for l_name in list(label_set.keys()):
image_paths = preprocess_util.list_images(
os.path.join(path, l_name))
for p in image_paths:
......
......@@ -14,7 +14,7 @@
import os
import math
import cPickle as pickle
import six.moves.cPickle as pickle
import random
import collections
......@@ -169,7 +169,7 @@ class Dataset:
random.shuffle(keyvalue_indices[k])
num_data_per_key_batch = \
math.ceil(num_per_batch / float(len(keyvalue_indices.keys())))
math.ceil(num_per_batch / float(len(list(keyvalue_indices.keys()))))
if num_data_per_key_batch < 2:
raise Exception("The number of data in a batch is too small")
......@@ -182,8 +182,8 @@ class Dataset:
end_idx = int(
min(begin_idx + num_data_per_key_batch,
len(keyvalue_indices[k])))
print "begin_idx, end_idx"
print begin_idx, end_idx
print("begin_idx, end_idx")
print(begin_idx, end_idx)
for idx in range(begin_idx, end_idx):
permuted_data.append(self.data[keyvalue_indices[k][idx]])
keyvalue_readpointer[k] = end_idx
......@@ -357,6 +357,6 @@ class DatasetCreater(object):
data_batcher.create_batches_and_list(
self.output_path, self.train_list_name, self.test_list_name,
self.label_set_name)
self.num_classes = len(train_label_set.keys())
self.num_classes = len(list(train_label_set.keys()))
self.create_meta_file(train_data)
return out_path
......@@ -15,6 +15,8 @@
Show the content of proto buffer data file of PADDLE
"""
from __future__ import print_function
import os
import sys
from google.protobuf.internal.decoder import _DecodeVarint
......@@ -39,7 +41,7 @@ def read_proto(file, message):
def usage():
print >> sys.stderr, "Usage: python show_pb.py PROTO_DATA_FILE"
print("Usage: python show_pb.py PROTO_DATA_FILE", file=sys.stderr)
exit(1)
......@@ -50,8 +52,8 @@ if __name__ == '__main__':
f = open(sys.argv[1])
header = DataFormat.DataHeader()
read_proto(f, header)
print header
print(header)
sample = DataFormat.DataSample()
while read_proto(f, sample):
print sample
print(sample)
......@@ -24,7 +24,7 @@ import sys
import struct
import numpy as np
import torchfile
import cPickle as pickle
import six.moves.cPickle as pickle
import argparse
......@@ -48,7 +48,7 @@ def save_net_parameters(layers, params, output_path):
biases = params[i * 2 + 1]
weight_file = os.path.join(output_path, '_%s.w0' % layers[i])
biases_file = os.path.join(output_path, '_%s.wbias' % layers[i])
print "Saving for layer %s." % layers[i]
print("Saving for layer %s." % layers[i])
save_layer_parameters(weight_file, [weight])
save_layer_parameters(biases_file, biases)
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册