From ee1d08abba7a2565c2ec1f4791254abdc32b95f8 Mon Sep 17 00:00:00 2001 From: minqiyang Date: Thu, 9 Aug 2018 00:52:25 +0800 Subject: [PATCH] Fix CI issues --- python/paddle/dataset/movielens.py | 11 ++++++-- python/paddle/dataset/wmt16.py | 2 +- python/paddle/fluid/compat.py | 28 +++++++++---------- python/paddle/fluid/op.py | 3 ++ .../unittests/test_conv3d_transpose_op.py | 2 +- .../fluid/tests/unittests/test_maxout_op.py | 2 +- 6 files changed, 27 insertions(+), 21 deletions(-) diff --git a/python/paddle/dataset/movielens.py b/python/paddle/dataset/movielens.py index 056ec2178..354b7d4ae 100644 --- a/python/paddle/dataset/movielens.py +++ b/python/paddle/dataset/movielens.py @@ -27,6 +27,8 @@ import paddle.dataset.common import re import random import functools +import six +import paddle.fluid.compat as cpt __all__ = [ 'train', 'test', 'get_movie_title_dict', 'max_movie_id', 'max_user_id', @@ -112,6 +114,7 @@ def __initialize_meta_info__(): categories_set = set() with package.open('ml-1m/movies.dat') as movie_file: for i, line in enumerate(movie_file): + line = cpt.to_literal_str(line, encoding='latin') movie_id, title, categories = line.strip().split('::') categories = categories.split('|') for c in categories: @@ -136,6 +139,7 @@ def __initialize_meta_info__(): USER_INFO = dict() with package.open('ml-1m/users.dat') as user_file: for line in user_file: + line = cpt.to_literal_str(line, encoding='latin') uid, gender, age, job, _ = line.strip().split("::") USER_INFO[int(uid)] = UserInfo( index=uid, gender=gender, age=age, job_id=job) @@ -148,6 +152,7 @@ def __reader__(rand_seed=0, test_ratio=0.1, is_test=False): with zipfile.ZipFile(file=fn) as package: with package.open('ml-1m/ratings.dat') as rating: for line in rating: + line = cpt.to_literal_str(line, encoding='latin') if (rand.random() < test_ratio) == is_test: uid, mov_id, rating, _ = line.strip().split("::") uid = int(uid) @@ -187,7 +192,7 @@ def max_movie_id(): Get the maximum value of movie id. """ __initialize_meta_info__() - return reduce(__max_index_info__, list(MOVIE_INFO.values())).index + return six.moves.reduce(__max_index_info__, list(MOVIE_INFO.values())).index def max_user_id(): @@ -195,7 +200,7 @@ def max_user_id(): Get the maximum value of user id. """ __initialize_meta_info__() - return reduce(__max_index_info__, list(USER_INFO.values())).index + return six.moves.reduce(__max_index_info__, list(USER_INFO.values())).index def __max_job_id_impl__(a, b): @@ -210,7 +215,7 @@ def max_job_id(): Get the maximum value of job id. """ __initialize_meta_info__() - return reduce(__max_job_id_impl__, list(USER_INFO.values())).job_id + return six.moves.reduce(__max_job_id_impl__, list(USER_INFO.values())).job_id def movie_categories(): diff --git a/python/paddle/dataset/wmt16.py b/python/paddle/dataset/wmt16.py index cd34b523e..3e453a647 100644 --- a/python/paddle/dataset/wmt16.py +++ b/python/paddle/dataset/wmt16.py @@ -62,7 +62,7 @@ def __build_dict(tar_file, dict_size, save_path, lang): word_dict = defaultdict(int) with tarfile.open(tar_file, mode="r") as f: for line in f.extractfile("wmt16/train"): - line_split = line.strip().split("\t") + line_split = line.strip().split(six.b("\t")) if len(line_split) != 2: continue sen = line_split[0] if lang == "en" else line_split[1] for w in sen.split(): diff --git a/python/paddle/fluid/compat.py b/python/paddle/fluid/compat.py index 6cb59d50a..32f567253 100644 --- a/python/paddle/fluid/compat.py +++ b/python/paddle/fluid/compat.py @@ -13,39 +13,40 @@ # limitations under the License. import six +import math # str and bytes related functions -def to_literal_str(obj): +def to_literal_str(obj, encoding='utf-8'): if isinstance(obj, list): - return [_to_literal_str(item) for item in obj] + return [_to_literal_str(item, encoding) for item in obj] elif isinstance(obj, set): - return set([_to_literal_str(item) for item in obj]) + return set([_to_literal_str(item, encoding) for item in obj]) else: - return _to_literal_str(obj) + return _to_literal_str(obj, encoding) -def _to_literal_str(obj): +def _to_literal_str(obj, encoding): if isinstance(obj, six.binary_type): - return obj.decode('utf-8') + return obj.decode(encoding) elif isinstance(obj, six.text_type): return obj else: return six.u(obj) -def to_bytes(obj): +def to_bytes(obj, encoding='utf-8'): if isinstance(obj, list): - return [_to_bytes(item) for item in obj] + return [_to_bytes(item, encoding) for item in obj] elif isinstance(obj, set): - return set([_to_bytes(item) for item in obj]) + return set([_to_bytes(item, encoding) for item in obj]) else: - return _to_bytes(obj) + return _to_bytes(obj, encoding) -def _to_bytes(obj): +def _to_bytes(obj, encoding): if isinstance(obj, six.text_type): - return obj.encode('utf-8') + return obj.encode(encoding) elif isinstance(obj, six.binary_type): return obj else: @@ -53,9 +54,6 @@ def _to_bytes(obj): # math related functions -import math - - def round(x, d=0): """ Compatible round which act the same behaviour in Python3. diff --git a/python/paddle/fluid/op.py b/python/paddle/fluid/op.py index 93f021a36..a2db5bad5 100644 --- a/python/paddle/fluid/op.py +++ b/python/paddle/fluid/op.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import numpy as np import six import paddle.fluid.core as core @@ -99,6 +100,8 @@ class OpDescCreationMethod(object): new_attr = op_desc.attrs.add() new_attr.name = attr.name new_attr.type = attr.type + if isinstance(user_defined_attr, np.ndarray): + user_defined_attr = user_defined_attr.tolist() if attr.type == framework_pb2.INT: new_attr.i = user_defined_attr elif attr.type == framework_pb2.FLOAT: diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py index 300fa5e8b..2e55b8939 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py @@ -25,7 +25,7 @@ def conv3dtranspose_forward_naive(input_, filter_, attrs): groups = attrs['groups'] assert in_c == f_c out_c = f_out_c * groups - sub_in_c = in_c / groups + sub_in_c = in_c // groups stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ 'dilations'] diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index f5ddf7251..2151853ae 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -19,7 +19,7 @@ from op_test import OpTest def maxout_forward_naive(input, groups): s0, s1, s2, s3 = input.shape - return np.ndarray([s0, s1 / groups, groups, s2, s3], \ + return np.ndarray([s0, s1 // groups, groups, s2, s3], \ buffer = input, dtype=input.dtype).max(axis=(2)) -- GitLab