From 4bf3c8c5a154b234648fa1d64160f799ef614018 Mon Sep 17 00:00:00 2001 From: minqiyang Date: Thu, 26 Jul 2018 22:02:07 +0800 Subject: [PATCH] Use six.moves to migrate zip, map, reduce and pickle in Python2 to Python3 --- python/paddle/dataset/cifar.py | 3 ++- python/paddle/dataset/common.py | 3 +-- python/paddle/dataset/conll05.py | 1 + python/paddle/dataset/flowers.py | 3 ++- python/paddle/dataset/image.py | 2 +- python/paddle/fluid/framework.py | 6 +++--- python/paddle/fluid/layer_helper.py | 1 + .../image_classification/cifar10_small_test_set.py | 3 ++- .../fluid/transpiler/memory_optimization_transpiler.py | 1 + python/paddle/reader/creator.py | 2 +- python/paddle/reader/decorator.py | 2 ++ python/paddle/trainer/PyDataProviderWrapper.py | 2 +- python/paddle/trainer_config_helpers/data_sources.py | 2 +- python/paddle/trainer_config_helpers/layers.py | 2 +- 14 files changed, 20 insertions(+), 13 deletions(-) diff --git a/python/paddle/dataset/cifar.py b/python/paddle/dataset/cifar.py index cc6384e74c4..f6b4ff8fbd0 100644 --- a/python/paddle/dataset/cifar.py +++ b/python/paddle/dataset/cifar.py @@ -28,11 +28,12 @@ images per class. """ -import pickle import itertools import numpy import paddle.dataset.common import tarfile +from six.moves import zip +from six.moves import cPickle as pickle __all__ = ['train100', 'test100', 'train10', 'test10', 'convert'] diff --git a/python/paddle/dataset/common.py b/python/paddle/dataset/common.py index 771577c89d9..a892287dbc8 100644 --- a/python/paddle/dataset/common.py +++ b/python/paddle/dataset/common.py @@ -20,9 +20,8 @@ import shutil import sys import importlib import paddle.dataset -import pickle +import six.moves.cPickle as pickle import glob -import pickle as pickle __all__ = [ 'DATA_HOME', diff --git a/python/paddle/dataset/conll05.py b/python/paddle/dataset/conll05.py index 466344cc9db..25623feabb8 100644 --- a/python/paddle/dataset/conll05.py +++ b/python/paddle/dataset/conll05.py @@ -24,6 +24,7 @@ import tarfile import gzip import itertools import paddle.dataset.common +from six.moves import zip __all__ = ['test, get_dict', 'get_embedding', 'convert'] diff --git a/python/paddle/dataset/flowers.py b/python/paddle/dataset/flowers.py index db9be0c04bb..914dae348bc 100644 --- a/python/paddle/dataset/flowers.py +++ b/python/paddle/dataset/flowers.py @@ -28,7 +28,6 @@ Graphics and Image Processing (2008) http://www.robots.ox.ac.uk/~vgg/publications/papers/nilsback08.{pdf,ps.gz}. """ -import pickle import itertools import functools from .common import download @@ -39,6 +38,8 @@ from paddle.reader import * import os import numpy as np from multiprocessing import cpu_count +from six.moves import cPickle as pickle +from six.moves import zip __all__ = ['train', 'test', 'valid'] DATA_URL = 'http://www.robots.ox.ac.uk/~vgg/data/flowers/102/102flowers.tgz' diff --git a/python/paddle/dataset/image.py b/python/paddle/dataset/image.py index 3048dfd5182..3b3d89c93c4 100644 --- a/python/paddle/dataset/image.py +++ b/python/paddle/dataset/image.py @@ -36,7 +36,7 @@ except ImportError: cv2 = None import os import tarfile -import pickle +import six.moves.cPickle as pickle __all__ = [ "load_image_bytes", "load_image", "resize_short", "to_chw", "center_crop", diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 53658610e55..3cfbe8cebf8 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -963,9 +963,9 @@ class Block(object): raise ValueError("Var {0} is not found recursively".format(name)) def all_parameters(self): - return list(self.iter_parameters()) + return list(self._iter_parameters()) - def iter_parameters(self): + def _iter_parameters(self): return (item[1] for item in list(self.vars.items()) if isinstance(item[1], Parameter)) @@ -1199,7 +1199,7 @@ class Block(object): if not isinstance(other, Block): raise TypeError( "_copy_param_info_from should be invoked with Block") - for p in other.iter_parameters(): + for p in other._iter_parameters(): assert isinstance(p, Parameter) v = self.vars.get(p.name, None) if v is None: diff --git a/python/paddle/fluid/layer_helper.py b/python/paddle/fluid/layer_helper.py index 5f2ff3c9c87..715c562516b 100644 --- a/python/paddle/fluid/layer_helper.py +++ b/python/paddle/fluid/layer_helper.py @@ -20,6 +20,7 @@ from . import unique_name from paddle.fluid.initializer import Constant, Xavier from .param_attr import ParamAttr, WeightNormParamAttr from . import core +from six.moves import zip class LayerHelper(object): diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py index ebbe09d0988..9e4c384d929 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py @@ -28,11 +28,12 @@ images per class. """ -import pickle import itertools import numpy import paddle.v2.dataset.common import tarfile +from six.moves import cPickle as pickle +from six.moves import zip __all__ = ['train10'] diff --git a/python/paddle/fluid/transpiler/memory_optimization_transpiler.py b/python/paddle/fluid/transpiler/memory_optimization_transpiler.py index 907acb1cac1..20ba7ed2b0b 100644 --- a/python/paddle/fluid/transpiler/memory_optimization_transpiler.py +++ b/python/paddle/fluid/transpiler/memory_optimization_transpiler.py @@ -17,6 +17,7 @@ from .. import core from ..framework import Program, default_main_program, Parameter from ..backward import _rename_arg_ from functools import reduce +from six.moves import range dtype_to_size = { core.VarDesc.VarType.FP16: 2, diff --git a/python/paddle/reader/creator.py b/python/paddle/reader/creator.py index 12c3afbcb05..025338a2ae0 100644 --- a/python/paddle/reader/creator.py +++ b/python/paddle/reader/creator.py @@ -67,7 +67,7 @@ def recordio(paths, buf_size=100): import recordio as rec import paddle.reader.decorator as dec - import pickle as pickle + import six.moves.cPickle as pickle def reader(): if isinstance(paths, str): diff --git a/python/paddle/reader/decorator.py b/python/paddle/reader/decorator.py index 7faca28e400..ce410e61b92 100644 --- a/python/paddle/reader/decorator.py +++ b/python/paddle/reader/decorator.py @@ -22,6 +22,8 @@ import subprocess from six.moves.queue import Queue from six.moves import zip_longest +from six.moves import map +from six.moves import zip import itertools import random import zlib diff --git a/python/paddle/trainer/PyDataProviderWrapper.py b/python/paddle/trainer/PyDataProviderWrapper.py index 6af25077285..374976db9f1 100644 --- a/python/paddle/trainer/PyDataProviderWrapper.py +++ b/python/paddle/trainer/PyDataProviderWrapper.py @@ -42,7 +42,7 @@ except ImportError: try: import cPickle as pickle except ImportError: - import pickle + import six.moves.cPickle as pickle import io diff --git a/python/paddle/trainer_config_helpers/data_sources.py b/python/paddle/trainer_config_helpers/data_sources.py index ab9a2562dcc..a2a32d848cb 100644 --- a/python/paddle/trainer_config_helpers/data_sources.py +++ b/python/paddle/trainer_config_helpers/data_sources.py @@ -20,7 +20,7 @@ from .utils import deprecated try: import cPickle as pickle except ImportError: - import pickle + import six.moves.cPickle as pickle __all__ = ['define_py_data_sources2'] diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index d9787ef42a3..ee34c157334 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -28,7 +28,7 @@ from .default_decorators import * try: import cPickle as pickle except ImportError: - import pickle + import six.moves.cPickle as pickle import copy __all__ = [ -- GitLab