未验证 提交 9930a582 编写于 作者: Z zhiboniu 提交者: GitHub

update 2.0 public api in dataset&framework (#31985)

上级 f1bc322c
...@@ -265,6 +265,7 @@ from .framework import DataParallel #DEFINE_ALIAS ...@@ -265,6 +265,7 @@ from .framework import DataParallel #DEFINE_ALIAS
from .framework import set_default_dtype #DEFINE_ALIAS from .framework import set_default_dtype #DEFINE_ALIAS
from .framework import get_default_dtype #DEFINE_ALIAS from .framework import get_default_dtype #DEFINE_ALIAS
from .framework import set_grad_enabled #DEFINE_ALIAS
from .tensor.search import index_sample #DEFINE_ALIAS from .tensor.search import index_sample #DEFINE_ALIAS
from .tensor.stat import mean #DEFINE_ALIAS from .tensor.stat import mean #DEFINE_ALIAS
......
...@@ -15,18 +15,18 @@ ...@@ -15,18 +15,18 @@
Dataset package. Dataset package.
""" """
import paddle.dataset.mnist import paddle.dataset.mnist # noqa: F401
import paddle.dataset.imikolov import paddle.dataset.imikolov # noqa: F401
import paddle.dataset.imdb import paddle.dataset.imdb # noqa: F401
import paddle.dataset.cifar import paddle.dataset.cifar # noqa: F401
import paddle.dataset.movielens import paddle.dataset.movielens # noqa: F401
import paddle.dataset.conll05 import paddle.dataset.conll05 # noqa: F401
import paddle.dataset.uci_housing import paddle.dataset.uci_housing # noqa: F401
import paddle.dataset.wmt14 import paddle.dataset.wmt14 # noqa: F401
import paddle.dataset.wmt16 import paddle.dataset.wmt16 # noqa: F401
import paddle.dataset.flowers import paddle.dataset.flowers # noqa: F401
import paddle.dataset.voc2012 import paddle.dataset.voc2012 # noqa: F401
import paddle.dataset.image import paddle.dataset.image # noqa: F401
# set __all__ as empty for not showing APIs under paddle.dataset # set __all__ as empty for not showing APIs under paddle.dataset
__all__ = [] __all__ = []
...@@ -37,8 +37,6 @@ import tarfile ...@@ -37,8 +37,6 @@ import tarfile
import six import six
from six.moves import cPickle as pickle from six.moves import cPickle as pickle
__all__ = ['train100', 'test100', 'train10', 'test10']
URL_PREFIX = 'https://dataset.bj.bcebos.com/cifar/' URL_PREFIX = 'https://dataset.bj.bcebos.com/cifar/'
CIFAR10_URL = URL_PREFIX + 'cifar-10-python.tar.gz' CIFAR10_URL = URL_PREFIX + 'cifar-10-python.tar.gz'
CIFAR10_MD5 = 'c58f30108f718f92721af3b95e74349a' CIFAR10_MD5 = 'c58f30108f718f92721af3b95e74349a'
......
...@@ -26,14 +26,6 @@ import paddle.dataset ...@@ -26,14 +26,6 @@ import paddle.dataset
import six.moves.cPickle as pickle import six.moves.cPickle as pickle
import glob import glob
__all__ = [
'DATA_HOME',
'download',
'md5file',
'split',
'cluster_files_reader',
]
HOME = os.path.expanduser('~') HOME = os.path.expanduser('~')
DATA_HOME = os.path.join(HOME, '.cache', 'paddle', 'dataset') DATA_HOME = os.path.join(HOME, '.cache', 'paddle', 'dataset')
......
...@@ -30,8 +30,6 @@ import paddle.compat as cpt ...@@ -30,8 +30,6 @@ import paddle.compat as cpt
import paddle.utils.deprecated as deprecated import paddle.utils.deprecated as deprecated
from six.moves import zip, range from six.moves import zip, range
__all__ = ['test, get_dict', 'get_embedding']
DATA_URL = 'http://paddlemodels.bj.bcebos.com/conll05st/conll05st-tests.tar.gz' DATA_URL = 'http://paddlemodels.bj.bcebos.com/conll05st/conll05st-tests.tar.gz'
DATA_MD5 = '387719152ae52d60422c016e92a742fc' DATA_MD5 = '387719152ae52d60422c016e92a742fc'
WORDDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FwordDict.txt' WORDDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FwordDict.txt'
......
...@@ -35,7 +35,12 @@ import itertools ...@@ -35,7 +35,12 @@ import itertools
import functools import functools
from .common import download from .common import download
import tarfile import tarfile
from paddle.dataset.image import *
from paddle.dataset.image import load_image_bytes
from paddle.dataset.image import load_image
from paddle.dataset.image import simple_transform
from paddle.dataset.image import batch_images_from_tar
from paddle.reader import map_readers, xmap_readers from paddle.reader import map_readers, xmap_readers
from paddle import compat as cpt from paddle import compat as cpt
import paddle.utils.deprecated as deprecated import paddle.utils.deprecated as deprecated
...@@ -45,7 +50,6 @@ from multiprocessing import cpu_count ...@@ -45,7 +50,6 @@ from multiprocessing import cpu_count
import six import six
from six.moves import cPickle as pickle from six.moves import cPickle as pickle
from paddle.utils import try_import from paddle.utils import try_import
__all__ = ['train', 'test', 'valid']
DATA_URL = 'http://paddlemodels.bj.bcebos.com/flowers/102flowers.tgz' DATA_URL = 'http://paddlemodels.bj.bcebos.com/flowers/102flowers.tgz'
LABEL_URL = 'http://paddlemodels.bj.bcebos.com/flowers/imagelabels.mat' LABEL_URL = 'http://paddlemodels.bj.bcebos.com/flowers/imagelabels.mat'
......
...@@ -58,12 +58,6 @@ import os ...@@ -58,12 +58,6 @@ import os
import tarfile import tarfile
import six.moves.cPickle as pickle import six.moves.cPickle as pickle
__all__ = [
"load_image_bytes", "load_image", "resize_short", "to_chw", "center_crop",
"random_crop", "left_right_flip", "simple_transform", "load_and_transform",
"batch_images_from_tar"
]
def _check_cv2(): def _check_cv2():
if cv2 is None: if cv2 is None:
......
...@@ -30,8 +30,6 @@ import re ...@@ -30,8 +30,6 @@ import re
import string import string
import six import six
__all__ = ['build_dict', 'train', 'test']
#URL = 'http://ai.stanford.edu/%7Eamaas/data/sentiment/aclImdb_v1.tar.gz' #URL = 'http://ai.stanford.edu/%7Eamaas/data/sentiment/aclImdb_v1.tar.gz'
URL = 'https://dataset.bj.bcebos.com/imdb%2FaclImdb_v1.tar.gz' URL = 'https://dataset.bj.bcebos.com/imdb%2FaclImdb_v1.tar.gz'
MD5 = '7c2ac02c03563afcf9b574c7e56c153a' MD5 = '7c2ac02c03563afcf9b574c7e56c153a'
......
...@@ -27,8 +27,6 @@ import collections ...@@ -27,8 +27,6 @@ import collections
import tarfile import tarfile
import six import six
__all__ = ['train', 'test', 'build_dict']
#URL = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz' #URL = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz'
URL = 'https://dataset.bj.bcebos.com/imikolov%2Fsimple-examples.tgz' URL = 'https://dataset.bj.bcebos.com/imikolov%2Fsimple-examples.tgz'
MD5 = '30177ea32e27c525793142b6bf2c8e2d' MD5 = '30177ea32e27c525793142b6bf2c8e2d'
......
...@@ -26,7 +26,6 @@ import gzip ...@@ -26,7 +26,6 @@ import gzip
import numpy import numpy
import struct import struct
from six.moves import range from six.moves import range
__all__ = ['train', 'test']
URL_PREFIX = 'https://dataset.bj.bcebos.com/mnist/' URL_PREFIX = 'https://dataset.bj.bcebos.com/mnist/'
TEST_IMAGE_URL = URL_PREFIX + 't10k-images-idx3-ubyte.gz' TEST_IMAGE_URL = URL_PREFIX + 't10k-images-idx3-ubyte.gz'
......
...@@ -34,11 +34,6 @@ import functools ...@@ -34,11 +34,6 @@ import functools
import six import six
import paddle.compat as cpt import paddle.compat as cpt
__all__ = [
'train', 'test', 'get_movie_title_dict', 'max_movie_id', 'max_user_id',
'age_table', 'movie_categories', 'max_job_id', 'user_info', 'movie_info'
]
age_table = [1, 18, 25, 35, 45, 50, 56] age_table = [1, 18, 25, 35, 45, 50, 56]
#URL = 'http://files.grouplens.org/datasets/movielens/ml-1m.zip' #URL = 'http://files.grouplens.org/datasets/movielens/ml-1m.zip'
......
...@@ -29,8 +29,6 @@ import os ...@@ -29,8 +29,6 @@ import os
import paddle.dataset.common import paddle.dataset.common
import paddle.utils.deprecated as deprecated import paddle.utils.deprecated as deprecated
__all__ = ['train', 'test']
URL = 'http://paddlemodels.bj.bcebos.com/uci_housing/housing.data' URL = 'http://paddlemodels.bj.bcebos.com/uci_housing/housing.data'
MD5 = 'd4accdce7a25600298819f8e28e8d593' MD5 = 'd4accdce7a25600298819f8e28e8d593'
feature_names = [ feature_names = [
......
...@@ -29,8 +29,6 @@ from paddle.dataset.image import * ...@@ -29,8 +29,6 @@ from paddle.dataset.image import *
import paddle.utils.deprecated as deprecated import paddle.utils.deprecated as deprecated
from PIL import Image from PIL import Image
__all__ = ['train', 'test', 'val']
VOC_URL = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/\ VOC_URL = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/\
VOCtrainval_11-May-2012.tar' VOCtrainval_11-May-2012.tar'
......
...@@ -30,12 +30,6 @@ import paddle.dataset.common ...@@ -30,12 +30,6 @@ import paddle.dataset.common
import paddle.compat as cpt import paddle.compat as cpt
import paddle.utils.deprecated as deprecated import paddle.utils.deprecated as deprecated
__all__ = [
'train',
'test',
'get_dict',
]
URL_DEV_TEST = ('http://www-lium.univ-lemans.fr/~schwenk/' URL_DEV_TEST = ('http://www-lium.univ-lemans.fr/~schwenk/'
'cslm_joint_paper/data/dev+test.tgz') 'cslm_joint_paper/data/dev+test.tgz')
MD5_DEV_TEST = '7d7897317ddd8ba0ae5c5fa7248d3ff5' MD5_DEV_TEST = '7d7897317ddd8ba0ae5c5fa7248d3ff5'
......
...@@ -40,14 +40,6 @@ import paddle ...@@ -40,14 +40,6 @@ import paddle
import paddle.compat as cpt import paddle.compat as cpt
import paddle.utils.deprecated as deprecated import paddle.utils.deprecated as deprecated
__all__ = [
"train",
"test",
"validation",
"fetch",
"get_dict",
]
DATA_URL = ("http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gz") DATA_URL = ("http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gz")
DATA_MD5 = "0c38be43600334966403524a40dcd81e" DATA_MD5 = "0c38be43600334966403524a40dcd81e"
......
...@@ -12,35 +12,25 @@ ...@@ -12,35 +12,25 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# TODO: import framework api under this directory # TODO: import framework api under this directory
__all__ = [
'create_parameter', 'ParamAttr', 'CPUPlace', 'CUDAPlace', 'CUDAPinnedPlace',
'NPUPlace', 'get_default_dtype', 'set_default_dtype'
]
__all__ += [ from . import random # noqa: F401
'grad', 'set_grad_enabled', 'LayerList', 'load', 'save', 'no_grad', from .random import seed # noqa: F401
'DataParallel' from .framework import get_default_dtype # noqa: F401
] from .framework import set_default_dtype # noqa: F401
from .framework import set_grad_enabled # noqa: F401
from . import random from ..fluid.param_attr import ParamAttr # noqa: F401
from .random import seed from ..fluid.layers.tensor import create_parameter # noqa: F401
from .framework import get_default_dtype from ..fluid.core import CPUPlace # noqa: F401
from .framework import set_default_dtype from ..fluid.core import CUDAPlace # noqa: F401
from .framework import set_grad_enabled from ..fluid.core import CUDAPinnedPlace # noqa: F401
from ..fluid.core import NPUPlace # noqa: F401
from ..fluid.core import VarBase # noqa: F401
from ..fluid.param_attr import ParamAttr #DEFINE_ALIAS from paddle.fluid import core # noqa: F401
# from ..fluid.layers.tensor import create_global_var #DEFINE_ALIAS from ..fluid.dygraph.base import no_grad_ as no_grad # noqa: F401
from ..fluid.layers.tensor import create_parameter #DEFINE_ALIAS from ..fluid.dygraph.base import grad # noqa: F401
from ..fluid.core import CPUPlace #DEFINE_ALIAS from .io import save # noqa: F401
from ..fluid.core import CUDAPlace #DEFINE_ALIAS from .io import load # noqa: F401
from ..fluid.core import CUDAPinnedPlace #DEFINE_ALIAS from ..fluid.dygraph.parallel import DataParallel # noqa: F401
from ..fluid.core import NPUPlace #DEFINE_ALIAS
from ..fluid.core import VarBase #DEFINE_ALIAS
from paddle.fluid import core #DEFINE_ALIAS
from ..fluid.dygraph.base import no_grad_ as no_grad #DEFINE_ALIAS
from ..fluid.dygraph.base import grad #DEFINE_ALIAS
from .io import save
from .io import load
from ..fluid.dygraph.parallel import DataParallel #DEFINE_ALIAS
...@@ -19,8 +19,6 @@ from paddle.fluid.framework import _dygraph_tracer ...@@ -19,8 +19,6 @@ from paddle.fluid.framework import _dygraph_tracer
import numpy as np import numpy as np
from contextlib import contextmanager from contextlib import contextmanager
__all__ = ['set_default_dtype', 'get_default_dtype']
def set_default_dtype(d): def set_default_dtype(d):
""" """
......
...@@ -38,11 +38,6 @@ from paddle.fluid.dygraph.jit import _SaveLoadConfig ...@@ -38,11 +38,6 @@ from paddle.fluid.dygraph.jit import _SaveLoadConfig
from paddle.fluid.dygraph.io import _construct_program_holders, _construct_params_and_buffers from paddle.fluid.dygraph.io import _construct_program_holders, _construct_params_and_buffers
from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX, INFER_PARAMS_INFO_SUFFIX from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX, INFER_PARAMS_INFO_SUFFIX
__all__ = [
'save',
'load',
]
def _build_saved_state_dict(state_dict): def _build_saved_state_dict(state_dict):
save_dict = {} save_dict = {}
......
...@@ -16,8 +16,6 @@ ...@@ -16,8 +16,6 @@
import paddle.fluid as fluid import paddle.fluid as fluid
from paddle.fluid import core from paddle.fluid import core
__all__ = ['seed', 'get_cuda_rng_state', 'set_cuda_rng_state']
def seed(seed): def seed(seed):
""" """
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册