diff --git a/python/paddle/v2/dataset/cifar.py b/python/paddle/v2/dataset/cifar.py index 2ac71c6effe9d5f1140d1f574db9c9848b56433a..77c54bd268b5d988b0802a3edca91605e56f730e 100644 --- a/python/paddle/v2/dataset/cifar.py +++ b/python/paddle/v2/dataset/cifar.py @@ -1,82 +1,61 @@ """ -CIFAR Dataset. - -URL: https://www.cs.toronto.edu/~kriz/cifar.html - -the default train_creator, test_creator used for CIFAR-10 dataset. +CIFAR dataset: https://www.cs.toronto.edu/~kriz/cifar.html """ import cPickle import itertools -import tarfile - import numpy +import paddle.v2.dataset.common +import tarfile -from config import download - -__all__ = [ - 'cifar_100_train_creator', 'cifar_100_test_creator', 'train_creator', - 'test_creator' -] +__all__ = ['train100', 'test100', 'train10', 'test10'] -CIFAR10_URL = 'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz' +URL_PREFIX = 'https://www.cs.toronto.edu/~kriz/' +CIFAR10_URL = URL_PREFIX + 'cifar-10-python.tar.gz' CIFAR10_MD5 = 'c58f30108f718f92721af3b95e74349a' -CIFAR100_URL = 'https://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz' +CIFAR100_URL = URL_PREFIX + 'cifar-100-python.tar.gz' CIFAR100_MD5 = 'eb9058c3a382ffc7106e4002c42a8d85' -def __read_batch__(filename, sub_name): - def reader(): - def __read_one_batch_impl__(batch): - data = batch['data'] - labels = batch.get('labels', batch.get('fine_labels', None)) - assert labels is not None - for sample, label in itertools.izip(data, labels): - yield (sample / 255.0).astype(numpy.float32), int(label) +def reader_creator(filename, sub_name): + def read_batch(batch): + data = batch['data'] + labels = batch.get('labels', batch.get('fine_labels', None)) + assert labels is not None + for sample, label in itertools.izip(data, labels): + yield (sample / 255.0).astype(numpy.float32), int(label) + def reader(): with tarfile.open(filename, mode='r') as f: names = (each_item.name for each_item in f if sub_name in each_item.name) for name in names: batch = cPickle.load(f.extractfile(name)) - for item in __read_one_batch_impl__(batch): + for item in read_batch(batch): yield item return reader -def cifar_100_train_creator(): - fn = download(url=CIFAR100_URL, md5=CIFAR100_MD5) - return __read_batch__(fn, 'train') - - -def cifar_100_test_creator(): - fn = download(url=CIFAR100_URL, md5=CIFAR100_MD5) - return __read_batch__(fn, 'test') - - -def train_creator(): - """ - Default train reader creator. Use CIFAR-10 dataset. - """ - fn = download(url=CIFAR10_URL, md5=CIFAR10_MD5) - return __read_batch__(fn, 'data_batch') +def train100(): + return reader_creator( + paddle.v2.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5), + 'train') -def test_creator(): - """ - Default test reader creator. Use CIFAR-10 dataset. - """ - fn = download(url=CIFAR10_URL, md5=CIFAR10_MD5) - return __read_batch__(fn, 'test_batch') +def test100(): + return reader_creator( + paddle.v2.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5), + 'test') -def unittest(): - for _ in train_creator()(): - pass - for _ in test_creator()(): - pass +def train10(): + return reader_creator( + paddle.v2.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5), + 'data_batch') -if __name__ == '__main__': - unittest() +def test10(): + return reader_creator( + paddle.v2.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5), + 'test_batch') diff --git a/python/paddle/v2/dataset/common.py b/python/paddle/v2/dataset/common.py new file mode 100644 index 0000000000000000000000000000000000000000..a5ffe25a116e9be039bdebaaaad435685e23d372 --- /dev/null +++ b/python/paddle/v2/dataset/common.py @@ -0,0 +1,34 @@ +import requests +import hashlib +import os +import shutil + +__all__ = ['DATA_HOME', 'download', 'md5file'] + +DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset') + +if not os.path.exists(DATA_HOME): + os.makedirs(DATA_HOME) + + +def md5file(fname): + hash_md5 = hashlib.md5() + f = open(fname, "rb") + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + f.close() + return hash_md5.hexdigest() + + +def download(url, module_name, md5sum): + dirname = os.path.join(DATA_HOME, module_name) + if not os.path.exists(dirname): + os.makedirs(dirname) + + filename = os.path.join(dirname, url.split('/')[-1]) + if not (os.path.exists(filename) and md5file(filename) == md5sum): + r = requests.get(url, stream=True) + with open(filename, 'w') as f: + shutil.copyfileobj(r.raw, f) + + return filename diff --git a/python/paddle/v2/dataset/config.py b/python/paddle/v2/dataset/config.py deleted file mode 100644 index 02a009f09c71ccf6a5292a188565adeeb3f875f6..0000000000000000000000000000000000000000 --- a/python/paddle/v2/dataset/config.py +++ /dev/null @@ -1,36 +0,0 @@ -import hashlib -import os -import shutil -import urllib2 - -__all__ = ['DATA_HOME', 'download'] - -DATA_HOME = os.path.expanduser('~/.cache/paddle_data_set') - -if not os.path.exists(DATA_HOME): - os.makedirs(DATA_HOME) - - -def download(url, md5): - filename = os.path.split(url)[-1] - assert DATA_HOME is not None - filepath = os.path.join(DATA_HOME, md5) - if not os.path.exists(filepath): - os.makedirs(filepath) - __full_file__ = os.path.join(filepath, filename) - - def __file_ok__(): - if not os.path.exists(__full_file__): - return False - md5_hash = hashlib.md5() - with open(__full_file__, 'rb') as f: - for chunk in iter(lambda: f.read(4096), b""): - md5_hash.update(chunk) - - return md5_hash.hexdigest() == md5 - - while not __file_ok__(): - response = urllib2.urlopen(url) - with open(__full_file__, mode='wb') as of: - shutil.copyfileobj(fsrc=response, fdst=of) - return __full_file__ diff --git a/python/paddle/v2/dataset/mnist.py b/python/paddle/v2/dataset/mnist.py index db84f37aa4fc3477b17599a48a4de9b45cfb6c1f..a36c20e3fa3734bdc14c1f47779a61375f298511 100644 --- a/python/paddle/v2/dataset/mnist.py +++ b/python/paddle/v2/dataset/mnist.py @@ -1,39 +1,66 @@ -import sklearn.datasets.mldata -import sklearn.model_selection +""" +MNIST dataset. +""" import numpy -from config import DATA_HOME +import paddle.v2.dataset.common +import subprocess -__all__ = ['train_creator', 'test_creator'] +__all__ = ['train', 'test'] +URL_PREFIX = 'http://yann.lecun.com/exdb/mnist/' +TEST_IMAGE_URL = URL_PREFIX + 't10k-images-idx3-ubyte.gz' +TEST_IMAGE_MD5 = '25e3cc63507ef6e98d5dc541e8672bb6' +TEST_LABEL_URL = URL_PREFIX + 't10k-labels-idx1-ubyte.gz' +TEST_LABEL_MD5 = '4e9511fe019b2189026bd0421ba7b688' +TRAIN_IMAGE_URL = URL_PREFIX + 'train-images-idx3-ubyte.gz' +TRAIN_IMAGE_MD5 = 'f68b3c2dcbeaaa9fbdd348bbdeb94873' +TRAIN_LABEL_URL = URL_PREFIX + 'train-labels-idx1-ubyte.gz' +TRAIN_LABEL_MD5 = 'd53e105ee54ea40749a09fcbcd1e9432' -def __mnist_reader_creator__(data, target): + +def reader_creator(image_filename, label_filename, buffer_size): def reader(): - n_samples = data.shape[0] - for i in xrange(n_samples): - yield (data[i] / 255.0).astype(numpy.float32), int(target[i]) + # According to http://stackoverflow.com/a/38061619/724872, we + # cannot use standard package gzip here. + m = subprocess.Popen(["zcat", image_filename], stdout=subprocess.PIPE) + m.stdout.read(16) # skip some magic bytes - return reader + l = subprocess.Popen(["zcat", label_filename], stdout=subprocess.PIPE) + l.stdout.read(8) # skip some magic bytes + while True: + labels = numpy.fromfile( + l.stdout, 'ubyte', count=buffer_size).astype("int") -TEST_SIZE = 10000 + if labels.size != buffer_size: + break # numpy.fromfile returns empty slice after EOF. -data = sklearn.datasets.mldata.fetch_mldata( - "MNIST original", data_home=DATA_HOME) -X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - data.data, data.target, test_size=TEST_SIZE, random_state=0) + images = numpy.fromfile( + m.stdout, 'ubyte', count=buffer_size * 28 * 28).reshape( + (buffer_size, 28 * 28)).astype('float32') + images = images / 255.0 * 2.0 - 1.0 -def train_creator(): - return __mnist_reader_creator__(X_train, y_train) + for i in xrange(buffer_size): + yield images[i, :], int(labels[i]) + m.terminate() + l.terminate() -def test_creator(): - return __mnist_reader_creator__(X_test, y_test) + return reader -def unittest(): - assert len(list(test_creator()())) == TEST_SIZE +def train(): + return reader_creator( + paddle.v2.dataset.common.download(TRAIN_IMAGE_URL, 'mnist', + TRAIN_IMAGE_MD5), + paddle.v2.dataset.common.download(TRAIN_LABEL_URL, 'mnist', + TRAIN_LABEL_MD5), 100) -if __name__ == '__main__': - unittest() +def test(): + return reader_creator( + paddle.v2.dataset.common.download(TEST_IMAGE_URL, 'mnist', + TEST_IMAGE_MD5), + paddle.v2.dataset.common.download(TEST_LABEL_URL, 'mnist', + TEST_LABEL_MD5), 100) diff --git a/python/paddle/v2/dataset/movielens.py b/python/paddle/v2/dataset/movielens.py index 314329e91cadf8a74466ed9f385cd596c0ba6f9f..dcffcff2f58c63d451761d37f14127d730faf621 100644 --- a/python/paddle/v2/dataset/movielens.py +++ b/python/paddle/v2/dataset/movielens.py @@ -1,5 +1,5 @@ import zipfile -from config import download +from common import download import re import random import functools diff --git a/python/paddle/v2/dataset/tests/cifar_test.py b/python/paddle/v2/dataset/tests/cifar_test.py new file mode 100644 index 0000000000000000000000000000000000000000..a2af45ecf508462fe4b596b5d8d6401c5b974eff --- /dev/null +++ b/python/paddle/v2/dataset/tests/cifar_test.py @@ -0,0 +1,42 @@ +import paddle.v2.dataset.cifar +import unittest + + +class TestCIFAR(unittest.TestCase): + def check_reader(self, reader): + sum = 0 + label = 0 + for l in reader(): + self.assertEqual(l[0].size, 3072) + if l[1] > label: + label = l[1] + sum += 1 + return sum, label + + def test_test10(self): + instances, max_label_value = self.check_reader( + paddle.v2.dataset.cifar.test10()) + self.assertEqual(instances, 10000) + self.assertEqual(max_label_value, 9) + + def test_train10(self): + instances, max_label_value = self.check_reader( + paddle.v2.dataset.cifar.train10()) + self.assertEqual(instances, 50000) + self.assertEqual(max_label_value, 9) + + def test_test100(self): + instances, max_label_value = self.check_reader( + paddle.v2.dataset.cifar.test100()) + self.assertEqual(instances, 10000) + self.assertEqual(max_label_value, 99) + + def test_train100(self): + instances, max_label_value = self.check_reader( + paddle.v2.dataset.cifar.train100()) + self.assertEqual(instances, 50000) + self.assertEqual(max_label_value, 99) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/dataset/tests/common_test.py b/python/paddle/v2/dataset/tests/common_test.py new file mode 100644 index 0000000000000000000000000000000000000000..7d8406171b8478e4a8331637c5e867c18d5eb3d8 --- /dev/null +++ b/python/paddle/v2/dataset/tests/common_test.py @@ -0,0 +1,23 @@ +import paddle.v2.dataset.common +import unittest +import tempfile + + +class TestCommon(unittest.TestCase): + def test_md5file(self): + _, temp_path = tempfile.mkstemp() + with open(temp_path, 'w') as f: + f.write("Hello\n") + self.assertEqual('09f7e02f1290be211da707a266f153b3', + paddle.v2.dataset.common.md5file(temp_path)) + + def test_download(self): + yi_avatar = 'https://avatars0.githubusercontent.com/u/1548775?v=3&s=460' + self.assertEqual( + paddle.v2.dataset.common.DATA_HOME + '/test/1548775?v=3&s=460', + paddle.v2.dataset.common.download( + yi_avatar, 'test', 'f75287202d6622414c706c36c16f8e0d')) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/v2/dataset/tests/mnist_test.py b/python/paddle/v2/dataset/tests/mnist_test.py new file mode 100644 index 0000000000000000000000000000000000000000..b4408cc2f590d4d8da4ce5e98213cf7b208cfc15 --- /dev/null +++ b/python/paddle/v2/dataset/tests/mnist_test.py @@ -0,0 +1,30 @@ +import paddle.v2.dataset.mnist +import unittest + + +class TestMNIST(unittest.TestCase): + def check_reader(self, reader): + sum = 0 + label = 0 + for l in reader(): + self.assertEqual(l[0].size, 784) + if l[1] > label: + label = l[1] + sum += 1 + return sum, label + + def test_train(self): + instances, max_label_value = self.check_reader( + paddle.v2.dataset.mnist.train()) + self.assertEqual(instances, 60000) + self.assertEqual(max_label_value, 9) + + def test_test(self): + instances, max_label_value = self.check_reader( + paddle.v2.dataset.mnist.test()) + self.assertEqual(instances, 10000) + self.assertEqual(max_label_value, 9) + + +if __name__ == '__main__': + unittest.main()