uci_housing.py 3.6 KB
Newer Older
D
dangqingqing 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Y
Yu Yang 已提交
14 15 16
"""
UCI Housing dataset.

R
root 已提交
17
This module will paddle.v2.dataset.common.download dataset from
Q
qijun 已提交
18
https://archive.ics.uci.edu/ml/machine-learning-databases/housing/ and
Q
qijun 已提交
19
parse training set and test set into paddle reader creators.
Y
Yu Yang 已提交
20
"""
D
dangqingqing 已提交
21 22 23

import numpy as np
import os
R
root 已提交
24
import paddle.v2.dataset.common
D
dangqingqing 已提交
25 26 27 28 29 30 31

__all__ = ['train', 'test']

URL = 'https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data'
MD5 = 'd4accdce7a25600298819f8e28e8d593'
feature_names = [
    'CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX',
Y
Your Name 已提交
32
    'PTRATIO', 'B', 'LSTAT', 'convert'
D
dangqingqing 已提交
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
]

UCI_TRAIN_DATA = None
UCI_TEST_DATA = None


def feature_range(maximums, minimums):
    import matplotlib
    matplotlib.use('Agg')
    import matplotlib.pyplot as plt
    fig, ax = plt.subplots()
    feature_num = len(maximums)
    ax.bar(range(feature_num), maximums - minimums, color='r', align='center')
    ax.set_title('feature scale')
    plt.xticks(range(feature_num), feature_names)
    plt.xlim([-1, feature_num])
    fig.set_figheight(6)
    fig.set_figwidth(10)
    if not os.path.exists('./image'):
        os.makedirs('./image')
    fig.savefig('image/ranges.png', dpi=48)
    plt.close(fig)


def load_data(filename, feature_num=14, ratio=0.8):
    global UCI_TRAIN_DATA, UCI_TEST_DATA
    if UCI_TRAIN_DATA is not None and UCI_TEST_DATA is not None:
        return

    data = np.fromfile(filename, sep=' ')
    data = data.reshape(data.shape[0] / feature_num, feature_num)
    maximums, minimums, avgs = data.max(axis=0), data.min(axis=0), data.sum(
        axis=0) / data.shape[0]
    feature_range(maximums[:-1], minimums[:-1])
    for i in xrange(feature_num - 1):
        data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i])
    offset = int(data.shape[0] * ratio)
    UCI_TRAIN_DATA = data[:offset]
    UCI_TEST_DATA = data[offset:]


def train():
Q
qijun 已提交
75
    """
Q
qijun 已提交
76
    UCI_HOUSING training set creator.
Q
qijun 已提交
77

Q
qijun 已提交
78 79
    It returns a reader creator, each sample in the reader is features after
    normalization and price number.
Q
qijun 已提交
80

Q
qijun 已提交
81
    :return: Training reader creator
Q
qijun 已提交
82 83
    :rtype: callable
    """
D
dangqingqing 已提交
84
    global UCI_TRAIN_DATA
R
root 已提交
85
    load_data(paddle.v2.dataset.common.download(URL, 'uci_housing', MD5))
D
dangqingqing 已提交
86 87 88 89 90 91 92 93 94

    def reader():
        for d in UCI_TRAIN_DATA:
            yield d[:-1], d[-1:]

    return reader


def test():
Q
qijun 已提交
95 96 97
    """
    UCI_HOUSING test set creator.

Q
qijun 已提交
98 99
    It returns a reader creator, each sample in the reader is features after
    normalization and price number.
Q
qijun 已提交
100 101 102 103

    :return: Test reader creator
    :rtype: callable
    """
D
dangqingqing 已提交
104
    global UCI_TEST_DATA
R
root 已提交
105
    load_data(paddle.v2.dataset.common.download(URL, 'uci_housing', MD5))
D
dangqingqing 已提交
106 107 108 109 110 111

    def reader():
        for d in UCI_TEST_DATA:
            yield d[:-1], d[-1:]

    return reader
Y
Yancey1989 已提交
112 113


114
def fetch():
R
root 已提交
115 116 117 118 119 120 121 122 123
    paddle.v2.dataset.common.download(URL, 'uci_housing', MD5)


def convert(path):
    """
    Converts dataset to recordio format
    """
    paddle.v2.dataset.common.convert(path, train(), 10, "uci_housing_train")
    paddle.v2.dataset.common.convert(path, test(), 10, "uci_houseing_test")