mnist.py 7.5 KB
Newer Older
Q
qiaolongfei 已提交
1 2 3
import paddle.v2.framework.core as core
from paddle.v2.framework.op import Operator
import numpy
Q
qiaolongfei 已提交
4
import paddle.v2 as paddle
Q
qiaolongfei 已提交
5

Q
qiaolongfei 已提交
6
BATCH_SIZE = 100
Q
qiaolongfei 已提交
7 8 9

scope = core.Scope()
place = core.CPUPlace()
Q
qiaolongfei 已提交
10 11
# if you want to test GPU training, you can use gpu place
# place = core.GPUPlace(0)
Q
qiaolongfei 已提交
12 13
dev_ctx = core.DeviceContext.create(place)

14 15
init_net = core.Net.create()
forward_net = core.Net.create()
Q
qiaolongfei 已提交
16 17 18 19
backward_net = None
optimize_net = core.Net.create()


Q
qiaolongfei 已提交
20
def atomic_id():
Q
qiaolongfei 已提交
21 22 23 24 25 26
    id = 0
    while True:
        yield id
        id += 1


Q
qiaolongfei 已提交
27
uniq_id = atomic_id().next
Q
qiaolongfei 已提交
28 29 30 31 32 33 34 35 36 37


def data_layer(name, dims):
    var = scope.new_var(name)
    tensor = var.get_tensor()
    tensor.set_dims(dims)  # 1 is batch size holder.
    return name


def feed_data(name, data):
Q
qiaolongfei 已提交
38
    assert isinstance(data, numpy.ndarray)
Q
qiaolongfei 已提交
39 40
    tensor = scope.find_var(name).get_tensor()
    tensor.set_dims(data.shape)
41
    if data.dtype == numpy.dtype("int32"):
Q
qiaolongfei 已提交
42
        tensor.alloc_int(place)
43
    elif data.dtype == numpy.dtype("float32"):
Q
qiaolongfei 已提交
44
        tensor.alloc_float(place)
Q
qiaolongfei 已提交
45 46
    else:
        raise ValueError("data type not supported")
Q
qiaolongfei 已提交
47 48 49 50 51 52 53
    tensor.set(data, place)


def grad_var_name(var_name):
    return var_name + "@GRAD"


Q
qiaolongfei 已提交
54
def sgd_optimizer(net, param_name, learning_rate=0.005):
Q
qiaolongfei 已提交
55 56
    grad_name = grad_var_name(param_name)
    optimize_op = Operator(
Q
qiaolongfei 已提交
57 58 59 60 61
        "sgd",
        param=param_name,
        grad=grad_name,
        param_out=param_name,
        learning_rate=learning_rate)
Q
qiaolongfei 已提交
62
    net.append_op(optimize_op)
Q
qiaolongfei 已提交
63 64 65


# should use operator and add these to the init_network
66 67 68 69 70 71
def init_param(net, param_name, dims):
    scope.new_var(param_name)
    op = Operator(
        "uniform_random", Out=param_name, dims=dims, min=-0.5, max=0.5, seed=10)
    op.infer_shape(scope)
    net.append_op(op)
Q
qiaolongfei 已提交
72 73 74


# fc_layer
Q
qiaolongfei 已提交
75
def fc_layer(net, input, size, act="softmax", bias=True, param=None, name=None):
Q
qiaolongfei 已提交
76
    """
77
    The fully connected layer.
Q
qiaolongfei 已提交
78

79
    :param input: The name of input variable.
Q
qiaolongfei 已提交
80
    :type input: str
81 82 83 84 85 86 87 88 89
    :param size: The size of fully connected layer.
    :param act: The name of activation.
    :param param: The attribute of learnable parameter which can be used to
                  modify initialization mean and std of the parameter.
    :param bias: The attribute of bias. If set False, this layer does not have
                 a bias.
    :param name: The name of this layer. If it is not set explictly, a name
                 will be generated automatically.
    :return: The name of the output variable.
Q
qiaolongfei 已提交
90
    """
91

Q
qiaolongfei 已提交
92
    if name is None:
93
        name = "fc_%d" % uniq_id()
Q
qiaolongfei 已提交
94
    if not isinstance(name, str):
95
        raise ValueError("The name of a layer should be a string.")
Q
qiaolongfei 已提交
96 97 98 99

    input_dims = scope.find_var(input).get_tensor().get_dims()

    w_name = param or name + ".w"
100
    init_param(net=init_net, param_name=w_name, dims=[input_dims[1], size])
Q
qiaolongfei 已提交
101 102 103 104 105
    sgd_optimizer(net=optimize_net, param_name=w_name, learning_rate=0.01)

    pre_activation = name + ".mul.out"
    scope.new_var(pre_activation)
    mul_op = Operator("mul", X=input, Y=w_name, Out=pre_activation)
Q
qiaolongfei 已提交
106
    net.append_op(mul_op)
Q
qiaolongfei 已提交
107 108 109 110

    # create bias variable if needed
    if bias:
        bias_name = name + ".b"
111
        init_param(net=init_net, param_name=bias_name, dims=[size])
Q
qiaolongfei 已提交
112
        sgd_optimizer(
Q
qiaolongfei 已提交
113
            net=optimize_net, param_name=bias_name, learning_rate=0.001)
Q
qiaolongfei 已提交
114 115
        bias_out = name + ".rowwise_add.out"
        scope.new_var(bias_out)
Q
qiaolongfei 已提交
116
        rowwise_append_op = Operator(
Q
qiaolongfei 已提交
117
            "rowwise_add", X=pre_activation, b=bias_name, Out=bias_out)
Q
qiaolongfei 已提交
118
        net.append_op(rowwise_append_op)
Q
qiaolongfei 已提交
119 120 121
        pre_activation = bias_out

    activation_op = Operator(act, X=pre_activation, Y=name)
Q
qiaolongfei 已提交
122
    net.append_op(activation_op)
Q
qiaolongfei 已提交
123 124 125 126 127 128
    scope.new_var(name)
    net.infer_shape(scope)
    return name


def cross_entropy_layer(net, input, label):
129
    cost_name = "cross_entropy_%d" % uniq_id()
Q
qiaolongfei 已提交
130 131
    cross_entropy_op = Operator(
        "onehot_cross_entropy", X=input, label=label, Y=cost_name)
Q
qiaolongfei 已提交
132
    net.append_op(cross_entropy_op)
Q
qiaolongfei 已提交
133 134 135 136 137
    scope.new_var(cost_name)
    net.infer_shape(scope)
    return cost_name


Q
qiaolongfei 已提交
138
def create_backward_net(forward_net):
Q
qiaolongfei 已提交
139 140 141 142 143 144 145 146 147 148
    net = core.Operator.backward(forward_net, set())
    for input in net.inputs()["all"]:
        var = scope.new_var(input)
        var.get_tensor()
    for output in net.outputs()["all"]:
        var = scope.new_var(output)
        var.get_tensor()
    return net


Q
qiaolongfei 已提交
149
def debug_print_op(op):
Q
qiaolongfei 已提交
150 151 152 153
    print("===============" + op.type() + "==============")
    print("***inputs:***")
    for input in op.inputs()["all"]:
        print input, scope.find_var(input).get_tensor().get_dims()
Q
qiaolongfei 已提交
154
    print("\n***outputs:***")
Q
qiaolongfei 已提交
155 156 157 158 159 160
    for output in op.outputs()["all"]:
        print output, scope.find_var(output).get_tensor().get_dims()
    print("")
    print("")


Q
qiaolongfei 已提交
161 162 163 164
def set_cost(cost):
    cost_shape = numpy.array(scope.find_var(cost).get_tensor()).shape
    cost_grad = \
        scope.find_var(grad_var_name(cost)).get_tensor()
Q
qiaolongfei 已提交
165 166 167 168 169
    cost_grad.set_dims(cost_shape)
    cost_grad.alloc_float(place)
    cost_grad.set(numpy.ones(cost_shape).astype("float32"), place)


Q
qiaolongfei 已提交
170
def get_cost_mean(cost):
Q
qiaolongfei 已提交
171
    cost_data = numpy.array(scope.find_var(cost).get_tensor())
Q
qiaolongfei 已提交
172
    return cost_data.sum() / len(cost_data)
Q
qiaolongfei 已提交
173

Q
qiaolongfei 已提交
174

Q
qiaolongfei 已提交
175 176 177 178 179
def error_rate(predict, label):
    predict_var = numpy.array(scope.find_var(predict).get_tensor()).argmax(
        axis=1)
    label = numpy.array(scope.find_var(label).get_tensor())
    error_num = numpy.sum(predict_var != label)
Q
qiaolongfei 已提交
180
    return error_num / float(len(label))
Q
qiaolongfei 已提交
181 182


183 184
images = data_layer(name="pixel", dims=[BATCH_SIZE, 784])
labels = data_layer(name="label", dims=[BATCH_SIZE])
185 186
fc1 = fc_layer(net=forward_net, input=images, size=100, act="sigmoid")
fc2 = fc_layer(net=forward_net, input=fc1, size=100, act="sigmoid")
187
predict = fc_layer(net=forward_net, input=fc2, size=10, act="softmax")
188 189 190 191 192
cost = cross_entropy_layer(net=forward_net, input=predict, label=labels)

init_net.complete_add_op(True)
forward_net.complete_add_op(True)
backward_net = create_backward_net(forward_net)
Q
qiaolongfei 已提交
193
optimize_net.complete_add_op(True)
Q
qiaolongfei 已提交
194

195 196
print(init_net)
print(forward_net)
Q
qiaolongfei 已提交
197
print(backward_net)
Q
qiaolongfei 已提交
198
print(optimize_net)
Q
qiaolongfei 已提交
199

200
debug_print_op(forward_net)
Q
qiaolongfei 已提交
201 202
debug_print_op(backward_net)
debug_print_op(optimize_net)
Q
qiaolongfei 已提交
203

Q
qiaolongfei 已提交
204
train_reader = paddle.batch(
Q
qiaolongfei 已提交
205 206 207 208
    paddle.reader.shuffle(
        paddle.dataset.mnist.train(), buf_size=8192),
    batch_size=BATCH_SIZE)

Q
qiaolongfei 已提交
209

Q
qiaolongfei 已提交
210
def test(cost_name):
Q
qiaolongfei 已提交
211 212
    test_reader = paddle.batch(
        paddle.dataset.mnist.test(), batch_size=BATCH_SIZE)
Q
qiaolongfei 已提交
213 214 215
    cost = []
    error = []
    for data in test_reader():
Q
qiaolongfei 已提交
216 217 218 219
        image_data = numpy.array(map(lambda x: x[0], data)).astype("float32")
        label_data = numpy.array(map(lambda x: x[1], data)).astype("int32")
        feed_data(images, image_data)
        feed_data(labels, label_data)
Q
qiaolongfei 已提交
220

221 222
        forward_net.infer_shape(scope)
        forward_net.run(scope, dev_ctx)
Q
qiaolongfei 已提交
223
        cost.append(get_cost_mean(cost_name))
Q
qiaolongfei 已提交
224 225 226 227 228
        error.append(error_rate(predict, "label"))
    print("cost=" + str(sum(cost) / float(len(cost))) + " error_rate=" + str(
        sum(error) / float(len(error))))


Q
qiaolongfei 已提交
229
PASS_NUM = 1
230 231

init_net.run(scope, dev_ctx)
Q
qiaolongfei 已提交
232
for pass_id in range(PASS_NUM):
Q
qiaolongfei 已提交
233
    batch_id = 0
Q
qiaolongfei 已提交
234

Q
qiaolongfei 已提交
235
    for data in train_reader():
Q
qiaolongfei 已提交
236 237 238 239
        image_data = numpy.array(map(lambda x: x[0], data)).astype("float32")
        label_data = numpy.array(map(lambda x: x[1], data)).astype("int32")
        feed_data(images, image_data)
        feed_data(labels, label_data)
Q
qiaolongfei 已提交
240

241 242
        forward_net.infer_shape(scope)
        forward_net.run(scope, dev_ctx)
Q
qiaolongfei 已提交
243
        set_cost(cost)
Q
qiaolongfei 已提交
244 245
        backward_net.infer_shape(scope)
        backward_net.run(scope, dev_ctx)
Q
qiaolongfei 已提交
246

Q
qiaolongfei 已提交
247
        optimize_net.run(scope, dev_ctx)
Q
qiaolongfei 已提交
248 249
        if batch_id % 100 == 0:
            print("pass[" + str(pass_id) + "] batch_id[" + str(batch_id) + "]")
Q
qiaolongfei 已提交
250
            test(cost)
Q
qiaolongfei 已提交
251 252

        batch_id = batch_id + 1