diff --git a/.travis.yml b/.travis.yml index 5d82d9729b75ef493a0bd03921c453f9a519c8cd..4fb2ca938795bb6a69f7d7991aee9f7386947bf2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ cache: - $HOME/third_party - $HOME/.ccache - $HOME/.cache/pip + - $HOME/Library/Caches/Homebrew sudo: required dist: trusty os: diff --git a/demo/mnist/api_train_v2.py b/demo/mnist/api_train_v2.py index 59486ed1b3ba494a20b06b7ef7027fc3e86c043c..b5cc74ce67dfc8e1afa65bd52f5ec600260032ce 100644 --- a/demo/mnist/api_train_v2.py +++ b/demo/mnist/api_train_v2.py @@ -25,8 +25,7 @@ def main(): act=paddle.activation.Softmax()) cost = paddle.layer.classification_cost(input=inference, label=label) - topology = paddle.layer.parse_network(cost) - parameters = paddle.parameters.create(topology) + parameters = paddle.parameters.create(cost) for param_name in parameters.keys(): array = parameters.get(param_name) array[:] = numpy.random.uniform(low=-1.0, high=1.0, size=array.shape) @@ -46,7 +45,7 @@ def main(): trainer = paddle.trainer.SGD(update_equation=adam_optimizer) trainer.train(train_data_reader=train_reader, - topology=topology, + topology=cost, parameters=parameters, event_handler=event_handler, batch_size=32, # batch size should be refactor in Data reader diff --git a/doc/design/reader/README.md b/doc/design/reader/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8f7abf12f733542734efe91111f365a34aa4b15b --- /dev/null +++ b/doc/design/reader/README.md @@ -0,0 +1,144 @@ +# Python Data Reader Design Doc + +Paddle reads data from data reader during training. It will be passed into `paddle.train` as a parameter. + +## Data Reader Interface + +Data reader is a function with no parameter that creates a iterable (anything can be used in `for x in iterable`): + +``` +iterable = data_reader() +``` + +Element produced for the iterable should be a **single** entry of data, **not** a mini batch. That entry of data could be a single item, or a tuple of items. Item should be of [supported type](http://www.paddlepaddle.org/doc/ui/data_provider/pydataprovider2.html?highlight=dense_vector#input-types) (e.g., numpy 1d array of float32, int, list of int) + +An example implementation for single item data reader: + +```python +def data_reader_fake_image(): + while True: + yield numpy.random.uniform(-1, 1, size=20*20) +``` + +An example implementation for multiple item data reader: +```python +def data_reader_fake_image_and_label(): + while True: + yield numpy.random.uniform(-1, 1, size=20*20), False +``` + +## Usage + +data reader, mapping from item(s) read to data layer, batch size and number of total pass will be passed into `paddle.train`: + +```python +# two data layer is created: +image_layer = paddle.layer.data("image", ...) +label_layer = paddle.layer.data("label", ...) + +# ... + +paddle.train(paddle.dataset.mnist, {"image":0, "label":1}, 128, 10, ...) +``` + +## Data Reader Decorators + +Data reader decorators takes a single or multiple data reader, returns a new data reader. It is similar to a [python decorator](https://wiki.python.org/moin/PythonDecorators), but it does not use `@` syntax. + +Since we have a strict interface for data readers (no parameter, return a single data item). Data reader can be used flexiable via data reader decorators. Following are a few examples: + +### Prefetch Data + +Since reading data may take time and training can not proceed without data. It is generally a good idea to prefetch data. + +Use `paddle.reader.buffered` to prefetch data: + +```python +buffered_reader = paddle.reader.buffered(paddle.dataset.mnist, 100) +``` + +`buffered_reader` will try to buffer (prefetch) `100` data entries. + +### Compose Multiple Data Readers + +For example, we want to use a source of real images (reusing mnist dataset), and a source of fake images as input for [Generative Adversarial Networks](https://arxiv.org/abs/1406.2661). + +We can do: + +```python +def data_reader_fake_image(): + while True: + yield numpy.random.uniform(-1, 1, size=20*20) + +def data_reader_bool(t): + while True: + yield t + +true_reader = lambda : data_reader_bool(True) +false_reader = lambda : data_reader_bool(False) + +reader = paddle.reader.combine(paddle.dataset.mnist, data_reader_fake_image, true_reader, false_reader) +# Skipped 1 because paddle.dataset.mnist produces two items per data entry. +# And we don't care second item at this time. +paddle.train(reader, {"true_image":0, "fake_image": 2, "true_label": 3, "false_label": 4}, ...) +``` + +### Shuffle + +Given shuffle buffer size `n`, `paddle.reader.shuffle` will return a data reader that buffers `n` data entries and shuffle them before a data entry is read. + +Example: +```python +reader = paddle.reader.shuffle(paddle.dataset.mnist, 512) +``` + +## Q & A + +### Why return only a single entry, but not a mini batch? + +If a mini batch is returned, data reader need to take care of batch size. But batch size is a concept for training, it makes more sense for user to specify batch size as a parameter for `train`. + +Practically, always return a single entry make reusing existing data reader much easier (e.g., if existing data reader return not a single entry but 3 entries, training code will be more complex because it need to handle cases like batch size 2). + +### Why use a dictionary but not a list to provide mapping? + +We decided to use dictionary (`{"image":0, "label":1}`) instead of list (`["image", "label"]`) is because that user can easily resue item (e.g., using `{"image_a":0, "image_b":0, "label":1}`) or skip item (e.g., using `{"image_a":0, "label":2}`). + +### How to create custom data reader + +```python +def image_reader(image_path, label_path): + f = open(image_path) + l = open(label_path) + images = numpy.fromfile( + f, 'ubyte', count=n * 28 * 28).reshape((n, 28 * 28)).astype('float32') + images = images / 255.0 * 2.0 - 1.0 + labels = numpy.fromfile(l, 'ubyte', count=n).astype("int") + for i in xrange(n): + yield images[i, :], labels[i] # a single entry of data is created each time + f.close() + +# use python lambda to change image_reader into a function with no parameters. +reader = lambda : image_reader("/path/to/image_file", "/path/to/label_file") +paddle.train(reader, {"image":0, "label":1}, ...) +``` + +### How is `paddle.train` implemented + +An example implementation of paddle.train could be: + +```python +def minibatch_decorater(reader, minibatch_size): + def ret(): + r = reader() + buf = [r.next() for x in xrange(minibatch_size)] + while len(buf) > 0: + yield buf + buf = [r.next() for x in xrange(minibatch_size)] + return ret + +def train(reader, mapping, batch_size, total_pass): + for pass_idx in range(total_pass): + for mini_batch in minibatch_decorater(reader): # this loop will never end in online learning. + do_forward_backward(mini_batch, mapping) +``` diff --git a/paddle/gserver/evaluators/Evaluator.cpp b/paddle/gserver/evaluators/Evaluator.cpp index ae7508e2bb117a60492e0c28230f2fbb4b14915e..a2a5028e8418fd2884a436394a05903e1fdd795c 100644 --- a/paddle/gserver/evaluators/Evaluator.cpp +++ b/paddle/gserver/evaluators/Evaluator.cpp @@ -866,21 +866,20 @@ void PnpairEvaluator::calc(std::vector& predictArray) { ClassRegistrar Evaluator::registrar_; Evaluator* Evaluator::create(const EvaluatorConfig& config) { - Evaluator* evaluator = nullptr; - if (config.type() == "classification_error") { - evaluator = new ClassificationErrorEvaluator(); - } else if (config.type() == "sum") { - evaluator = new SumEvaluator(); - } else if (config.type() == "last-column-sum") { - evaluator = new ColumnSumEvaluator(-1); - } else if (config.type() == "last-column-auc") { - evaluator = new AucEvaluator(-1); - } else { - evaluator = registrar_.createByType(config.type()); - } + Evaluator* evaluator = registrar_.createByType(config.type()); evaluator->init(config); return evaluator; } + +REGISTER_EVALUATOR(classification_error, ClassificationErrorEvaluator); +REGISTER_EVALUATOR(sum, SumEvaluator); +static InitFunction __reg_type_auc_sum__([]() { + Evaluator::registrar_.registerClass( + "last-column-sum", [] { return new ColumnSumEvaluator(-1); }); + Evaluator::registrar_.registerClass("last-column-auc", + [] { return new AucEvaluator(-1); }); +}); + /** * @brief print value of each layer. * diff --git a/paddle/gserver/layers/SequenceConcatLayer.cpp b/paddle/gserver/layers/SequenceConcatLayer.cpp index b4677687a6cc7755fdb7584a9524de9b65a0c627..599706eb419ede72dbd6f4c8c74e57f5f9965388 100644 --- a/paddle/gserver/layers/SequenceConcatLayer.cpp +++ b/paddle/gserver/layers/SequenceConcatLayer.cpp @@ -168,13 +168,17 @@ void SequenceConcatLayer::backward(const UpdateCallback& callback) { size_t rightNumIns = 0; for (size_t seqId = 0; seqId < numSequences1; ++seqId) { leftNumIns = starts1[seqId + 1] - starts1[seqId]; - inputGrad1->subMatrix(starts1[seqId], leftNumIns) - ->add(*(outputGrad->subMatrix(offset, leftNumIns))); + if (inputGrad1) { + inputGrad1->subMatrix(starts1[seqId], leftNumIns) + ->add(*(outputGrad->subMatrix(offset, leftNumIns))); + } offset += leftNumIns; rightNumIns = starts2[seqId + 1] - starts2[seqId]; - inputGrad2->subMatrix(starts2[seqId], rightNumIns) - ->add(*(outputGrad->subMatrix(offset, rightNumIns))); + if (inputGrad2) { + inputGrad2->subMatrix(starts2[seqId], rightNumIns) + ->add(*(outputGrad->subMatrix(offset, rightNumIns))); + } offset += rightNumIns; } } diff --git a/python/paddle/v2/parameters.py b/python/paddle/v2/parameters.py index e5b7dabcb8eb3a845dedea663f978e7a9820495d..ea504d5104716d157add87ed3f6e31ea69e0a3f0 100644 --- a/python/paddle/v2/parameters.py +++ b/python/paddle/v2/parameters.py @@ -1,27 +1,27 @@ import numpy as np - -from paddle.proto.ModelConfig_pb2 import ModelConfig -from paddle.proto.ParameterConfig_pb2 import ParameterConfig +from . import layer as v2_layer import py_paddle.swig_paddle as api +from paddle.proto.ParameterConfig_pb2 import ParameterConfig __all__ = ['Parameters', 'create'] -def create(*topologies): +def create(*layers): """ - Create parameter pool by topologies. + Create parameter pool by layers. In paddle, layer can be represent a + model config. - :param topologies: + :param layers: :return: """ - pool = Parameters() - for topo in topologies: - if not isinstance(topo, ModelConfig): + for layer in layers: + if not isinstance(layer, v2_layer.Layer): raise ValueError( - 'create must pass a topologies which type is ModelConfig') - - for param in topo.parameters: - pool.__append_config__(param) + 'create must pass a topologies which type is paddle.layer.Layer') + model_config = v2_layer.parse_network(*layers) + pool = Parameters() + for param in model_config.parameters: + pool.__append_config__(param) return pool diff --git a/python/paddle/v2/trainer.py b/python/paddle/v2/trainer.py index 9ba13dc5c8a81f8dcf39260d1a44dcdcc7c22ad5..4365bd41e7073bce4112e5813dbf1517856c06f5 100644 --- a/python/paddle/v2/trainer.py +++ b/python/paddle/v2/trainer.py @@ -1,12 +1,13 @@ import collections import py_paddle.swig_paddle as api +from paddle.proto.ModelConfig_pb2 import ModelConfig from py_paddle import DataProviderConverter -from paddle.proto.ModelConfig_pb2 import ModelConfig +from . import event as v2_event +from . import layer as v2_layer from . import optimizer as v2_optimizer from . import parameters as v2_parameters -from . import event as v2_event __all__ = ['ITrainer', 'SGD'] @@ -73,7 +74,7 @@ class SGD(ITrainer): Training method. Will train num_passes of input data. :param train_data_reader: - :param topology: Network Topology, a protobuf ModelConfig message. + :param topology: Network Topology, use one or more Layers to represent it. :param parameters: The parameter pools. :param num_passes: The total train passes. :param test_data_reader: @@ -87,6 +88,8 @@ class SGD(ITrainer): if event_handler is None: event_handler = default_event_handler + topology = v2_layer.parse_network(topology) + __check_train_args__(**locals()) gm = api.GradientMachine.createFromConfigProto(