diff --git a/doc/fluid/getstarted/quickstart_cn.rst b/doc/fluid/getstarted/quickstart_cn.rst deleted file mode 120000 index 93a9e4e37a8495c553cec257c27363ca8d062d39..0000000000000000000000000000000000000000 --- a/doc/fluid/getstarted/quickstart_cn.rst +++ /dev/null @@ -1 +0,0 @@ -../../v2/getstarted/quickstart_cn.rst \ No newline at end of file diff --git a/doc/fluid/getstarted/quickstart_cn.rst b/doc/fluid/getstarted/quickstart_cn.rst new file mode 100644 index 0000000000000000000000000000000000000000..135beb75d0330f39d062753aa2aa83a077f36bb1 --- /dev/null +++ b/doc/fluid/getstarted/quickstart_cn.rst @@ -0,0 +1,45 @@ +快速开始 +======== + +快速安装 +-------- + +PaddlePaddle支持使用pip快速安装，目前支持CentOS 6以上, Ubuntu 14.04以及MacOS 10.12，并安装有Python2.7。 +执行下面的命令完成快速安装，版本为cpu_avx_openblas： + + .. code-block:: bash + + pip install paddlepaddle + +如果需要安装支持GPU的版本（cuda7.5_cudnn5_avx_openblas），需要执行： + + .. code-block:: bash + + pip install paddlepaddle-gpu + +更详细的安装和编译方法参考： :ref:`install_steps` 。 + +快速使用 +-------- + +创建一个 housing.py 并粘贴此Python代码： + + .. code-block:: python + + import paddle.dataset.uci_housing as uci_housing + import paddle.fluid as fluid + + with fluid.scope_guard(fluid.core.Scope()): + # initialize executor with cpu + exe = fluid.Executor(place=fluid.CPUPlace()) + # load inference model + [inference_program, feed_target_names,fetch_targets] = \ + fluid.io.load_inference_model(uci_housing.fluid_model(), exe) + # run inference + result = exe.run(inference_program, + feed={feed_target_names[0]: uci_housing.predict_reader()}, + fetch_list=fetch_targets) + # print predicted price is $12,273.97 + print 'Predicted price: ${:,.2f}'.format(result[0][0][0] * 1000) + +执行 :code:`python housing.py` 瞧！ 它应该打印出预测住房数据的清单。 diff --git a/doc/fluid/getstarted/quickstart_en.rst b/doc/fluid/getstarted/quickstart_en.rst deleted file mode 120000 index 6e1894faa1176bb9e77f616e07df36191e54b782..0000000000000000000000000000000000000000 --- a/doc/fluid/getstarted/quickstart_en.rst +++ /dev/null @@ -1 +0,0 @@ -../../v2/getstarted/quickstart_en.rst \ No newline at end of file diff --git a/doc/fluid/getstarted/quickstart_en.rst b/doc/fluid/getstarted/quickstart_en.rst new file mode 100644 index 0000000000000000000000000000000000000000..df6619cfd039fc1fdca8cde57db9cc6aebf8f029 --- /dev/null +++ b/doc/fluid/getstarted/quickstart_en.rst @@ -0,0 +1,49 @@ +Quick Start +============ + +Quick Install +------------- + +You can use pip to install PaddlePaddle with a single command, supports +CentOS 6 above, Ubuntu 14.04 above or MacOS 10.12, with Python 2.7 installed. +Simply run the following command to install, the version is cpu_avx_openblas: + + .. code-block:: bash + + pip install paddlepaddle + +If you need to install GPU version (cuda7.5_cudnn5_avx_openblas), run: + + .. code-block:: bash + + pip install paddlepaddle-gpu + +For more details about installation and build: :ref:`install_steps` . + +Quick Use +--------- + +Create a new file called housing.py, and paste this Python +code: + + + .. code-block:: python + + import paddle.dataset.uci_housing as uci_housing + import paddle.fluid as fluid + + with fluid.scope_guard(fluid.core.Scope()): + # initialize executor with cpu + exe = fluid.Executor(place=fluid.CPUPlace()) + # load inference model + [inference_program, feed_target_names,fetch_targets] = \ + fluid.io.load_inference_model(uci_housing.fluid_model(), exe) + # run inference + result = exe.run(inference_program, + feed={feed_target_names[0]: uci_housing.predict_reader()}, + fetch_list=fetch_targets) + # print predicted price is $12,273.97 + print 'Predicted price: ${:,.2f}'.format(result[0][0][0] * 1000) + +Run :code:`python housing.py` and voila! It should print out a list of predictions +for the test housing data. diff --git a/doc/v2/faq/local/index_en.rst b/doc/v2/faq/local/index_en.rst index 4cb43031933a8bbe9aebae04bc3e9c74a6d21b95..fa95b1753dbe293811d7a8601497ad521fa3ecda 100644 --- a/doc/v2/faq/local/index_en.rst +++ b/doc/v2/faq/local/index_en.rst @@ -1,5 +1,248 @@ ############################# -Local Training and Prediction +Parameter Setting ############################# -TBD +.. contents:: + +1. Reduce Memory Consumption +------------------- + +The training procedure of neural networks demands dozens of gigabytes of host memory or serval gigabytes of device memory, which is a rather memory consuming work. The memory consumed by PaddlePaddle framework mainly includes: +\: + +* Cache memory for DataProvider (only on host memory), +* Memory for neurons' activation information (on both host memory and device memory), +* Memory for parameters (on both host memory and device memory), +* Other memory demands. + +Other memory demands is mainly used to support the running demand of PaddlePaddle framework itself, such as string allocation，temporary variables, which are not considered currently. + +Reduce DataProvider Cache Memory +++++++++++++++++++++++++++ + +PyDataProvider works under asynchronous mechanism, it loads together with the data fetch and shuffle procedure in host memory: + +.. graphviz:: + + digraph { + rankdir=LR; + Data Files -> Host Memory Pool -> PaddlePaddle Training + } + +Thus the reduction of the DataProvider cache memory can reduce memory occupancy, meanwhile speed up the data loading procedure before training. However, the size of the memory pool can actually affect the granularity of shuffle，which means a shuffle operation is needed before each data file reading process to ensure the randomness of data when try to reduce the size of the memory pool. + +.. literalinclude:: src/reduce_min_pool_size.py + +In this way, the memory consumption can be significantly reduced and hence the training procedure can be accelerated. More details are demonstrated in :ref:`api_pydataprovider2`. + +The Neurons Activation Memory +++++++++++++++ + +Each neuron activation operating in a neural network training process contains certain amount of temporary data such as the activation data (like the output value of a neuron). These data will be used to update parameters in back propagation period. The scale of memory consumed by these data is mainly related with two parameters, which are batch size and the length of each Sequence. Therefore, the neurons activation memory consuming is actually in proportion to the information contains in each mini-batch training. + +Two practical ways: + +* Reduce batch size. Set a smaller value in network configuration settings(batch_size=1000) can be helpful. But setting batch size to a smaller value may affect the training result due to it is a super parameter of the neural network itself. +* Shorten the sequence length or cut off those excessively long sequences. For example, if the length of sequences in a dataset are mostly varies between 100 and 200, but there is sequence lengthen out to 10,000, then it’s quite potentially leads to OOM (out of memory), especially in RNN models such as LSTM. + +The Parameters Memory +++++++++ + +The PaddlePaddle framework supports almost all popular optimizers. Different optimizers have different memory requirement. For example, the :code:`adadelta` consumes approximately 5 times memory + +space than the weights parameter’s scale, which means the :code:`adadelta` needs at least :code:`500M` memory if the model file contains all + +parameters needs :code:`100M`. + +Some optimization algorithms such as :code:`momentum` are worth giving a shot. + +2. Tricks To Speed Up Training +------------------- + +The training procedure of PaddlePaddle may be speed up when considering following aspects:\： + +* Reduce the time consumption of data loading +* Speed up training epochs +* Introduce more computing resources with the utilization of distribute training frameworks + +Reduce The Time Consumption of Data Loading +++++++++++++++++++ + + +The \ :code:`pydataprovider`\ holds big potential to speed up the data loading procedure if the cache pool and enable memory cache when use it. The principle of the reduction of :code:`DataProvider` cache pool is basically the same with the method which reduct the memory occupation with the set of a smaller cache pool. + +.. literalinclude:: src/reduce_min_pool_size.py + +Beside, the interface :code:`@provider` provides a parameter :code:`cache` to control cache. If set it to :code:`CacheType.CACHE_PASS_IN_MEM`, the data after the first :code:`pass` ( a pass means all data have be fed into the network for training) will be cached in memory and no new data will be read from the :code:`python` side in following :code:`pass` , instead from the cached data in memory. This strategy can also drop the time consuming in data loading process. + + +Accelerating Training Epochs +++++++++++++ + +Sparse training is supported in PaddlePaddle. The features needs to be trained is any of :code:`sparse_binary_vector`, :code:`sparse_vector` and :code:`integer_value` . Meanwhile, the Layer interacts with the training data need to turn the Parameter to sparse updating mode by setting :code:`sparse_update=True`. +Take :code:`word2vec` as an example, to train a language distance, one needs to predict the middle word with two words prior to it and next to it. The DataProvider of this task is: + +.. literalinclude:: src/word2vec_dataprovider.py + +The configuration of this task is: + +.. literalinclude:: src/word2vec_config.py + +Introduce More Computing Resources +++++++++++++++++++ + +More computing resources can be introduced with following manners: +* Single CPU platform training + + * Use multi-threading by set :code:`trainer_count`。 + +* Single GPU platform training + + * Set :code:`use_gpu` to train on single GPU. + * Set :code:`use_gpu` and :code:`trainer_count` to enable multiple GPU training support. + +* Cluster Training + + * Refer to :ref:`cluster_train` 。 + +3. Assign GPU Devices +------------------ + +Assume a computing platform consists of 4 GPUs which serial number from 0 to 3: + +* Method1: specify a GPU as computing device by set: + `CUDA_VISIBLE_DEVICES `_ + +.. code-block:: bash + + env CUDA_VISIBLE_DEVICES=2,3 paddle train --use_gpu=true --trainer_count=2 + +* Method2: Assign by —gpu_id: + +.. code-block:: bash + + paddle train --use_gpu=true --trainer_count=2 --gpu_id=2 + + +4. How to Fix Training Termination Caused By :code:`Floating point exception` During Training. +------------------------------------------------------------------------ + +Paddle binary catches floating exceptions during runtime, it will be terminated when NaN or Inf occurs. Floating exceptions are mostly caused by float overflow, divide by zero. There are three main reasons may raise such exception: + +* Parameters or gradients during training are oversize, which leads to float overflow during calculation. +* The model failed to converge and diverges to a big value. +* Parameters may converge to a singular value due to bad training data. If the scale of input data is too big and contains millions of parameter values, float overflow error may arise when operating matrix multiplication. + +Two ways to solve this problem: + +1. Set :code:`gradient_clipping_threshold` as: + +.. code-block:: python + + optimizer = paddle.optimizer.RMSProp( + learning_rate=1e-3, + gradient_clipping_threshold=10.0, + regularization=paddle.optimizer.L2Regularization(rate=8e-4)) + +Details can refer to example `nmt_without_attention `_ 示例。 + +2. Set :code:`error_clipping_threshold` as: + +.. code-block:: python + + decoder_inputs = paddle.layer.fc( + act=paddle.activation.Linear(), + size=decoder_size * 3, + bias_attr=False, + input=[context, current_word], + layer_attr=paddle.attr.ExtraLayerAttribute( + error_clipping_threshold=100.0)) + +Details can refer to example `machine translation `_ 。 + +The main difference between these two methods are: + +1. They both block the gradient, but happen in different occasions，the former one happens when then :code:`optimzier` updates the network parameters while the latter happens when the back propagation computing of activation functions. +2. The block target are different, the former blocks the trainable parameters’ gradient while the later blocks the gradient to be propagated to prior layers. + +Moreover, Such problems may be fixed with smaller learning rates or data normalization. + +5. Fetch Multi Layers’ Prediction Result With Infer Interface +----------------------------------------------- + +* Join the layer to be used as :code:`output_layer` layer to the input parameters of :code:`paddle.inference.Inference()` interface with: + +.. code-block:: python + + inferer = paddle.inference.Inference(output_layer=[layer1, layer2], parameters=parameters) + +* Assign certain fields to output. Take :code:`value` as example, it can be down with following code: + +.. code-block:: python + + out = inferer.infer(input=data_batch, field=["value"]) + +It is important to note that: + +* If 2 layers are assigned as output layer, then the output results consists of 2 matrixes. +* Assume the output of first layer A is a matrix sizes N1 * M1, the output of second layer B is a matrix sizes N2 * M2； +* By default, paddle.v2 will transverse join A and B, when N1 not equal to N2, it will raise following error: + +.. code-block:: python + + ValueError: all the input array dimensions except for the concatenation axis must match exactly + +The transverse of different matrixes of multi layers mainly happens when: + +* Output sequence layer and non sequence layer; +* Multiple output layers process multiple sequence with different length; + +Such issue can be avoided by calling infer interface and set :code:`flatten_result=False`. Thus, the infer interface returns a python list, in which + +* The number of elements equals to the number of output layers in the network; +* Each element in list is a result matrix of a layer, which type is numpy.ndarray; +* The height of each matrix outputted by each layer equals to the number of samples under non sequential mode or equals to the number of elements in the input sequence under sequential mode. Their width are both equal to the layer size in configuration. + +6. Fetch the Output of A Certain Layer During Training +----------------------------------------------- + +In event_handler, the interface :code:`event.gm.getLayerOutputs("layer_name")` gives the forward output value organized in :code:`numpy.ndarray` corresponding to :code:`layer_name` in the mini-batch. +The output can be used in custom measurements in following way: + +.. code-block:: python + + def score_diff(right_score, left_score): + return np.average(np.abs(right_score - left_score)) + + def event_handler(event): + if isinstance(event, paddle.event.EndIteration): + if event.batch_id % 25 == 0: + diff = score_diff( + event.gm.getLayerOutputs("right_score")["right_score"][ + "value"], + event.gm.getLayerOutputs("left_score")["left_score"][ + "value"]) + logger.info(("Pass %d Batch %d : Cost %.6f, " + "average absolute diff scores: %.6f") % + (event.pass_id, event.batch_id, event.cost, diff)) + +Note: this function can not get content of :code:`paddle.layer.recurrent_group` step, but output of :code:`paddle.layer.recurrent_group` can be fetched. + +7. Fetch Parameters’ Weight and Gradient During Training +----------------------------------------------- + +Under certain situations, knowing the weights of currently training mini-batch can provide more inceptions of many problems. Their value can be acquired by printing values in :code:`event_handler` (note that to gain such parameters when training on GPU, you should set :code:`paddle.event.EndForwardBackward`). Detailed code is as following: + +.. code-block:: python + + ... + parameters = paddle.parameters.create(cost) + ... + def event_handler(event): + if isinstance(event, paddle.event.EndForwardBackward): + if event.batch_id % 25 == 0: + for p in parameters.keys(): + logger.info("Param %s, Grad %s", + parameters.get(p), parameters.get_grad(p)) + +Note that “acquire the output of a certain layer during training” or “acquire the weights and gradients of parameters during training ” both needs to copy training data from C++ environment to numpy, which have certain degree of influence on training performance. Don’t use these two functions when the training procedure cares about the performance. diff --git a/doc/v2/howto/rnn/index_en.rst b/doc/v2/howto/rnn/index_en.rst index e1b20ef2e7bf4c521b613e54577ff6a3feaa8936..6e8b5c61b23ca2725dc0c9761c8dd4165033973c 100644 --- a/doc/v2/howto/rnn/index_en.rst +++ b/doc/v2/howto/rnn/index_en.rst @@ -1,10 +1,32 @@ RNN Models ========== +Recurrent neural networks(RNN) are an important tool to model sequential data. PaddlePaddle provides flexible interface for building complex recurrent neural network. We will demonstrate how to use PaddlePaddle to build RNN models in the following 4 parts. + +In the first part, we will guide you how to configure recurrent neural network in PaddlePaddle from simple to complex. First, we will use a vanilla recurrent neural network as an example to show how to configure recurrent neural network architecture. Then We will use the sequence to sequence model as an example to demonstrate how you can configure complex recurrent neural network models gradually. .. toctree:: :maxdepth: 1 rnn_config_en.rst + +Recurrent Group is the key unit to build complex recurrent neural network models. The second part describes related concepts and Basic principles of Recurrent Group, and give a detailed description of Recurrent Group API interface. In addition, it also introduces Sequence-level RNN(hierarchical sequence as input) and the usage of Recurrent Group in it. + +.. toctree:: + :maxdepth: 1 + recurrent_group_en.md + +In the third part, two-level sequence is demonstrated briefly and then layers supporting two-level sequence as input are listed and described respectively. + +.. toctree:: + :maxdepth: 1 + hierarchical_layer_en.rst + +In the last part, the unit test of hierarchical RNN is presented as an example to explain how to use hierarchical RNN. We will use two-level sequence RNN and single-layer sequence RNN which have same effects with former as the network configuration seperately in unit test. + +.. toctree:: + :maxdepth: 1 + hrnn_rnn_api_compare_en.rst + diff --git a/paddle/fluid/framework/data_type.h b/paddle/fluid/framework/data_type.h index 4c1b3e7581fe716271c62389c6053a24158913d2..2a528eb3aa562568c92059250f2c9bc5a75ec103 100644 --- a/paddle/fluid/framework/data_type.h +++ b/paddle/fluid/framework/data_type.h @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once +#include #include #include "paddle/fluid/framework/framework.pb.h" #include "paddle/fluid/platform/enforce.h" @@ -22,18 +23,21 @@ namespace paddle { namespace framework { inline proto::VarType::Type ToDataType(std::type_index type) { - using namespace paddle::framework::proto; if (typeid(platform::float16).hash_code() == type.hash_code()) { return proto::VarType::FP16; - } else if (typeid(float).hash_code() == type.hash_code()) { + } else if (typeid(const float).hash_code() == type.hash_code()) { + // CPPLint complains Using C-style cast. Use static_cast() instead + // One fix to this is to replace float with const float because + // typeid(T) == typeid(const T) + // http://en.cppreference.com/w/cpp/language/typeid return proto::VarType::FP32; - } else if (typeid(double).hash_code() == type.hash_code()) { + } else if (typeid(const double).hash_code() == type.hash_code()) { return proto::VarType::FP64; - } else if (typeid(int).hash_code() == type.hash_code()) { + } else if (typeid(const int).hash_code() == type.hash_code()) { return proto::VarType::INT32; - } else if (typeid(int64_t).hash_code() == type.hash_code()) { + } else if (typeid(const int64_t).hash_code() == type.hash_code()) { return proto::VarType::INT64; - } else if (typeid(bool).hash_code() == type.hash_code()) { + } else if (typeid(const bool).hash_code() == type.hash_code()) { return proto::VarType::BOOL; } else { PADDLE_THROW("Not supported"); @@ -41,7 +45,6 @@ inline proto::VarType::Type ToDataType(std::type_index type) { } inline std::type_index ToTypeIndex(proto::VarType::Type type) { - using namespace paddle::framework::proto; switch (type) { case proto::VarType::FP16: return typeid(platform::float16); @@ -62,7 +65,6 @@ inline std::type_index ToTypeIndex(proto::VarType::Type type) { template inline void VisitDataType(proto::VarType::Type type, Visitor visitor) { - using namespace paddle::framework::proto; switch (type) { case proto::VarType::FP16: visitor.template operator()(); @@ -88,7 +90,6 @@ inline void VisitDataType(proto::VarType::Type type, Visitor visitor) { } inline std::string DataTypeToString(const proto::VarType::Type type) { - using namespace paddle::framework::proto; switch (type) { case proto::VarType::FP16: return "float16"; diff --git a/paddle/fluid/framework/details/multi_devices_graph_builder.cc b/paddle/fluid/framework/details/multi_devices_graph_builder.cc index 002952436e58eecfcecf5c9fa40c01b795170681..10d39e779336e2001d66a55ac6d01ee768ddd4ff 100644 --- a/paddle/fluid/framework/details/multi_devices_graph_builder.cc +++ b/paddle/fluid/framework/details/multi_devices_graph_builder.cc @@ -34,7 +34,7 @@ MultiDevSSAGraphBuilder::MultiDevSSAGraphBuilder( const std::vector &places, const std::string &loss_var_name, const std::unordered_set ¶ms, - const std::vector &local_scopes, + const std::vector &local_scopes, bool skip_scale_loss, platform::NCCLContextMap *nccl_ctxs) : loss_var_name_(loss_var_name), places_(places), @@ -45,7 +45,7 @@ MultiDevSSAGraphBuilder::MultiDevSSAGraphBuilder( const std::vector &places, const std::string &loss_var_name, const std::unordered_set ¶ms, - const std::vector &local_scopes) + const std::vector &local_scopes, bool skip_scale_loss) : loss_var_name_(loss_var_name), places_(places), local_scopes_(local_scopes) { @@ -53,6 +53,7 @@ MultiDevSSAGraphBuilder::MultiDevSSAGraphBuilder( for (auto &p : params) { grad_names_.insert(GradVarName(p)); } + skip_scale_loss_ = skip_scale_loss; } void MultiDevSSAGraphBuilder::CreateOpHandleIOs(SSAGraph *result, @@ -95,7 +96,9 @@ std::unique_ptr MultiDevSSAGraphBuilder::Build( // always use the first device CreateSendOp(&result, *op); } else if (IsScaleLossOp(*op)) { - CreateScaleLossGradOp(&result); + if (!skip_scale_loss_) { + CreateScaleLossGradOp(&result); + } is_forwarding = false; } else { CreateComputationalOps(&result, *op); diff --git a/paddle/fluid/framework/details/multi_devices_graph_builder.h b/paddle/fluid/framework/details/multi_devices_graph_builder.h index b5ba2dbd3c00f23fabd993d7908664db38a31941..009c31b40c279ae3e924d2d7c67933e7444ed85c 100644 --- a/paddle/fluid/framework/details/multi_devices_graph_builder.h +++ b/paddle/fluid/framework/details/multi_devices_graph_builder.h @@ -34,12 +34,14 @@ class MultiDevSSAGraphBuilder : public SSAGraphBuilder { const std::string &loss_var_name, const std::unordered_set ¶ms, const std::vector &local_scopes, + bool skip_scale_loss, platform::NCCLContextMap *nccl_ctxs); #else MultiDevSSAGraphBuilder(const std::vector &places, const std::string &loss_var_name, const std::unordered_set ¶ms, - const std::vector &local_scopes); + const std::vector &local_scopes, + bool skip_scale_loss); #endif std::unique_ptr Build(const ProgramDesc &program) const override; @@ -57,6 +59,7 @@ class MultiDevSSAGraphBuilder : public SSAGraphBuilder { #ifdef PADDLE_WITH_CUDA platform::NCCLContextMap *nccl_ctxs_; #endif + bool skip_scale_loss_; bool IsScaleLossOp(const OpDesc &op) const; diff --git a/paddle/fluid/framework/details/send_op_handle.cc b/paddle/fluid/framework/details/send_op_handle.cc index 549b9d9abbe5bfd17df3509e0442bfa19b7ecd61..0763f92171e7813ec0ee8ca4f3aa42b76205130a 100644 --- a/paddle/fluid/framework/details/send_op_handle.cc +++ b/paddle/fluid/framework/details/send_op_handle.cc @@ -34,7 +34,10 @@ void SendOpHandle::RunImpl() { } in->generated_op_->Wait(dev_ctxes_[p]); } - this->RunAndRecordEvent([&] { op_->Run(*local_scope_, place_); }); + auto &tmp_scope = local_scope_->FindVar(kLocalExecScopeName)->Get(); + // FIXME(wuyi): can not use RunAndRecordEvent here, for it will cause dead + // lock. + op_->Run(*tmp_scope, place_); } std::string SendOpHandle::Name() const { return "send"; } diff --git a/paddle/fluid/framework/feed_fetch_type.h b/paddle/fluid/framework/feed_fetch_type.h index b0d1e9f0a7074da33af1cd279ab913ab604150b1..fae792ad9fa766f456ed706cc9adeb4e34d20123 100644 --- a/paddle/fluid/framework/feed_fetch_type.h +++ b/paddle/fluid/framework/feed_fetch_type.h @@ -13,7 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#include #include #include "paddle/fluid/framework/lod_tensor.h" @@ -22,7 +21,8 @@ namespace framework { using FeedFetchType = LoDTensor; using FeedFetchList = std::vector; -static const std::string kFeedOpType = "feed"; -static const std::string kFetchOpType = "fetch"; +static const char kFeedOpType[] = "feed"; +static const char kFetchOpType[] = "fetch"; + } // namespace framework } // namespace paddle diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc index 67e02e2f119707bba376056510a8ca1034590b55..a673fa52880f3f14cdf11a39d2272880a97be19c 100644 --- a/paddle/fluid/framework/parallel_executor.cc +++ b/paddle/fluid/framework/parallel_executor.cc @@ -57,7 +57,8 @@ ParallelExecutor::ParallelExecutor( const std::unordered_set ¶ms, const std::unordered_set &bcast_vars, const ProgramDesc &main_program, const std::string &loss_var_name, - Scope *scope, const std::vector &local_scopes, bool allow_op_delay) + Scope *scope, const std::vector &local_scopes, bool allow_op_delay, + bool customize_scale_loss) : member_(new ParallelExecutorPrivate(places)) { member_->global_scope_ = scope; @@ -90,12 +91,13 @@ ParallelExecutor::ParallelExecutor( // Step 2. Convert main_program to SSA form and dependency graph. Also, insert // ncclOp #ifdef PADDLE_WITH_CUDA - details::MultiDevSSAGraphBuilder builder(member_->places_, loss_var_name, - params, member_->local_scopes_, - member_->nccl_ctxs_.get()); + details::MultiDevSSAGraphBuilder builder( + member_->places_, loss_var_name, params, member_->local_scopes_, + customize_scale_loss, member_->nccl_ctxs_.get()); #else details::MultiDevSSAGraphBuilder builder(member_->places_, loss_var_name, - params, member_->local_scopes_); + params, member_->local_scopes_, + customize_scale_loss); #endif auto graph = builder.Build(main_program); diff --git a/paddle/fluid/framework/parallel_executor.h b/paddle/fluid/framework/parallel_executor.h index f4f283bb4b5eafc33619c98b5f30e1e8f453ece3..49da123d98181c3d3abcdd64d14c5583142eba58 100644 --- a/paddle/fluid/framework/parallel_executor.h +++ b/paddle/fluid/framework/parallel_executor.h @@ -40,7 +40,7 @@ class ParallelExecutor { const ProgramDesc& main_program, const std::string& loss_var_name, Scope* scope, const std::vector& local_scopes, - bool allow_op_delay); + bool allow_op_delay, bool customize_scale_loss); ~ParallelExecutor(); diff --git a/paddle/fluid/framework/tensor_util.cc b/paddle/fluid/framework/tensor_util.cc index d1b01ae05b808b229309e9689165483a11530c84..d2e60ab1dd16758a91d22ef6872edc5053ef88b3 100644 --- a/paddle/fluid/framework/tensor_util.cc +++ b/paddle/fluid/framework/tensor_util.cc @@ -20,7 +20,7 @@ namespace paddle { namespace framework { void TensorCopy(const Tensor& src, const platform::Place& dst_place, - const platform::DeviceContext& ctx, Tensor* dst) { + const platform::DeviceContext& ctx, Tensor* dst, bool sync) { VLOG(3) << "TensorCopy " << src.dims() << " from " << src.place() << " to " << dst_place; src.check_memory_size(); @@ -47,9 +47,11 @@ void TensorCopy(const Tensor& src, const platform::Place& dst_place, PADDLE_ENFORCE(platform::is_gpu_place(ctx_place)); auto ctx_gpu_place = boost::get(ctx_place); PADDLE_ENFORCE_EQ(src_gpu_place, ctx_gpu_place); - memory::Copy( - dst_cpu_place, dst_ptr, src_gpu_place, src_ptr, size, - reinterpret_cast(ctx).stream()); + auto stream = + sync ? nullptr + : reinterpret_cast(ctx) + .stream(); + memory::Copy(dst_cpu_place, dst_ptr, src_gpu_place, src_ptr, size, stream); } else if (platform::is_cpu_place(src_place) && platform::is_gpu_place(dst_place)) { auto src_cpu_place = boost::get(src_place); @@ -58,18 +60,22 @@ void TensorCopy(const Tensor& src, const platform::Place& dst_place, PADDLE_ENFORCE(platform::is_gpu_place(ctx_place)); auto ctx_gpu_place = boost::get(ctx_place); PADDLE_ENFORCE_EQ(dst_gpu_place, ctx_gpu_place); - memory::Copy( - dst_gpu_place, dst_ptr, src_cpu_place, src_ptr, size, - reinterpret_cast(ctx).stream()); + auto stream = + sync ? nullptr + : reinterpret_cast(ctx) + .stream(); + memory::Copy(dst_gpu_place, dst_ptr, src_cpu_place, src_ptr, size, stream); } else if (platform::is_gpu_place(src_place) && platform::is_gpu_place(dst_place)) { auto src_gpu_place = boost::get(src_place); auto dst_gpu_place = boost::get(dst_place); auto ctx_place = ctx.GetPlace(); PADDLE_ENFORCE(platform::is_gpu_place(ctx_place)); - memory::Copy( - dst_gpu_place, dst_ptr, src_gpu_place, src_ptr, size, - reinterpret_cast(ctx).stream()); + auto stream = + sync ? nullptr + : reinterpret_cast(ctx) + .stream(); + memory::Copy(dst_gpu_place, dst_ptr, src_gpu_place, src_ptr, size, stream); } #endif } diff --git a/paddle/fluid/framework/tensor_util.h b/paddle/fluid/framework/tensor_util.h index 78b165ebed13cbae791b922e8820cd9551dfd198..3af68402dc56230171e858bf8f8f8c89c2bfe760 100644 --- a/paddle/fluid/framework/tensor_util.h +++ b/paddle/fluid/framework/tensor_util.h @@ -24,7 +24,8 @@ namespace paddle { namespace framework { void TensorCopy(const Tensor& src, const platform::Place& dst_place, - const platform::DeviceContext& ctx, Tensor* dst); + const platform::DeviceContext& ctx, Tensor* dst, + bool sync = false); void TensorCopy(const Tensor& src, const platform::Place& dst_place, Tensor* dst); diff --git a/paddle/fluid/memory/memcpy.cc b/paddle/fluid/memory/memcpy.cc index eddcaab8befda84dd14ed46c31ac025dfbcc7ca9..a177d4985fd0e2cca983b6873af89c60f526b811 100644 --- a/paddle/fluid/memory/memcpy.cc +++ b/paddle/fluid/memory/memcpy.cc @@ -32,7 +32,11 @@ void Copy( platform::CPUPlace dst_place, void* dst, platform::CUDAPlace src_place, const void* src, size_t num, cudaStream_t stream) { platform::SetDeviceId(src_place.device); - platform::GpuMemcpyAsync(dst, src, num, cudaMemcpyDeviceToHost, stream); + if (stream) { + platform::GpuMemcpyAsync(dst, src, num, cudaMemcpyDeviceToHost, stream); + } else { + platform::GpuMemcpySync(dst, src, num, cudaMemcpyDeviceToHost); + } } template <> @@ -40,7 +44,11 @@ void Copy( platform::CUDAPlace dst_place, void* dst, platform::CPUPlace src_place, const void* src, size_t num, cudaStream_t stream) { platform::SetDeviceId(dst_place.device); - platform::GpuMemcpyAsync(dst, src, num, cudaMemcpyHostToDevice, stream); + if (stream) { + platform::GpuMemcpyAsync(dst, src, num, cudaMemcpyHostToDevice, stream); + } else { + platform::GpuMemcpySync(dst, src, num, cudaMemcpyHostToDevice); + } } template <> @@ -49,10 +57,19 @@ void Copy( const void* src, size_t num, cudaStream_t stream) { if (dst_place == src_place) { platform::SetDeviceId(src_place.device); - platform::GpuMemcpyAsync(dst, src, num, cudaMemcpyDeviceToDevice, stream); + if (stream) { + platform::GpuMemcpyAsync(dst, src, num, cudaMemcpyDeviceToDevice, stream); + } else { + platform::GpuMemcpySync(dst, src, num, cudaMemcpyDeviceToDevice); + } } else { - platform::GpuMemcpyPeer(dst, dst_place.device, src, src_place.device, num, - stream); + if (stream) { + platform::GpuMemcpyPeerAsync(dst, dst_place.device, src, src_place.device, + num, stream); + } else { + platform::GpuMemcpyPeerSync(dst, dst_place.device, src, src_place.device, + num); + } } } @@ -83,7 +100,11 @@ void Copy( platform::CUDAPlace src_place, const void* src, size_t num, cudaStream_t stream) { platform::SetDeviceId(src_place.device); - platform::GpuMemcpyAsync(dst, src, num, cudaMemcpyDeviceToHost, stream); + if (stream) { + platform::GpuMemcpyAsync(dst, src, num, cudaMemcpyDeviceToHost, stream); + } else { + platform::GpuMemcpySync(dst, src, num, cudaMemcpyDeviceToHost); + } } template <> @@ -92,7 +113,11 @@ void Copy( platform::CUDAPinnedPlace src_place, const void* src, size_t num, cudaStream_t stream) { platform::SetDeviceId(dst_place.device); - platform::GpuMemcpyAsync(dst, src, num, cudaMemcpyHostToDevice, stream); + if (stream) { + platform::GpuMemcpyAsync(dst, src, num, cudaMemcpyHostToDevice, stream); + } else { + platform::GpuMemcpySync(dst, src, num, cudaMemcpyHostToDevice); + } } #endif diff --git a/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc b/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc index 0b7c1d6af714558d35918dac62d92d9e0f86c970..4372f23fc1dbd85e43b04a9d644977392316c2e9 100644 --- a/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc +++ b/paddle/fluid/operators/reader/create_double_buffer_reader_op.cc @@ -180,7 +180,8 @@ void DoubleBufferReader::PrefetchThreadFunc() { auto* gpu_ctx = ctxs_[cached_tensor_id].get(); gpu_batch.resize(cpu_batch.size()); for (size_t i = 0; i < cpu_batch.size(); ++i) { - framework::TensorCopy(cpu_batch[i], place_, *gpu_ctx, &gpu_batch[i]); + framework::TensorCopy(cpu_batch[i], place_, *gpu_ctx, &gpu_batch[i], + true); gpu_batch[i].set_lod(cpu_batch[i].lod()); } } diff --git a/paddle/fluid/platform/gpu_info.cc b/paddle/fluid/platform/gpu_info.cc index aaebeb1353a13ab16fcf98f10da59d41fd2f5b48..4cee93f3a4224cb97327254cd1679021d197a1b1 100644 --- a/paddle/fluid/platform/gpu_info.cc +++ b/paddle/fluid/platform/gpu_info.cc @@ -127,11 +127,24 @@ void GpuMemcpyAsync(void *dst, const void *src, size_t count, "cudaMemcpyAsync failed in paddle::platform::GpuMemcpyAsync"); } -void GpuMemcpyPeer(void *dst, int dst_device, const void *src, int src_device, - size_t count, cudaStream_t stream) { +void GpuMemcpySync(void *dst, const void *src, size_t count, + enum cudaMemcpyKind kind) { + PADDLE_ENFORCE(cudaMemcpy(dst, src, count, kind), + "cudaMemcpy failed in paddle::platform::GpuMemcpySync"); +} + +void GpuMemcpyPeerAsync(void *dst, int dst_device, const void *src, + int src_device, size_t count, cudaStream_t stream) { PADDLE_ENFORCE( cudaMemcpyPeerAsync(dst, dst_device, src, src_device, count, stream), - "cudaMemcpyPeerAsync failed in paddle::platform::GpuMemcpyPeer"); + "cudaMemcpyPeerAsync failed in paddle::platform::GpuMemcpyPeerAsync"); +} + +void GpuMemcpyPeerSync(void *dst, int dst_device, const void *src, + int src_device, size_t count) { + PADDLE_ENFORCE( + cudaMemcpyPeer(dst, dst_device, src, src_device, count), + "cudaMemcpyPeer failed in paddle::platform::GpuMemcpyPeerSync"); } void GpuMemsetAsync(void *dst, int value, size_t count, cudaStream_t stream) { diff --git a/paddle/fluid/platform/gpu_info.h b/paddle/fluid/platform/gpu_info.h index 36345e17406e22970806fa274d5a73a703517c43..f4640d3eaa2165c35e8e14690d83e9e7e7168c0b 100644 --- a/paddle/fluid/platform/gpu_info.h +++ b/paddle/fluid/platform/gpu_info.h @@ -57,9 +57,17 @@ size_t GpuMaxChunkSize(); void GpuMemcpyAsync(void *dst, const void *src, size_t count, enum cudaMemcpyKind kind, cudaStream_t stream); -//! Copy memory from one device to another device. -void GpuMemcpyPeer(void *dst, int dst_device, const void *src, int src_device, - size_t count, cudaStream_t stream); +//! Copy memory from address src to dst synchronously. +void GpuMemcpySync(void *dst, const void *src, size_t count, + enum cudaMemcpyKind kind); + +//! Copy memory from one device to another device asynchronously. +void GpuMemcpyPeerAsync(void *dst, int dst_device, const void *src, + int src_device, size_t count, cudaStream_t stream); + +//! Copy memory from one device to another device synchronously. +void GpuMemcpyPeerSync(void *dst, int dst_device, const void *src, + int src_device, size_t count); //! Set memory dst with value count size asynchronously void GpuMemsetAsync(void *dst, int value, size_t count, cudaStream_t stream); diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 1f21e7abe76b2a32d6c18e5c26c4f25b65daef5b..b20b514fcdd0b41fefa0933bc2d22645e7d4b6d6 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -502,11 +502,11 @@ All parameter, weight, gradient are variables in Paddle. const std::unordered_set &bcast_vars, const ProgramDesc &main_program, const std::string &loss_var_name, Scope *scope, std::vector &local_scopes, - bool allow_op_delay) { - new (&self) - ParallelExecutor(num_threads, use_event, places, params, - bcast_vars, main_program, loss_var_name, - scope, local_scopes, allow_op_delay); + bool allow_op_delay, bool customize_loss_grad) { + new (&self) ParallelExecutor(num_threads, use_event, places, + params, bcast_vars, main_program, + loss_var_name, scope, local_scopes, + allow_op_delay, customize_loss_grad); }) .def("bcast_params", &ParallelExecutor::BCastParamsToGPUs) // NOTE: even we return a vec* to Python use reference policy. diff --git a/paddle/fluid/recordio/README.md b/paddle/fluid/recordio/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ef99c0cf0fa71d807a95898454d8fabb287324e9 --- /dev/null +++ b/paddle/fluid/recordio/README.md @@ -0,0 +1,13 @@ +## Background + +The RecordIO file format is a container for records. This package is a C++ implementation of https://github.com/paddlepaddle/recordio, which originates from https://github.com/wangkuiyi/recordio. + +## Fault-tolerant Writing + +For the initial design purpose of RecordIO within Google, which was logging, RecordIO groups record into *chunks*, whose header contains an MD5 hash of the chunk. A process that writes logs is supposed to call the Writer interface to add records. Once the writer accumulates a handful of them, it groups a chunk, put the MD5 into the chunk header, and appends the chunk to the file. In the event the process crashes unexpected, the last chunk in the RecordIO file could be incomplete/corrupt. The RecordIO reader is able to recover from these errors when the process restarts by identifying incomplete chucks and skipping over them. + +## Reading Ranges + +A side-effect of chunks is to make it easy to indexing records while reading, thus allows us to read a range of successive records. This is good for distributed log process, where each MapReduce task handles only part of records in a big RecordIO file. + +The procedure that creates the index starts from reading the header of the first chunk. It indexes the offset (0) and the size of the chunk, and skips to the header of the next chunk by calling the `fseek` API. Please be aware that most distributed filesystems and all POSIX-compatible local filesystem provides `fseek`, and makes sure that `fseek` runs much faster than `fread`. This procedure generates a map from chunks to their offsets, which allows the readers is to locate and read a range of records. diff --git a/python/paddle/dataset/uci_housing.py b/python/paddle/dataset/uci_housing.py index 6a56e9d5563c76ab6f524ccea9191693dc227010..fbfa477d055eb5f484989eacce38cee8d617d729 100644 --- a/python/paddle/dataset/uci_housing.py +++ b/python/paddle/dataset/uci_housing.py @@ -19,7 +19,11 @@ https://archive.ics.uci.edu/ml/machine-learning-databases/housing/ and parse training set and test set into paddle reader creators. """ +import os + import numpy as np +import tempfile +import tarfile import os import paddle.dataset.common @@ -34,8 +38,9 @@ feature_names = [ UCI_TRAIN_DATA = None UCI_TEST_DATA = None -URL_MODEL = 'https://github.com/PaddlePaddle/book/raw/develop/01.fit_a_line/fit_a_line.tar' -MD5_MODEL = '52fc3da8ef3937822fcdd87ee05c0c9b' + +FLUID_URL_MODEL = 'https://github.com/PaddlePaddle/book/raw/develop/01.fit_a_line/fluid/fit_a_line.fluid.tar' +FLUID_MD5_MODEL = '6e6dd637ccd5993961f68bfbde46090b' def feature_range(maximums, minimums): @@ -113,6 +118,29 @@ def test(): return reader +def fluid_model(): + parameter_tar = paddle.dataset.common.download( + FLUID_URL_MODEL, 'uci_housing', FLUID_MD5_MODEL, 'fit_a_line.fluid.tar') + + tar = tarfile.TarFile(parameter_tar, mode='r') + dirpath = tempfile.mkdtemp() + tar.extractall(path=dirpath) + + return dirpath + + +def predict_reader(): + """ + It returns just one tuple data to do inference. + + :return: one tuple data + :rtype: tuple + """ + global UCI_TEST_DATA + load_data(paddle.dataset.common.download(URL, 'uci_housing', MD5)) + return (UCI_TEST_DATA[0][:-1], ) + + def fetch(): paddle.dataset.common.download(URL, 'uci_housing', MD5) diff --git a/python/paddle/fluid/parallel_executor.py b/python/paddle/fluid/parallel_executor.py index fbdd6fd449625a21f91758dc12490b02070aea1a..4adbb2ea99b58c78c5c08c7ac8a556ca1de1615e 100644 --- a/python/paddle/fluid/parallel_executor.py +++ b/python/paddle/fluid/parallel_executor.py @@ -29,7 +29,8 @@ class ParallelExecutor(object): main_program=None, num_threads=None, allow_op_delay=False, - share_vars_from=None): + share_vars_from=None, + customize_loss_grad=False): """ ParallelExecutor can run program in parallel. @@ -78,7 +79,7 @@ class ParallelExecutor(object): else: for i in xrange(multiprocessing.cpu_count()): p = core.Place() - self._act_places.append(core.CPUPlace(i)) + self._act_places.append(core.CPUPlace()) p.set_place(self._act_places[-1]) self._places.append(p) assert self._places, "no place for execution" @@ -122,7 +123,8 @@ class ParallelExecutor(object): loss_name if loss_name else '', scope, local_scopes, - allow_op_delay) + allow_op_delay, + customize_loss_grad) self.scope = scope def run(self, fetch_list, feed=None, feed_dict=None): diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py index c7fada2077ca2e1d75870d44a5759c43b51c3579..c6f45381af8ac64d117eb27325f25763fbf6cae7 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py @@ -55,7 +55,8 @@ class TestElementWiseAddOp(unittest.TestCase): "X": block.var('x'), "Y": block.var('y'), }, - outputs={"Out": block.var('out'), }) + outputs={"Out": block.var('out'), }, + attrs={"axis": self.axis, }) # generate backward op_desc grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc( @@ -79,7 +80,7 @@ class TestElementWiseAddOp(unittest.TestCase): }, fetch_list=['x@GRAD', 'y@GRAD']) self.__assert_close(x_grad, out[0], "x@GRAD") - self.__assert_close(y_grad, out[1], "y@GRAD", atol=2.0) + self.__assert_close(y_grad, out[1], "y@GRAD", atol=1.4) places = [core.CPUPlace()] if core.is_compiled_with_cuda() and core.op_support_gpu(