Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into fix_prelu

.gitignore

@@ -28,4 +28,3 @@ cmake_install.cmake
 paddle/.timestamp
 python/paddlepaddle.egg-info/
 paddle/pybind/pybind.h
-python/paddle/v2/framework/tests/tmp/*

cmake/external/openblas.cmake

@@ -98,7 +98,7 @@ IF(NOT ${CBLAS_FOUND})
         ENDIF()
         INSTALL(CODE "execute_process(
             COMMAND ${CMAKE_COMMAND} -E copy_directory ${CBLAS_INSTALL_DIR}/lib
-                    destination ${CMAKE_INSTALL_PREFIX}/${TMP_INSTALL_DIR}
+                    ${CMAKE_INSTALL_PREFIX}/${TMP_INSTALL_DIR}
             )"
         )
         INSTALL(CODE "MESSAGE(STATUS \"Installing: \"

doc/design/evaluator.md

+## Evaluator Design
+
+### The Problem
+
+During training or serving, we provide the evaluation function to measure the model performance, e.g., accuracy, precision. In the operator based framework design, the data go through the network pipeline batch by batch. As a result, inside the operator, we only can calculate one minibatch metrics. We need to provide a mechanism to calculate the metrics for each N pass/batch the user wanted.
+
+### Evaluator Design
+Currently, every operation is expressed in the graph. we divide the evaluator process into three steps.
+
+1. Initialize the metric state and add it into the block.
+
+2. Calculate the statistic of the metric state in every mini-batch. The single operator is only responsible for calculating necessary statistics for one mini-batch. For example, accuracy operator only calculate a minibatch data if run once.
+
+
+3. Merge the mini-batch statistics to form the evaluation result for multiple mini-batches. When it comes to distributed training/Multi-GPU training, aggregate the value from different devices.
+
+### Implementation
+This design is shown in python API. 
+Each metric operator need to caculate the metric statistic and return the batch aware states, Python side responsible for accumulate the states for each pass. 
+
+    
+```python
+class Evaluator(object):
+    """
+    Evaluator Base class.
+    """
+    def __init__(self, name, **kwargs):
+       """
+       Different evaluator may has different metric states. E.g, Accuracy need two variables, total and right sample counts.
+       Auc need four variables, `true_positives`,
+         `true_negatives`, `false_positives` and `false_negatives`. So every evaluator should create its needed variables and append to main_program
+
+       The initialization of Evaluator should be responsible for:
+       create metric states and append to the main_program
+       """ 
+       pass
+
+    def _update_ops(self, input, label, **kwargs)
+       """
+       Add mini-batch evaluator caculate operators to the main_program.
+       Add increment operator to accumulate the metric states.
+       """
+    
+
+    def reset(self, executor, reset_program=None):
+      """
+      Reset metric states at the begin of each pass/user specified batch number.
+      Execute the reset_program to reset the states.
+      """
+      
+
+    def eval(self, executor, eval_program=None):
+      """
+      Merge the mini-batch statistics to form the evaluation result for multiple mini-batches.
+      Execute the eval_program and return the result.
+      """
+      return eval_result
+```

paddle/capi/Matrix.cpp

@@ -121,6 +121,7 @@ paddle_error paddle_matrix_get_shape(paddle_matrix mat,
 
 paddle_matrix paddle_matrix_create_sparse(
     uint64_t height, uint64_t width, uint64_t nnz, bool isBinary, bool useGpu) {
+#ifndef PADDLE_MOBILE_INFERENCE
   auto ptr = new paddle::capi::CMatrix();
   ptr->mat = paddle::Matrix::createSparseMatrix(
       height,
@@ -131,6 +132,9 @@ paddle_matrix paddle_matrix_create_sparse(
       false,
       useGpu);
   return ptr;
+#else
+  return nullptr;
+#endif
 }
 
 paddle_error paddle_matrix_sparse_copy_from(paddle_matrix mat,
@@ -140,6 +144,7 @@ paddle_error paddle_matrix_sparse_copy_from(paddle_matrix mat,
                                             uint64_t colSize,
                                             float* valueArray,
                                             uint64_t valueSize) {
+#ifndef PADDLE_MOBILE_INFERENCE
   if (mat == nullptr) return kPD_NULLPTR;
   auto ptr = cast(mat);
   if (rowArray == nullptr || colArray == nullptr ||
@@ -160,4 +165,7 @@ paddle_error paddle_matrix_sparse_copy_from(paddle_matrix mat,
   } else {
     return kPD_NOT_SUPPORTED;
   }
+#else
+  return kPD_NOT_SUPPORTED;
+#endif
 }

paddle/capi/matrix.h

@@ -48,6 +48,7 @@ PD_API paddle_matrix paddle_matrix_create(uint64_t height,
  * @param isBinary is binary (either 1 or 0 in matrix) or not.
  * @param useGpu is using GPU or not.
  * @return paddle_matrix.
+ * @note Mobile inference does not support this interface.
  */
 PD_API paddle_matrix paddle_matrix_create_sparse(
     uint64_t height, uint64_t width, uint64_t nnz, bool isBinary, bool useGpu);
@@ -129,6 +130,7 @@ PD_API paddle_error paddle_matrix_get_shape(paddle_matrix mat,
  * NULL if the matrix is binary.
  * @param [in] valueSize length of value array. Zero if the matrix is binary.
  * @return paddle_error
+ * @note Mobile inference does not support this interface.
  */
 PD_API paddle_error paddle_matrix_sparse_copy_from(paddle_matrix mat,
                                                    int* rowArray,

paddle/cuda/CMakeLists.txt

@@ -27,7 +27,9 @@ if(WITH_GPU)
     set_source_files_properties(${CUDA_CXX_SOURCES}
                                 PROPERTIES COMPILE_FLAGS "-D__NVCC__")
 else()
+    if (NOT MOBILE_INFERENCE)
     set(CUDA_CXX_SOURCES src/hl_warpctc_wrap.cc)
+    endif()
 endif()
 
 set(CUDA_CU_SOURCES

paddle/framework/CMakeLists.txt

@@ -38,9 +38,9 @@ py_proto_compile(framework_py_proto SRCS framework.proto)
 add_custom_target(framework_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
 add_dependencies(framework_py_proto framework_py_proto_init)
 add_custom_command(TARGET framework_py_proto POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SOURCE_DIR}/python/paddle/v2/framework/proto
-    COMMAND cp *.py ${PADDLE_SOURCE_DIR}/python/paddle/v2/framework/proto/
-    COMMENT "Copy generated python proto into directory paddle/v2/framework/proto."
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SOURCE_DIR}/python/paddle/v2/fluid/proto
+    COMMAND cp *.py ${PADDLE_SOURCE_DIR}/python/paddle/v2/fluid/proto/
+    COMMENT "Copy generated python proto into directory paddle/v2/fluid/proto."
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 
 cc_library(backward SRCS backward.cc DEPS net_op)

paddle/gserver/CMakeLists.txt

@@ -85,9 +85,49 @@ if(MOBILE_INFERENCE)
          gradientmachines/GradientMachineMode.cpp
          gradientmachines/MultiGradientMachine.cpp)
 
-    # Remove useless layers
+    # Remove layers that used in training
     list(REMOVE_ITEM GSERVER_SOURCES
-    	 layers/RecurrentLayerGroup.cpp)
+    	 layers/RecurrentLayerGroup.cpp
+         layers/CostLayer.cpp
+         layers/MultiBoxLossLayer.cpp
+         layers/WarpCTCLayer.cpp
+         layers/CTCLayer.cpp
+         layers/LinearChainCTC.cpp
+         layers/PrintLayer.cpp)
+    list(REMOVE_ITEM GSERVER_SOURCES
+         layers/OuterProdLayer.cpp
+         layers/SumToOneNormLayer.cpp
+         layers/ConvShiftLayer.cpp
+         layers/InterpolationLayer.cpp
+         layers/AgentLayer.cpp
+         layers/DotMulOperator.cpp
+         layers/GruStepLayer.cpp
+         layers/LstmStepLayer.cpp
+         layers/ConvexCombinationLayer.cpp
+         layers/Conv3DLayer.cpp
+         layers/DeConv3DLayer.cpp
+         layers/CropLayer.cpp
+         layers/CrossEntropyOverBeam.cpp
+         layers/DataNormLayer.cpp
+         layers/FeatureMapExpandLayer.cpp
+         layers/HierarchicalSigmoidLayer.cpp
+         layers/MultinomialSampler.cpp
+         layers/NCELayer.cpp
+         layers/KmaxSeqScoreLayer.cpp
+         layers/MDLstmLayer.cpp
+         layers/MultiplexLayer.cpp
+         layers/PadLayer.cpp
+         layers/Pool3DLayer.cpp
+         layers/ResizeLayer.cpp
+         layers/RotateLayer.cpp
+         layers/RowConvLayer.cpp
+         layers/RowL2NormLayer.cpp
+         layers/SamplingIdLayer.cpp
+         layers/ScaleShiftLayer.cpp
+         layers/SelectiveFullyConnectedLayer.cpp
+         layers/SpatialPyramidPoolLayer.cpp
+         layers/BilinearInterpLayer.cpp
+         layers/ClipLayer.cpp)
 endif()
 
 if(WITH_GPU)

paddle/gserver/gradientmachines/NeuralNetwork.cpp

@@ -16,7 +16,6 @@ limitations under the License. */
 
 #include "NeuralNetwork.h"
 #include "hl_gpu.h"
-#include "paddle/gserver/layers/AgentLayer.h"
 #include "paddle/utils/CustomStackTrace.h"
 #include "paddle/utils/Logging.h"
 #include "paddle/utils/Stat.h"
@@ -28,6 +27,7 @@ limitations under the License. */
 #ifndef PADDLE_MOBILE_INFERENCE
 #include "MultiNetwork.h"
 #include "RecurrentGradientMachine.h"
+#include "paddle/gserver/layers/AgentLayer.h"
 #endif
 
 namespace paddle {
@@ -192,9 +192,11 @@ void NeuralNetwork::init(const ModelConfig& config,
 void NeuralNetwork::connect(LayerPtr agentLayer,
                             LayerPtr realLayer,
                             int height) {
+#ifndef PADDLE_MOBILE_INFERENCE
   AgentLayer* agent = dynamic_cast<AgentLayer*>(agentLayer.get());
   CHECK_NOTNULL(agent);
   agent->setRealLayer(realLayer, height);
+#endif
 }
 
 void NeuralNetwork::connect(std::string agentLayerName,

paddle/gserver/layers/Layer.cpp

@@ -98,6 +98,7 @@ ClassRegistrar<Layer, LayerConfig> Layer::registrar_;
 LayerPtr Layer::create(const LayerConfig& config) {
   std::string type = config.type();
 
+#ifndef PADDLE_MOBILE_INFERENCE
   // NOTE: As following types have illegal character '-',
   // they can not use REGISTER_LAYER to registrar.
   // Besides, to fit with old training models,
@@ -106,7 +107,6 @@ LayerPtr Layer::create(const LayerConfig& config) {
     return LayerPtr(new MultiClassCrossEntropy(config));
   else if (type == "rank-cost")
     return LayerPtr(new RankingCost(config));
-#ifndef PADDLE_MOBILE_INFERENCE
   else if (type == "auc-validation")
     return LayerPtr(new AucValidation(config));
   else if (type == "pnpair-validation")

paddle/gserver/layers/ROIPoolLayer.cpp

@@ -100,8 +100,9 @@ void ROIPoolLayer::forward(PassType passType) {
     size_t roiEndH = round(bottomROIs[4] * spatialScale_);
     CHECK_GE(roiBatchIdx, 0UL);
     CHECK_LT(roiBatchIdx, batchSize);
-    size_t roiHeight = std::max(roiEndH - roiStartH + 1, 1UL);
-    size_t roiWidth = std::max(roiEndW - roiStartW + 1, 1UL);
+    size_t roiHeight =
+        std::max(roiEndH - roiStartH + 1, static_cast<size_t>(1));
+    size_t roiWidth = std::max(roiEndW - roiStartW + 1, static_cast<size_t>(1));
     real binSizeH =
         static_cast<real>(roiHeight) / static_cast<real>(pooledHeight_);
     real binSizeW =
@@ -114,10 +115,14 @@ void ROIPoolLayer::forward(PassType passType) {
           size_t wstart = static_cast<size_t>(std::floor(pw * binSizeW));
           size_t hend = static_cast<size_t>(std::ceil((ph + 1) * binSizeH));
           size_t wend = static_cast<size_t>(std::ceil((pw + 1) * binSizeW));
-          hstart = std::min(std::max(hstart + roiStartH, 0UL), height_);
-          wstart = std::min(std::max(wstart + roiStartW, 0UL), width_);
-          hend = std::min(std::max(hend + roiStartH, 0UL), height_);
-          wend = std::min(std::max(wend + roiStartW, 0UL), width_);
+          hstart = std::min(
+              std::max(hstart + roiStartH, static_cast<size_t>(0)), height_);
+          wstart = std::min(
+              std::max(wstart + roiStartW, static_cast<size_t>(0)), width_);
+          hend = std::min(std::max(hend + roiStartH, static_cast<size_t>(0)),
+                          height_);
+          wend = std::min(std::max(wend + roiStartW, static_cast<size_t>(0)),
+                          width_);
 
           bool isEmpty = (hend <= hstart) || (wend <= wstart);
           size_t poolIndex = ph * pooledWidth_ + pw;

paddle/gserver/tests/CMakeLists.txt

 # gserver pacakge unittests
 
 add_simple_unittest(test_LinearChainCRF)
-add_simple_unittest(test_MultinomialSampler)
 add_simple_unittest(test_RecurrentLayer)
 
+if(NOT MOBILE_INFERENCE)
+  add_simple_unittest(test_MultinomialSampler)
+endif()
+
 function(gserver_test TARGET)
   add_unittest_without_exec(${TARGET}
       ${TARGET}.cpp
@@ -49,7 +52,7 @@ if(WITH_PYTHON)
 endif()
 
 ############### test_WarpCTCLayer #######################
-if(NOT WITH_DOUBLE)
+if(NOT WITH_DOUBLE AND NOT MOBILE_INFERENCE)
     add_unittest_without_exec(test_WarpCTCLayer
         test_WarpCTCLayer.cpp)
 

paddle/math/BaseMatrix.cu

@@ -1902,5 +1902,52 @@ void BaseMatrixT<real>::sumOfProducts(BaseMatrixT& b,
 }
 
 template class BaseMatrixT<real>;
+
+#ifndef PADDLE_MOBILE_INFERENCE
+
 template class BaseMatrixT<int>;
+
+#else
+
+template <>
+void BaseMatrixT<int>::zero() {
+  applyUnary(unary::Zero<int>());
+}
+
+template <>
+void BaseMatrixT<int>::assign(int p) {
+  applyUnary(unary::Assign<int>(p));
+}
+
+template <>
+void BaseMatrixT<int>::isEqualTo(BaseMatrixT& b, int value) {
+  applyBinary(binary::IsEqual<int>(value), b);
+}
+
+template <>
+void BaseMatrixT<int>::neg() {
+  applyUnary(unary::Neg<int>());
+}
+
+template <>
+void BaseMatrixT<int>::abs2() {
+  applyUnary(unary::Abs<int>());
+}
+
+template <>
+void BaseMatrixT<int>::add(int p) {
+  applyUnary(unary::Add<int>(p));
+}
+
+template <>
+void BaseMatrixT<int>::add(int p1, int p2) {
+  applyUnary(unary::Add2<int>(p1, p2));
+}
+
+template <>
+void BaseMatrixT<int>::applyL1(int learningRate, int decayRate) {
+  applyUnary(unary::ApplyL1<int>(learningRate * decayRate));
+}
+
+#endif
 }  // namespace paddle

paddle/math/CMakeLists.txt

@@ -25,6 +25,19 @@ else()
     message(STATUS "Compile with MKLDNNMatrix")
 endif()
 
+if(MOBILE_INFERENCE)
+    list(REMOVE_ITEM MATH_SOURCES
+         ${CMAKE_CURRENT_SOURCE_DIR}/SIMDFunctions.cpp)
+    # Remove sparse
+    list(REMOVE_ITEM MATH_HEADERS
+         ${CMAKE_CURRENT_SOURCE_DIR}/CpuSparseMatrix.h
+         ${CMAKE_CURRENT_SOURCE_DIR}/SparseMatrix.h
+         ${CMAKE_CURRENT_SOURCE_DIR}/SparseRowMatrix.h)
+    list(REMOVE_ITEM MATH_SOURCES
+         ${CMAKE_CURRENT_SOURCE_DIR}/CpuSparseMatrix.cpp
+         ${CMAKE_CURRENT_SOURCE_DIR}/SparseMatrix.cpp
+         ${CMAKE_CURRENT_SOURCE_DIR}/SparseRowMatrix.cpp)
+endif()
 set(MATH_SOURCES
     "${PADDLE_SOURCE_DIR}/paddle/math/BaseMatrix.cu"
     "${PADDLE_SOURCE_DIR}/paddle/math/TrainingAlgorithmOp.cu"

paddle/math/CpuSparseMatrix.h

@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #pragma once
+
+#ifndef PADDLE_MOBILE_INFERENCE
+
 #include <cstddef>
 #include "Matrix.h"
 
@@ -309,3 +312,57 @@ private:
   using Matrix::subMatrix;
 };
 }  // namespace paddle
+
+#else
+
+#include "Matrix.h"
+
+namespace paddle {
+
+class CpuSparseMatrix : public Matrix {
+public:
+  CpuSparseMatrix(size_t height,
+                  size_t width,
+                  size_t nnz, /* used to allocate space */
+                  SparseValueType valueType = FLOAT_VALUE,
+                  SparseFormat format = SPARSE_CSR,
+                  bool trans = false)
+      : Matrix(NULL, height, width, trans, false) {}
+
+  CpuSparseMatrix(real* data,
+                  int* rows,
+                  int* cols,
+                  size_t height,
+                  size_t width,
+                  size_t nnz,
+                  SparseValueType valueType,
+                  SparseFormat format,
+                  bool trans)
+      : Matrix(NULL, height, width, trans, false) {}
+
+  real* getValue() const { return nullptr; }
+  size_t getColStartIdx(size_t i) const { return 0; }
+  size_t getRowStartIdx(size_t i) const { return 0; }
+  size_t getColNum(size_t i) const { return 0; }
+  int* getRowCols(size_t i) const { return nullptr; }
+
+  CpuSparseMatrixPtr getTmpSparseMatrix(size_t height, size_t width) {
+    return nullptr;
+  }
+
+  void resize(size_t newHeight,
+              size_t newWidth,
+              size_t newNnz, /* used to allocate space */
+              SparseValueType valueType,
+              SparseFormat format) {}
+  void resize(size_t newHeight, size_t newWidth) {}
+  MatrixPtr getTranspose() { return nullptr; }
+  void setRow(size_t row,
+              size_t colNum,
+              const unsigned int* cols,
+              const real* values) {}
+};
+
+}  // namespace paddle
+
+#endif

paddle/math/Matrix.cpp

@@ -451,6 +451,7 @@ void GpuMatrix::addSharedBias(Matrix& b, real scale) {
 }
 
 void GpuMatrix::collectBias(Matrix& a, real scale) {
+#ifdef PADDLE_WITH_CUDA
   CHECK_EQ(getHeight(), (size_t)1);
   CHECK_EQ(width_, a.getWidth());
   GpuSparseMatrix* sMatPtr = dynamic_cast<GpuSparseMatrix*>(&a);
@@ -461,6 +462,7 @@ void GpuMatrix::collectBias(Matrix& a, real scale) {
     hl_sparse_matrix_s A_d = sMatPtr->sMatrix_.get();
     hl_sparse_matrix_column_sum(data, A_d, sMatPtr->getHeight(), width_, scale);
   }
+#endif
 }
 
 void GpuMatrix::collectSharedBias(Matrix& a, real scale) {
@@ -552,6 +554,7 @@ void GpuMatrix::mul(const GpuSparseMatrix& a,
                     const GpuMatrix& b,
                     real scaleAB,
                     real scaleT) {
+#ifdef PADDLE_WITH_CUDA
   CHECK(isContiguous());
   CHECK(b.isContiguous());
   CHECK(b.useGpu_ == true) << "Matrix type are not equal";
@@ -578,12 +581,14 @@ void GpuMatrix::mul(const GpuSparseMatrix& a,
                           b.height_,
                           scaleAB,
                           scaleT);
+#endif
 }
 
 void GpuMatrix::mul(const GpuMatrix& a,
                     const GpuSparseMatrix& b,
                     real scaleAB,
                     real scaleT) {
+#ifdef PADDLE_WITH_CUDA
   CHECK(isContiguous());
   CHECK(a.isContiguous());
   CHECK(a.useGpu_ == true) << "Matrix type are not equal";
@@ -622,6 +627,7 @@ void GpuMatrix::mul(const GpuMatrix& a,
                             scaleAB,
                             scaleT);
   }
+#endif
 }
 
 /* this = a*b */
@@ -1557,6 +1563,7 @@ void GpuMatrix::bilinearBackward(const Matrix& out,
 }
 
 void GpuMatrix::multiBinaryLabelCrossEntropy(Matrix& output, Matrix& label) {
+#ifdef PADDLE_WITH_CUDA
   GpuMatrix* outputPtr = dynamic_cast<GpuMatrix*>(&output);
   auto labelPtr = dynamic_cast<GpuSparseMatrix*>(&label);
 
@@ -1572,9 +1579,11 @@ void GpuMatrix::multiBinaryLabelCrossEntropy(Matrix& output, Matrix& label) {
   hl_sparse_matrix_s mat_d = labelPtr->sMatrix_.get();
   hl_matrix_multi_binary_cross_entropy(
       output_d, entropy_d, mat_d, height_, outputPtr->width_);
+#endif
 }
 
 void GpuMatrix::multiBinaryLabelCrossEntropyBp(Matrix& output, Matrix& label) {
+#ifdef PADDLE_WITH_CUDA
   GpuMatrix* outputPtr = dynamic_cast<GpuMatrix*>(&output);
   auto labelPtr = dynamic_cast<GpuSparseMatrix*>(&label);
 
@@ -1590,6 +1599,7 @@ void GpuMatrix::multiBinaryLabelCrossEntropyBp(Matrix& output, Matrix& label) {
   hl_sparse_matrix_s mat_d = labelPtr->sMatrix_.get();
   hl_matrix_multi_binary_cross_entropy_bp(
       output_d, grad_d, mat_d, height_, width_);
+#endif
 }
 
 void GpuMatrix::vol2Col(real* dataSrc,
@@ -3255,6 +3265,7 @@ template void CpuMatrix::mul<CpuMatrix, CacheRowCpuMatrix>(CpuSparseMatrix* a,
                                                            real scaleAB,
                                                            real scaleT);
 
+#ifndef PADDLE_MOBILE_INFERENCE
 void SharedCpuMatrix::mul(CpuSparseMatrix* a,
                           CpuMatrix* b,
                           real scaleAB,
@@ -3383,6 +3394,7 @@ void SharedCpuMatrix::initBlock(int blockNum) {
   }
 }
 
+#endif
 /* Add a (column) vector b to matrix a, column by column */
 void CpuMatrix::addColumnVector(const Matrix& b) {
   BaseMatrix::addColVector(const_cast<Matrix&>(b));

paddle/math/Matrix.h

@@ -2070,6 +2070,7 @@ public:
 
 class SharedCpuMatrix : public CpuMatrix {
 public:
+#ifndef PADDLE_MOBILE_INFERENCE
   /* blockNum is number of partitions of the matrix  */
   SharedCpuMatrix(int blockNum, size_t height, size_t width, bool trans = false)
       : CpuMatrix(height, width, trans) {
@@ -2115,6 +2116,7 @@ private:
   ThreadLocal<CpuMatrixPtr> localBuf_;
   ThreadLocal<std::vector<int>> localBufRows_;
   ThreadLocal<std::vector<int>> blockSeq_;
+#endif
 };
 
 typedef struct { unsigned int col; } sparse_non_value_t;

paddle/math/SparseMatrix.h

@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #pragma once
+
+#ifndef PADDLE_MOBILE_INFERENCE
+
 #include <cstddef>
 #include "CpuSparseMatrix.h"
 #include "Matrix.h"
@@ -237,3 +240,47 @@ private:
 };
 
 }  // namespace paddle
+
+#else
+
+#include "CpuSparseMatrix.h"
+
+namespace paddle {
+
+class GpuSparseMatrix : public Matrix {
+public:
+  GpuSparseMatrix(size_t height,
+                  size_t width,
+                  size_t nnz, /* used to allocate space */
+                  SparseValueType valueType = FLOAT_VALUE,
+                  SparseFormat format_ = SPARSE_CSR,
+                  bool trans = false)
+      : Matrix(NULL, height, width, trans, false) {}
+
+  GpuSparseMatrix(real* value,
+                  int* rows,
+                  int* cols,
+                  size_t height,
+                  size_t width,
+                  size_t nnz,
+                  SparseValueType valueType,
+                  SparseFormat format,
+                  bool trans)
+      : Matrix(NULL, height, width, trans, true) {}
+
+  void resize(size_t newHeight,
+              size_t newWidth,
+              size_t newNnz, /* used to allocate space */
+              SparseValueType valueType,
+              SparseFormat format) {}
+  void resize(size_t newHeight, size_t newWidth) {}
+  MatrixPtr getTranspose() { return nullptr; }
+  void setRow(size_t row,
+              size_t colNum,
+              const unsigned int* cols,
+              const real* values) {}
+};
+
+}  // namespace paddle
+
+#endif

paddle/math/SparseRowMatrix.h

@@ -14,6 +14,8 @@ limitations under the License. */
 
 #pragma once
 
+#ifndef PADDLE_MOBILE_INFERENCE
+
 #include <gflags/gflags.h>
 #include <string.h>
 #include <algorithm>
@@ -313,3 +315,27 @@ private:
 };
 
 }  // namespace paddle
+
+#else
+namespace paddle {
+
+class SparseRowCpuMatrix : public CpuMatrix {
+public:
+  void reserveStore() {}
+  void clearIndices() {}
+};
+
+class SparsePrefetchRowCpuMatrix : public SparseRowCpuMatrix {
+public:
+  void setupIndices() {}
+  void addRows(MatrixPtr input) {}
+  void addRows(IVectorPtr ids) {}
+};
+
+class SparseAutoGrowRowCpuMatrix : public SparseRowCpuMatrix {};
+class CacheRowCpuMatrix : public SparseAutoGrowRowCpuMatrix {};
+class SparseRowIdsCpuMatrix : public CpuMatrix {};
+
+}  // namespace paddle
+
+#endif

paddle/math/tests/CMakeLists.txt

@@ -3,8 +3,10 @@
 add_simple_unittest(test_ExecViaCpu)
 add_simple_unittest(test_SIMDFunctions)
 add_simple_unittest(test_TrainingAlgorithm)
-add_simple_unittest(test_SparseMatrix)
 add_simple_unittest(test_RowBuffer)
+if(NOT MOBILE_INFERENCE)
+    add_simple_unittest(test_SparseMatrix)
+endif()
 
 # TODO(yuyang18): Refactor TestUtil.cpp. Remove this cross module reference.
 add_unittest(test_matrixCompare

paddle/operators/accuracy_op.cc

@@ -30,6 +30,10 @@ class AccuracyOp : public framework::OperatorWithKernel {
                    "Input (Label) of accuracy op should not be null.");
     PADDLE_ENFORCE(ctx->HasOutput("Accuracy"),
                    "Output (Accuracy) of AccuracyOp should not be null.");
+    PADDLE_ENFORCE(ctx->HasOutput("Correct"),
+                   "Output (Correct) of AccuracyOp should not be null.");
+    PADDLE_ENFORCE(ctx->HasOutput("Total"),
+                   "Output (Total) of AccuracyOp should not be null.");
 
     auto inference_dim = ctx->GetInputDim("Out");
     auto label_dim = ctx->GetInputDim("Label");
@@ -43,6 +47,8 @@ class AccuracyOp : public framework::OperatorWithKernel {
                       " the same as label.");
 
     ctx->SetOutputDim("Accuracy", {1});
+    ctx->SetOutputDim("Correct", {1});
+    ctx->SetOutputDim("Total", {1});
     ctx->ShareLoD("Out", /*->*/ "Accuracy");
   }
 
@@ -66,6 +72,8 @@ class AccuracyOpMaker : public framework::OpProtoAndCheckerMaker {
     AddInput("Label", "Label of the training data");
     // TODO(typhoonzero): AddInput("Weight", ...
     AddOutput("Accuracy", "The accuracy of current batch");
+    AddOutput("Correct", "The correct samples count of current batch");
+    AddOutput("Total", "The samples count of current batch");
 
     AddComment(R"DOC(
 Accuracy Operator. 

paddle/operators/accuracy_op.cu

@@ -24,7 +24,8 @@ using platform::PADDLE_CUDA_NUM_THREADS;
 template <int BlockSize>
 __global__ void AccuracyCudaKernel(const int N, const int D,
                                    const int64_t* Xdata,
-                                   const int64_t* labeldata, float* accuracy) {
+                                   const int64_t* labeldata, int* correct_data,
+                                   float* accuracy) {
   int count = 0;
   __shared__ int total[BlockSize];
 
@@ -43,6 +44,7 @@ __global__ void AccuracyCudaKernel(const int N, const int D,
   // reduce the count with init value 0, and output accuracy.
   int result = thrust::reduce(thrust::device, total, total + BlockSize, 0);
   if (threadIdx.x == 0) {
+    *correct_data = result;
     *accuracy = static_cast<float>(result) / static_cast<float>(N);
   }
 }
@@ -56,31 +58,48 @@ class AccuracyOpCUDAKernel : public framework::OpKernel<T> {
     auto* inference = ctx.Input<Tensor>("Out");
     auto* indices = ctx.Input<Tensor>("Indices");
     auto* label = ctx.Input<Tensor>("Label");
+
     auto* accuracy = ctx.Output<Tensor>("Accuracy");
+    auto* correct = ctx.Output<Tensor>("Correct");
+    auto* total = ctx.Output<Tensor>("Total");
     // FIXME(typhoonzero): only support indices currently
     // if add support for output values, how to detect the data type?
     const int64_t* indices_data = indices->data<int64_t>();
     const int64_t* label_data = label->data<int64_t>();
+
+    int* correct_data = correct->mutable_data<int>(ctx.GetPlace());
+    int* total_data = total->mutable_data<int>(ctx.GetPlace());
     float* accuracy_data = accuracy->mutable_data<float>(ctx.GetPlace());
 
-    size_t num_samples = inference->dims()[0];
+    int num_samples = static_cast<int>(inference->dims()[0]);
     size_t infer_width = inference->dims()[1];
     PADDLE_ENFORCE(cudaMemset(accuracy_data, 0, sizeof(float)));
+    // cudaMemset((void**)&correct_data, 0, sizeof(float));
 
     if (num_samples == 0) {
       return;
     }
+    cudaMemcpy(total_data, &num_samples, sizeof(int), cudaMemcpyHostToDevice);
 
     AccuracyCudaKernel<PADDLE_CUDA_NUM_THREADS><<<
         1, PADDLE_CUDA_NUM_THREADS, 0, ctx.cuda_device_context().stream()>>>(
-        num_samples, infer_width, indices_data, label_data, accuracy_data);
+        num_samples, infer_width, indices_data, label_data, correct_data,
+        accuracy_data);
+
+    int d_num_samples, d_num_correct;
+    float d_accuracy;
+    cudaMemcpy(&d_num_correct, correct_data, sizeof(int),
+               cudaMemcpyDeviceToHost);
+    cudaMemcpy(&d_num_samples, total_data, sizeof(int), cudaMemcpyDeviceToHost);
+    cudaMemcpy(&d_accuracy, accuracy_data, sizeof(float),
+               cudaMemcpyDeviceToHost);
   }
 };
 
 }  // namespace operators
 }  // namespace paddle
 
-// FIXME(typhoonzero): types of T is for infernece data.
-// label data is always int
+// FIXME(typhoonzero): types of T is for inference data.
+// label data is always int64
 REGISTER_OP_GPU_KERNEL(accuracy, paddle::operators::AccuracyOpCUDAKernel<float>,
                        paddle::operators::AccuracyOpCUDAKernel<double>);

paddle/operators/accuracy_op.h

@@ -29,7 +29,11 @@ class AccuracyKernel : public framework::OpKernel<T> {
     auto* indices = ctx.Input<Tensor>("Indices");
     auto* label = ctx.Input<Tensor>("Label");
     auto* accuracy = ctx.Output<Tensor>("Accuracy");
+    auto* correct = ctx.Output<Tensor>("Correct");
+    auto* total = ctx.Output<Tensor>("Total");
 
+    int* correct_data = correct->mutable_data<int>(ctx.GetPlace());
+    int* total_data = total->mutable_data<int>(ctx.GetPlace());
     float* accuracy_data = accuracy->mutable_data<float>(ctx.GetPlace());
 
     const int64_t* indices_data = indices->data<int64_t>();
@@ -55,7 +59,8 @@ class AccuracyKernel : public framework::OpKernel<T> {
       }
     }
 
-    // FIXME(typhoonzero): we don't accumulate the accuracy for now.
+    *correct_data = num_correct;
+    *total_data = num_samples;
     *accuracy_data =
         static_cast<float>(num_correct) / static_cast<float>(num_samples);
   }

paddle/operators/compare_op.cc

@@ -94,5 +94,13 @@ class CompareOp : public framework::OperatorWithKernel {
 
 REGISTER_LOGICAL_OP(less_than, "Out = X < Y");
 REGISTER_LOGICAL_KERNEL(less_than, CPU, paddle::operators::LessThanFunctor);
+REGISTER_LOGICAL_OP(less_equal, "Out = X <= Y");
+REGISTER_LOGICAL_KERNEL(less_equal, CPU, paddle::operators::LessEqualFunctor);
+REGISTER_LOGICAL_OP(greater_than, "Out = X > Y");
+REGISTER_LOGICAL_KERNEL(greater_than, CPU,
+                        paddle::operators::GreaterThanFunctor);
+REGISTER_LOGICAL_OP(greater_equal, "Out = X >= Y");
+REGISTER_LOGICAL_KERNEL(greater_equal, CPU,
+                        paddle::operators::GreaterEqualFunctor);
 REGISTER_LOGICAL_OP(equal, "Out = X == Y");
 REGISTER_LOGICAL_KERNEL(equal, CPU, paddle::operators::EqualFunctor);

paddle/operators/compare_op.cu

@@ -15,4 +15,9 @@
 #include "paddle/operators/compare_op.h"
 
 REGISTER_LOGICAL_KERNEL(less_than, GPU, paddle::operators::LessThanFunctor);
+REGISTER_LOGICAL_KERNEL(less_equal, GPU, paddle::operators::LessEqualFunctor);
+REGISTER_LOGICAL_KERNEL(greater_than, GPU,
+                        paddle::operators::GreaterThanFunctor);
+REGISTER_LOGICAL_KERNEL(greater_equal, GPU,
+                        paddle::operators::GreaterEqualFunctor);
 REGISTER_LOGICAL_KERNEL(equal, GPU, paddle::operators::EqualFunctor);

paddle/operators/compare_op.h

@@ -27,6 +27,24 @@ struct LessThanFunctor {
   HOSTDEVICE bool operator()(const T& a, const T& b) const { return a < b; }
 };
 
+template <typename T>
+struct LessEqualFunctor {
+  using ELEM_TYPE = T;
+  HOSTDEVICE bool operator()(const T& a, const T& b) const { return a <= b; }
+};
+
+template <typename T>
+struct GreaterThanFunctor {
+  using ELEM_TYPE = T;
+  HOSTDEVICE bool operator()(const T& a, const T& b) const { return a > b; }
+};
+
+template <typename T>
+struct GreaterEqualFunctor {
+  using ELEM_TYPE = T;
+  HOSTDEVICE bool operator()(const T& a, const T& b) const { return a >= b; }
+};
+
 template <typename T>
 struct EqualFunctor {
   using ELEM_TYPE = T;

paddle/operators/elementwise_add_op.cc

@@ -34,7 +34,13 @@ REGISTER_OP(elementwise_add, ops::ElementwiseOp, ops::ElementwiseAddOpMaker,
             elementwise_add_grad, ops::ElementwiseOpGrad);
 REGISTER_OP_CPU_KERNEL(
     elementwise_add,
-    ops::ElementwiseAddKernel<paddle::platform::CPUPlace, float>);
+    ops::ElementwiseAddKernel<paddle::platform::CPUPlace, float>,
+    ops::ElementwiseAddKernel<paddle::platform::CPUPlace, double>,
+    ops::ElementwiseAddKernel<paddle::platform::CPUPlace, int>,
+    ops::ElementwiseAddKernel<paddle::platform::CPUPlace, int64_t>);
 REGISTER_OP_CPU_KERNEL(
     elementwise_add_grad,
-    ops::ElementwiseAddGradKernel<paddle::platform::CPUPlace, float>);
+    ops::ElementwiseAddGradKernel<paddle::platform::CPUPlace, float>,
+    ops::ElementwiseAddGradKernel<paddle::platform::CPUPlace, double>,
+    ops::ElementwiseAddGradKernel<paddle::platform::CPUPlace, int>,
+    ops::ElementwiseAddGradKernel<paddle::platform::CPUPlace, int64_t>);

paddle/operators/elementwise_div_op.cc

@@ -35,7 +35,13 @@ REGISTER_OP(elementwise_div, ops::ElementwiseOp, ops::ElementwiseDivOpMaker,
             elementwise_div_grad, ops::ElementwiseOpGrad);
 REGISTER_OP_CPU_KERNEL(
     elementwise_div,
-    ops::ElementwiseDivKernel<paddle::platform::CPUPlace, float>);
+    ops::ElementwiseDivKernel<paddle::platform::CPUPlace, float>,
+    ops::ElementwiseDivKernel<paddle::platform::CPUPlace, double>,
+    ops::ElementwiseDivKernel<paddle::platform::CPUPlace, int>,
+    ops::ElementwiseDivKernel<paddle::platform::CPUPlace, int64_t>);
 REGISTER_OP_CPU_KERNEL(
     elementwise_div_grad,
-    ops::ElementwiseDivGradKernel<paddle::platform::CPUPlace, float>);
+    ops::ElementwiseDivGradKernel<paddle::platform::CPUPlace, float>,
+    ops::ElementwiseDivGradKernel<paddle::platform::CPUPlace, double>,
+    ops::ElementwiseDivGradKernel<paddle::platform::CPUPlace, int>,
+    ops::ElementwiseDivGradKernel<paddle::platform::CPUPlace, int64_t>);

paddle/operators/elementwise_mul_op.cc

@@ -37,8 +37,12 @@ REGISTER_OP(elementwise_mul, ops::ElementwiseOp, ops::ElementwiseMulOpMaker,
 REGISTER_OP_CPU_KERNEL(
     elementwise_mul,
     ops::ElementwiseMulKernel<paddle::platform::CPUPlace, float>,
-    ops::ElementwiseMulKernel<paddle::platform::CPUPlace, double>);
+    ops::ElementwiseMulKernel<paddle::platform::CPUPlace, double>,
+    ops::ElementwiseMulKernel<paddle::platform::CPUPlace, int>,
+    ops::ElementwiseMulKernel<paddle::platform::CPUPlace, int64_t>);
 REGISTER_OP_CPU_KERNEL(
     elementwise_mul_grad,
     ops::ElementwiseMulGradKernel<paddle::platform::CPUPlace, float>,
-    ops::ElementwiseMulGradKernel<paddle::platform::CPUPlace, double>);
+    ops::ElementwiseMulGradKernel<paddle::platform::CPUPlace, double>,
+    ops::ElementwiseMulGradKernel<paddle::platform::CPUPlace, int>,
+    ops::ElementwiseMulGradKernel<paddle::platform::CPUPlace, int64_t>);

paddle/operators/elementwise_sub_op.cc

@@ -34,7 +34,13 @@ REGISTER_OP(elementwise_sub, ops::ElementwiseOp, ops::ElementwiseSubOpMaker,
             elementwise_sub_grad, ops::ElementwiseOpGrad);
 REGISTER_OP_CPU_KERNEL(
     elementwise_sub,
-    ops::ElementwiseSubKernel<paddle::platform::CPUPlace, float>);
+    ops::ElementwiseSubKernel<paddle::platform::CPUPlace, float>,
+    ops::ElementwiseSubKernel<paddle::platform::CPUPlace, double>,
+    ops::ElementwiseSubKernel<paddle::platform::CPUPlace, int>,
+    ops::ElementwiseSubKernel<paddle::platform::CPUPlace, int64_t>);
 REGISTER_OP_CPU_KERNEL(
     elementwise_sub_grad,
-    ops::ElementwiseSubGradKernel<paddle::platform::CPUPlace, float>);
+    ops::ElementwiseSubGradKernel<paddle::platform::CPUPlace, float>,
+    ops::ElementwiseSubGradKernel<paddle::platform::CPUPlace, double>,
+    ops::ElementwiseSubGradKernel<paddle::platform::CPUPlace, int>,
+    ops::ElementwiseSubGradKernel<paddle::platform::CPUPlace, int64_t>);

paddle/operators/reduce_op.h

@@ -14,6 +14,7 @@
 
 #pragma once
 
+#include "glog/logging.h"
 #include "paddle/framework/eigen.h"
 #include "paddle/framework/op_registry.h"
 
@@ -26,6 +27,10 @@ template <typename T, size_t D, int MajorType = Eigen::RowMajor,
           typename IndexType = Eigen::DenseIndex>
 using EigenTensor = framework::EigenTensor<T, D, MajorType, IndexType>;
 
+template <typename T, int MajorType = Eigen::RowMajor,
+          typename IndexType = Eigen::DenseIndex>
+using EigenScalar = framework::EigenScalar<T, MajorType, IndexType>;
+
 struct SumFunctor {
   template <typename Place, typename X, typename Y, typename Dim>
   void operator()(const Place& place, X& x, Y& y, const Dim& dim) {
@@ -133,11 +138,18 @@ class ReduceKernel : public framework::OpKernel<T> {
       dims_vector.erase(dims_vector.begin() + dim);
       dims = framework::make_ddim(dims_vector);
     }
-    auto out = EigenTensor < T, D == 1 ? 1 : (D - 1) > ::From(*output, dims);
+
     auto& place = context.GetEigenDevice<Place>();
     Functor functor;
+
+    if (D == 1) {
+      auto out = EigenScalar<T>::From(*output);
+      functor(place, x, out, reduce_dim);
+    } else {
+      auto out = EigenTensor<T, (D - 1)>::From(*output, dims);
       functor(place, x, out, reduce_dim);
     }
+  }
 };
 
 template <typename Place, typename T, typename Functor>
@@ -186,13 +198,13 @@ class ReduceGradKernel : public framework::OpKernel<T> {
     auto x_reduce = EigenTensor<T, D>::From(*input1, dims);
     auto x_reduce_grad = EigenTensor<T, D>::From(*input2, dims);
 
-    Eigen::array<int, D> braodcast_dim;
-    for (size_t i = 0; i < D; ++i) braodcast_dim[i] = 1;
-    braodcast_dim[dim] = input0->dims()[dim];
+    Eigen::array<int, D> broadcast_dim;
+    for (size_t i = 0; i < D; ++i) broadcast_dim[i] = 1;
+    broadcast_dim[dim] = input0->dims()[dim];
     auto& place = context.GetEigenDevice<Place>();
     Functor functor;
-    functor(place, x, x_reduce, x_grad, x_reduce_grad, braodcast_dim,
-            braodcast_dim[dim]);
+    functor(place, x, x_reduce, x_grad, x_reduce_grad, broadcast_dim,
+            broadcast_dim[dim]);
   }
 };
 

paddle/parameter/Parameter.cpp

@@ -200,7 +200,10 @@ void Parameter::setMat(ParameterType pType, int matType) {
                                      false,
                                      useGpu_);
     }
-  } else if (matType == MAT_NORMAL_SHARED) {
+  }
+#ifndef PADDLE_MOBILE_INFERENCE
+  // NOLINTNEXTLINE
+  else if (matType == MAT_NORMAL_SHARED) {
     CHECK_EQ(height * width, bufs_[pType]->getSize());
     size_t blockNum = 0;
     CHECK(isGradShared(&blockNum));
@@ -259,7 +262,10 @@ void Parameter::setMat(ParameterType pType, int matType) {
   } else if (matType == MAT_SPARSE_ROW_AUTO_GROW) {
     CHECK(isGradSparseUpdate());
     mats_[pType] = std::make_shared<SparseAutoGrowRowCpuMatrix>(height, width);
-  } else {
+  }
+#endif
+  // NOLINTNEXTLINE
+  else {
     LOG(FATAL) << "Unsupported mat type" << matType;
   }
 }

paddle/scripts/deb/postinst

-#!/bin/bash
-set -e
-echo "Post install paddle debian package."
-echo "Install some python package used for paddle. You can run "
-echo "  pip install /usr/opt/paddle/share/wheels/*.whl to install them."
-find /usr/ -name '*paddle*.whl' | xargs pip install

paddle/scripts/docker/README.md

@@ -2,178 +2,198 @@
 
 ## Goals
 
-We want the building procedure generates Docker images so that we can run PaddlePaddle applications on Kubernetes clusters.
+We want to make the building procedures:
 
-We want to build .deb packages so that enterprise users can run PaddlePaddle applications without Docker.
+1. Static, can reproduce easily.
+1. Generate python `whl` packages that can be widely use cross many distributions.
+1. Build different binaries per release to satisfy different environments:
+    - Binaries for different CUDA and CUDNN versions, like CUDA 7.5, 8.0, 9.0
+    - Binaries containing only capi
+    - Binaries for python with wide unicode support or not.
+1. Build docker images with PaddlePaddle pre-installed, so that we can run
+PaddlePaddle applications directly in docker or on Kubernetes clusters.
 
-We want to minimize the size of generated Docker images and .deb packages so to reduce the download time.
+To achieve this, we created a repo: https://github.com/PaddlePaddle/buildtools
+which gives several docker images that are `manylinux1` sufficient. Then we
+can build PaddlePaddle using these images to generate corresponding `whl`
+binaries.
 
-We want to encapsulate building tools and dependencies in a *development* Docker image so to ease the tools installation for developers.
+## Run The Build
 
-Developers use various editors (emacs, vim, Eclipse, Jupyter Notebook), so the development Docker image contains only building tools, not editing tools, and developers are supposed to git clone source code into their development computers and map the code into the development container.
+### Build Evironments
 
-We want the procedure and tools also work with testing, continuous integration, and releasing.
+The pre-built build environment images are:
 
+| Image | Tag |
+| ----- | --- |
+| paddlepaddle/paddle_manylinux_devel | cuda7.5_cudnn5 |
+| paddlepaddle/paddle_manylinux_devel | cuda8.0_cudnn5 |
+| paddlepaddle/paddle_manylinux_devel | cuda7.5_cudnn7 |
+| paddlepaddle/paddle_manylinux_devel | cuda9.0_cudnn7 |
 
-## Docker Images
-
-So we need two Docker images for each version of PaddlePaddle:
-
-1. `paddle:<version>-dev`
-
-   This a development image contains only the development tools and standardizes the building procedure.  Users include:
+### Start Build
 
-   - developers -- no longer need to install development tools on the host, and can build their current work on the host (development computer).
-   - release engineers -- use this to build the official release from certain branch/tag on Github.com.
-   - document writers / Website developers -- Our documents are in the source repo in the form of .md/.rst files and comments in source code.  We need tools to extract the information, typeset, and generate Web pages.
+Choose one docker image that suit your environment and run the following
+command to start a build:
 
-   Of course, developers can install building tools on their development computers.  But different versions of PaddlePaddle might require different set or version of building tools.  Also, it makes collaborative debugging easier if all developers use a unified development environment.
-
-  The development image should include the following tools:
-
-   - gcc/clang
-   - nvcc
-   - Python
-   - sphinx
-   - woboq
-   - sshd
+```bash
+git clone https://github.com/PaddlePaddle/Paddle.git
+cd Paddle
+docker run --rm -v $PWD:/paddle -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TESTING=OFF" -e "RUN_TEST=OFF" -e "PYTHON_ABI=cp27-cp27mu" paddlepaddle/paddle_manylinux_devel /paddle/paddle/scripts/docker/build.sh
+```
 
-   Many developers work on a remote computer with GPU; they could ssh into the computer and  `docker exec` into the development container. However, running `sshd` in the container allows developers to ssh into the container directly.
+After the build finishes, you can get output `whl` package under
+`build/python/dist`.
 
-1. `paddle:<version>`
+This command mounts the source directory on the host into `/paddle` in the container, then run the build script `/paddle/paddle/scripts/docker/build.sh`
+in the container. When it writes to `/paddle/build` in the container, it writes to `$PWD/build` on the host indeed.
 
-   This is the production image, generated using the development image. This image might have multiple variants:
+### Build Options
 
-   - GPU/AVX   `paddle:<version>-gpu`
-   - GPU/no-AVX  `paddle:<version>-gpu-noavx`
-   - no-GPU/AVX  `paddle:<version>`
-   - no-GPU/no-AVX  `paddle:<version>-noavx`
+Users can specify the following Docker build arguments with either "ON" or "OFF" value:
 
-   We allow users to choose between GPU and no-GPU because the GPU version image is much larger than then the no-GPU version.
+| Option | Default | Description |
+| ------ | -------- | ----------- |
+| `WITH_GPU` | OFF | Generates NVIDIA CUDA GPU code and relies on CUDA libraries. |
+| `WITH_AVX` | OFF | Set to "ON" to enable AVX support. |
+| `WITH_TESTING` | ON | Build unit tests binaries. |
+| `WITH_MKLDNN` | ON | Build with [Intel® MKL DNN](https://github.com/01org/mkl-dnn) support. |
+| `WITH_MKLML` | ON | Build with [Intel® MKL](https://software.intel.com/en-us/mkl) support. |
+| `WITH_GOLANG` | ON | Build fault-tolerant parameter server written in go. |
+| `WITH_SWIG_PY` | ON | Build with SWIG python API support. |
+| `WITH_C_API` | OFF | Build capi libraries for inference. |
+| `WITH_PYTHON` | ON | Build with python support. Turn this off if build is only for capi. |
+| `WITH_STYLE_CHECK` | ON | Check the code style when building. |
+| `PYTHON_ABI` | "" | Build for different python ABI support, can be cp27-cp27m or cp27-cp27mu |
+| `RUN_TEST` | OFF | Run unit test immediently after the build. |
+| `WITH_DOC` | OFF | Build docs after build binaries. |
+| `WOBOQ` | OFF | Generate WOBOQ code viewer under `build/woboq_out` |
 
-   We allow users the choice between AVX and no-AVX, because some cloud providers don't provide AVX-enabled VMs.
 
+## Docker Images
 
-## Development Environment
+You can get the latest PaddlePaddle docker images by
+`docker pull paddlepaddle/paddle:<version>` or build one by yourself.
 
-Here we describe how to use above two images.  We start from considering our daily development environment.
+### Official Docker Releases
 
-Developers work on a computer, which is usually a laptop or desktop:
+Official docker images at
+[here](https://hub.docker.com/r/paddlepaddle/paddle/tags/),
+you can choose either latest or images with a release tag like `0.10.0`,
+Currently available tags are:
 
-<img src="doc/paddle-development-environment.png" width=500 />
+|   Tag  | Description |
+| ------ | --------------------- |
+| latest | latest CPU only image |
+| latest-gpu | latest binary with GPU support |
+| 0.10.0 | release 0.10.0 CPU only binary image |
+| 0.10.0-gpu | release 0.10.0 with GPU support |
 
-or, they might rely on a more sophisticated box (like with GPUs):
+### Build Your Own Image
 
-<img src="doc/paddle-development-environment-gpu.png" width=500 />
+Build PaddlePaddle docker images are quite simple since PaddlePaddle can
+be installed by just running `pip install`. A sample `Dockerfile` is:
 
-A principle here is that source code lies on the development computer (host) so that editors like Eclipse can parse the source code to support auto-completion.
+```dockerfile
+FROM nvidia/cuda:7.5-cudnn5-runtime-centos6
+RUN yum install -y centos-release-SCL
+RUN yum install -y python27
+# This whl package is generated by previous build steps.
+ADD python/dist/paddlepaddle-0.10.0-cp27-cp27mu-linux_x86_64.whl /
+RUN pip install /paddlepaddle-0.10.0-cp27-cp27mu-linux_x86_64.whl && rm -f /*.whl
+```
 
+Then build the image by running `docker build -t [REPO]/paddle:[TAG] .` under
+the directory containing your own `Dockerfile`.
 
-## Usages
+- NOTE: note that you can choose different base images for your environment, you can find all the versions [here](https://hub.docker.com/r/nvidia/cuda/).
 
-### Build the Development Docker Image
+### Use Docker Images
 
-The following commands check out the source code to the host and build the development image `paddle:dev`:
+Suppose that you have written an application program `train.py` using
+PaddlePaddle, we can test and run it using docker:
 
 ```bash
-git clone https://github.com/PaddlePaddle/Paddle paddle
-cd paddle
-docker build -t paddle:dev .
+docker run --rm -it -v $PWD:/work paddlepaddle/paddle /work/a.py
 ```
 
-The `docker build` command assumes that `Dockerfile` is in the root source tree.  Note that in this design, this `Dockerfile` is this only one in our repo.
-
-Users can specify a Ubuntu mirror server for faster downloading:
-
-```bash
-docker build -t paddle:dev --build-arg UBUNTU_MIRROR=mirror://mirrors.ubuntu.com/mirrors.txt .
-```
+But this works only if all dependencies of `train.py` are in the production image. If this is not the case, we need to build a new Docker image from the production image and with more dependencies installs.
 
-### Build PaddlePaddle from Source Code
+### Run PaddlePaddle Book In Docker
 
-Given the development image `paddle:dev`, the following command builds PaddlePaddle from the source tree on the development computer (host):
+Our [book repo](https://github.com/paddlepaddle/book) also provide a docker
+image to start a jupiter notebook inside docker so that you can run this book
+using docker:
 
 ```bash
-docker run --rm -v $PWD:/paddle -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TESTING=OFF" -e "RUN_TEST=OFF" paddle:dev
+docker run -d -p 8888:8888 paddlepaddle/book
 ```
 
-This command mounts the source directory on the host into `/paddle` in the container, so the default entry point of `paddle:dev`, `build.sh`, could build the source code with possible local changes.  When it writes to `/paddle/build` in the container, it writes to `$PWD/build` on the host indeed.
-
-`build.sh` builds the following:
-
-- PaddlePaddle binaries,
-- `$PWD/build/paddle-<version>.deb` for production installation, and
-- `$PWD/build/Dockerfile`, which builds the production Docker image.
+Please refer to https://github.com/paddlepaddle/book if you want to build this
+docker image by your self.
 
-Users can specify the following Docker build arguments with either "ON" or "OFF" value:
-- `WITH_GPU`: ***Required***. Generates NVIDIA CUDA GPU code and relies on CUDA libraries.
-- `WITH_AVX`: ***Required***. Set to "OFF" prevents from generating AVX instructions. If you don't know what is AVX, you might want to set "ON".
-- `WITH_TEST`: ***Optional, default OFF***. Build unit tests binaries. Once you've built the unit tests, you can run these test manually by the following command:
-  ```bash
-    docker run --rm -v $PWD:/paddle -e "WITH_GPU=OFF" -e "WITH_AVX=ON" paddle:dev sh -c "cd /paddle/build; make coverall"
-  ```
-- `RUN_TEST`: ***Optional, default OFF***. Run unit tests after building. You can't run unit tests without building it.
+### Run Distributed Applications
 
-### Build the Production Docker Image
+In our [API design doc](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/api.md#distributed-training), we proposed an API that starts a distributed training job on a cluster.  This API need to build a PaddlePaddle application into a Docker image as above and calls kubectl to run it on the cluster.  This API might need to generate a Dockerfile look like above and call `docker build`.
 
-The following command builds the production image:
+Of course, we can manually build an application image and launch the job using the kubectl tool:
 
 ```bash
-docker build -t paddle -f build/Dockerfile ./build
+docker build -f some/Dockerfile -t myapp .
+docker tag myapp me/myapp
+docker push
+kubectl ...
 ```
 
-This production image is minimal -- it includes binary `paddle`, the shared library `libpaddle.so`, and Python runtime.
+## Docker Images for Developers
 
-### Run PaddlePaddle Applications
+We have a special docker image for developers:
+`paddlepaddle/paddle:<version>-dev`. This image is also generated from
+https://github.com/PaddlePaddle/buildtools
 
-Again the development happens on the host.  Suppose that we have a simple application program in `a.py`, we can test and run it using the production image:
+This a development image contains only the
+development tools and standardizes the building procedure.  Users include:
 
-```bash
-docker run --rm -it -v $PWD:/work paddle /work/a.py
-```
+- developers -- no longer need to install development tools on the host, and can build their current work on the host (development computer).
+- release engineers -- use this to build the official release from certain branch/tag on Github.com.
+- document writers / Website developers -- Our documents are in the source repo in the form of .md/.rst files and comments in source code.  We need tools to extract the information, typeset, and generate Web pages.
 
-But this works only if all dependencies of `a.py` are in the production image. If this is not the case, we need to build a new Docker image from the production image and with more dependencies installs.
+Of course, developers can install building tools on their development computers.  But different versions of PaddlePaddle might require different set or version of building tools.  Also, it makes collaborative debugging easier if all developers use a unified development environment.
 
-### Build and Run PaddlePaddle Applications
+The development image contains the following tools:
 
-We need a Dockerfile in https://github.com/paddlepaddle/book that builds Docker image `paddlepaddle/book:<version>`, basing on the PaddlePaddle production image:
+   - gcc/clang
+   - nvcc
+   - Python
+   - sphinx
+   - woboq
+   - sshd
 
-```
-FROM paddlepaddle/paddle:<version>
-RUN pip install -U matplotlib jupyter ...
-COPY . /book
-EXPOSE 8080
-CMD ["jupyter"]
-```
+Many developers work on a remote computer with GPU; they could ssh into the computer and  `docker exec` into the development container. However, running `sshd` in the container allows developers to ssh into the container directly.
 
-The book image is an example of PaddlePaddle application image.  We can build it
 
-```bash
-git clone https://github.com/paddlepaddle/book
-cd book
-docker build -t book .
-```
+### Development Workflow
 
-### Build and Run Distributed Applications
+Here we describe how the workflow goes on.  We start from considering our daily development environment.
 
-In our [API design doc](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/api.md#distributed-training), we proposed an API that starts a distributed training job on a cluster.  This API need to build a PaddlePaddle application into a Docker image as above and calls kubectl to run it on the cluster.  This API might need to generate a Dockerfile look like above and call `docker build`.
+Developers work on a computer, which is usually a laptop or desktop:
 
-Of course, we can manually build an application image and launch the job using the kubectl tool:
+<img src="doc/paddle-development-environment.png" width=500 />
 
-```bash
-docker build -f some/Dockerfile -t myapp .
-docker tag myapp me/myapp
-docker push
-kubectl ...
-```
+or, they might rely on a more sophisticated box (like with GPUs):
+
+<img src="doc/paddle-development-environment-gpu.png" width=500 />
+
+A principle here is that source code lies on the development computer (host) so that editors like Eclipse can parse the source code to support auto-completion.
 
 ### Reading source code with woboq codebrowser
+
 For developers who are interested in the C++ source code, please use -e "WOBOQ=ON" to enable the building of C++ source code into HTML pages using [Woboq codebrowser](https://github.com/woboq/woboq_codebrowser).
 
 - The following command builds PaddlePaddle, generates HTML pages from C++ source code, and writes HTML pages into `$HOME/woboq_out` on the host:
 
 ```bash
-docker run -v $PWD:/paddle -v $HOME/woboq_out:/woboq_out -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TEST=ON" -e "WOBOQ=ON" paddle:dev
+docker run -v $PWD:/paddle -v $HOME/woboq_out:/woboq_out -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TEST=ON" -e "WOBOQ=ON" paddlepaddle/paddle:latest-dev
 ```
 
 - You can open the generated HTML files in your Web browser. Or, if you want to run a Nginx container to serve them for a wider audience, you can run:

paddle/scripts/docker/build.sh

 #!/bin/bash
 
-set -xe
-
-
 function cmake_gen() {
-    # Set BASE_IMAGE according to env variables
-    if [[ ${WITH_GPU} == "ON" ]]; then
-    BASE_IMAGE="nvidia/cuda:8.0-cudnn5-runtime-ubuntu16.04"
-    else
-    BASE_IMAGE="ubuntu:16.04"
-    fi
-
-    DOCKERFILE_GPU_ENV=""
-    DOCKERFILE_CUDNN_DSO=""
-    if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then
-        DOCKERFILE_GPU_ENV="ENV LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}"
-        DOCKERFILE_CUDNN_DSO="RUN ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.5 /usr/lib/x86_64-linux-gnu/libcudnn.so"
-    fi
-
     mkdir -p /paddle/build
     cd /paddle/build
 
@@ -26,10 +9,29 @@ function cmake_gen() {
     # delete previous built whl packages
     rm -rf /paddle/paddle/dist 2>/dev/null || true
 
+    # Support build for all python versions, currently
+    # including cp27-cp27m and cp27-cp27mu.
+    PYTHON_FLAGS=""
+    if [ "$1" != "" ]; then
+        echo "using python abi: $1"
+        if [ "$1" == "cp27-cp27m" ]; then
+            export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs2/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs4/lib:}
+            PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27m/bin/python
+        -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27m/include/python2.7
+        -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs2/lib/libpython2.7.so"
+        elif [ "$1" == "cp27-cp27mu" ]; then
+            export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs2/lib:}
+            PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27mu/bin/python
+        -DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27mu/include/python2.7
+        -DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs4/lib/libpython2.7.so"
+        fi
+    fi
+
     cat <<EOF
     ========================================
     Configuring cmake in /paddle/build ...
         -DCMAKE_BUILD_TYPE=Release
+        ${PYTHON_FLAGS}
         -DWITH_DOC=OFF
         -DWITH_GPU=${WITH_GPU:-OFF}
         -DWITH_MKLDNN=${WITH_MKLDNN:-ON}
@@ -46,12 +48,12 @@ function cmake_gen() {
         -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
     ========================================
 EOF
-
     # Disable UNITTEST_USE_VIRTUALENV in docker because
     # docker environment is fully controlled by this script.
     # See /Paddle/CMakeLists.txt, UNITTEST_USE_VIRTUALENV option.
     cmake .. \
         -DCMAKE_BUILD_TYPE=Release \
+        ${PYTHON_FLAGS} \
         -DWITH_DOC=OFF \
         -DWITH_GPU=${WITH_GPU:-OFF} \
         -DWITH_MKLDNN=${WITH_MKLDNN:-ON} \
@@ -134,6 +136,19 @@ EOF
 
 
 function gen_dockerfile() {
+    # Set BASE_IMAGE according to env variables
+    if [[ ${WITH_GPU} == "ON" ]]; then
+    BASE_IMAGE="nvidia/cuda:8.0-cudnn5-runtime-ubuntu16.04"
+    else
+    BASE_IMAGE="ubuntu:16.04"
+    fi
+
+    DOCKERFILE_GPU_ENV=""
+    DOCKERFILE_CUDNN_DSO=""
+    if [[ ${WITH_GPU:-OFF} == 'ON' ]]; then
+        DOCKERFILE_GPU_ENV="ENV LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}"
+        DOCKERFILE_CUDNN_DSO="RUN ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.5 /usr/lib/x86_64-linux-gnu/libcudnn.so"
+    fi
 
     cat <<EOF
     ========================================
@@ -168,13 +183,14 @@ EOF
     ${DOCKERFILE_GPU_ENV}
     ADD go/cmd/pserver/pserver /usr/bin/
     ADD go/cmd/master/master /usr/bin/
-    ADD paddle/pybind/print_operators_doc /usr/bin/
     # default command shows the paddle version and exit
     CMD ["paddle", "version"]
 EOF
 }
 
-cmake_gen
+set -xe
+
+cmake_gen ${PYTHON_ABI:-""}
 run_build
 run_test
 gen_docs

paddle/scripts/docker/build_android.sh

@@ -44,7 +44,7 @@ if [ $ANDROID_ABI == "armeabi-v7a" ]; then
         -DHOST_C_COMPILER=/usr/bin/gcc \
         -DHOST_CXX_COMPILER=/usr/bin/g++ \
         -DCMAKE_INSTALL_PREFIX=$DEST_ROOT \
-        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_BUILD_TYPE=MinSizeRel \
         -DUSE_EIGEN_FOR_BLAS=ON \
         -DWITH_C_API=ON \
         -DWITH_SWIG_PY=OFF \
@@ -58,7 +58,7 @@ elif [ $ANDROID_ABI == "arm64-v8a" ]; then
         -DHOST_C_COMPILER=/usr/bin/gcc \
         -DHOST_CXX_COMPILER=/usr/bin/g++ \
         -DCMAKE_INSTALL_PREFIX=$DEST_ROOT \
-        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_BUILD_TYPE=MinSizeRel \
         -DUSE_EIGEN_FOR_BLAS=OFF \
         -DWITH_C_API=ON \
         -DWITH_SWIG_PY=OFF \
@@ -72,7 +72,7 @@ elif [ $ANDROID_ABI == "armeabi" ]; then
         -DHOST_C_COMPILER=/usr/bin/gcc \
         -DHOST_CXX_COMPILER=/usr/bin/g++ \
         -DCMAKE_INSTALL_PREFIX=$DEST_ROOT \
-        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_BUILD_TYPE=MinSizeRel \
         -DWITH_C_API=ON \
         -DWITH_SWIG_PY=OFF \
         -DWITH_STYLE_CHECK=OFF \

paddle/testing/TestUtil.cpp

@@ -33,6 +33,7 @@ MatrixPtr makeRandomSparseMatrix(size_t height,
                                  bool withValue,
                                  bool useGpu,
                                  bool equalNnzPerSample) {
+#ifndef PADDLE_MOBILE_INFERENCE
   std::vector<int64_t> ids(height);
   std::vector<int64_t> indices(height + 1);
   indices[0] = 0;
@@ -84,6 +85,8 @@ MatrixPtr makeRandomSparseMatrix(size_t height,
     }
     return mat;
   }
+#endif
+  return nullptr;
 }
 
 void generateSequenceStartPositions(size_t batchSize,

paddle/trainer/MergeModel.cpp

@@ -27,6 +27,9 @@ using namespace paddle;  // NOLINT
 using namespace std;     // NOLINT
 
 int main(int argc, char** argv) {
+  initMain(argc, argv);
+  initPython(argc, argv);
+
   if (FLAGS_model_dir.empty() || FLAGS_config_file.empty() ||
       FLAGS_model_file.empty()) {
     LOG(INFO) << "Usage: ./paddle_merge_model --model_dir=pass-00000 "
@@ -34,9 +37,6 @@ int main(int argc, char** argv) {
     return 0;
   }
 
-  initMain(argc, argv);
-  initPython(argc, argv);
-
   string confFile = FLAGS_config_file;
 #ifndef PADDLE_WITH_CUDA
   FLAGS_use_gpu = false;

python/paddle/v2/__init__.py

@@ -37,6 +37,8 @@ import model
 import paddle.trainer.config_parser as cp
 
 __all__ = [
+    'default_startup_program',
+    'default_main_program',
     'optimizer',
     'layer',
     'activation',

python/paddle/v2/fluid/evaluator.py

-import paddle.v2.fluid.op as op
 import numpy as np
+from paddle.v2.fluid.framework import Program, g_main_program, unique_name, Variable
 import paddle.v2.fluid.core as core
 
 
-def avg_accumulate(accumulated_var, per_eval, num_batches, place):
-    t = np.array(accumulated_var.get_tensor())
-    t[0] += per_eval[0]
-    accumulated_var.get_tensor().set([t[0] / float(num_batches)], place)
+def _clone_var_in_block_(block, var):
+    assert isinstance(var, Variable)
+    return block.create_var(
+        name=var.name,
+        shape=var.shape,
+        dtype=var.data_type,
+        type=var.type,
+        lod_level=var.lod_level,
+        persistable=True)
 
 
 class Evaluator(object):
-    def __init__(self,
-                 scope,
-                 operator='accuracy',
-                 input='Inference',
-                 label='Label',
-                 output='Output',
-                 place=core.CPUPlace()):
-        """
-        create an evaluator for evaluating the inference.
-        NOTE: default run on CPUPlace(), running on GPUPlace doesn't improve performance much.
-
-        :param scope: the scope instance contains the input.
-        :type scope: paddle.v2.fluid.core.scope
-        :param operator: operator name for caculating the evaluation for each mini-batch.
-        :type operator: string
-        :param input: output variable name of forward network.
-        :type input: string
-        :param label: variable name of label
-        :type label: string
-        """
-        self.scope = scope
-        self.place = place
-        self.output_name = output
-        self.num_batches = 0
-        # create variable to store accumulated evaluator output
-        eval_name = ''.join([operator, "@Eval"])
-        if scope.find_var(eval_name):
-            raise Exception("evaluator already exist in scope: %s" % eval_name)
-        self.accumulated_var = scope.var(eval_name)
-        t = self.accumulated_var.get_tensor()
-        t.set_dims((1, ))
-        t.set([0.0], place)
-        # self.accumulated_var = block.create_var(block, name=eval_name, shape=(1,))
-        # self.accumulated_var.get_tensor().set([0.0])
-        # create operator of evaluation
-        var_map = dict()  # var name -> variable
-        var_map[input] = [input]
-        var_map[label] = [label]
-        var_map[output] = [output]
-        self.op = op.Operator(operator, **var_map)
-
-    def evaluate(self, ctx, accumulator=avg_accumulate):
-        self.op.run(self.scope, ctx)
-        per_eval = np.array(self.scope.find_var(self.output_name).get_tensor())
-        self.num_batches += 1
-        accumulator(self.accumulated_var, per_eval, self.num_batches,
-                    self.place)
+    """
+    Evalutor Base class.
+
+    create metric states
+    add mini-batch evaluator caculate operator
+    add increment operator to accumulate the metric states
+    """
+
+    def __init__(self, name, **kwargs):
+        """
+        init the global states
+        """
+        self._states = {}
+        if kwargs.has_key("main_program"):
+            self._main_program = kwargs.get("main_program")
+        else:
+            self._main_program = g_main_program
+
+    def _update_ops(self, *args, **kwargs):
+        """
+        append update ops to the global states
+        """
+        raise NotImplementedError()
+
+    def reset(self, executor, reset_program=None):
+        """
+        Clear metric states at the begin of each pass/user specified batch
+        """
+        if reset_program == None:
+            reset_program = Program()
+        else:
+            reset_program = program
+        block = reset_program.global_block()
+        for k, var in self._states.iteritems():
+            g_var = _clone_var_in_block_(block, var)
+            zeros = block.create_var(dtype="float32", persistable=True)
+            block.append_op(
+                type="fill_constant",
+                outputs={"Out": [zeros]},
+                attrs={
+                    "shape": g_var.shape,
+                    "value": .0,
+                    "data_type": 5,
+                })
+            block.append_op(
+                type="scale", inputs={"X": zeros}, outputs={"Out": g_var})
+        executor.run(reset_program, fetch_list=self._states.values())
+
+    def eval(self, executor, eval_program=None):
+        """
+        Merge the mini-batch statistics to form the evaluation result for multiple mini-batches.
+        """
+        raise NotImplementedError()
+
+
+class Accuracy(Evaluator):
+    """
+    Accuracy need two state variable Total, Correct
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(Accuracy, self).__init__("accuracy", **kwargs)
+        block = self._main_program.global_block()
+        g_total = block.create_var(
+            name=unique_name("Total"),
+            persistable=True,
+            dtype="int64",
+            shape=[1])
+        g_correct = block.create_var(
+            name=unique_name("Correct"),
+            persistable=True,
+            dtype="int64",
+            shape=[1])
+        self._states["Total"] = g_total
+        self._states["Correct"] = g_correct
+
+    def _update_ops(self, input, label, k=1, **kwargs):
+        block = self._main_program.global_block()
+        topk_out = block.create_var(dtype=input.data_type)
+        topk_indices = block.create_var(dtype="int64")
+        block.append_op(
+            type="top_k",
+            inputs={"X": [input]},
+            outputs={"Out": [topk_out],
+                     "Indices": [topk_indices]},
+            attrs={"k": k})
+        acc_out = block.create_var(dtype=kwargs.get("out_dtype", "float32"))
+        correct = block.create_var(dtype="int64", persistable=True)
+        total = block.create_var(dtype="int64", persistable=True)
+        block.append_op(
+            type="accuracy",
+            inputs={
+                "Out": [topk_out],
+                "Indices": [topk_indices],
+                "Label": [label]
+            },
+            outputs={
+                "Accuracy": [acc_out],
+                "Correct": [correct],
+                "Total": [total],
+            })
+
+        block.append_op(
+            type="cast",
+            inputs={"X": [self._states["Total"]]},
+            outputs={"Out": [self._states["Total"]]},
+            attrs={
+                "in_data_type": 5,  # float32
+                "out_data_type": 2,  #int32
+            })
+        block.append_op(
+            type="cast",
+            inputs={"X": [self._states["Correct"]]},
+            outputs={"Out": [self._states["Correct"]]},
+            attrs={
+                "in_data_type": 5,
+                "out_data_type": 2,
+            })
+
+        block.append_op(
+            type="elementwise_add",
+            inputs={"X": [self._states["Total"]],
+                    "Y": [total]},
+            outputs={"Out": [self._states["Total"]]})
+        block.append_op(
+            type="elementwise_add",
+            inputs={"X": [self._states["Correct"]],
+                    "Y": [correct]},
+            outputs={"Out": [self._states["Correct"]]})
+
+        return acc_out
+
+    def eval(self, executor, eval_program=None):
+        if eval_program != None:
+            eval_program = eval_program
+        else:
+            eval_program = Program()
+        block = eval_program.global_block()
+        eval_out = block.create_var(dtype=self._states["Total"].data_type)
+        e_total = _clone_var_in_block_(block, self._states["Total"])
+        e_correct = _clone_var_in_block_(block, self._states["Correct"])
+        block.append_op(
+            type="cast",
+            inputs={"X": [e_total]},
+            outputs={"Out": [e_total]},
+            attrs={
+                "in_data_type": 2,  #int32
+                "out_data_type": 5,  #float32
+            })
+        block.append_op(
+            type="cast",
+            inputs={"X": [e_correct]},
+            outputs={"Out": [e_correct]},
+            attrs={
+                "in_data_type": 2,
+                "out_data_type": 5,
+            })
+        block.append_op(
+            type="elementwise_div",
+            inputs={"X": e_correct,
+                    "Y": e_total},
+            outputs={"Out": eval_out})
+        out = executor.run(eval_program, fetch_list=[eval_out])
+        return np.array(out[0])
+
+
+def accuracy(*args, **kwargs):
+    cls = Accuracy(*args, **kwargs)
+    out = cls._update_ops(*args, **kwargs)
+    return cls, out

python/paddle/v2/fluid/framework.py

@@ -4,7 +4,7 @@ import collections
 import numpy as np
 import copy
 
-__all__ = ['Block', 'Variable', 'Program', 'Operator']
+__all__ = ['Block', 'Variable', 'Program', 'Operator', 'default_startup_program', 'default_main_program']
 
 
 def unique_name(prefix):
@@ -562,3 +562,9 @@ class Parameter(Variable):
 # program is a global instance.
 g_main_program = Program()
 g_startup_program = Program()
+
+def default_startup_program():
+    return g_startup_program
+
+def default_main_program():
+    return g_main_program

python/paddle/v2/fluid/layers.py

@@ -574,7 +574,9 @@ def accuracy(input, label, k=1, **kwargs):
                  "Indices": [topk_indices]},
         attrs={"k": k})
     acc_out_dtype = kwargs.get("out_dtype", "float32")
-    acc_out = helper.create_tmp_variable(dtype=acc_out_dtype)
+    acc_out = helper.create_tmp_variable(dtype="float32")
+    correct = helper.create_tmp_variable(dtype="int64")
+    total = helper.create_tmp_variable(dtype="int64")
     helper.append_op(
         type="accuracy",
         inputs={
@@ -582,7 +584,11 @@ def accuracy(input, label, k=1, **kwargs):
             "Indices": [topk_indices],
             "Label": [label]
         },
-        outputs={"Accuracy": [acc_out]})
+        outputs={
+            "Accuracy": [acc_out],
+            "Correct": [correct],
+            "Total": [total],
+        })
     return acc_out
 
 

python/paddle/v2/fluid/optimizer.py

@@ -9,7 +9,7 @@ from paddle.v2.fluid.layer_helper import LayerHelper
 
 __all__ = [
     'SGDOptimizer', 'MomentumOptimizer', 'AdagradOptimizer', 'AdamOptimizer',
-    'AdamaxOptimizer'
+    'AdamaxOptimizer', 'DecayedAdagradOptimizer'
 ]
 
 
@@ -85,7 +85,7 @@ class Optimizer(object):
         """
         if (name in self._accumulators and
                 param.name in self._accumulators[name]):
-            raise Exception("Accumulator {} already exists for parmeter {}".
+            raise Exception("Accumulator {} already exists for parameter {}".
                             format(name, param.name))
 
         assert isinstance(self.helper, LayerHelper)
@@ -307,7 +307,7 @@ class AdagradOptimizer(Optimizer):
         moment_acc = self._get_accumulator(self._moment_acc_str,
                                            param_and_grad[0])
 
-        # create the adagrad optimizer op
+        # Create the adagrad optimizer op
         adagrad_op = block.append_op(
             type=self.type,
             inputs={
@@ -510,3 +510,51 @@ class AdamaxOptimizer(Optimizer):
             attrs={"scale": self._beta1})
 
         return [scale_beta1]
+
+
+class DecayedAdagradOptimizer(Optimizer):
+    """Simple Decayed Adagrad optimizer with moment state
+    """
+    _moment_acc_str = "moment"
+
+    def __init__(self,
+                 learning_rate,
+                 decay=0.95,
+                 epsilon=1.0e-6,
+                 global_step=None):
+        assert learning_rate is not None
+        assert decay is not None
+        assert epsilon is not None
+
+        super(DecayedAdagradOptimizer, self).__init__(global_step)
+        self.type = "decayed_adagrad"
+        self._learning_rate = learning_rate
+        self._decay = decay
+        self._epsilon = epsilon
+
+    def _create_accumulators(self, block, parameters):
+        assert isinstance(block, framework.Block)
+
+        for p in parameters:
+            self._add_accumulator(self._moment_acc_str, p)
+
+    def _append_optimize_op(self, block, param_and_grad):
+        assert isinstance(block, framework.Block)
+
+        moment_acc = self._get_accumulator(self._moment_acc_str,
+                                           param_and_grad[0])
+
+        # Create the decayed adagrad optimizer op
+        decayed_adagrad_op = block.append_op(
+            type=self.type,
+            inputs={
+                "Param": param_and_grad[0],
+                "Grad": param_and_grad[1],
+                "Moment": moment_acc,
+                "LearningRate": self._create_param_lr(param_and_grad)
+            },
+            outputs={"ParamOut": param_and_grad[0],
+                     "MomentOut": moment_acc},
+            attrs={"epsilon": self._epsilon})
+
+        return decayed_adagrad_op

python/paddle/v2/fluid/tests/book/test_fit_a_line.py

@@ -2,45 +2,33 @@ import paddle.v2 as paddle
 import paddle.v2.fluid.layers as layers
 import paddle.v2.fluid.core as core
 import paddle.v2.fluid.optimizer as optimizer
-
-from paddle.v2.fluid.framework import Program
+import paddle.v2.fluid.framework as framework
 from paddle.v2.fluid.io import save_persistables, load_persistables
 from paddle.v2.fluid.executor import Executor
 
 import numpy as np
 
-startup_program = Program()
-main_program = Program()
 x = layers.data(
     name='x',
     shape=[13],
-    data_type='float32',
-    main_program=main_program,
-    startup_program=startup_program)
+    data_type='float32')
 
 y_predict = layers.fc(input=x,
                       size=1,
-                      act=None,
-                      main_program=main_program,
-                      startup_program=startup_program)
+                      act=None)
 
 y = layers.data(
     name='y',
     shape=[1],
-    data_type='float32',
-    main_program=main_program,
-    startup_program=startup_program)
+    data_type='float32')
 
 cost = layers.square_error_cost(
     input=y_predict,
-    label=y,
-    main_program=main_program,
-    startup_program=startup_program)
-avg_cost = layers.mean(
-    x=cost, main_program=main_program, startup_program=startup_program)
+    label=y)
+avg_cost = layers.mean(x=cost)
 
 sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001)
-opts = sgd_optimizer.minimize(avg_cost, startup_program)
+opts = sgd_optimizer.minimize(avg_cost)
 
 BATCH_SIZE = 20
 
@@ -52,12 +40,12 @@ train_reader = paddle.batch(
 place = core.CPUPlace()
 exe = Executor(place)
 
-exe.run(startup_program, feed={}, fetch_list=[])
+exe.run(framework.default_startup_program())
 
 PASS_NUM = 100
 for pass_id in range(PASS_NUM):
-    save_persistables(exe, "./fit_a_line.model/", main_program=main_program)
-    load_persistables(exe, "./fit_a_line.model/", main_program=main_program)
+    save_persistables(exe, "./fit_a_line.model/")
+    load_persistables(exe, "./fit_a_line.model/")
     for data in train_reader():
         x_data = np.array(map(lambda x: x[0], data)).astype("float32")
         y_data = np.array(map(lambda x: x[1], data)).astype("float32")
@@ -69,7 +57,7 @@ for pass_id in range(PASS_NUM):
         tensor_y = core.LoDTensor()
         tensor_y.set(y_data, place)
         # print tensor_y.get_dims()
-        outs = exe.run(main_program,
+        outs = exe.run(framework.default_main_program(),
                        feed={'x': tensor_x,
                              'y': tensor_y},
                        fetch_list=[avg_cost])

python/paddle/v2/fluid/tests/book/test_image_classification_train.py

@@ -5,19 +5,17 @@ import paddle.v2.fluid.layers as layers
 import paddle.v2.fluid.nets as nets
 import paddle.v2.fluid.optimizer as optimizer
 from paddle.v2.fluid.executor import Executor
-from paddle.v2.fluid.framework import g_startup_program, g_main_program
+import paddle.v2.fluid.framework as framework
 from paddle.v2.fluid.initializer import XavierInitializer
 
 
-def resnet_cifar10(input, depth=32, main_program=None, startup_program=None):
+def resnet_cifar10(input, depth=32):
     def conv_bn_layer(input,
                       ch_out,
                       filter_size,
                       stride,
                       padding,
-                      act='relu',
-                      main_program=None,
-                      startup_program=None):
+                      act='relu'):
         tmp = layers.conv2d(
             input=input,
             filter_size=filter_size,
@@ -25,14 +23,10 @@ def resnet_cifar10(input, depth=32, main_program=None, startup_program=None):
             stride=stride,
             padding=padding,
             act=None,
-            bias_attr=False,
-            main_program=main_program,
-            startup_program=startup_program)
+            bias_attr=False)
         return layers.batch_norm(
             input=tmp,
-            act=act,
-            main_program=main_program,
-            startup_program=startup_program)
+            act=act)
 
     def shortcut(input, ch_in, ch_out, stride, program, init_program):
         if ch_in != ch_out:
@@ -44,40 +38,30 @@ def resnet_cifar10(input, depth=32, main_program=None, startup_program=None):
     def basicblock(input,
                    ch_in,
                    ch_out,
-                   stride,
-                   main_program=main_program,
-                   startup_program=startup_program):
+                   stride):
         tmp = conv_bn_layer(
             input,
             ch_out,
             3,
             stride,
-            1,
-            main_program=main_program,
-            startup_program=startup_program)
+            1)
         tmp = conv_bn_layer(
             tmp,
             ch_out,
             3,
             1,
             1,
-            act=None,
-            main_program=main_program,
-            startup_program=startup_program)
-        short = shortcut(input, ch_in, ch_out, stride, main_program,
-                         startup_program)
+            act=None)
+        short = shortcut(input, ch_in, ch_out, stride)
         return layers.elementwise_add(
             x=tmp,
             y=short,
-            act='relu',
-            main_program=main_program,
-            startup_program=startup_program)
+            act='relu')
 
-    def layer_warp(block_func, input, ch_in, ch_out, count, stride, program,
-                   startup_program):
-        tmp = block_func(input, ch_in, ch_out, stride, program, startup_program)
+    def layer_warp(block_func, input, ch_in, ch_out, count, stride):
+        tmp = block_func(input, ch_in, ch_out, stride)
         for i in range(1, count):
-            tmp = block_func(tmp, ch_out, ch_out, 1, program, startup_program)
+            tmp = block_func(tmp, ch_out, ch_out, 1)
         return tmp
 
     assert (depth - 2) % 6 == 0
@@ -87,53 +71,41 @@ def resnet_cifar10(input, depth=32, main_program=None, startup_program=None):
         ch_out=16,
         filter_size=3,
         stride=1,
-        padding=1,
-        main_program=main_program,
-        startup_program=startup_program)
+        padding=1)
     res1 = layer_warp(
         basicblock,
         conv1,
         16,
         16,
         n,
-        1,
-        main_program=main_program,
-        startup_program=startup_program)
+        1)
     res2 = layer_warp(
         basicblock,
         res1,
         16,
         32,
         n,
-        2,
-        main_program=main_program,
-        startup_program=startup_program)
+        2)
     res3 = layer_warp(
         basicblock,
         res2,
         32,
         64,
         n,
-        2,
-        main_program=main_program,
-        startup_program=startup_program)
+        2)
     pool = layers.pool2d(
         input=res3,
         pool_size=8,
         pool_type='avg',
-        pool_stride=1,
-        main_program=main_program,
-        startup_program=startup_program)
+        pool_stride=1)
     return pool
 
 
-def vgg16_bn_drop(input, main_program=None, startup_program=None):
+def vgg16_bn_drop(input):
     def conv_block(input,
                    num_filter,
                    groups,
-                   dropouts,
-                   main_program=None,
-                   startup_program=None):
+                   dropouts):
         return nets.img_conv_group(
             input=input,
             pool_size=2,
@@ -143,51 +115,34 @@ def vgg16_bn_drop(input, main_program=None, startup_program=None):
             conv_act='relu',
             conv_with_batchnorm=True,
             conv_batchnorm_drop_rate=dropouts,
-            pool_type='max',
-            main_program=main_program,
-            startup_program=startup_program)
+            pool_type='max')
 
-    conv1 = conv_block(input, 64, 2, [0.3, 0], main_program, startup_program)
-    conv2 = conv_block(conv1, 128, 2, [0.4, 0], main_program, startup_program)
-    conv3 = conv_block(conv2, 256, 3, [0.4, 0.4, 0], main_program,
-                       startup_program)
-    conv4 = conv_block(conv3, 512, 3, [0.4, 0.4, 0], main_program,
-                       startup_program)
-    conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0], main_program,
-                       startup_program)
+    conv1 = conv_block(input, 64, 2, [0.3, 0])
+    conv2 = conv_block(conv1, 128, 2, [0.4, 0])
+    conv3 = conv_block(conv2, 256, 3, [0.4, 0.4, 0])
+    conv4 = conv_block(conv3, 512, 3, [0.4, 0.4, 0])
+    conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0])
 
     drop = layers.dropout(
         x=conv5,
-        dropout_prob=0.5,
-        main_program=main_program,
-        startup_program=startup_program)
+        dropout_prob=0.5)
     fc1 = layers.fc(input=drop,
                     size=512,
                     act=None,
-                    param_attr={"initializer": XavierInitializer()},
-                    main_program=main_program,
-                    startup_program=startup_program)
+                    param_attr={"initializer": XavierInitializer()})
     reshape1 = layers.reshape(
         x=fc1,
-        shape=list(fc1.shape + (1, 1)),
-        main_program=main_program,
-        startup_program=startup_program)
+        shape=list(fc1.shape + (1, 1)))
     bn = layers.batch_norm(
         input=reshape1,
-        act='relu',
-        main_program=main_program,
-        startup_program=startup_program)
+        act='relu')
     drop2 = layers.dropout(
         x=bn,
-        dropout_prob=0.5,
-        main_program=main_program,
-        startup_program=startup_program)
+        dropout_prob=0.5)
     fc2 = layers.fc(input=drop2,
                     size=512,
                     act=None,
-                    param_attr={"initializer": XavierInitializer()},
-                    main_program=main_program,
-                    startup_program=startup_program)
+                    param_attr={"initializer": XavierInitializer()})
     return fc2
 
 
@@ -225,7 +180,7 @@ train_reader = paddle.batch(
 place = core.CPUPlace()
 exe = Executor(place)
 
-exe.run(g_startup_program, feed={}, fetch_list=[])
+exe.run(framework.default_startup_program())
 
 for pass_id in range(PASS_NUM):
     batch_id = 0
@@ -243,7 +198,7 @@ for pass_id in range(PASS_NUM):
         tensor_img.set(img_data, place)
         tensor_y.set(y_data, place)
 
-        outs = exe.run(g_main_program,
+        outs = exe.run(framework.default_main_program(),
                        feed={"pixel": tensor_img,
                              "label": tensor_y},
                        fetch_list=[avg_cost, accuracy])

python/paddle/v2/fluid/tests/book/test_recognize_digits_conv.py

@@ -3,67 +3,46 @@ import paddle.v2.fluid.layers as layers
 import paddle.v2.fluid.nets as nets
 import paddle.v2.fluid.core as core
 import paddle.v2.fluid.optimizer as optimizer
-
-from paddle.v2.fluid.framework import Program
+import paddle.v2.fluid.evaluator as evaluator
+import paddle.v2.fluid.framework as framework
 from paddle.v2.fluid.executor import Executor
 
 import numpy as np
 
-startup_program = Program()
-main_program = Program()
-
 images = layers.data(
     name='pixel',
     shape=[1, 28, 28],
-    data_type='float32',
-    main_program=main_program,
-    startup_program=startup_program)
+    data_type='float32')
 label = layers.data(
     name='label',
     shape=[1],
-    data_type='int64',
-    main_program=main_program,
-    startup_program=startup_program)
+    data_type='int64')
 conv_pool_1 = nets.simple_img_conv_pool(
     input=images,
     filter_size=5,
     num_filters=20,
     pool_size=2,
     pool_stride=2,
-    act="relu",
-    main_program=main_program,
-    startup_program=startup_program)
+    act="relu")
 conv_pool_2 = nets.simple_img_conv_pool(
     input=conv_pool_1,
     filter_size=5,
     num_filters=50,
     pool_size=2,
     pool_stride=2,
-    act="relu",
-    main_program=main_program,
-    startup_program=startup_program)
+    act="relu")
 
 predict = layers.fc(input=conv_pool_2,
                     size=10,
-                    act="softmax",
-                    main_program=main_program,
-                    startup_program=startup_program)
-cost = layers.cross_entropy(
-    input=predict,
-    label=label,
-    main_program=main_program,
-    startup_program=startup_program)
-avg_cost = layers.mean(x=cost, main_program=main_program)
-accuracy = layers.accuracy(
-    input=predict,
-    label=label,
-    main_program=main_program,
-    startup_program=startup_program)
-
-# optimizer = optimizer.MomentumOptimizer(learning_rate=0.1 / 128.0,
-# momentum=0.9)
+                    act="softmax")
+cost = layers.cross_entropy(input=predict, label=label)
+avg_cost = layers.mean(x=cost)
 optimizer = optimizer.AdamOptimizer(learning_rate=0.01, beta1=0.9, beta2=0.999)
-opts = optimizer.minimize(avg_cost, startup_program)
+opts = optimizer.minimize(avg_cost)
+
+accuracy, acc_out = evaluator.accuracy(
+    input=predict,
+    label=label)
 
 BATCH_SIZE = 50
 PASS_NUM = 3
@@ -75,10 +54,11 @@ train_reader = paddle.batch(
 place = core.CPUPlace()
 exe = Executor(place)
 
-exe.run(startup_program, feed={}, fetch_list=[])
+exe.run(framework.default_startup_program())
 
 for pass_id in range(PASS_NUM):
     count = 0
+    accuracy.reset(exe)
     for data in train_reader():
         img_data = np.array(map(lambda x: x[0].reshape([1, 28, 28]),
                                 data)).astype("float32")
@@ -90,14 +70,20 @@ for pass_id in range(PASS_NUM):
         tensor_img.set(img_data, place)
         tensor_y.set(y_data, place)
 
-        outs = exe.run(main_program,
+        outs = exe.run(framework.default_main_program(),
                        feed={"pixel": tensor_img,
                              "label": tensor_y},
-                       fetch_list=[avg_cost, accuracy])
+                       fetch_list=[avg_cost, acc_out])
         loss = np.array(outs[0])
         acc = np.array(outs[1])
-
+        pass_acc = accuracy.eval(exe)
+        print "pass id : ", pass_id, pass_acc
+        # print loss, acc
         if loss < 10.0 and acc > 0.9:
             # if avg cost less than 10.0 and accuracy is larger than 0.9, we think our code is good.
             exit(0)
+
+    pass_acc = accuracy.eval(exe)
+    print "pass id : ", pass_id, pass_acc
+
 exit(1)

python/paddle/v2/fluid/tests/book/test_recognize_digits_mlp.py

@@ -2,8 +2,7 @@ import paddle.v2 as paddle
 import paddle.v2.fluid.layers as layers
 import paddle.v2.fluid.core as core
 import paddle.v2.fluid.optimizer as optimizer
-
-from paddle.v2.fluid.framework import Program
+import paddle.v2.fluid.framework as framework
 from paddle.v2.fluid.executor import Executor
 from paddle.v2.fluid.regularizer import L2DecayRegularizer
 from paddle.v2.fluid.initializer import UniformInitializer
@@ -11,14 +10,10 @@ from paddle.v2.fluid.initializer import UniformInitializer
 import numpy as np
 
 BATCH_SIZE = 128
-startup_program = Program()
-main_program = Program()
 image = layers.data(
     name='x',
     shape=[784],
-    data_type='float32',
-    main_program=main_program,
-    startup_program=startup_program)
+    data_type='float32')
 
 param_attr = {
     'name': None,
@@ -30,45 +25,30 @@ param_attr = {
 hidden1 = layers.fc(input=image,
                     size=128,
                     act='relu',
-                    main_program=main_program,
-                    startup_program=startup_program,
                     param_attr=param_attr)
 hidden2 = layers.fc(input=hidden1,
                     size=64,
                     act='relu',
-                    main_program=main_program,
-                    startup_program=startup_program,
                     param_attr=param_attr)
 
 predict = layers.fc(input=hidden2,
                     size=10,
                     act='softmax',
-                    main_program=main_program,
-                    startup_program=startup_program,
                     param_attr=param_attr)
 
 label = layers.data(
     name='y',
     shape=[1],
-    data_type='int64',
-    main_program=main_program,
-    startup_program=startup_program)
+    data_type='int64')
 
-cost = layers.cross_entropy(
-    input=predict,
-    label=label,
-    main_program=main_program,
-    startup_program=startup_program)
-avg_cost = layers.mean(
-    x=cost, main_program=main_program, startup_program=startup_program)
+cost = layers.cross_entropy(input=predict, label=label)
+avg_cost = layers.mean(x=cost)
 accuracy = layers.accuracy(
     input=predict,
-    label=label,
-    main_program=main_program,
-    startup_program=startup_program)
+    label=label)
 
 optimizer = optimizer.MomentumOptimizer(learning_rate=0.001, momentum=0.9)
-opts = optimizer.minimize(avg_cost, startup_program)
+opts = optimizer.minimize(avg_cost)
 
 train_reader = paddle.batch(
     paddle.reader.shuffle(
@@ -78,7 +58,7 @@ train_reader = paddle.batch(
 place = core.CPUPlace()
 exe = Executor(place)
 
-exe.run(startup_program, feed={}, fetch_list=[])
+exe.run(framework.default_startup_program())
 
 PASS_NUM = 100
 for pass_id in range(PASS_NUM):
@@ -93,7 +73,7 @@ for pass_id in range(PASS_NUM):
         tensor_y = core.LoDTensor()
         tensor_y.set(y_data, place)
 
-        outs = exe.run(main_program,
+        outs = exe.run(framework.default_main_program(),
                        feed={'x': tensor_x,
                              'y': tensor_y},
                        fetch_list=[avg_cost, accuracy])

python/paddle/v2/fluid/tests/book/test_recommender_system.py

@@ -3,16 +3,13 @@ import paddle.v2.fluid.layers as layers
 import paddle.v2.fluid.nets as nets
 import paddle.v2.fluid.core as core
 import paddle.v2.fluid.optimizer as optimizer
-
-from paddle.v2.fluid.framework import Program
+import paddle.v2.fluid.framework as framework
 from paddle.v2.fluid.executor import Executor
 
 import numpy as np
 
-startup_program = Program()
-main_program = Program()
-is_sparse = True
-use_gpu = False
+IS_SPARSE = True
+USE_GPU = False
 BATCH_SIZE = 256
 
 
@@ -25,99 +22,71 @@ def get_usr_combined_features():
     uid = layers.data(
         name='user_id',
         shape=[1],
-        data_type='int64',
-        main_program=main_program,
-        startup_program=startup_program)
+        data_type='int64')
 
     usr_emb = layers.embedding(
         input=uid,
         data_type='float32',
         size=[USR_DICT_SIZE, 32],
         param_attr={'name': 'user_table'},
-        is_sparse=is_sparse,
-        main_program=main_program,
-        startup_program=startup_program)
+        is_sparse=IS_SPARSE)
 
     usr_fc = layers.fc(input=usr_emb,
-                       size=32,
-                       main_program=main_program,
-                       startup_program=startup_program)
+                       size=32)
 
     USR_GENDER_DICT_SIZE = 2
 
     usr_gender_id = layers.data(
         name='gender_id',
         shape=[1],
-        data_type='int64',
-        main_program=main_program,
-        startup_program=startup_program)
+        data_type='int64')
 
     usr_gender_emb = layers.embedding(
         input=usr_gender_id,
         size=[USR_GENDER_DICT_SIZE, 16],
         param_attr={'name': 'gender_table'},
-        is_sparse=is_sparse,
-        main_program=main_program,
-        startup_program=startup_program)
+        is_sparse=IS_SPARSE)
 
     usr_gender_fc = layers.fc(input=usr_gender_emb,
-                              size=16,
-                              main_program=main_program,
-                              startup_program=startup_program)
+                              size=16)
 
     USR_AGE_DICT_SIZE = len(paddle.dataset.movielens.age_table)
     usr_age_id = layers.data(
         name='age_id',
         shape=[1],
-        data_type="int64",
-        main_program=main_program,
-        startup_program=startup_program)
+        data_type="int64")
 
     usr_age_emb = layers.embedding(
         input=usr_age_id,
         size=[USR_AGE_DICT_SIZE, 16],
-        is_sparse=is_sparse,
-        param_attr={'name': 'age_table'},
-        main_program=main_program,
-        startup_program=startup_program)
+        is_sparse=IS_SPARSE,
+        param_attr={'name': 'age_table'})
 
     usr_age_fc = layers.fc(input=usr_age_emb,
-                           size=16,
-                           main_program=main_program,
-                           startup_program=startup_program)
+                           size=16)
 
     USR_JOB_DICT_SIZE = paddle.dataset.movielens.max_job_id() + 1
     usr_job_id = layers.data(
         name='job_id',
         shape=[1],
-        data_type="int64",
-        main_program=main_program,
-        startup_program=startup_program)
+        data_type="int64")
 
     usr_job_emb = layers.embedding(
         input=usr_job_id,
         size=[USR_JOB_DICT_SIZE, 16],
         param_attr={'name': 'job_table'},
-        is_sparse=is_sparse,
-        main_program=main_program,
-        startup_program=startup_program)
+        is_sparse=IS_SPARSE)
 
     usr_job_fc = layers.fc(input=usr_job_emb,
-                           size=16,
-                           main_program=main_program,
-                           startup_program=startup_program)
+                           size=16)
 
     concat_embed = layers.concat(
         input=[usr_fc, usr_gender_fc, usr_age_fc, usr_job_fc],
-        axis=1,
-        main_program=main_program,
-        startup_program=startup_program)
+        axis=1)
 
     usr_combined_features = layers.fc(input=concat_embed,
                                       size=200,
-                                      act="tanh",
-                                      main_program=main_program,
-                                      startup_program=startup_program)
+                                      act="tanh")
 
     return usr_combined_features
 
@@ -129,83 +98,61 @@ def get_mov_combined_features():
     mov_id = layers.data(
         name='movie_id',
         shape=[1],
-        data_type='int64',
-        main_program=main_program,
-        startup_program=startup_program)
+        data_type='int64')
 
     mov_emb = layers.embedding(
         input=mov_id,
         data_type='float32',
         size=[MOV_DICT_SIZE, 32],
         param_attr={'name': 'movie_table'},
-        is_sparse=is_sparse,
-        main_program=main_program,
-        startup_program=startup_program)
+        is_sparse=IS_SPARSE)
 
     mov_fc = layers.fc(input=mov_emb,
-                       size=32,
-                       main_program=main_program,
-                       startup_program=startup_program)
+                       size=32)
 
     CATEGORY_DICT_SIZE = len(paddle.dataset.movielens.movie_categories())
 
     category_id = layers.data(
         name='category_id',
         shape=[1],
-        data_type='int64',
-        main_program=main_program,
-        startup_program=startup_program)
+        data_type='int64')
 
     mov_categories_emb = layers.embedding(
         input=category_id,
         size=[CATEGORY_DICT_SIZE, 32],
-        is_sparse=is_sparse,
-        main_program=main_program,
-        startup_program=startup_program)
+        is_sparse=IS_SPARSE)
 
     mov_categories_hidden = layers.sequence_pool(
         input=mov_categories_emb,
-        pool_type="sum",
-        main_program=main_program,
-        startup_program=startup_program)
+        pool_type="sum")
 
     MOV_TITLE_DICT_SIZE = len(paddle.dataset.movielens.get_movie_title_dict())
 
     mov_title_id = layers.data(
         name='movie_title',
         shape=[1],
-        data_type='int64',
-        main_program=main_program,
-        startup_program=startup_program)
+        data_type='int64')
 
     mov_title_emb = layers.embedding(
         input=mov_title_id,
         size=[MOV_TITLE_DICT_SIZE, 32],
-        is_sparse=is_sparse,
-        main_program=main_program,
-        startup_program=startup_program)
+        is_sparse=IS_SPARSE)
 
     mov_title_conv = nets.sequence_conv_pool(
         input=mov_title_emb,
         num_filters=32,
         filter_size=3,
         act="tanh",
-        pool_type="sum",
-        main_program=main_program,
-        startup_program=startup_program)
+        pool_type="sum")
 
     concat_embed = layers.concat(
         input=[mov_fc, mov_categories_hidden, mov_title_conv],
-        axis=1,
-        main_program=main_program,
-        startup_program=startup_program)
+        axis=1)
 
     # FIXME(dzh) : need tanh operator
     mov_combined_features = layers.fc(input=concat_embed,
                                       size=200,
-                                      act="tanh",
-                                      main_program=main_program,
-                                      startup_program=startup_program)
+                                      act="tanh")
 
     return mov_combined_features
 
@@ -217,27 +164,18 @@ def model():
     # need cos sim
     inference = layers.cos_sim(
         X=usr_combined_features,
-        Y=mov_combined_features,
-        main_program=main_program,
-        startup_program=startup_program)
+        Y=mov_combined_features)
 
     label = layers.data(
         name='score',
         shape=[1],
-        data_type='float32',
-        main_program=main_program,
-        startup_program=startup_program)
+        data_type='float32')
 
     square_cost = layers.square_error_cost(
         input=inference,
-        label=label,
-        main_program=main_program,
-        startup_program=startup_program)
+        label=label)
 
-    avg_cost = layers.mean(
-        x=square_cost,
-        main_program=main_program,
-        startup_program=startup_program)
+    avg_cost = layers.mean(x=square_cost)
 
     return avg_cost
 
@@ -245,16 +183,15 @@ def model():
 def main():
     cost = model()
     sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.2)
-    opts = sgd_optimizer.minimize(cost, startup_program=startup_program)
-    block = main_program.block(0)
+    opts = sgd_optimizer.minimize(cost)
 
-    if use_gpu:
+    if USE_GPU:
         place = core.GPUPlace(0)
     else:
         place = core.CPUPlace()
 
     exe = Executor(place)
-    exe.run(startup_program, feed={}, fetch_list=[])
+    exe.run(framework.default_startup_program())
 
     train_reader = paddle.batch(
         paddle.reader.shuffle(
@@ -303,7 +240,7 @@ def main():
     PASS_NUM = 100
     for pass_id in range(PASS_NUM):
         for data in train_reader():
-            outs = exe.run(main_program,
+            outs = exe.run(framework.default_main_program(),
                            feed=func_feed(feeding, data),
                            fetch_list=[cost])
             out = np.array(outs[0])

python/paddle/v2/fluid/tests/book/test_understand_sentiment_conv.py

@@ -3,8 +3,7 @@ import paddle.v2.fluid.layers as layers
 import paddle.v2.fluid.nets as nets
 import paddle.v2.fluid.core as core
 import paddle.v2.fluid.optimizer as optimizer
-
-from paddle.v2.fluid.framework import Program, g_main_program, g_startup_program
+import paddle.v2.fluid.framework as framework
 from paddle.v2.fluid.executor import Executor
 
 import numpy as np
@@ -70,7 +69,7 @@ def main():
     place = core.CPUPlace()
     exe = Executor(place)
 
-    exe.run(g_startup_program)
+    exe.run(framework.default_startup_program())
 
     for pass_id in xrange(PASS_NUM):
         for data in train_data():
@@ -82,7 +81,7 @@ def main():
             tensor_label = core.LoDTensor()
             tensor_label.set(label, place)
 
-            outs = exe.run(g_main_program,
+            outs = exe.run(framework.default_main_program(),
                            feed={"words": tensor_words,
                                  "label": tensor_label},
                            fetch_list=[cost, acc])

python/paddle/v2/fluid/tests/book/test_understand_sentiment_dynamic_lstm.py

@@ -3,8 +3,7 @@ import paddle.v2.fluid.layers as layers
 import paddle.v2.fluid.nets as nets
 import paddle.v2.fluid.core as core
 import paddle.v2.fluid.optimizer as optimizer
-
-from paddle.v2.fluid.framework import Program, g_main_program, g_startup_program
+import paddle.v2.fluid.framework as framework
 from paddle.v2.fluid.executor import Executor
 
 import numpy as np
@@ -81,7 +80,7 @@ def main():
     place = core.CPUPlace()
     exe = Executor(place)
 
-    exe.run(g_startup_program)
+    exe.run(framework.default_startup_program())
 
     for pass_id in xrange(PASS_NUM):
         for data in train_data():
@@ -93,7 +92,7 @@ def main():
             tensor_label = core.LoDTensor()
             tensor_label.set(label, place)
 
-            outs = exe.run(g_main_program,
+            outs = exe.run(framework.default_main_program(),
                            feed={"words": tensor_words,
                                  "label": tensor_label},
                            fetch_list=[cost, acc])

python/paddle/v2/fluid/tests/book/test_understand_sentiment_lstm.py

@@ -2,8 +2,7 @@ import paddle.v2 as paddle
 import paddle.v2.fluid.layers as layers
 import paddle.v2.fluid.core as core
 import paddle.v2.fluid.optimizer as optimizer
-
-from paddle.v2.fluid.framework import g_main_program, g_startup_program
+import paddle.v2.fluid.framework as framework
 from paddle.v2.fluid.executor import Executor
 
 import numpy as np
@@ -88,10 +87,10 @@ def main():
     place = core.CPUPlace()
     tensor_words, tensor_label = prepare_feed_data(data, place)
     exe = Executor(place)
-    exe.run(g_startup_program)
+    exe.run(framework.default_startup_program())
 
     while True:
-        outs = exe.run(g_main_program,
+        outs = exe.run(framework.default_main_program(),
                        feed={"words": tensor_words,
                              "label": tensor_label},
                        fetch_list=[cost, acc])

python/paddle/v2/fluid/tests/book/test_word2vec.py

@@ -2,20 +2,17 @@ import paddle.v2 as paddle
 import paddle.v2.fluid.layers as layers
 import paddle.v2.fluid.core as core
 import paddle.v2.fluid.optimizer as optimizer
-
-from paddle.v2.fluid.framework import Program
+import paddle.v2.fluid.framework as framework
 from paddle.v2.fluid.executor import Executor
 
 import numpy as np
 
-startup_program = Program()
-main_program = Program()
-
-embed_size = 32
-hidden_size = 256
+PASS_NUM = 100
+EMBED_SIZE = 32
+HIDDEN_SIZE = 256
 N = 5
-batch_size = 32
-is_sparse = True
+BATCH_SIZE = 32
+IS_SPARSE = True
 
 word_dict = paddle.dataset.imikolov.build_dict()
 dict_size = len(word_dict)
@@ -23,97 +20,67 @@ dict_size = len(word_dict)
 first_word = layers.data(
     name='firstw',
     shape=[1],
-    data_type='int64',
-    main_program=main_program,
-    startup_program=startup_program)
+    data_type='int64')
 second_word = layers.data(
     name='secondw',
     shape=[1],
-    data_type='int64',
-    main_program=main_program,
-    startup_program=startup_program)
+    data_type='int64')
 third_word = layers.data(
     name='thirdw',
     shape=[1],
-    data_type='int64',
-    main_program=main_program,
-    startup_program=startup_program)
+    data_type='int64')
 forth_word = layers.data(
     name='forthw',
     shape=[1],
-    data_type='int64',
-    main_program=main_program,
-    startup_program=startup_program)
+    data_type='int64')
 next_word = layers.data(
     name='nextw',
     shape=[1],
-    data_type='int64',
-    main_program=main_program,
-    startup_program=startup_program)
+    data_type='int64')
 
 embed_first = layers.embedding(
     input=first_word,
-    size=[dict_size, embed_size],
+    size=[dict_size, EMBED_SIZE],
     data_type='float32',
-    is_sparse=is_sparse,
-    param_attr={'name': 'shared_w'},
-    main_program=main_program,
-    startup_program=startup_program)
+    is_sparse=IS_SPARSE,
+    param_attr={'name': 'shared_w'})
 embed_second = layers.embedding(
     input=second_word,
-    size=[dict_size, embed_size],
+    size=[dict_size, EMBED_SIZE],
     data_type='float32',
-    is_sparse=is_sparse,
-    param_attr={'name': 'shared_w'},
-    main_program=main_program,
-    startup_program=startup_program)
-
+    is_sparse=IS_SPARSE,
+    param_attr={'name': 'shared_w'})
 embed_third = layers.embedding(
     input=third_word,
-    size=[dict_size, embed_size],
+    size=[dict_size, EMBED_SIZE],
     data_type='float32',
-    is_sparse=is_sparse,
-    param_attr={'name': 'shared_w'},
-    main_program=main_program,
-    startup_program=startup_program)
+    is_sparse=IS_SPARSE,
+    param_attr={'name': 'shared_w'})
 embed_forth = layers.embedding(
     input=forth_word,
-    size=[dict_size, embed_size],
+    size=[dict_size, EMBED_SIZE],
     data_type='float32',
-    is_sparse=is_sparse,
-    param_attr={'name': 'shared_w'},
-    main_program=main_program,
-    startup_program=startup_program)
+    is_sparse=IS_SPARSE,
+    param_attr={'name': 'shared_w'})
 
 concat_embed = layers.concat(
     input=[embed_first, embed_second, embed_third, embed_forth],
-    axis=1,
-    main_program=main_program,
-    startup_program=startup_program)
-
+    axis=1)
 hidden1 = layers.fc(input=concat_embed,
-                    size=hidden_size,
-                    act='sigmoid',
-                    main_program=main_program,
-                    startup_program=startup_program)
+                    size=HIDDEN_SIZE,
+                    act='sigmoid')
 predict_word = layers.fc(input=hidden1,
                          size=dict_size,
-                         act='softmax',
-                         main_program=main_program,
-                         startup_program=startup_program)
+                         act='softmax')
 cost = layers.cross_entropy(
     input=predict_word,
-    label=next_word,
-    main_program=main_program,
-    startup_program=startup_program)
-avg_cost = layers.mean(
-    x=cost, main_program=main_program, startup_program=startup_program)
-
+    label=next_word)
+avg_cost = layers.mean(x=cost)
 sgd_optimizer = optimizer.SGDOptimizer(learning_rate=0.001)
-opts = sgd_optimizer.minimize(avg_cost, startup_program)
+opts = sgd_optimizer.minimize(avg_cost)
 
 train_reader = paddle.batch(
-    paddle.dataset.imikolov.train(word_dict, N), batch_size)
+    paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE)
 
 place = core.CPUPlace()
 exe = Executor(place)
@@ -122,8 +89,8 @@ exe = Executor(place)
 # below exit line.
 exit(0)
 
-exe.run(startup_program, feed={}, fetch_list=[])
-PASS_NUM = 100
+exe.run(framework.default_startup_program())
+
 for pass_id in range(PASS_NUM):
     for data in train_reader():
         input_data = [[data_idx[idx] for data_idx in data] for idx in xrange(5)]
@@ -150,7 +117,7 @@ for pass_id in range(PASS_NUM):
         next_tensor = core.LoDTensor()
         next_tensor.set(next_data, place)
 
-        outs = exe.run(main_program,
+        outs = exe.run(framework.default_main_program(),
                        feed={
                            'firstw': first_tensor,
                            'secondw': second_tensor,

python/paddle/v2/fluid/tests/test_accuracy_op.py

@@ -18,7 +18,9 @@ class TestAccuracyOp(OpTest):
                     num_correct += 1
                     break
         self.outputs = {
-            'Accuracy': np.array([num_correct / float(n)]).astype("float32")
+            'Accuracy': np.array([num_correct / float(n)]).astype("float32"),
+            'Correct': np.array([num_correct]).astype("int32"),
+            'Total': np.array([n]).astype("int32")
         }
 
     def test_check_output(self):

python/paddle/v2/framework/tests/test_beam_search_decode_op.py → python/paddle/v2/fluid/tests/test_beam_search_decode_op.py

 import unittest
 
 import numpy as np
-import paddle.v2.framework.core as core
-from paddle.v2.framework.op import Operator
+import paddle.v2.fluid.core as core
+from paddle.v2.fluid.op import Operator
 
 
 class TestBeamSearchDecodeOp(unittest.TestCase):

python/paddle/v2/fluid/tests/test_compare_op.py

@@ -23,6 +23,9 @@ def create_test_class(op_type, typename, callback):
 
 for _type_name in {'float32', 'float64', 'int32', 'int64'}:
     create_test_class('less_than', _type_name, lambda _a, _b: _a < _b)
+    create_test_class('less_equal', _type_name, lambda _a, _b: _a <= _b)
+    create_test_class('greater_than', _type_name, lambda _a, _b: _a > _b)
+    create_test_class('greater_equal', _type_name, lambda _a, _b: _a >= _b)
     create_test_class('equal', _type_name, lambda _a, _b: _a == _b)
 
 if __name__ == '__main__':

python/paddle/v2/fluid/tests/test_evaluator.py

-from paddle.v2.fluid.evaluator import Evaluator
-from paddle.v2.fluid.op import Operator
-import paddle.v2.fluid.core as core
-import unittest
-import op_test
-import numpy as np
-
-
-class TestEvaluator(unittest.TestCase):
-    def setup(self, scope, inputs, outputs):
-        def __create_var__(var_name, arr):
-            np_arr = np.array(arr)
-            scope.var(var_name)
-            # tensor = var.get_tensor()
-            # tensor.set_dims(np_arr.shape)
-
-        for var_name, arr in inputs.iteritems():
-            __create_var__(var_name, arr)
-
-        for var_name, arr in outputs.iteritems():
-            __create_var__(var_name, arr)
-
-    def test_evaluator(self):
-
-        inputs = {
-            'Inference': np.array([[1, 1, 1, 1, 1, 0, 0, 0, 0, 1]]).T,
-            'Label': np.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0])
-        }
-        outputs = {'Accuracy': np.array([0.9])}
-        out_name = 'Accuracy'
-
-        places = [core.CPUPlace()]
-        if core.is_compile_gpu():
-            places.append(core.GPUPlace(0))
-
-        for place in places:
-            scope = core.Scope()
-            self.setup(scope, inputs, outputs)
-
-            evaluator = Evaluator(
-                scope,
-                operator='accuracy',
-                input='Inference',
-                label='Label',
-                output=out_name,
-                place=place)
-            op_test.set_input(scope, evaluator.op, inputs, place)
-            ctx = core.DeviceContext.create(place)
-
-            for i in range(10):  # simulate 10 mini-batches
-                evaluator.evaluate(ctx)
-
-            actual = np.array(scope.find_var(out_name).get_tensor())
-            print actual
-
-            self.assertTrue(
-                np.allclose(
-                    actual, outputs[out_name], atol=1e-5),
-                "output name: " + out_name + " has diff.")
-
-
-if __name__ == '__main__':
-    exit(0)
-    unittest.main()

python/paddle/v2/fluid/tests/test_optimizer.py

@@ -198,7 +198,7 @@ class TestAdagradOptimizer(unittest.TestCase):
         adagrad_op = opts[0]
         self.assertEqual(adagrad_op.type, "adagrad")
 
-        # check accumulators
+        # Check accumulators
         accumulators = adagrad_optimizer.get_accumulators()
         self.assertEqual(len(accumulators), 1)
         self.assertTrue(adagrad_optimizer.get_moment_str() in accumulators)
@@ -331,5 +331,59 @@ class TestAdamaxOptimizer(unittest.TestCase):
         self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate)
 
 
+class TestDecayedAdagradOptimizer(unittest.TestCase):
+    class MockDecayedAdagrad(optimizer.DecayedAdagradOptimizer):
+        def get_accumulators(self):
+            return self._accumulators
+
+        def get_moment_str(self):
+            return self._moment_acc_str
+
+    def test_decayed_adagrad_optimizer(self):
+        init_program = framework.Program()
+        program = framework.Program()
+        block = program.global_block()
+        mul_x = block.create_parameter(
+            dtype="float32", shape=[5, 10], lod_level=0, name="mul.x")
+        mul_y = block.create_var(
+            dtype="float32", shape=[10, 8], lod_level=0, name="mul.y")
+        mul_out = block.create_var(
+            dtype="float32", shape=[5, 8], lod_level=0, name="mul.out")
+        block.append_op(
+            type="mul",
+            inputs={"X": mul_x,
+                    "Y": mul_y},
+            outputs={"Out": mul_out},
+            attrs={"x_num_col_dims": 1})
+        learning_rate = 0.01
+        decayed_adagrad_optimizer = self.MockDecayedAdagrad(
+            learning_rate=learning_rate, decay=0.95, epsilon=1.0e-6)
+        params_grads = append_backward_ops(mul_out)
+        self.assertEqual(len(params_grads), 1)
+        self.assertEqual(len(decayed_adagrad_optimizer.get_accumulators()), 0)
+        opts = decayed_adagrad_optimizer.create_optimization_pass(
+            params_grads, mul_out, init_program)
+        self.assertEqual(len(opts), 1)
+        decayed_adagrad_op = opts[0]
+        self.assertEqual(decayed_adagrad_op.type, "decayed_adagrad")
+
+        # Check accumulators
+        accumulators = decayed_adagrad_optimizer.get_accumulators()
+        self.assertEqual(len(accumulators), 1)
+        self.assertTrue(
+            decayed_adagrad_optimizer.get_moment_str() in accumulators)
+        moment_acc = accumulators[decayed_adagrad_optimizer.get_moment_str()]
+        self.assertEqual(len(moment_acc), 1)
+        self.assertTrue(mul_x.name in moment_acc)
+
+        # Check init_program
+        init_ops = init_program.global_block().ops
+        self.assertEqual(len(init_ops), 2)
+        self.assertEqual(init_ops[0].type, "fill_constant")
+        self.assertAlmostEqual(init_ops[0].attr('value'), learning_rate)
+        self.assertEqual(init_ops[1].type, "fill_constant")
+        self.assertAlmostEqual(init_ops[1].attr('value'), 0.0)
+
+
 if __name__ == '__main__':
     unittest.main()