Merge branch 'clang-format' of https://github.com/gangliao/Paddle; branch...

Merge branch 'clang-format' of https://github.com/gangliao/Paddle; branch 'develop' of https://github.com/PaddlePaddle/Paddle into clang-format

CMakeLists.txt

@@ -36,8 +36,8 @@ include(simd)
 ################################ Configurations #######################################
 option(WITH_GPU         "Compile PaddlePaddle with NVIDIA GPU"          ${CUDA_FOUND})
 option(WITH_AVX         "Compile PaddlePaddle with AVX intrinsics"      ${AVX_FOUND})
-option(WITH_MKLDNN      "Compile PaddlePaddle with mkl-dnn support."    ${AVX_FOUND})
-option(WITH_MKLML       "Compile PaddlePaddle with mklml package."      ${AVX_FOUND})
+option(WITH_MKLDNN      "Compile PaddlePaddle with mkl-dnn support."    OFF)
+option(WITH_MKLML       "Compile PaddlePaddle with mklml package."      OFF)
 option(WITH_DSO         "Compile PaddlePaddle with dynamic linked CUDA" ON)
 option(WITH_TESTING     "Compile PaddlePaddle with unit testing"        ON)
 option(WITH_SWIG_PY     "Compile PaddlePaddle with inference api"       ON)

cmake/cpplint.cmake

@@ -56,11 +56,14 @@ macro(add_style_check_target TARGET_NAME)
                 # cpplint code style
                 get_filename_component(base_filename ${filename} NAME)
                 set(CUR_GEN ${CMAKE_CURRENT_BINARY_DIR}/${base_filename}.cpplint)
-                add_custom_command(TARGET ${TARGET_NAME} PRE_BUILD
+                add_custom_command(OUTPUT ${CUR_GEN} PRE_BUILD
                     COMMAND "${PYTHON_EXECUTABLE}" "${PROJ_ROOT}/paddle/scripts/cpplint.py"
                             "--filter=${STYLE_FILTER}"
                             "--write-success=${CUR_GEN}" ${filename}
+                    DEPENDS ${filename} ${PROJ_ROOT}/paddle/scripts/cpplint.py
                     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+                add_custom_target(${base_filename}.cpplint DEPENDS ${CUR_GEN})
+                add_dependencies(${TARGET_NAME} ${base_filename}.cpplint)
             endif()
         endforeach()
     endif()

cmake/generic.cmake

@@ -187,7 +187,13 @@ function(cc_library TARGET_NAME)
     endif()
     
     # cpplint code style
-    add_style_check_target(${TARGET_NAME} ${cc_library_SRCS})
+    foreach(source_file ${cc_library_SRCS})
+      string(REGEX REPLACE "\\.[^.]*$" "" source ${source_file})
+      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${source}.h)
+        list(APPEND cc_library_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/${source}.h)
+      endif()
+    endforeach()
+    add_style_check_target(${TARGET_NAME} ${cc_library_SRCS} ${cc_library_HEADERS})
 
   else(cc_library_SRCS)
     if (cc_library_DEPS)
@@ -239,6 +245,14 @@ function(nv_library TARGET_NAME)
         add_dependencies(${TARGET_NAME} ${nv_library_DEPS})
         target_link_libraries(${TARGET_NAME} ${nv_library_DEPS})
       endif()
+      # cpplint code style
+      foreach(source_file ${nv_library_SRCS})
+        string(REGEX REPLACE "\\.[^.]*$" "" source ${source_file})
+        if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${source}.h)
+          list(APPEND cc_library_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/${source}.h)
+        endif()
+      endforeach()
+      add_style_check_target(${TARGET_NAME} ${nv_library_SRCS} ${nv_library_HEADERS})
     else(nv_library_SRCS)
       if (nv_library_DEPS)
         merge_static_libs(${TARGET_NAME} ${nv_library_DEPS})

cmake/util.cmake

@@ -118,7 +118,6 @@ endfunction()
 macro(add_unittest_without_exec TARGET_NAME)
     add_executable(${TARGET_NAME} ${ARGN})
     link_paddle_test(${TARGET_NAME})
-    add_style_check_target(${TARGET_NAME} ${ARGN})
 endmacro()
 
 # add_unittest

paddle/framework/CMakeLists.txt

@@ -12,13 +12,15 @@ cc_test(variable_test SRCS variable_test.cc)
 cc_library(scope SRCS scope.cc)
 cc_test(scope_test SRCS scope_test.cc DEPS scope)
 
-proto_library(attr_type SRCS attr_type.proto)
-proto_library(op_proto SRCS op_proto.proto DEPS attr_type)
-proto_library(op_desc SRCS op_desc.proto DEPS attr_type)
+proto_library(attribute_proto SRCS attribute.proto)
+proto_library(op_proto SRCS op_proto.proto DEPS attribute_proto)
+proto_library(op_desc SRCS op_desc.proto DEPS attribute_proto)
 cc_test(op_proto_test SRCS op_proto_test.cc DEPS op_proto protobuf)
 cc_test(op_desc_test SRCS op_desc_test.cc DEPS op_desc protobuf)
 
-cc_library(operator SRCS operator.cc DEPS op_desc device_context tensor scope)
+cc_library(attribute SRCS attribute.cc DEPS op_desc op_proto)
+
+cc_library(operator SRCS operator.cc DEPS op_desc device_context tensor scope attribute)
 cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry)
 
 cc_library(grad_op_builder SRCS grad_op_builder.cc DEPS op_proto operator)
@@ -26,7 +28,7 @@ cc_library(op_registry SRCS op_registry.cc DEPS op_desc grad_op_builder)
 cc_test(op_registry_test SRCS op_registry_test.cc DEPS op_registry)
 cc_test(grad_op_builder_test SRCS grad_op_builder_test.cc DEPS grad_op_builder op_registry add_op)
 
-py_proto_compile(framework_py_proto SRCS attr_type.proto op_proto.proto op_desc.proto)
+py_proto_compile(framework_py_proto SRCS attribute.proto op_proto.proto op_desc.proto)
 # Generate an empty __init__.py to make framework_py_proto as a valid python module.
 add_custom_target(framework_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
 add_dependencies(framework_py_proto framework_py_proto_init)

paddle/framework/attribute.cc

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/framework/attribute.h"
+
+#include <vector>
+
+namespace paddle {
+namespace framework {
+
+template <>
+AttrType AttrTypeID<int>() {
+  return INT;
+}
+template <>
+AttrType AttrTypeID<float>() {
+  return FLOAT;
+}
+template <>
+AttrType AttrTypeID<std::string>() {
+  return STRING;
+}
+template <>
+AttrType AttrTypeID<std::vector<int>>() {
+  return INTS;
+}
+template <>
+AttrType AttrTypeID<std::vector<float>>() {
+  return FLOATS;
+}
+template <>
+AttrType AttrTypeID<std::vector<std::string>>() {
+  return STRINGS;
+}
+
+Attribute GetAttrValue(const AttrDesc& attr_desc) {
+  switch (attr_desc.type()) {
+    case paddle::framework::AttrType::INT: {
+      return attr_desc.i();
+    }
+    case paddle::framework::AttrType::FLOAT: {
+      return attr_desc.f();
+    }
+    case paddle::framework::AttrType::STRING: {
+      return attr_desc.s();
+    }
+    case paddle::framework::AttrType::INTS: {
+      std::vector<int> val(attr_desc.ints_size());
+      for (int i = 0; i < attr_desc.ints_size(); ++i) {
+        val[i] = attr_desc.ints(i);
+      }
+      return val;
+    }
+    case paddle::framework::AttrType::FLOATS: {
+      std::vector<float> val(attr_desc.floats_size());
+      for (int i = 0; i < attr_desc.floats_size(); ++i) {
+        val[i] = attr_desc.floats(i);
+      }
+      return val;
+    }
+    case paddle::framework::AttrType::STRINGS: {
+      std::vector<std::string> val(attr_desc.strings_size());
+      for (int i = 0; i < attr_desc.strings_size(); ++i) {
+        val[i] = attr_desc.strings(i);
+      }
+      return val;
+    }
+  }
+  PADDLE_ENFORCE(false, "Unknown OpDesc::AttrDesc::type !");
+  return boost::blank();
+}
+
+}  // namespace framework
+}  // namespace paddle

paddle/framework/attr_checker.h → paddle/framework/attribute.h

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
 #pragma once
 
 #include <boost/variant.hpp>
@@ -6,6 +20,9 @@
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
+
+#include "paddle/framework/attribute.pb.h"
+#include "paddle/framework/op_desc.pb.h"
 #include "paddle/platform/enforce.h"
 
 namespace paddle {
@@ -14,13 +31,19 @@ namespace framework {
 typedef boost::variant<boost::blank, int, float, std::string, std::vector<int>,
                        std::vector<float>, std::vector<std::string>>
     Attribute;
+
 typedef std::unordered_map<std::string, Attribute> AttributeMap;
 
+template <typename T>
+AttrType AttrTypeID();
+
+Attribute GetAttrValue(const AttrDesc& attr_desc);
+
 // check whether a value(attribute) fit a certain limit
 template <typename T>
 class LargerThanChecker {
  public:
-  LargerThanChecker(T lower_bound) : lower_bound_(lower_bound) {}
+  explicit LargerThanChecker(T lower_bound) : lower_bound_(lower_bound) {}
   void operator()(T& value) const {
     PADDLE_ENFORCE(value > lower_bound_, "larger_than check fail");
   }
@@ -35,7 +58,8 @@ class LargerThanChecker {
 template <typename T>
 class DefaultValueSetter {
  public:
-  DefaultValueSetter(T default_value) : default_value_(default_value) {}
+  explicit DefaultValueSetter(T default_value)
+      : default_value_(default_value) {}
   void operator()(T& value) const { value = default_value_; }
 
  private:
@@ -78,7 +102,8 @@ class TypedAttrChecker {
   typedef std::function<void(T&)> ValueChecker;
 
  public:
-  TypedAttrChecker(const std::string& attr_name) : attr_name_(attr_name) {}
+  explicit TypedAttrChecker(const std::string& attr_name)
+      : attr_name_(attr_name) {}
 
   TypedAttrChecker& InEnum(const std::unordered_set<T>& range) {
     value_checkers_.push_back(EnumInContainer<T>(range));

paddle/framework/backward.cc

@@ -59,19 +59,17 @@ std::shared_ptr<OperatorBase> BackwardRecursive(
   //  If all input gradients of forwarding operator do not need to calculate,
   //  just return an NOP. Not return null ptr because NOP does not take
   //  too much time for calculation, but it is useful for simplifying logic.
-  if (AllInSet(forwardOp.inputs_, OperatorBase::GRAD_VAR_SUFFIX(),
-               no_grad_names)) {
+  if (AllInSet(forwardOp.inputs_, kGradVarSuffix, no_grad_names)) {
     return NOP();
   }
 
   //  All output gradients of forwarding operator do not need to calculate.
   //  Then all input gradients cannot be computed at all, and we put them into
   //  `no_grad_names` set. Return an NOP.
-  if (AllInSet(forwardOp.outputs_, OperatorBase::GRAD_VAR_SUFFIX(),
-               no_grad_names)) {
+  if (AllInSet(forwardOp.outputs_, kGradVarSuffix, no_grad_names)) {
     for (auto& name : forwardOp.inputs_) {
       // Mark all input is not need
-      no_grad_names.insert(name + OperatorBase::GRAD_VAR_SUFFIX());
+      no_grad_names.insert(name + kGradVarSuffix);
     }
     return NOP();
   }
@@ -134,9 +132,9 @@ std::shared_ptr<OperatorBase> BackwardRecursive(
     std::shared_ptr<OperatorBase> grad_op = OpRegistry::CreateGradOp(forwardOp);
     for (std::string& grad_input : grad_op->inputs_) {
       if (no_grad_names.count(grad_input)) {
-        std::string prefix = grad_input.substr(
-            0, grad_input.size() - OperatorBase::GRAD_VAR_SUFFIX().size());
-        grad_input = prefix + OperatorBase::ZERO_VAR_SUFFIX();
+        std::string prefix =
+            grad_input.substr(0, grad_input.size() - kGradVarSuffix.size());
+        grad_input = prefix + kZeroVarSuffix;
 
         // If part of input gradient of that operator is not calculated, fill
         // zero variables to that input gradient.
@@ -147,7 +145,7 @@ std::shared_ptr<OperatorBase> BackwardRecursive(
 
     for (std::string& grad_output : grad_op->outputs_) {
       if (no_grad_names.count(grad_output)) {
-        grad_output = OperatorBase::EMPTY_VAR_NAME();
+        grad_output = kEmptyVarName;
       }
     }
 
@@ -168,14 +166,14 @@ std::shared_ptr<OperatorBase> Backward(
   std::unordered_set<std::string> no_grad_names;
   no_grad_names.reserve(no_grad_vars.size());
 
-  no_grad_names.insert(OperatorBase::EMPTY_VAR_NAME() +
-                       OperatorBase::GRAD_VAR_SUFFIX());
+  no_grad_names.insert(kEmptyVarName + kGradVarSuffix);
 
   for (auto& name : no_grad_vars) {
-    no_grad_names.insert(name + OperatorBase::GRAD_VAR_SUFFIX());
+    no_grad_names.insert(name + kGradVarSuffix);
   }
   size_t uid = 0;
   return BackwardRecursive(forwardOp, no_grad_names, uid);
 }
+
 }  // namespace framework
 }  // namespace paddle

paddle/framework/backward_test.cc

@@ -78,14 +78,14 @@ class FcOp : public ops::NetOp {
                                {Output("mul_result")}, {}));
     auto b_name = Input("b");
     std::string before_act = "mul_result";
-    if (b_name != EMPTY_VAR_NAME()) {
+    if (b_name != kEmptyVarName) {
       AddOp(OpRegistry::CreateOp("rowwise_add", {Output("mul_result"), b_name},
                                  {Output("add_result")}, {}));
       before_act = "add_result";
     } else {
       auto out_varname = Output("add_result");
-      if (out_varname != EMPTY_VAR_NAME()) {
-        this->Rename(out_varname, EMPTY_VAR_NAME());
+      if (out_varname != kEmptyVarName) {
+        this->Rename(out_varname, kEmptyVarName);
       }
     }
 
@@ -163,13 +163,12 @@ TEST(Backward, simple_op_grad) {
   ASSERT_NE(fwd, nullptr);
   auto gop = f::OpRegistry::CreateGradOp(*fwd);
   ASSERT_EQ(4UL, gop->inputs_.size());
-  ASSERT_EQ(f::OperatorBase::EMPTY_VAR_NAME(), gop->inputs_[0]);
+  ASSERT_EQ(f::kEmptyVarName, gop->inputs_[0]);
   ASSERT_EQ("rowwise_add_grad", gop->type_);
-  ASSERT_EQ("X" + f::OperatorBase::GRAD_VAR_SUFFIX(), gop->outputs_[0]);
-  ASSERT_EQ("b" + f::OperatorBase::GRAD_VAR_SUFFIX(), gop->outputs_[1]);
+  ASSERT_EQ("X" + f::kGradVarSuffix, gop->outputs_[0]);
+  ASSERT_EQ("b" + f::kGradVarSuffix, gop->outputs_[1]);
 
-  ASSERT_EQ("X" + f::OperatorBase::GRAD_VAR_SUFFIX(),
-            gop->Output("X" + f::OperatorBase::GRAD_VAR_SUFFIX()));
+  ASSERT_EQ("X" + f::kGradVarSuffix, gop->Output("X" + f::kGradVarSuffix));
 }
 
 TEST(Backward, simple_op_not_need_grad) {
@@ -177,7 +176,7 @@ TEST(Backward, simple_op_not_need_grad) {
   ASSERT_NE(fwd, nullptr);
   auto gop = f::Backward(*fwd, {"X"});
   ASSERT_EQ(std::find(gop->outputs_.begin(), gop->outputs_.end(),
-                      "X" + f::OperatorBase::GRAD_VAR_SUFFIX()),
+                      "X" + f::kGradVarSuffix),
             gop->outputs_.end());
 
   auto no_input_gop = f::Backward(*fwd, {"X", "b"});
@@ -210,8 +209,8 @@ TEST(Backward, net_fc_backward_normal) {
 }
 
 TEST(Backward, net_fc_backward_not_have_b) {
-  std::shared_ptr<f::OperatorBase> fwd = f::OpRegistry::CreateOp(
-      "fc", {"X", "w", f::OperatorBase::EMPTY_VAR_NAME()},
+  std::shared_ptr<f::OperatorBase> fwd =
+      f::OpRegistry::CreateOp("fc", {"X", "w", f::kEmptyVarName},
                               {"mul_result", "add_result", "tmp"}, {});
   ASSERT_NE(fwd, nullptr);
   std::shared_ptr<f::OperatorBase> gop = f::Backward(*fwd, {});
@@ -242,24 +241,21 @@ TEST(Backward, net_input_of_network_not_need_grad) {
 
   std::unordered_set<std::string> all_output = std::unordered_set<std::string>(
       bwd_net->outputs_.begin(), bwd_net->outputs_.end());
-  all_output.erase(f::OperatorBase::EMPTY_VAR_NAME());
+  all_output.erase(f::kEmptyVarName);
 
   for (auto &out : {"W1", "b1", "hidden0", "W2", "b2"}) {
-    ASSERT_NE(all_output.find(out + f::OperatorBase::GRAD_VAR_SUFFIX()),
-              all_output.end());
+    ASSERT_NE(all_output.find(out + f::kGradVarSuffix), all_output.end());
   }
 
   // Not Generated X
-  ASSERT_EQ(all_output.find("X" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-            all_output.end());
+  ASSERT_EQ(all_output.find("X" + f::kGradVarSuffix), all_output.end());
 
   ASSERT_EQ(2UL, bwd_net->ops_.size());
   ASSERT_TRUE(bwd_net->ops_[1]->IsNetOp());
   auto first_fc_grad = static_cast<ops::NetOp *>(bwd_net->ops_[1].get());
   ASSERT_EQ(3UL, first_fc_grad->ops_.size());
-  ASSERT_EQ(
-      f::OperatorBase::EMPTY_VAR_NAME(),
-      first_fc_grad->ops_[2]->Output("A" + f::OperatorBase::GRAD_VAR_SUFFIX()));
+  ASSERT_EQ(f::kEmptyVarName,
+            first_fc_grad->ops_[2]->Output("A" + f::kGradVarSuffix));
 }
 
 TEST(Backward, net_shared_weight) {
@@ -311,17 +307,15 @@ TEST(Backward, op_part_of_output_are_not_need) {
   ASSERT_EQ(1UL, fill_zero.inputs_.size());
   ASSERT_EQ("Z", fill_zero.inputs_[0]);
   ASSERT_EQ(1UL, fill_zero.outputs_.size());
-  ASSERT_EQ("Z" + f::OperatorBase::ZERO_VAR_SUFFIX(), fill_zero.outputs_[0]);
+  ASSERT_EQ("Z" + f::kZeroVarSuffix, fill_zero.outputs_[0]);
 
   auto &d_many_out = *net->ops_[1];
   ASSERT_EQ("many_output_op_grad", d_many_out.type_);
   ASSERT_EQ(1UL + 2UL + 2UL, d_many_out.inputs_.size());  // I/O/OG
-  ASSERT_EQ("Z" + f::OperatorBase::ZERO_VAR_SUFFIX(),
-            d_many_out.Input("z" + f::OperatorBase::GRAD_VAR_SUFFIX()));
-  ASSERT_EQ("Y" + f::OperatorBase::GRAD_VAR_SUFFIX(),
-            d_many_out.Input("y" + f::OperatorBase::GRAD_VAR_SUFFIX()));
-  ASSERT_EQ("X" + f::OperatorBase::GRAD_VAR_SUFFIX(),
-            d_many_out.Output("x" + f::OperatorBase::GRAD_VAR_SUFFIX()));
+  ASSERT_EQ("Z" + f::kZeroVarSuffix, d_many_out.Input("z" + f::kGradVarSuffix));
+  ASSERT_EQ("Y" + f::kGradVarSuffix, d_many_out.Input("y" + f::kGradVarSuffix));
+  ASSERT_EQ("X" + f::kGradVarSuffix,
+            d_many_out.Output("x" + f::kGradVarSuffix));
 }
 
 TEST(Backward, op_part_of_input_are_not_need) {
@@ -331,12 +325,10 @@ TEST(Backward, op_part_of_input_are_not_need) {
   ASSERT_EQ(grad_mul.type_, "mul_grad");
   ASSERT_EQ(grad_mul.inputs_.size(), 2UL + 1UL + 1UL);
   ASSERT_EQ(grad_mul.outputs_.size(), 2UL);
-  ASSERT_EQ(grad_mul.Output("A" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-            f::OperatorBase::EMPTY_VAR_NAME());
-  ASSERT_EQ(grad_mul.Output("B" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-            "b" + f::OperatorBase::GRAD_VAR_SUFFIX());
-  ASSERT_EQ(grad_mul.Input("Out" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-            "out" + f::OperatorBase::GRAD_VAR_SUFFIX());
+  ASSERT_EQ(grad_mul.Output("A" + f::kGradVarSuffix), f::kEmptyVarName);
+  ASSERT_EQ(grad_mul.Output("B" + f::kGradVarSuffix), "b" + f::kGradVarSuffix);
+  ASSERT_EQ(grad_mul.Input("Out" + f::kGradVarSuffix),
+            "out" + f::kGradVarSuffix);
   ASSERT_EQ(grad_mul.Input("A"), "a");
   ASSERT_EQ(grad_mul.Input("B"), "b");
   ASSERT_EQ(grad_mul.Input("Out"), "out");
@@ -368,23 +360,4 @@ TEST(Backward, linear_net_intermediate_variable_has_no_grad) {
   EXPECT_EQ(bwd_net->ops_[1]->outputs_.size(), 0UL);
   EXPECT_EQ(bwd_net->ops_[2]->inputs_.size(), 0UL);
   EXPECT_EQ(bwd_net->ops_[2]->outputs_.size(), 0UL);
-
-  /*
-    EXPECT_EQ(grad_fc.Output("X" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-              f::OperatorBase::EMPTY_VAR_NAME());
-  EXPECT_EQ(grad_fc.Output("W" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-    "w3" + f::OperatorBase::GRAD_VAR_SUFFIX());
-  EXPECT_EQ(grad_fc.Output("b" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-    "b3" + f::OperatorBase::GRAD_VAR_SUFFIX());
-  EXPECT_EQ(grad_fc.Output("mul_result" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-  "mul_out3" + f::OperatorBase::GRAD_VAR_SUFFIX());
-
-  EXPECT_EQ(grad_fc.Input("Out" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-  "out3" + f::OperatorBase::GRAD_VAR_SUFFIX());
-  EXPECT_EQ(grad_fc.Input("X"), "out2");
-  EXPECT_EQ(grad_fc.Input("W"), "w3");
-  EXPECT_EQ(grad_fc.Input("mul_result"), "mul_out3");
-  EXPECT_EQ(grad_fc.Input("add_result"), "tmp_out3");
-  EXPECT_EQ(grad_fc.Input("Out"), "out3");
-  */
 }

paddle/framework/ddim.h

@@ -25,18 +25,15 @@ limitations under the License. */
 namespace paddle {
 namespace framework {
 
-namespace {
-typedef boost::variant<Dim<1>, Dim<2>, Dim<3>, Dim<4>, Dim<5>, Dim<6>, Dim<7>,
-                       Dim<8>, Dim<9>>
-    DDimVar;
-}
-
 /**
  * \brief A dynamically sized dimension.
  *
  * The number of dimensions must be between [1, 9].
  */
 struct DDim {
+  typedef boost::variant<Dim<1>, Dim<2>, Dim<3>, Dim<4>, Dim<5>, Dim<6>, Dim<7>,
+                         Dim<8>, Dim<9>>
+      DDimVar;
   DDimVar var;
 
   DDim() : var(Dim<1>()) {}

paddle/framework/grad_op_builder.cc

@@ -56,8 +56,7 @@ static void TransOpArg(const OperatorBase* src_op, OperatorBase* dst_op,
 
   for (const auto& arg : src_arg_list) {
     std::string src_name = arg.name();
-    std::string dst_name =
-        is_grad ? src_name + OperatorBase::GRAD_VAR_SUFFIX() : src_name;
+    std::string dst_name = is_grad ? src_name + kGradVarSuffix : src_name;
     (*dst_op->in_out_idxs_)[dst_name] = idx++;
     int src_arg_idx = src_op->in_out_idxs_->at(src_name);
     int src_begin =
@@ -65,10 +64,9 @@ static void TransOpArg(const OperatorBase* src_op, OperatorBase* dst_op,
     int src_end = src_format == nullptr ? src_arg_idx + 1
                                         : src_format->at(src_arg_idx + 1);
     for (int i = src_begin; i < src_end; ++i) {
-      std::string s = is_grad ? src_inout[i] + OperatorBase::GRAD_VAR_SUFFIX()
-                              : arg.ignore_gradient()
-                                    ? OperatorBase::EMPTY_VAR_NAME()
-                                    : src_inout[i];
+      std::string s =
+          is_grad ? src_inout[i] + kGradVarSuffix
+                  : (arg.ignore_gradient() ? kEmptyVarName : src_inout[i]);
       dst_inout.emplace_back(s);
     }
     if (dst_format != nullptr) {

paddle/framework/grad_op_builder.h

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
 #pragma once
 
 #include "paddle/framework/operator.h"

paddle/framework/grad_op_builder_test.cc

@@ -83,24 +83,21 @@ TEST(GradOpBuilder, MutiInOut) {
   EXPECT_EQ(grad_test_op->Input("Out1"), "out1");
   EXPECT_EQ(grad_test_op->Inputs("Out2_mult"),
             std::vector<std::string>({"out2_1", "out2_2"}));
-  EXPECT_EQ(grad_test_op->Input("Out1" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-            "out1" + f::OperatorBase::GRAD_VAR_SUFFIX());
-  EXPECT_EQ(
-      grad_test_op->Inputs("Out2_mult" + f::OperatorBase::GRAD_VAR_SUFFIX()),
+  EXPECT_EQ(grad_test_op->Input("Out1" + f::kGradVarSuffix),
+            "out1" + f::kGradVarSuffix);
+  EXPECT_EQ(grad_test_op->Inputs("Out2_mult" + f::kGradVarSuffix),
             std::vector<std::string>(
-          {"out2_1" + f::OperatorBase::GRAD_VAR_SUFFIX(),
-           "out2_2" + f::OperatorBase::GRAD_VAR_SUFFIX()}));
+                {"out2_1" + f::kGradVarSuffix, "out2_2" + f::kGradVarSuffix}));
 
   ASSERT_EQ(grad_test_op->outputs_.size(), 5UL);
-  EXPECT_EQ(grad_test_op->Output("In1" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-            "in1" + f::OperatorBase::GRAD_VAR_SUFFIX());
-  EXPECT_EQ(
-      grad_test_op->Outputs("In2_mult" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-      std::vector<std::string>({"in2_1" + f::OperatorBase::GRAD_VAR_SUFFIX(),
-                                "in2_2" + f::OperatorBase::GRAD_VAR_SUFFIX(),
-                                "in2_3" + f::OperatorBase::GRAD_VAR_SUFFIX()}));
-  EXPECT_EQ(grad_test_op->Output("In3" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-            "in3" + f::OperatorBase::GRAD_VAR_SUFFIX());
+  EXPECT_EQ(grad_test_op->Output("In1" + f::kGradVarSuffix),
+            "in1" + f::kGradVarSuffix);
+  EXPECT_EQ(grad_test_op->Outputs("In2_mult" + f::kGradVarSuffix),
+            std::vector<std::string>({"in2_1" + f::kGradVarSuffix,
+                                      "in2_2" + f::kGradVarSuffix,
+                                      "in2_3" + f::kGradVarSuffix}));
+  EXPECT_EQ(grad_test_op->Output("In3" + f::kGradVarSuffix),
+            "in3" + f::kGradVarSuffix);
 }
 
 TEST(GradOpBuilder, IOIgnoredInGradient) {
@@ -116,30 +113,25 @@ TEST(GradOpBuilder, IOIgnoredInGradient) {
   ASSERT_EQ(grad_test_op->inputs_.size(), 5UL + 3UL + 3UL);
   EXPECT_EQ(grad_test_op->Input("In1"), "in1");
   EXPECT_EQ(grad_test_op->Inputs("In2_mult"),
-            std::vector<std::string>({f::OperatorBase::EMPTY_VAR_NAME(),
-                                      f::OperatorBase::EMPTY_VAR_NAME()}));
+            std::vector<std::string>({f::kEmptyVarName, f::kEmptyVarName}));
   EXPECT_EQ(grad_test_op->Inputs("In3_mult"),
             std::vector<std::string>({"in3_1", "in3_2"}));
   EXPECT_EQ(grad_test_op->Inputs("Out1_mult"),
             std::vector<std::string>({"out1_1", "out1_2"}));
-  EXPECT_EQ(grad_test_op->Input("Out2"), f::OperatorBase::EMPTY_VAR_NAME());
-  EXPECT_EQ(
-      grad_test_op->Inputs("Out1_mult" + f::OperatorBase::GRAD_VAR_SUFFIX()),
+  EXPECT_EQ(grad_test_op->Input("Out2"), f::kEmptyVarName);
+  EXPECT_EQ(grad_test_op->Inputs("Out1_mult" + f::kGradVarSuffix),
             std::vector<std::string>(
-          {"out1_1" + f::OperatorBase::GRAD_VAR_SUFFIX(),
-           "out1_2" + f::OperatorBase::GRAD_VAR_SUFFIX()}));
-  EXPECT_EQ(grad_test_op->Input("Out2" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-            "out2" + f::OperatorBase::GRAD_VAR_SUFFIX());
+                {"out1_1" + f::kGradVarSuffix, "out1_2" + f::kGradVarSuffix}));
+  EXPECT_EQ(grad_test_op->Input("Out2" + f::kGradVarSuffix),
+            "out2" + f::kGradVarSuffix);
 
   ASSERT_EQ(grad_test_op->outputs_.size(), 5UL);
-  EXPECT_EQ(grad_test_op->Output("In1" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-            "in1" + f::OperatorBase::GRAD_VAR_SUFFIX());
-  EXPECT_EQ(
-      grad_test_op->Outputs("In2_mult" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-      std::vector<std::string>({"in2_1" + f::OperatorBase::GRAD_VAR_SUFFIX(),
-                                "in2_2" + f::OperatorBase::GRAD_VAR_SUFFIX()}));
-  EXPECT_EQ(
-      grad_test_op->Outputs("In3_mult" + f::OperatorBase::GRAD_VAR_SUFFIX()),
-      std::vector<std::string>({"in3_1" + f::OperatorBase::GRAD_VAR_SUFFIX(),
-                                "in3_2" + f::OperatorBase::GRAD_VAR_SUFFIX()}));
+  EXPECT_EQ(grad_test_op->Output("In1" + f::kGradVarSuffix),
+            "in1" + f::kGradVarSuffix);
+  EXPECT_EQ(grad_test_op->Outputs("In2_mult" + f::kGradVarSuffix),
+            std::vector<std::string>(
+                {"in2_1" + f::kGradVarSuffix, "in2_2" + f::kGradVarSuffix}));
+  EXPECT_EQ(grad_test_op->Outputs("In3_mult" + f::kGradVarSuffix),
+            std::vector<std::string>(
+                {"in3_1" + f::kGradVarSuffix, "in3_2" + f::kGradVarSuffix}));
 }

paddle/framework/op_desc.proto

@@ -15,7 +15,7 @@ limitations under the License. */
 syntax = "proto2";
 package paddle.framework;
 
-import "attr_type.proto";
+import "attribute.proto";
 
 // AttrDesc is used to describe Attributes of an Operator. It contain's
 // name, type, and value of Attribute.

paddle/framework/op_proto.proto

@@ -22,7 +22,7 @@ limitations under the License. */
 syntax = "proto2";
 package paddle.framework;
 
-import "attr_type.proto";
+import "attribute.proto";
 
 // Attribute protocol message for 3rd-party language binding.
 // It will store the Op support what attribute and what type.

paddle/framework/op_registry.cc

@@ -14,37 +14,8 @@ limitations under the License. */
 
 #include <paddle/framework/op_registry.h>
 
-namespace paddle {
-namespace framework {
-
-template <>
-void AttrTypeHelper::SetAttrType<int>(AttrProto* attr) {
-  attr->set_type(paddle::framework::AttrType::INT);
-}
-
-template <>
-void AttrTypeHelper::SetAttrType<float>(AttrProto* attr) {
-  attr->set_type(paddle::framework::AttrType::FLOAT);
-}
-
-template <>
-void AttrTypeHelper::SetAttrType<std::string>(AttrProto* attr) {
-  attr->set_type(paddle::framework::AttrType::STRING);
-}
+#include <vector>
 
-template <>
-void AttrTypeHelper::SetAttrType<std::vector<int>>(AttrProto* attr) {
-  attr->set_type(paddle::framework::AttrType::INTS);
-}
-
-template <>
-void AttrTypeHelper::SetAttrType<std::vector<float>>(AttrProto* attr) {
-  attr->set_type(paddle::framework::AttrType::FLOATS);
-}
-
-template <>
-void AttrTypeHelper::SetAttrType<std::vector<std::string>>(AttrProto* attr) {
-  attr->set_type(paddle::framework::AttrType::STRINGS);
-}
-}  // namespace framework
+namespace paddle {
+namespace framework {}  // namespace framework
 }  // namespace paddle

paddle/framework/op_registry.h

@@ -19,7 +19,7 @@ limitations under the License. */
 #include <type_traits>
 #include <unordered_map>
 #include <unordered_set>
-#include "paddle/framework/attr_checker.h"
+#include "paddle/framework/attribute.h"
 #include "paddle/framework/grad_op_builder.h"
 #include "paddle/framework/op_desc.pb.h"
 #include "paddle/framework/scope.h"
@@ -27,49 +27,6 @@ limitations under the License. */
 namespace paddle {
 namespace framework {
 
-// helper class to set attribute type
-struct AttrTypeHelper {
-  template <typename T>
-  static void SetAttrType(AttrProto* attr);
-
-  static Attribute GetAttrValue(const AttrDesc& attr_desc) {
-    switch (attr_desc.type()) {
-      case paddle::framework::AttrType::INT: {
-        return attr_desc.i();
-      }
-      case paddle::framework::AttrType::FLOAT: {
-        return attr_desc.f();
-      }
-      case paddle::framework::AttrType::STRING: {
-        return attr_desc.s();
-      }
-      case paddle::framework::AttrType::INTS: {
-        std::vector<int> val(attr_desc.ints_size());
-        for (int i = 0; i < attr_desc.ints_size(); ++i) {
-          val[i] = attr_desc.ints(i);
-        }
-        return val;
-      }
-      case paddle::framework::AttrType::FLOATS: {
-        std::vector<float> val(attr_desc.floats_size());
-        for (int i = 0; i < attr_desc.floats_size(); ++i) {
-          val[i] = attr_desc.floats(i);
-        }
-        return val;
-      }
-      case paddle::framework::AttrType::STRINGS: {
-        std::vector<std::string> val(attr_desc.strings_size());
-        for (int i = 0; i < attr_desc.strings_size(); ++i) {
-          val[i] = attr_desc.strings(i);
-        }
-        return val;
-      }
-    }
-    PADDLE_ENFORCE(false, "Unknown OpDesc::AttrDesc::type !");
-    return boost::blank();
-  }
-};
-
 // this class not only make proto but also init attribute checkers.
 class OpProtoAndCheckerMaker {
  public:
@@ -136,7 +93,7 @@ class OpProtoAndCheckerMaker {
     *attr->mutable_name() = name;
     *attr->mutable_comment() = comment;
     attr->set_generated(generated);
-    AttrTypeHelper::SetAttrType<T>(attr);
+    attr->set_type(AttrTypeID<T>());
     return op_checker_->AddAttrChecker<T>(name);
   }
 
@@ -297,7 +254,7 @@ class OpRegistry {
 
     AttributeMap attrs;
     for (auto& attr : op_desc.attrs()) {
-      attrs[attr.name()] = AttrTypeHelper::GetAttrValue(attr);
+      attrs[attr.name()] = GetAttrValue(attr);
     }
 
     return CreateOp(op_desc.type(), inputs, outputs, attrs);
@@ -314,7 +271,7 @@ class OpRegistry {
   static std::unordered_map<std::string, OpProto>& protos() {
     static std::unordered_map<std::string, OpProto> protos_;
     return protos_;
-  };
+  }
 
   static std::unordered_map<std::string, std::string>& grad_ops() {
     static std::unordered_map<std::string, std::string> grad_ops_;
@@ -336,12 +293,12 @@ class OpRegistry {
   static std::unordered_map<std::string, OpAttrChecker>& op_checkers() {
     static std::unordered_map<std::string, OpAttrChecker> op_checkers_;
     return op_checkers_;
-  };
+  }
 
   static void GenerateTempVariableName(OperatorBase* op) {
     static std::atomic<size_t> gUniqId(0UL);
     for (auto& outname : op->outputs_) {
-      if (outname == OperatorBase::TMP_VAR_NAME()) {
+      if (outname == kTempVarName) {
         outname += op->type_;
         outname += "@";
         outname += std::to_string(gUniqId.fetch_add(1));
@@ -353,7 +310,7 @@ class OpRegistry {
 template <typename OpType, typename ProtoMakerType>
 class OpRegisterHelper {
  public:
-  OpRegisterHelper(const char* op_type) {
+  explicit OpRegisterHelper(const char* op_type) {
     OpRegistry::RegisterOp<OpType, ProtoMakerType>(op_type);
   }
 };

paddle/framework/operator.h

@@ -20,7 +20,7 @@ limitations under the License. */
 #include <unordered_map>
 #include <vector>
 
-#include "paddle/framework/attr_checker.h"
+#include "paddle/framework/attribute.h"
 #include "paddle/framework/op_desc.pb.h"
 #include "paddle/framework/op_proto.pb.h"
 #include "paddle/framework/scope.h"
@@ -32,9 +32,29 @@ limitations under the License. */
 namespace paddle {
 namespace framework {
 
+/// If a variable is a empty variable, that name will be used.
+const std::string kEmptyVarName = "@EMPTY@";
+
+/// If a variable is a temporary variable, that name will be set in Python,
+/// but it will be convert to a unique name in scope after OpCreator.
+const std::string kTempVarName = "@TEMP@";
+
+/// If a variable's name has a certain suffix, it means that the
+/// variable is the gradient of another varibale.
+/// e.g. Variable "x@GRAD" is the gradient of varibale "x".
+const std::string kGradVarSuffix = "@GRAD";
+
+/// Variables with this suffix are supposed to be filled up with zeros.
+const std::string kZeroVarSuffix = "@ZERO";
+
+inline std::string GradVarName(const std::string& var_name) {
+  return var_name + kGradVarSuffix;
+}
+
 class OperatorBase;
 class InferShapeContext;
 class ExecutionContext;
+
 /**
  * OperatorBase has the basic element that Net will call to do computation.
  * Only CreateOperator from OpRegistry will new Operator directly. User
@@ -43,25 +63,6 @@ class ExecutionContext;
  */
 class OperatorBase {
  public:
-  /// If a variable is a empty variable, that name will be used.
-  static std::string EMPTY_VAR_NAME() { return "@EMPTY@"; }
-
-  /// If a variable is a temporary variable, that name will be set in Python,
-  /// but it will be convert to a unique name in scope after OpCreator.
-  static std::string TMP_VAR_NAME() { return "@TEMP@"; }
-
-  /// If a variable's name has a certain suffix, it means that the
-  /// variable is the gradient of another varibale.
-  /// e.g. Variable "x@GRAD" is the gradient of varibale "x".
-  static std::string GRAD_VAR_SUFFIX() { return "@GRAD"; }
-
-  static std::string GRAD_VAR_NAME(const std::string& name) {
-    return name + GRAD_VAR_SUFFIX();
-  }
-
-  /// Variables with this suffix are supposed to be filled up with zeros.
-  static std::string ZERO_VAR_SUFFIX() { return "@ZERO"; }
-
   virtual ~OperatorBase() {}
 
   template <typename T>
@@ -284,7 +285,7 @@ class OperatorWithKernel : public OperatorBase {
     platform::Place place_;
 
     OpKernelKey() = default;
-    OpKernelKey(const platform::DeviceContext& dev_ctx) {
+    explicit OpKernelKey(const platform::DeviceContext& dev_ctx) {
       place_ = dev_ctx.GetPlace();
     }
 

paddle/framework/pybind.cc

@@ -105,7 +105,16 @@ PYBIND11_PLUGIN(core) {
       .def("set", PyCUDATensorSetFromArray<float>)
       .def("set", PyCUDATensorSetFromArray<int>)
 #endif
-      .def("shape", [](Tensor &self) { return vectorize(self.dims()); });
+      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
+      .def("set_float_element",
+           [](Tensor &self, size_t offset, float f) {
+             // TODO(yuyang18): Only support GPU now.
+             self.data<float>()[offset] = f;
+           })
+      .def("get_float_element", [](Tensor &self, size_t offset) -> float {
+        // TODO(yuyang18): Only support GPU now.
+        return self.data<float>()[offset];
+      });
 
   py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
 
@@ -154,8 +163,8 @@ All parameter, weight, gradient are variables in Paddle.
   m.def_submodule(
        "var_names",
        "The module will return special predefined variable name in Paddle")
-      .def("empty", OperatorBase::EMPTY_VAR_NAME)
-      .def("temp", OperatorBase::TMP_VAR_NAME);
+      .def("empty", []() { return kEmptyVarName; })
+      .def("temp", []() { return kTempVarName; });
   // clang-format off
   py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
       .def_static("create",

paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp

@@ -967,8 +967,9 @@ void RecurrentGradientMachine::generateSequence() {
   size_t numSequences = getGenBatchSize();
 
   resizeBootFrame(numSequences);
-  // We create only two sub-network in generation for alternate use.
-  // Thus, we can reduce total memory of output_ in layer forward.
+  // We create only two sub-network in generation, one stores states of all
+  // layers in previous time step and the other storing the states at current
+  // time step.
   resizeOrCreateFrames(2);
 
   // outFrameLines_.size() > 1UL
@@ -1001,10 +1002,9 @@ void RecurrentGradientMachine::generateSequence() {
 
   // init outArg
   size_t resultNum = generator_.config.num_results_per_sample();
-  IVector::resizeOrCreate(
-      generator_.outArg.ids,
-      generator_.config.max_num_frames() * numSequences * resultNum,
-      false);
+  size_t maxGenWordCount =
+      generator_.config.max_num_frames() * numSequences * resultNum;
+  IVector::resizeOrCreate(generator_.outArg.ids, maxGenWordCount, false);
   if (resultNum > 1) {
     CHECK_LE(resultNum, static_cast<size_t>(generator_.config.beam_size()));
     Matrix::resizeOrCreate(generator_.outArg.in,
@@ -1012,6 +1012,11 @@ void RecurrentGradientMachine::generateSequence() {
                            /* width */ resultNum,
                            false,
                            /* useGpu */ false);
+    Matrix::resizeOrCreate(generator_.outArg.value,
+                           /* height */ maxGenWordCount,
+                           /* width */ 1,
+                           false,
+                           /* useGpu */ false);
   }
   ICpuGpuVector::resizeOrCreate(generator_.outArg.sequenceStartPositions,
                                 numSequences + 1,
@@ -1313,13 +1318,20 @@ void RecurrentGradientMachine::fillGenOutputs() {
   starts[0] = 0;
   if (numResults > 1) {
     real* probs = generator_.outArg.in->getData();
+    real* idsProb = generator_.outArg.value->getData();
+    size_t curPos = 0;
     for (size_t i = 0; i < finalPaths_.size(); ++i) {
       for (size_t j = 0; j < finalPaths_[i].size(); ++j) {
         Path& path = finalPaths_[i][j];
-        generator_.ids.push_back(path.ids.size());  // sequence size
+        size_t genLen = path.ids.size();
+        generator_.ids.push_back(genLen);  // sequence size
         generator_.ids.insert(
             generator_.ids.end(), path.ids.begin(), path.ids.end());
         generator_.ids.push_back(-1);  // end of sequence
+
+        memcpy(idsProb + curPos, path.idsProb.data(), sizeof(real) * genLen);
+        curPos += genLen;
+        idsProb[curPos++] = -1.0;
         probs[i * numResults + j] = path.logProb;
 
         if (!j && dataArgsSize_) {

paddle/gserver/gradientmachines/RecurrentGradientMachine.h

@@ -189,6 +189,11 @@ public:
      */
     std::vector<int> ids;
 
+    /**
+     * @brief idsProb, log probability of each generated words.
+     */
+    std::vector<real> idsProb;
+
     /**
      * @brief logProb, current probability of path.
      */
@@ -228,11 +233,13 @@ public:
      */
     Path(Path& old, int newId, real logProb, int machineId, int topIndex)
         : ids(old.ids),
+          idsProb(old.idsProb),
           logProb(old.logProb + logProb),
           machineId(machineId),
           topIndex(topIndex),
           seqId(old.seqId) {
       ids.push_back(newId);
+      idsProb.push_back(logProb);
       if (!old.probHistory.empty()) {
         this->probHistory = old.probHistory;
         // probHistory store current prob, not sum
@@ -412,6 +419,7 @@ protected:
   struct Generator {
     GeneratorConfig config;
     std::vector<int> ids;       // store generated sequences
+    std::vector<real> idsProb;  // log probability of each generated word
     Argument outArg;            // final output argument
   };
   bool generating_;

paddle/gserver/tests/CMakeLists.txt

 # gserver pacakge unittests
 
+file(GLOB_RECURSE GSERVER_HEADER RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.h")
+file(GLOB_RECURSE GSERVER_SOURCES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.cpp")
+add_style_check_target(paddle_gserver ${GSERVER_SOURCES})
+add_style_check_target(paddle_gserver ${GSERVER_HEADER})
+
 ################### test_ProtoDataProvider ############
 add_unittest_without_exec(test_ProtoDataProvider
     test_ProtoDataProvider.cpp)

paddle/gserver/tests/LayerGradUtil.cpp

@@ -400,7 +400,6 @@ void initDataLayer(TestConfig testConf,
         const std::vector<int>& labelSeqStartPositions =
             testConf.inputDefs[i].labelSeqStartPositions;
         if (labelSeqStartPositions.size() != 0) {
-          CHECK(!sequenceStartPositions);
           CHECK_GE(static_cast<int>(labelSeqStartPositions.size()), 2);
 
           sequenceStartPositions =
@@ -410,6 +409,19 @@ void initDataLayer(TestConfig testConf,
                                            useGpu);
           data.sequenceStartPositions = sequenceStartPositions;
         }
+
+        const std::vector<int>& labelSubSeqStartPositions =
+            testConf.inputDefs[i].labelSubSeqStartPositions;
+        if (labelSubSeqStartPositions.size() != 0) {
+          CHECK_GE(static_cast<int>(labelSubSeqStartPositions.size()), 2);
+
+          subSequenceStartPositions =
+              ICpuGpuVector::create(labelSubSeqStartPositions.size(), useGpu);
+          subSequenceStartPositions->copyFrom(labelSubSeqStartPositions.data(),
+                                              labelSubSeqStartPositions.size(),
+                                              useGpu);
+          data.subSequenceStartPositions = subSequenceStartPositions;
+        }
         break;
       }
       default:

paddle/gserver/tests/LayerGradUtil.h

@@ -67,6 +67,7 @@ struct InputDef {
   bool isStatic;
   std::vector<int> labelInitValue;
   std::vector<int> labelSeqStartPositions;
+  std::vector<int> labelSubSeqStartPositions;
   MatrixPtr selfDefinedData;
 
   InputDef(InputType type, string nameIn, size_t dimIn, size_t sizeIn) {
@@ -81,8 +82,10 @@ struct InputDef {
   InputDef(InputType type,
            string nameIn,
            MatrixPtr selfDefinedData,
-           std::vector<int> selfDefinedSeqStartPos = {})
+           std::vector<int> selfDefinedSeqStartPos = {},
+           std::vector<int> selfDefinedSubSeqStartPos = {})
       : labelSeqStartPositions(selfDefinedSeqStartPos),
+        labelSubSeqStartPositions(selfDefinedSubSeqStartPos),
         selfDefinedData(selfDefinedData) {
     inputType = type;
     name = nameIn;

paddle/math/MathUtils.cpp

@@ -25,7 +25,7 @@ namespace paddle {
  */
 void sparseRand(
     int* major, int* minor, int nnz, int majorLen, int minorMax, bool useGpu) {
-  CHECK(size_t(nnz) > size_t(1));
+  CHECK(size_t(nnz) >= size_t(1));
   int* cpuMajor;
   int* cpuMinor;
   CpuIVector cpuMinorVec(nnz);

paddle/math/tests/test_matrixCompare.cpp

@@ -79,8 +79,8 @@ void testMatrixMaxSequence(int batchSize, int inputDim) {
 }
 
 TEST(Matrix, maxSequence) {
-  for (auto batchSize : {1, 10, 128, 1000, 6000}) {
-    for (auto inputDim : {1, 32, 100, 512}) {
+  for (auto batchSize : {1, 3, 997}) {   // prime numbers close to 1, 4, 1024
+    for (auto inputDim : {1, 7, 131}) {  // prime numbers close to 1, 8, 128
       VLOG(3) << " batchSize=" << batchSize << " inputDim=" << inputDim;
       testMatrixMaxSequence(batchSize, inputDim);
     }
@@ -240,14 +240,10 @@ TEST(Matrix, unary) {
     // inverse matrix
     testMatrixInverse(height);
 #else
-    LOG(WARNING) << "Cannot run Matrix Inverse Unit Test.\n"
-                 << "Failed to find lapack library in current system.\n"
-                 << "To address this issue, Please adopt one of the following "
-                    "approaches: \n"
-                 << "1. Simply issue `sudo apt-get install liblapacke-dev` to "
-                    "avoid re-build source code. \n"
-                 << "2. Install MKL/Openblas/ATLAS and re-build PaddlePaddle "
-                    "source code.";
+    LOG(WARNING) << "This version of PaddlePaddle was not built with LAPACK"
+                 << "support so we cannot test matrix inverse. To test "
+                 << "matrix inverse, please install LAPACKE "
+                 << "and MKL/Openblas/ATLAS, and re-build PaddlePaddle.";
 #endif
   }
 }
@@ -341,8 +337,8 @@ void testMatrixSoftmaxBp(int height, int width) {
 }
 
 TEST(Matrix, softmax) {
-  for (auto height : {1, 11, 73, 128, 200}) {
-    for (auto width : {1, 32, 100, 512, 1000}) {
+  for (auto height : {1, 3, 131}) {    // prime numbers close to 1, 4, 127
+    for (auto width : {1, 17, 251}) {  // prime numbers close to 1, 16, 256
       VLOG(3) << " height=" << height << " width=" << width;
 
       testMatrixSoftmax(height, width);
@@ -527,7 +523,7 @@ void testVectorRowFunc(int size) {
 }
 
 TEST(Vector, rowFunc) {
-  for (auto size : {1, 5, 31, 90, 150, 500, 1000, 4000}) {
+  for (auto size : {1, 3, 997}) {  // prime numbers close to 1, 4, 1024
     VLOG(3) << " size=" << size;
     testVectorRowFunc(size);
   }
@@ -604,7 +600,7 @@ void testVectorIsEqual(int size) {
 }
 
 TEST(Vector, Equal) {
-  for (auto size : {1, 5, 31, 90, 150, 500, 1000, 4000}) {
+  for (auto size : {1, 3, 997}) {  // prime numbers close to 1, 4, 1024
     VLOG(3) << " size=" << size;
     testVectorReset<int>(size);
     testVectorReset<real>(size);
@@ -635,9 +631,8 @@ void testMatrixTopK(int samples, int dim, int beamSize) {
 }
 
 TEST(Matrix, topK) {
-  for (auto samples : {1, 5, 31, 90, 150, 500}) {
-    for (auto dim :
-         {1, 5, 8, 10, 15, 64, 80, 120, 256, 300, 1280, 5120, 50000}) {
+  for (auto samples : {1, 17, 131}) {  // prime numbers close to 1, 16, 127
+    for (auto dim : {1, 3, 997}) {     // prime numbers close to 1, 4, 1024
       for (auto beamSize : {1, 5, 10, 20, 40, (int)rand() % dim + 1}) {
         if (beamSize > dim) continue;
         VLOG(3) << " samples=" << samples << " beamSize=" << beamSize
@@ -650,6 +645,7 @@ TEST(Matrix, topK) {
 
 void testSMatrixTopK(int samples, int dim, int beamSize, real ratio) {
   int nnz = samples * dim * ratio;
+  if (nnz < 1) nnz = 1;  // Because sparseRand in MathUtil.cpp requires this.
   MatrixPtr cpuSrc = std::make_shared<CpuSparseMatrix>(samples, dim, nnz);
   MatrixPtr gpuSrc = std::make_shared<GpuSparseMatrix>(samples, dim, nnz);
   MatrixPtr cpuVal = std::make_shared<CpuMatrix>(samples, beamSize);
@@ -683,9 +679,9 @@ void testSMatrixTopK(int samples, int dim, int beamSize, real ratio) {
 }
 
 TEST(SMatrix, topK) {
-  for (auto samples : {1, 5, 100}) {
-    for (auto dim : {10000, 10000, 50000}) {
-      for (auto beamSize : {1, 5, 40, 100, 500}) {
+  for (auto samples : {1, 3, 61}) {
+    for (auto dim : {1, 3, 61}) {
+      for (auto beamSize : {1, 3, 61}) {
         for (auto ratio : {0.01, 0.001}) {
           if (beamSize > dim) continue;
           VLOG(3) << " samples=" << samples << " beamSize=" << beamSize
@@ -806,10 +802,9 @@ void testClassificationError(int numSamples, int dim, int topkSize) {
 }
 
 TEST(Matrix, classificationError) {
-  for (auto numSamples : {1, 5, 31, 90, 150, 300}) {
-    for (auto dim :
-         {1, 5, 8, 10, 15, 64, 80, 120, 256, 300, 1280, 5120, 50000}) {
-      for (auto topkSize : {1, 5, 10, 20, 40, (int)rand() % dim + 1}) {
+  for (auto numSamples : {1, 3, 31}) {
+    for (auto dim : {1, 3, 31}) {
+      for (auto topkSize : {1, 3, (int)rand() % dim + 1}) {
         if (topkSize > dim) continue;
         VLOG(3) << " sample= " << numSamples << " topkSize= " << topkSize
                 << " dim= " << dim;
@@ -1016,13 +1011,15 @@ void testAvgPoolFwdBwd(int numSamples,
   TensorCheckErr(*inputGrad, *inputGpuGrad);
 }
 
+// TODO(yi): I noticed many such blindly combinatorial tests in this
+// file.  They are no help to locate defects at all.
 TEST(Matrix, PoolFwdBwd) {
-  for (auto numSamples : {5, 32}) {
-    for (auto channels : {1, 9, 32}) {
-      for (auto imgSizeH : {14, 28}) {
-        for (auto imgSizeW : {16, 30}) {
-          for (auto sizeX : {2, 5}) {
-            for (auto sizeY : {2, 5}) {
+  for (auto numSamples : {1, 3}) {
+    for (auto channels : {1, 3}) {
+      for (auto imgSizeH : {13, 17}) {
+        for (auto imgSizeW : {17, 19}) {
+          for (auto sizeX : {2, 3}) {
+            for (auto sizeY : {2, 3}) {
               for (auto sH : {1, 2}) {
                 for (auto sW : {1, 2}) {
                   for (auto pH : {0, (sizeY - 1) / 2}) {
@@ -1128,8 +1125,8 @@ TEST(Matrix, MaxOutFwdBwd) {
 }
 
 TEST(CpuMatrix, copyFrom) {
-  const size_t height = 1000;
-  const size_t width = 1000;
+  const size_t height = 31;
+  const size_t width = 53;
   CpuMatrix cpu(height, width);
   GpuMatrix gpu(height, width);
   CpuMatrix copy(height, width);
@@ -1149,6 +1146,10 @@ void testBatch2seqPadding(int batchSize, int inputDim) {
 
   IVectorPtr cpuSequence;
   generateSequenceStartPositions(batchSize, cpuSequence);
+  for (int i = 0; i < cpuSequence->getSize(); ++i) {
+    (cpuSequence->getData())[i] += 1;  // so no way that maxSeqLen is 0;
+  }
+
   IVectorPtr gpuSequence = IVector::create(cpuSequence->getSize(), true);
   gpuSequence->copyFrom(*cpuSequence);
 
@@ -1156,45 +1157,46 @@ void testBatch2seqPadding(int batchSize, int inputDim) {
   size_t maxSeqLen = *std::max_element(cpuSequence->getData(),
                                        cpuSequence->getData() + numSeq);
 
+  printf("numSeq = %ld, maxSeqLen = %ld\n", numSeq, maxSeqLen);
   MatrixPtr cBatch = std::make_shared<CpuMatrix>(numSeq * maxSeqLen, inputDim);
   MatrixPtr gBatch = std::make_shared<GpuMatrix>(numSeq * maxSeqLen, inputDim);
   MatrixPtr cCheck = std::make_shared<CpuMatrix>(numSeq * maxSeqLen, inputDim);
 
-  hl_sequence2batch_copy_padding(gBatch->getData(),
-                                 gpuInput->getData(),
-                                 cpuSequence->getData(),
-                                 inputDim,
-                                 maxSeqLen,
-                                 numSeq,
-                                 false,
-                                 true);
-  cCheck->copyFrom(*gBatch);
-
-  int* seqStart = cpuSequence->getData();
-  float* batchData = cBatch->getData();
-  float* seqData = cpuInput->getData();
-  for (size_t i = 0; i < maxSeqLen; i++) {
-    for (size_t j = 0; j < numSeq; j++) {
-      size_t sequenceStart = seqStart[j];
-      size_t sequenceLength = seqStart[j + 1] - seqStart[j];
-      if (i < sequenceLength) {
-        memcpy(batchData + (i * numSeq + j) * inputDim,
-               seqData + (sequenceStart + i) * inputDim,
-               inputDim * sizeof(real));
-      } else {
-        memset(batchData + (i * numSeq + j) * inputDim,
-               0,
-               inputDim * sizeof(real));
-      }
-    }
-  }
-
-  TensorCheckErr(*cBatch, *cCheck);
+  // hl_sequence2batch_copy_padding(gBatch->getData(),
+  //                                gpuInput->getData(),
+  //                                cpuSequence->getData(),
+  //                                inputDim,
+  //                                maxSeqLen,
+  //                                numSeq,
+  //                                false,
+  //                                true);
+  // cCheck->copyFrom(*gBatch);
+
+  // int* seqStart = cpuSequence->getData();
+  // float* batchData = cBatch->getData();
+  // float* seqData = cpuInput->getData();
+  // for (size_t i = 0; i < maxSeqLen; i++) {
+  //   for (size_t j = 0; j < numSeq; j++) {
+  //     size_t sequenceStart = seqStart[j];
+  //     size_t sequenceLength = seqStart[j + 1] - seqStart[j];
+  //     if (i < sequenceLength) {
+  //       memcpy(batchData + (i * numSeq + j) * inputDim,
+  //              seqData + (sequenceStart + i) * inputDim,
+  //              inputDim * sizeof(real));
+  //     } else {
+  //       memset(batchData + (i * numSeq + j) * inputDim,
+  //              0,
+  //              inputDim * sizeof(real));
+  //     }
+  //   }
+  // }
+
+  // TensorCheckErr(*cBatch, *cCheck);
 }
 
 TEST(Matrix, warpCTC) {
-  for (auto batchSize : {51, 526, 2884}) {
-    for (auto inputDim : {32, 512, 2026}) {
+  for (auto batchSize : {1, 3, 17}) {
+    for (auto inputDim : {1, 3, 31}) {
       VLOG(3) << " batchSize=" << batchSize << " inputDim=" << inputDim;
       testBatch2seqPadding(batchSize, inputDim);
     }

paddle/memory/detail/buddy_allocator.h

@@ -39,7 +39,7 @@ class BuddyAllocator {
 
  public:
   void* Alloc(size_t unaligned_size);
-  void Free(void*);
+  void Free(void* ptr);
   size_t Used();
 
  public:

paddle/memory/detail/meta_cache.h

@@ -33,17 +33,17 @@ namespace detail {
  */
 class MetadataCache {
  public:
-  MetadataCache(bool uses_gpu);
+  explicit MetadataCache(bool uses_gpu);
 
  public:
   /*! \brief Load the associated metadata for the specified memory block. */
-  Metadata load(const MemoryBlock*);
+  Metadata load(const MemoryBlock* memory_block);
 
   /*! \brief Store the associated metadata for the specified memory block. */
-  void store(MemoryBlock*, const Metadata&);
+  void store(MemoryBlock* memory_block, const Metadata& meta_data);
 
   /*! \brief Indicate that the specified metadata will no longer be used. */
-  void invalidate(MemoryBlock*);
+  void invalidate(MemoryBlock* memory_block);
 
  public:
   MetadataCache(const MetadataCache&) = delete;

paddle/memory/memory.h

@@ -68,7 +68,7 @@ class PODDeleter {
   static_assert(std::is_pod<T>::value, "T must be POD");
 
  public:
-  PODDeleter(Place place) : place_(place) {}
+  explicit PODDeleter(Place place) : place_(place) {}
   void operator()(T* ptr) { Free(place_, static_cast<void*>(ptr)); }
 
  private:

paddle/operators/.clang-format

+---
+Language:        Cpp
+BasedOnStyle:  Google
+Standard:  Cpp11
+...

paddle/operators/add_op.cc

@@ -18,7 +18,7 @@ namespace paddle {
 namespace operators {
 
 class AddOp : public OperatorWithKernel {
-protected:
+ protected:
   void InferShape(const InferShapeContext &ctx) const override {
     PADDLE_ENFORCE(ctx.InputSize() == 2, "Input size of AddOp must be two");
     PADDLE_ENFORCE(ctx.OutputSize() == 1, "Output size of AddOp must be one");
@@ -33,7 +33,7 @@ protected:
 };
 
 class AddOpMaker : public OpProtoAndCheckerMaker {
-public:
+ public:
   AddOpMaker(OpProto *proto, OpAttrChecker *op_checker)
       : OpProtoAndCheckerMaker(proto, op_checker) {
     AddInput("X", "The first input of add op");
@@ -48,7 +48,7 @@ The equation is: Out = X + Y
 };
 
 class AddOpGrad : public OperatorWithKernel {
-protected:
+ protected:
   void InferShape(const InferShapeContext &ctx) const override {}
 };
 

paddle/operators/add_op.cu

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
 #define EIGEN_USE_GPU
 #include "paddle/framework/op_registry.h"
 #include "paddle/operators/add_op.h"

paddle/operators/add_op.h

@@ -20,7 +20,7 @@ namespace operators {
 
 template <typename Place, typename T>
 class AddKernel : public OpKernel {
-public:
+ public:
   void Compute(const ExecutionContext& context) const override {
     auto input0 = context.Input<Tensor>(0);
     auto input1 = context.Input<Tensor>(1);

paddle/operators/cross_entropy_op.cc

@@ -18,7 +18,7 @@ namespace paddle {
 namespace operators {
 
 class OnehotCrossEntropyOp : public OperatorWithKernel {
-protected:
+ protected:
   void InferShape(const InferShapeContext &ctx) const override {
     PADDLE_ENFORCE(ctx.InputSize() == 2,
                    "Input size of OnehotCrossEntropyOp must be two");
@@ -37,7 +37,7 @@ protected:
 };
 
 class OnehotCrossEntropyOpMaker : public OpProtoAndCheckerMaker {
-public:
+ public:
   OnehotCrossEntropyOpMaker(OpProto *proto, OpAttrChecker *op_checker)
       : OpProtoAndCheckerMaker(proto, op_checker) {
     AddInput("X", "The first input of OnehotCrossEntropyOp");
@@ -54,8 +54,7 @@ OnehotCrossEntropy Operator.
 }  // namespace operators
 }  // namespace paddle
 
-REGISTER_OP(onehot_cross_entropy,
-            ops::OnehotCrossEntropyOp,
+REGISTER_OP(onehot_cross_entropy, ops::OnehotCrossEntropyOp,
             ops::OnehotCrossEntropyOpMaker);
 REGISTER_OP_CPU_KERNEL(onehot_cross_entropy,
                        ops::OnehotCrossEntropyOpKernel<ops::CPUPlace, float>);

paddle/operators/cross_entropy_op.cu

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
 #define EIGEN_USE_GPU
 #include "paddle/operators/cross_entropy_op.h"
 

paddle/operators/cross_entropy_op.h

@@ -20,7 +20,7 @@ namespace operators {
 
 template <typename Place, typename T>
 class OnehotCrossEntropyOpKernel : public OpKernel {
-public:
+ public:
   constexpr T LOG_THRESHOLD() const { return static_cast<T>(1e-20); }
 
   void Compute(const ExecutionContext& ctx) const override {

paddle/operators/fc_op.cc

@@ -18,31 +18,29 @@ namespace paddle {
 namespace operators {
 
 class FullyConnectedOp : public NetOp {
-public:
+ public:
   void Init() override {
     AddOp(OpRegistry::CreateOp("mul",
                                {
                                    Input("X"), Input("W"),
                                },
-                               {Output("before_act")},
-                               {}));
+                               {Output("before_act")}, {}));
     auto b = Input("b");
-    if (b != EMPTY_VAR_NAME()) {
+    if (b != framework::kEmptyVarName) {
       AddOp(OpRegistry::CreateOp("rowwise_add",
                                  {Output("before_act"), Input("b")},
-                                 {Output("before_act")},
-                                 {}));
+                                 {Output("before_act")}, {}));
     }
 
     auto activation = GetAttr<std::string>("activation");
-    AddOp(OpRegistry::CreateOp(
-        activation, {Output("before_act")}, {Output("Y")}, {}));
+    AddOp(OpRegistry::CreateOp(activation, {Output("before_act")},
+                               {Output("Y")}, {}));
     CompleteAddOp(false);
   }
 };
 
 class FullyConnectedOpMaker : public OpProtoAndCheckerMaker {
-public:
+ public:
   FullyConnectedOpMaker(OpProto *proto, OpAttrChecker *op_checker)
       : OpProtoAndCheckerMaker(proto, op_checker) {
     AddInput("X", "the input of fc operator");

paddle/operators/fill_zeros_like_op.cc

@@ -20,7 +20,7 @@ namespace paddle {
 namespace operators {
 
 class FillZerosLikeOp : public framework::OperatorWithKernel {
-protected:
+ protected:
   void InferShape(const framework::InferShapeContext &ctx) const override {
     PADDLE_ENFORCE(ctx.InputSize() == 1UL,
                    "Input size of FillZerosLikeOp must be one.");
@@ -36,7 +36,7 @@ protected:
 };
 
 class FillZerosLikeOpMaker : public framework::OpProtoAndCheckerMaker {
-public:
+ public:
   FillZerosLikeOpMaker(framework::OpProto *proto,
                        framework::OpAttrChecker *op_checker)
       : framework::OpProtoAndCheckerMaker(proto, op_checker) {
@@ -52,8 +52,7 @@ The output will have the same size with input.
 }  // namespace operators
 }  // namespace paddle
 
-REGISTER_OP(fill_zeros_like,
-            paddle::operators::FillZerosLikeOp,
+REGISTER_OP(fill_zeros_like, paddle::operators::FillZerosLikeOp,
             paddle::operators::FillZerosLikeOpMaker);
 REGISTER_OP_CPU_KERNEL(
     fill_zeros_like,

paddle/operators/fill_zeros_like_op.cu

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
 #include "paddle/framework/op_registry.h"
 #include "paddle/operators/fill_zeros_like_op.h"
 

paddle/operators/fill_zeros_like_op.h

@@ -22,7 +22,7 @@ namespace operators {
 
 template <typename Place, typename T>
 class FillZerosLikeKernel : public framework::OpKernel {
-public:
+ public:
   void Compute(const framework::ExecutionContext& context) const override {
     auto* output = context.Output<framework::Tensor>(0);
     output->mutable_data<T>(context.GetPlace());

paddle/operators/mean_op.cc

@@ -18,7 +18,7 @@ namespace paddle {
 namespace operators {
 
 class MeanOp : public OperatorWithKernel {
-protected:
+ protected:
   void InferShape(const InferShapeContext &ctx) const override {
     PADDLE_ENFORCE(ctx.InputSize() == 1, "Input size of AddOp must be one");
     PADDLE_ENFORCE(ctx.OutputSize() == 1, "Output size of AddOp must be one");
@@ -29,7 +29,7 @@ protected:
 };
 
 class MeanOpMaker : public OpProtoAndCheckerMaker {
-public:
+ public:
   MeanOpMaker(OpProto *proto, OpAttrChecker *op_checker)
       : OpProtoAndCheckerMaker(proto, op_checker) {
     AddInput("X", "The input of mean op");
@@ -39,9 +39,9 @@ public:
 };
 
 class MeanGradOp : public OperatorWithKernel {
-protected:
+ protected:
   void InferShape(const InferShapeContext &ctx) const override {
-    ctx.Output<Tensor>("X" + GRAD_VAR_SUFFIX())
+    ctx.Output<Tensor>("X" + framework::kGradVarSuffix)
         ->Resize(ctx.Input<Tensor>("X")->dims());
   }
 };

paddle/operators/mean_op.cu

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
 #define EIGEN_USE_GPU
 
 #include "paddle/operators/mean_op.h"

paddle/operators/mean_op.h

@@ -20,7 +20,7 @@ namespace operators {
 
 template <typename Place, typename T>
 class MeanKernel : public OpKernel {
-public:
+ public:
   void Compute(const ExecutionContext& context) const override {
     auto input = context.Input<Tensor>(0);
     auto output = context.Output<Tensor>(0);
@@ -37,12 +37,12 @@ public:
 
 template <typename Place, typename T>
 class MeanGradKernel : public OpKernel {
-public:
+ public:
   void Compute(const ExecutionContext& context) const override {
-    auto OG = context.Input<Tensor>("Out" + OperatorBase::GRAD_VAR_SUFFIX());
+    auto OG = context.Input<Tensor>("Out" + framework::kGradVarSuffix);
     PADDLE_ENFORCE(framework::product(OG->dims()) == 1,
                    "Mean Gradient should be scalar");
-    auto IG = context.Output<Tensor>("X" + OperatorBase::GRAD_VAR_SUFFIX());
+    auto IG = context.Output<Tensor>("X" + framework::kGradVarSuffix);
     IG->mutable_data<T>(context.GetPlace());
 
     T ig_size = (T)framework::product(IG->dims());

paddle/operators/mul_op.cc

@@ -18,7 +18,7 @@ namespace paddle {
 namespace operators {
 
 class MulOp : public OperatorWithKernel {
-protected:
+ protected:
   void InferShape(const InferShapeContext &ctx) const override {
     PADDLE_ENFORCE(ctx.InputSize() == 2, "The mul op must take two inputs");
     auto dim0 = ctx.Input<Tensor>(0)->dims();
@@ -34,7 +34,7 @@ protected:
 };
 
 class MulOpMaker : public OpProtoAndCheckerMaker {
-public:
+ public:
   MulOpMaker(OpProto *proto, OpAttrChecker *op_checker)
       : OpProtoAndCheckerMaker(proto, op_checker) {
     AddInput("X", "The first input of mul op");
@@ -49,7 +49,7 @@ The equation is: Out = X * Y
 };
 
 class MulOpGrad : public OperatorWithKernel {
-protected:
+ protected:
   void InferShape(const InferShapeContext &ctx) const override {}
   std::string DebugString() const override {
     LOG(INFO) << "MulGrad";

paddle/operators/mul_op.h

@@ -21,7 +21,7 @@ namespace operators {
 
 template <typename Place, typename T>
 class MulKernel : public OpKernel {
-public:
+ public:
   void Compute(const ExecutionContext& context) const override {
     Eigen::array<Eigen::IndexPair<Eigen::DenseIndex>, 1> dim_pair = {
         {Eigen::IndexPair<Eigen::DenseIndex>(1, 0)}};

paddle/operators/net_op.h

@@ -40,7 +40,7 @@ namespace operators {
  * it defines.
  */
 class NetOp : public framework::OperatorBase {
-public:
+ public:
   /**
    * Infer all the operators' input and output variables' shapes, will be called
    * before every mini-batch
@@ -90,7 +90,7 @@ public:
 
   std::vector<std::shared_ptr<OperatorBase>> ops_;
 
-private:
+ private:
   bool add_op_done_{false};
 
   template <typename T, typename KeyType>

paddle/operators/net_op_test.cc

@@ -12,7 +12,7 @@ static int infer_shape_cnt = 0;
 static int run_cnt = 0;
 
 class TestOp : public OperatorBase {
-public:
+ public:
   void InferShape(const framework::Scope& scope) const override {
     ++infer_shape_cnt;
   }
@@ -23,7 +23,7 @@ public:
 };
 
 class EmptyOp : public OperatorBase {
-public:
+ public:
   void InferShape(const Scope& scope) const override {}
   void Run(const Scope& scope,
            const platform::DeviceContext& dev_ctx) const override {}

paddle/operators/recurrent_op.cc

@@ -28,20 +28,18 @@ namespace operators {
 namespace rnn {
 
 void SegmentInputs(const std::vector<Scope*>& step_scopes,
-                   const std::vector<Link>& inlinks,
-                   const size_t seq_len,
+                   const std::vector<Link>& inlinks, const size_t seq_len,
                    bool infer_shape_mode) {
   PADDLE_ENFORCE(!inlinks.empty(), "no in links are provided.");
   for (size_t i = 0; i < inlinks.size(); ++i) {
     auto input_var = step_scopes[0]->FindVar(inlinks[i].external);
-    PADDLE_ENFORCE(input_var != nullptr,
-                   "input link [%s] is not in scope.",
+    PADDLE_ENFORCE(input_var != nullptr, "input link [%s] is not in scope.",
                    inlinks[i].external);
     Tensor* input = input_var->GetMutable<Tensor>();
-    DDim dims = input->dims();
+    framework::DDim dims = input->dims();
     PADDLE_ENFORCE(static_cast<size_t>(dims[0]) == seq_len,
                    "all the inlinks must have same length");
-    DDim step_dims = slice_ddim(dims, 1, dims.size());
+    framework::DDim step_dims = slice_ddim(dims, 1, dims.size());
     for (size_t j = 0; j < seq_len; j++) {
       Tensor* step_input =
           step_scopes[j]->NewVar(inlinks[i].internal)->GetMutable<Tensor>();
@@ -54,23 +52,21 @@ void SegmentInputs(const std::vector<Scope*>& step_scopes,
 }
 
 void ConcatOutputs(const std::vector<Scope*>& step_scopes,
-                   const std::vector<Link>& outlinks,
-                   const size_t seq_len,
+                   const std::vector<Link>& outlinks, const size_t seq_len,
                    bool infer_shape_mode) {
   for (size_t i = 0; i < outlinks.size(); i++) {
     auto output_var = step_scopes[0]->FindVar(outlinks[i].external);
-    PADDLE_ENFORCE(output_var != nullptr,
-                   "output link [%s] is not in scope.",
+    PADDLE_ENFORCE(output_var != nullptr, "output link [%s] is not in scope.",
                    outlinks[i].external);
     Tensor* output = output_var->GetMutable<Tensor>();
     if (infer_shape_mode) {
-      DDim step_dims = step_scopes[0]
+      framework::DDim step_dims = step_scopes[0]
                                       ->FindVar(outlinks[i].internal)
                                       ->GetMutable<Tensor>()
                                       ->dims();
       std::vector<int> dims_vec = vectorize(step_dims);
       dims_vec.insert(dims_vec.begin(), seq_len);
-      output->Resize(make_ddim(dims_vec));
+      output->Resize(framework::make_ddim(dims_vec));
     } else {
       output->mutable_data<float>(platform::CPUPlace());
       for (size_t j = 0; j < seq_len; j++) {
@@ -87,22 +83,16 @@ void ConcatOutputs(const std::vector<Scope*>& step_scopes,
 
 void LinkMemories(const std::vector<Scope*>& scopes,
                   const std::vector<rnn::MemoryAttr>& memories,
-                  const size_t step_id,
-                  const int offset,
+                  const size_t step_id, const int offset,
                   bool infer_shape_mode) {
   PADDLE_ENFORCE(step_id < scopes.size(),
-                 "step [%d] is out of range of step scopes' size [%d]",
-                 step_id,
+                 "step [%d] is out of range of step scopes' size [%d]", step_id,
                  scopes.size());
   PADDLE_ENFORCE(static_cast<int>(step_id) + offset >= 0,
-                 "offset [%d] must be large than -[%d]",
-                 offset,
-                 step_id);
+                 "offset [%d] must be large than -[%d]", offset, step_id);
   PADDLE_ENFORCE(step_id + offset < scopes.size(),
                  "offset [%d] is out of range, it must be less than (%d - %d)",
-                 offset,
-                 scopes.size(),
-                 step_id);
+                 offset, scopes.size(), step_id);
   auto scope = scopes[step_id];
   auto linked_scope = scopes[step_id + offset];
   for (auto& attr : memories) {
@@ -116,8 +106,7 @@ void LinkMemories(const std::vector<Scope*>& scopes,
   }
 }
 
-void InitArgument(const ArgumentName& name,
-                  Argument* arg,
+void InitArgument(const ArgumentName& name, Argument* arg,
                   const OperatorBase& op) {
   arg->step_net = op.Input(name.step_net);
   arg->step_scopes = op.Output(name.step_scopes);
@@ -126,8 +115,7 @@ void InitArgument(const ArgumentName& name,
   auto inlink_alias = op.GetAttr<std::vector<std::string>>(name.inlink_alias);
   PADDLE_ENFORCE(inlinks.size() == inlink_alias.size(),
                  "the size of inlinks and inlink_alias don't match:%d,%d",
-                 inlinks.size(),
-                 inlink_alias.size());
+                 inlinks.size(), inlink_alias.size());
   for (size_t i = 0; i < inlinks.size(); ++i) {
     rnn::Link link;
     link.external = inlinks[i];
@@ -139,8 +127,7 @@ void InitArgument(const ArgumentName& name,
   auto outlink_alias = op.GetAttr<std::vector<std::string>>(name.outlink_alias);
   PADDLE_ENFORCE(outlinks.size() == outlink_alias.size(),
                  "the size of outlinks and outlink_alias don't match:%d,%d",
-                 outlinks.size(),
-                 outlink_alias.size());
+                 outlinks.size(), outlink_alias.size());
   for (size_t i = 0; i < outlinks.size(); ++i) {
     rnn::Link link;
     link.external = outlinks[i];
@@ -156,12 +143,10 @@ void InitArgument(const ArgumentName& name,
 
   PADDLE_ENFORCE(memories.size() == boot_memories.size(),
                  "the size of memories, boot_memories don't match:%d,%d",
-                 memories.size(),
-                 boot_memories.size());
+                 memories.size(), boot_memories.size());
   PADDLE_ENFORCE(pre_memories.size() == boot_memories.size(),
                  "the size of pre_memories, boot_memories don't match:%d,%d",
-                 pre_memories.size(),
-                 boot_memories.size());
+                 pre_memories.size(), boot_memories.size());
   PADDLE_ENFORCE(memories.size() > 0, "more than 1 memories should be set");
 
   for (size_t i = 0; i < memories.size(); ++i) {
@@ -181,39 +166,39 @@ void RecurrentAlgorithm::InferShape(const Scope& scope) const {
                  ->dims()[0];
   CreateScopes(scope);
   auto step_scopes = GetStepScopes(scope);
-  rnn::SegmentInputs(
-      step_scopes, arg_->inlinks, seq_len_, true /*infer_shape_mode*/);
+  rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_,
+                     true /*infer_shape_mode*/);
   InitMemories(step_scopes[0], true /*infer_shape_mode*/);
   Variable* net = scope.FindVar(arg_->step_net);
   PADDLE_ENFORCE(net != nullptr, "failed to get step net");
   for (size_t i = 0; i < seq_len_; i++) {
     if (i > 0) {
-      rnn::LinkMemories(
-          step_scopes, arg_->memories, i, -1, true /*infer_shape_mode*/);
+      rnn::LinkMemories(step_scopes, arg_->memories, i, -1,
+                        true /*infer_shape_mode*/);
     }
     net->GetMutable<NetOp>()->InferShape(*step_scopes[i]);
   }
-  rnn::ConcatOutputs(
-      step_scopes, arg_->outlinks, seq_len_, true /*infer_shape_mode*/);
+  rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_,
+                     true /*infer_shape_mode*/);
 }
 
 void RecurrentAlgorithm::Run(const Scope& scope,
                              const platform::DeviceContext& dev_ctx) const {
   auto step_scopes = GetStepScopes(scope);
-  rnn::SegmentInputs(
-      step_scopes, arg_->inlinks, seq_len_, false /*infer_shape_mode*/);
+  rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_,
+                     false /*infer_shape_mode*/);
   InitMemories(step_scopes[0], false /*infer_shape_mode*/);
   Variable* net = scope.FindVar(arg_->step_net);
 
   for (size_t step_id = 0; step_id < seq_len_; step_id++) {
     if (step_id > 0) {
-      rnn::LinkMemories(
-          step_scopes, arg_->memories, step_id, -1, false /*infer_shape_mode*/);
+      rnn::LinkMemories(step_scopes, arg_->memories, step_id, -1,
+                        false /*infer_shape_mode*/);
     }
     net->GetMutable<NetOp>()->Run(*step_scopes[step_id], dev_ctx);
   }
-  rnn::ConcatOutputs(
-      step_scopes, arg_->outlinks, seq_len_, false /*infer_shape_mode*/);
+  rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_,
+                     false /*infer_shape_mode*/);
 }
 
 void RecurrentAlgorithm::CreateScopes(const Scope& scope) const {
@@ -245,8 +230,7 @@ void RecurrentAlgorithm::InitMemories(Scope* step_scope,
   for (auto& attr : arg_->memories) {
     Tensor* pre_mem = step_scope->NewVar(attr.pre_var)->GetMutable<Tensor>();
     PADDLE_ENFORCE(step_scope->FindVar(attr.boot_var) != nullptr,
-                   "memory [%s]'s boot variable [%s] not exists",
-                   attr.var,
+                   "memory [%s]'s boot variable [%s] not exists", attr.var,
                    attr.boot_var);
     Tensor* boot_mem = step_scope->FindVar(attr.boot_var)->GetMutable<Tensor>();
     if (infer_shape_mode) {
@@ -257,25 +241,15 @@ void RecurrentAlgorithm::InitMemories(Scope* step_scope,
   }
 }
 
-const rnn::ArgumentName RecurrentOp::kArgName{"step_net",
-                                              "step_scopes",
-                                              "inlinks",
-                                              "outlinks",
-                                              "inlink_alias",
-                                              "outlink_alias",
-                                              "memories",
-                                              "pre_memories",
-                                              "boot_memories"};
-
-const rnn::ArgumentName RecurrentGradientOp::kArgName{"step_net",
-                                                      "step_scopes",
-                                                      "outlink@grad",
-                                                      "inlink@grad",
-                                                      "inlink_alias",
-                                                      "outlink_alias",
-                                                      "memories",
-                                                      "pre_memories",
-                                                      "boot_memories@grad"};
+const rnn::ArgumentName RecurrentOp::kArgName{
+    "step_net", "step_scopes",  "inlinks",
+    "outlinks", "inlink_alias", "outlink_alias",
+    "memories", "pre_memories", "boot_memories"};
+
+const rnn::ArgumentName RecurrentGradientOp::kArgName{
+    "step_net",    "step_scopes",  "outlink@grad",
+    "inlink@grad", "inlink_alias", "outlink_alias",
+    "memories",    "pre_memories", "boot_memories@grad"};
 
 void RecurrentOp::Init() {
   OperatorBase::Init();
@@ -285,7 +259,7 @@ void RecurrentOp::Init() {
 }
 
 class RecurrentAlgorithmProtoAndCheckerMaker : public OpProtoAndCheckerMaker {
-public:
+ public:
   RecurrentAlgorithmProtoAndCheckerMaker(OpProto* proto,
                                          OpAttrChecker* op_checker)
       : OpProtoAndCheckerMaker(proto, op_checker) {
@@ -316,31 +290,29 @@ public:
 void RecurrentGradientAlgorithm::Run(
     const Scope& scope, const platform::DeviceContext& dev_ctx) const {
   auto step_scopes = GetStepScopes(scope);
-  rnn::SegmentInputs(
-      step_scopes, arg_->inlinks, seq_len_, false /*infer_shape_mode*/);
+  rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_,
+                     false /*infer_shape_mode*/);
   Variable* net = scope.FindVar(arg_->step_net);
   PADDLE_ENFORCE(net != nullptr, "failed to get step net");
   for (int step_id = seq_len_ - 1; step_id >= 0; --step_id) {
     if (static_cast<size_t>(step_id) != seq_len_ - 1) {
-      rnn::LinkMemories(
-          step_scopes, arg_->memories, step_id, 1, false /*infer_shape_mode*/);
+      rnn::LinkMemories(step_scopes, arg_->memories, step_id, 1,
+                        false /*infer_shape_mode*/);
     }
     net->GetMutable<NetOp>()->Run(*step_scopes[step_id], dev_ctx);
   }
   LinkBootMemoryGradients(step_scopes[0], false);
-  rnn::ConcatOutputs(
-      step_scopes, arg_->outlinks, seq_len_, false /*infer_shape_mode*/);
+  rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_,
+                     false /*infer_shape_mode*/);
 }
 
 void RecurrentGradientAlgorithm::LinkBootMemoryGradients(
     Scope* step_scope, bool infer_shape_mode) const {
   for (auto& attr : arg_->memories) {
     PADDLE_ENFORCE(step_scope->FindVar(attr.var) != nullptr,
-                   "memory variable [%s] does not exists",
-                   attr.var);
+                   "memory variable [%s] does not exists", attr.var);
     PADDLE_ENFORCE(step_scope->FindVar(attr.boot_var) != nullptr,
-                   "boot variable [%s] does not exists",
-                   attr.boot_var);
+                   "boot variable [%s] does not exists", attr.boot_var);
     Tensor* mem_grad = step_scope->NewVar(attr.var)->GetMutable<Tensor>();
     Tensor* boot_mem_grad =
         step_scope->NewVar(attr.boot_var)->GetMutable<Tensor>();
@@ -357,19 +329,19 @@ void RecurrentGradientAlgorithm::InferShape(const Scope& scope) const {
                  ->GetMutable<Tensor>()
                  ->dims()[0];
   auto step_scopes = GetStepScopes(scope);
-  rnn::SegmentInputs(
-      step_scopes, arg_->inlinks, seq_len_, true /*infer_shape_mode*/);
+  rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_,
+                     true /*infer_shape_mode*/);
   Variable* net = scope.FindVar(arg_->step_net);
   PADDLE_ENFORCE(net != nullptr, "failed to get step net");
   for (int step_id = seq_len_ - 1; step_id >= 0; --step_id) {
     if (static_cast<size_t>(step_id) != seq_len_ - 1) {
-      rnn::LinkMemories(
-          step_scopes, arg_->memories, step_id, 1, true /*infer_shape_mode*/);
+      rnn::LinkMemories(step_scopes, arg_->memories, step_id, 1,
+                        true /*infer_shape_mode*/);
     }
     net->GetMutable<NetOp>()->InferShape(*step_scopes[step_id]);
   }
-  rnn::ConcatOutputs(
-      step_scopes, arg_->outlinks, seq_len_, true /*infer_shape_mode*/);
+  rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_,
+                     true /*infer_shape_mode*/);
   LinkBootMemoryGradients(step_scopes[0], true /*infer_shape_mode*/);
 }
 
@@ -383,6 +355,5 @@ void RecurrentGradientOp::Init() {
 }  // namespace operators
 }  // namespace paddle
 
-REGISTER_OP(recurrent_op,
-            paddle::operators::RecurrentOp,
+REGISTER_OP(recurrent_op, paddle::operators::RecurrentOp,
             paddle::operators::RecurrentAlgorithmProtoAndCheckerMaker);

paddle/operators/recurrent_op.h

@@ -19,8 +19,6 @@
 namespace paddle {
 namespace operators {
 
-using namespace paddle::framework;
-
 namespace rnn {
 
 /**
@@ -70,31 +68,27 @@ struct ArgumentName {
 /**
  * Prepare inputs for each step net.
  */
-void SegmentInputs(const std::vector<Scope*>& step_scopes,
-                   const std::vector<Link>& inlinks,
-                   const size_t seq_len,
+void SegmentInputs(const std::vector<framework::Scope*>& step_scopes,
+                   const std::vector<Link>& inlinks, const size_t seq_len,
                    bool infer_shape_mode);
 
 /**
  * Process outputs of step nets and merge to variables.
  */
-void ConcatOutputs(const std::vector<Scope*>& step_scopes,
-                   const std::vector<Link>& outlinks,
-                   const size_t seq_len,
+void ConcatOutputs(const std::vector<framework::Scope*>& step_scopes,
+                   const std::vector<Link>& outlinks, const size_t seq_len,
                    bool infer_shape_mode);
 
-void LinkMemories(const std::vector<Scope*>& step_scopes,
-                  const std::vector<MemoryAttr>& memories,
-                  const size_t step_id,
-                  const int offset,
-                  bool infer_shape_mode);
+void LinkMemories(const std::vector<framework::Scope*>& step_scopes,
+                  const std::vector<MemoryAttr>& memories, const size_t step_id,
+                  const int offset, bool infer_shape_mode);
 
 void InitArgument(const ArgumentName& name, Argument* arg);
 
 };  // namespace rnn
 
 // The sequence format in RecurrentOp is Tensor<seq_len, batch_size, dim> now.
-// TODO:
+// TODO(Yan Chunwei):
 // 1. No-padding computing for sequences with indifinite length in one batch.
 // 2. Hierarchical RNN for sequence with sub-sequence.
 // 3. Internal Memory.
@@ -102,32 +96,35 @@ void InitArgument(const ArgumentName& name, Argument* arg);
 //    Refer to: https://arxiv.org/pdf/1502.02367.pdf
 
 class RecurrentAlgorithm {
-public:
-  void Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const;
+ public:
+  void Run(const framework::Scope& scope,
+           const platform::DeviceContext& dev_ctx) const;
 
   void Init(std::unique_ptr<rnn::Argument> arg) { arg_ = std::move(arg); }
 
   /**
    * InferShape must be called before Run.
    */
-  void InferShape(const Scope& scope) const;
+  void InferShape(const framework::Scope& scope) const;
 
-protected:
+ protected:
   /*
    * The step scopes will be stored in the father scope as a variable.
    *
    * NOTE the scopes are reused in both the forward and backward, so just
    * create once and expand its size if more steps need.
    */
-  void CreateScopes(const Scope& scope) const;
+  void CreateScopes(const framework::Scope& scope) const;
 
-  const std::vector<Scope*>& GetStepScopes(const Scope& scope) const {
-    return *scope.FindVar(arg_->step_scopes)->GetMutable<std::vector<Scope*>>();
+  const std::vector<framework::Scope*>& GetStepScopes(
+      const framework::Scope& scope) const {
+    return *scope.FindVar(arg_->step_scopes)
+                ->GetMutable<std::vector<framework::Scope*>>();
   }
 
-  void InitMemories(Scope* step_scopes, bool infer_shape_mode) const;
+  void InitMemories(framework::Scope* step_scopes, bool infer_shape_mode) const;
 
-private:
+ private:
   std::unique_ptr<rnn::Argument> arg_;
   mutable size_t seq_len_;
 };
@@ -143,69 +140,73 @@ class RecurrentGradientAlgorithm {
    * lot, and the latter is a wrapper acts like an dapter for it to make RNN an
    * operator.
    */
-public:
+ public:
   void Init(std::unique_ptr<rnn::Argument> arg) { arg_ = std::move(arg); }
 
-  void Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const;
+  void Run(const framework::Scope& scope,
+           const platform::DeviceContext& dev_ctx) const;
 
-  void LinkBootMemoryGradients(Scope* step_scopes, bool infer_shape_mode) const;
+  void LinkBootMemoryGradients(framework::Scope* step_scopes,
+                               bool infer_shape_mode) const;
 
   /**
    * InferShape must be called before Run.
    */
-  void InferShape(const Scope& scope) const;
+  void InferShape(const framework::Scope& scope) const;
 
-protected:
-  inline const std::vector<Scope*>& GetStepScopes(const Scope& scope) const {
-    return *scope.FindVar(arg_->step_scopes)->GetMutable<std::vector<Scope*>>();
+ protected:
+  inline const std::vector<framework::Scope*>& GetStepScopes(
+      const framework::Scope& scope) const {
+    return *scope.FindVar(arg_->step_scopes)
+                ->GetMutable<std::vector<framework::Scope*>>();
   }
 
-private:
+ private:
   std::unique_ptr<rnn::Argument> arg_;
   mutable size_t seq_len_;
 };
 
-class RecurrentOp final : public OperatorBase {
-public:
+class RecurrentOp final : public framework::OperatorBase {
+ public:
   void Init() override;
 
   /**
    * InferShape must be called before Run.
    */
-  virtual void InferShape(const Scope& scope) const override {
+  void InferShape(const framework::Scope& scope) const override {
     alg_.InferShape(scope);
   }
 
-  virtual void Run(const Scope& scope,
+  void Run(const framework::Scope& scope,
            const platform::DeviceContext& dev_ctx) const override {
     alg_.Run(scope, dev_ctx);
   }
 
   static const rnn::ArgumentName kArgName;
 
-private:
+ private:
   RecurrentAlgorithm alg_;
 };
 
-class RecurrentGradientOp final : public OperatorBase {
-public:
+class RecurrentGradientOp final : public framework::OperatorBase {
+ public:
   void Init() override;
 
   /**
    * InferShape must be called before Run.
    */
-  virtual void InferShape(const Scope& scope) const override {
+  void InferShape(const framework::Scope& scope) const override {
     alg_.InferShape(scope);
   }
 
-  virtual void Run(const Scope& scope,
+  void Run(const framework::Scope& scope,
            const platform::DeviceContext& dev_ctx) const override {
     alg_.Run(scope, dev_ctx);
   }
 
   static const rnn::ArgumentName kArgName;
 
-private:
+ private:
   RecurrentGradientAlgorithm alg_;
 };
 

paddle/operators/recurrent_op_test.cc

@@ -16,6 +16,7 @@
 #include <glog/logging.h>
 #include <gtest/gtest.h>
 
+#include "paddle/framework/ddim.h"
 #include "paddle/framework/op_registry.h"
 #include "paddle/framework/operator.h"
 #include "paddle/framework/tensor.h"
@@ -24,8 +25,11 @@
 namespace paddle {
 namespace operators {
 
+using framework::make_ddim;
+using framework::DDim;
+
 class RecurrentOpTest : public ::testing::Test {
-protected:
+ protected:
   virtual void SetUp() override {
     CreateGlobalVariables();
     CreateStepNet();
@@ -72,7 +76,7 @@ protected:
   }
 
   void CreateRNNOp() {
-    OpDesc op_desc;
+    framework::OpDesc op_desc;
 
     op_desc.set_type("recurrent_op");
     // inlinks 0
@@ -170,7 +174,7 @@ TEST_F(RecurrentOpTest, Run) {
 }
 
 class RecurrentGradientAlgorithmTest : public ::testing::Test {
-protected:
+ protected:
   virtual void SetUp() override {
     CreateGlobalVariables();
     CreateStepScopes();
@@ -273,13 +277,11 @@ protected:
     LOG(INFO) << "create variable step_net";
     Variable* var = scope_.NewVar("step_net");
     auto net = var->GetMutable<NetOp>();
-    net->AddOp(OpRegistry::CreateOp("mul",
-                                    {"rnn/h_pre", "rnn/w", "rnn/s_grad"},
-                                    {"rnn/h_pre_grad", "rnn/w_grad"},
-                                    {}));
+    net->AddOp(OpRegistry::CreateOp("mul", {"rnn/h_pre", "rnn/w", "rnn/s_grad"},
+                                    {"rnn/h_pre_grad", "rnn/w_grad"}, {}));
 
-    net->AddOp(OpRegistry::CreateOp(
-        "add_two", {"rnn/h_grad"}, {"rnn/x_grad", "rnn/s_grad"}, {}));
+    net->AddOp(OpRegistry::CreateOp("add_two", {"rnn/h_grad"},
+                                    {"rnn/x_grad", "rnn/s_grad"}, {}));
     net->CompleteAddOp();
   }
 
@@ -293,9 +295,7 @@ protected:
     inlink.internal = "rnn/x";
     auto step_scopes =
         scope_.FindVar("step_scopes")->GetMutable<std::vector<Scope*>>();
-    rnn::SegmentInputs(*step_scopes,
-                       std::vector<rnn::Link>{inlink},
-                       10,
+    rnn::SegmentInputs(*step_scopes, std::vector<rnn::Link>{inlink}, 10,
                        true /*infer_shape_mode*/);
   }
 
@@ -310,8 +310,8 @@ protected:
     auto step_scopes =
         scope_.FindVar("step_scopes")->GetMutable<std::vector<Scope*>>();
     for (int i = 1; i < 10; ++i) {
-      rnn::LinkMemories(
-          *step_scopes, memories, i, -1, true /*infer_shape_mode*/);
+      rnn::LinkMemories(*step_scopes, memories, i, -1,
+                        true /*infer_shape_mode*/);
     }
   }
 

paddle/operators/rowwise_add_op.cc

@@ -17,7 +17,7 @@ namespace paddle {
 namespace operators {
 
 class RowWiseAddOp : public OperatorWithKernel {
-protected:
+ protected:
   void InferShape(const InferShapeContext &ctx) const override {
     PADDLE_ENFORCE(ctx.InputSize() == 2UL,
                    "Two inputs is needed by rowwise add");
@@ -33,7 +33,7 @@ protected:
 };
 
 class RowWiseAddOpMaker : public OpProtoAndCheckerMaker {
-public:
+ public:
   RowWiseAddOpMaker(OpProto *proto, OpAttrChecker *op_checker)
       : OpProtoAndCheckerMaker(proto, op_checker) {
     AddInput("X", "The left input of row-wise add op, must be matrix");

paddle/operators/rowwise_add_op.cu

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
 #define EIGEN_USE_GPU
 #include "paddle/operators/rowwise_add_op.h"
 

paddle/operators/rowwise_add_op.h

@@ -20,7 +20,7 @@ namespace operators {
 
 template <typename Place, typename T>
 class RowWiseAddKernel : public OpKernel {
-public:
+ public:
   void Compute(const ExecutionContext& context) const override {
     auto out = context.Output<Tensor>(0);
     out->mutable_data<T>(context.GetPlace());

paddle/operators/sgd_op.cc

@@ -18,7 +18,7 @@ namespace paddle {
 namespace operators {
 
 class SGDOp : public OperatorWithKernel {
-protected:
+ protected:
   void InferShape(const InferShapeContext &ctx) const override {
     PADDLE_ENFORCE(ctx.InputSize() == 2, "Input size of SGDOp must be two");
     PADDLE_ENFORCE(ctx.OutputSize() == 1, "Output size of SGDOp must be one");
@@ -32,7 +32,7 @@ protected:
 };
 
 class SGDOpMaker : public OpProtoAndCheckerMaker {
-public:
+ public:
   SGDOpMaker(OpProto *proto, OpAttrChecker *op_checker)
       : OpProtoAndCheckerMaker(proto, op_checker) {
     AddInput("param", "input parameter");

paddle/operators/sgd_op.cu

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
 #define EIGEN_USE_GPU
 #include "paddle/operators/sgd_op.h"
 

paddle/operators/sgd_op.h

@@ -20,7 +20,7 @@ namespace operators {
 
 template <typename Place, typename T>
 class SGDOpKernel : public OpKernel {
-public:
+ public:
   void Compute(const ExecutionContext& ctx) const override {
     auto param = ctx.Input<Tensor>("param");
     auto grad = ctx.Input<Tensor>("grad");

paddle/operators/sigmoid_op.cc

@@ -17,7 +17,7 @@ namespace paddle {
 namespace operators {
 
 class SigmoidOp : public OperatorWithKernel {
-protected:
+ protected:
   void InferShape(const InferShapeContext &ctx) const override {
     PADDLE_ENFORCE(ctx.InputSize() == 1, "Sigmoid Op only have one input");
     PADDLE_ENFORCE(ctx.OutputSize() == 1, "Sigmoid Op only have one output");
@@ -26,7 +26,7 @@ protected:
 };
 
 class SigmoidOpMaker : public OpProtoAndCheckerMaker {
-public:
+ public:
   SigmoidOpMaker(OpProto *proto, OpAttrChecker *op_checker)
       : OpProtoAndCheckerMaker(proto, op_checker) {
     AddInput("X", "sigmoid input");
@@ -36,7 +36,7 @@ public:
 };
 
 class SigmoidOpGrad : public OperatorWithKernel {
-protected:
+ protected:
   void InferShape(const InferShapeContext &ctx) const override {}
   std::string DebugString() const override {
     LOG(INFO) << "SigmoidGrad";

paddle/operators/sigmoid_op.cu

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
 #define EIGEN_USE_GPU
 #include "paddle/operators/sigmoid_op.h"
 

paddle/operators/sigmoid_op.h

@@ -21,7 +21,7 @@ namespace operators {
 
 template <typename Place, typename T>
 class SigmoidKernel : public OpKernel {
-public:
+ public:
   void Compute(const ExecutionContext& context) const override {
     auto input = context.Input<Tensor>(0);
     auto output = context.Output<Tensor>(0);

paddle/operators/softmax_op.cc

@@ -18,7 +18,7 @@ namespace paddle {
 namespace operators {
 
 class SoftmaxOp : public OperatorWithKernel {
-protected:
+ protected:
   void InferShape(const InferShapeContext &ctx) const override {
     PADDLE_ENFORCE(ctx.InputSize() == 1UL,
                    "Only one input is need for softmax");
@@ -31,7 +31,7 @@ protected:
 };
 
 class SoftmaxOpMaker : public OpProtoAndCheckerMaker {
-public:
+ public:
   SoftmaxOpMaker(OpProto *proto, OpAttrChecker *op_checker)
       : OpProtoAndCheckerMaker(proto, op_checker) {
     AddInput("X", "input of softmax");
@@ -41,19 +41,19 @@ public:
 };
 
 class SoftmaxOpGrad : public OperatorWithKernel {
-protected:
+ protected:
   void InferShape(const InferShapeContext &ctx) const override {
     PADDLE_ENFORCE(ctx.InputSize() == 3UL,
                    "Input of SoftmaxOpGrad should be 3, X, Y, YG");
     PADDLE_ENFORCE(ctx.OutputSize() == 1UL,
                    "Output of SoftmaxOpGrad should be 1");
     PADDLE_ENFORCE(ctx.InputVar("Y") != nullptr, "Input(Y) should not be null");
-    PADDLE_ENFORCE(ctx.InputVar(GRAD_VAR_NAME("Y")) != nullptr,
+    PADDLE_ENFORCE(ctx.InputVar(framework::GradVarName("Y")) != nullptr,
                    "Input(Y@GRAD) should not be null");
     PADDLE_ENFORCE(ctx.Input<Tensor>("Y")->dims() ==
-                       ctx.Input<Tensor>(GRAD_VAR_NAME("Y"))->dims(),
+                       ctx.Input<Tensor>(framework::GradVarName("Y"))->dims(),
                    "the shape of Input(0) and Input(1) should be the same");
-    ctx.Output<Tensor>(GRAD_VAR_NAME("X"))
+    ctx.Output<Tensor>(framework::GradVarName("X"))
         ->Resize(ctx.Input<Tensor>("Y")->dims());
   }
 };

paddle/operators/softmax_op.cu

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
 #define EIGEN_USE_GPU
 #include "paddle/framework/op_registry.h"
 #include "paddle/operators/softmax_op.h"

paddle/operators/softmax_op.h

@@ -24,7 +24,7 @@ namespace operators {
 
 template <typename Place, typename T>
 class SoftmaxKernel : public OpKernel {
-public:
+ public:
   void Compute(const ExecutionContext& context) const override {
     auto input = context.Input<Tensor>("X");
     auto output = context.Output<Tensor>("Y");
@@ -63,13 +63,13 @@ public:
 
 template <typename Place, typename T>
 class SoftmaxGradKernel : public OpKernel {
-public:
+ public:
   void Compute(const ExecutionContext& context) const override {
     std::shared_ptr<Tensor> scale_ = std::make_shared<Tensor>();
 
     auto Y = context.Input<Tensor>("Y");
-    auto dY = context.Input<Tensor>(OperatorBase::GRAD_VAR_NAME("Y"));
-    auto dX = context.Output<Tensor>(OperatorBase::GRAD_VAR_NAME("X"));
+    auto dY = context.Input<Tensor>(framework::GradVarName("Y"));
+    auto dX = context.Output<Tensor>(framework::GradVarName("X"));
     dX->mutable_data<T>(context.GetPlace());
 
     const int batch_size = Y->dims()[0];

paddle/operators/type_alias.h

@@ -26,21 +26,16 @@ using OperatorBase = framework::OperatorBase;
 using InferShapeContext = framework::InferShapeContext;
 using ExecutionContext = framework::ExecutionContext;
 using Variable = framework::Variable;
-template <typename T,
-          int MajorType = Eigen::RowMajor,
+template <typename T, int MajorType = Eigen::RowMajor,
           typename IndexType = Eigen::DenseIndex>
 using EigenScalar = framework::EigenScalar<T, MajorType, IndexType>;
-template <typename T,
-          int MajorType = Eigen::RowMajor,
+template <typename T, int MajorType = Eigen::RowMajor,
           typename IndexType = Eigen::DenseIndex>
 using EigenVector = framework::EigenVector<T, MajorType, IndexType>;
-template <typename T,
-          int MajorType = Eigen::RowMajor,
+template <typename T, int MajorType = Eigen::RowMajor,
           typename IndexType = Eigen::DenseIndex>
 using EigenMatrix = framework::EigenMatrix<T, MajorType, IndexType>;
-template <typename T,
-          size_t D,
-          int MajorType = Eigen::RowMajor,
+template <typename T, size_t D, int MajorType = Eigen::RowMajor,
           typename IndexType = Eigen::DenseIndex>
 using EigenTensor = framework::EigenTensor<T, D, MajorType, IndexType>;
 using Tensor = framework::Tensor;

paddle/platform/device_context.h

@@ -40,7 +40,7 @@ class DeviceContext {
 class CPUDeviceContext : public DeviceContext {
  public:
   CPUDeviceContext();
-  CPUDeviceContext(CPUPlace);
+  explicit CPUDeviceContext(CPUPlace);
   virtual ~CPUDeviceContext() {}
 
   Eigen::DefaultDevice* eigen_device() const;
@@ -69,10 +69,10 @@ class CUDADeviceContext : public DeviceContext {
 
   // clang-format off
   /*! \brief  Return cublas handle in the device context. */
-  cublasHandle_t    cublas_handle   ();
+  cublasHandle_t    cublas_handle();
 
   /*! \brief  Return cudnn  handle in the device context. */
-  cudnnHandle_t     cudnn_handle    ();
+  cudnnHandle_t     cudnn_handle();
 
   /*! \brief  Return curand handle in the device context. */
   curandGenerator_t curand_generator();

paddle/platform/device_context_test.cc

@@ -15,24 +15,28 @@ limitations under the License. */
 #include "paddle/platform/device_context.h"
 #include "gtest/gtest.h"
 
-using DEVICE_GPU = Eigen::GpuDevice;
 TEST(Device, Init) {
+  using paddle::platform::DeviceContext;
+  using paddle::platform::CUDADeviceContext;
+  using paddle::platform::GPUPlace;
+
   int count = paddle::platform::GetDeviceCount();
   for (int i = 0; i < count; i++) {
-    paddle::platform::DeviceContext* device_context =
-        new paddle::platform::CUDADeviceContext(i);
+    DeviceContext* device_context = new CUDADeviceContext(GPUPlace(i));
     Eigen::GpuDevice* gpu_device =
-        device_context->template get_eigen_device<DEVICE_GPU>();
+        device_context->template get_eigen_device<Eigen::GpuDevice>();
     ASSERT_NE(nullptr, gpu_device);
     delete device_context;
   }
 }
 
 TEST(Device, CUDADeviceContext) {
+  using paddle::platform::CUDADeviceContext;
+  using paddle::platform::GPUPlace;
+
   int count = paddle::platform::GetDeviceCount();
   for (int i = 0; i < count; i++) {
-    paddle::platform::CUDADeviceContext* device_context =
-        new paddle::platform::CUDADeviceContext(i);
+    CUDADeviceContext* device_context = new CUDADeviceContext(GPUPlace(i));
     Eigen::GpuDevice* gpu_device = device_context->eigen_device();
     ASSERT_NE(nullptr, gpu_device);
     cudnnHandle_t cudnn_handle = device_context->cudnn_handle();

paddle/platform/dynload/cublas.cc

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
 #include <paddle/platform/dynload/cublas.h>
 
 namespace paddle {

paddle/platform/dynload/cudnn.cc

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
 #include <paddle/platform/dynload/cudnn.h>
 
 namespace paddle {

paddle/platform/dynload/curand.cc

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
 #include <paddle/platform/dynload/curand.h>
 
 namespace paddle {
@@ -10,6 +24,7 @@ void *curand_dso_handle;
 #define DEFINE_WRAP(__name) DynLoad__##__name __name
 
 CURAND_RAND_ROUTINE_EACH(DEFINE_WRAP);
-}
-}
-}
+
+}  // namespace dynload
+}  // namespace platform
\ No newline at end of file
+}  // namespace paddle

paddle/platform/enforce.h

@@ -162,5 +162,50 @@ inline void throw_on_error(T e) {
     }                                                                   \
   } while (0)
 
+/*
+ * Some enforce helpers here, usage:
+ *    int a = 1;
+ *    int b = 2;
+ *    PADDLE_ENFORCE_EQ(a, b);
+ *
+ *    will raise an expression described as follows:
+ *    "enforce a == b failed, 1 != 2" with detailed stack infomation.
+ *
+ *    extra messages is also supported, for example:
+ *    PADDLE_ENFORCE(a, b, "some simple enforce failed between %d numbers", 2)
+ */
+
+#define PADDLE_ENFORCE_EQ(__VAL0, __VAL1, ...) \
+  __PADDLE_BINARY_COMPARE(__VAL0, __VAL1, ==, !=, __VA_ARGS__)
+#define PADDLE_ENFORCE_NE(__VAL0, __VAL1, ...) \
+  __PADDLE_BINARY_COMPARE(__VAL0, __VAL1, !=, ==, __VA_ARGS__)
+#define PADDLE_ENFORCE_GT(__VAL0, __VAL1, ...) \
+  __PADDLE_BINARY_COMPARE(__VAL0, __VAL1, >, <=, __VA_ARGS__)
+#define PADDLE_ENFORCE_GE(__VAL0, __VAL1, ...) \
+  __PADDLE_BINARY_COMPARE(__VAL0, __VAL1, >=, <, __VA_ARGS__)
+#define PADDLE_ENFORCE_LT(__VAL0, __VAL1, ...) \
+  __PADDLE_BINARY_COMPARE(__VAL0, __VAL1, <, >=, __VA_ARGS__)
+#define PADDLE_ENFORCE_LE(__VAL0, __VAL1, ...) \
+  __PADDLE_BINARY_COMPARE(__VAL0, __VAL1, <=, >, __VA_ARGS__)
+
+// if two values have different data types, choose a compatible type for them.
+template <typename T1, typename T2>
+struct CompatibleType {
+  static const bool t1_to_t2 = std::is_convertible<T1, T2>::value;
+  typedef typename std::conditional<t1_to_t2, T2, T1>::type type;
+};
+
+#define __PADDLE_BINARY_COMPARE(__VAL0, __VAL1, __CMP, __INV_CMP, ...)        \
+  PADDLE_ENFORCE(__COMPATIBLE_TYPE(__VAL0, __VAL1, __VAL0)                    \
+                     __CMP __COMPATIBLE_TYPE(__VAL0, __VAL1, __VAL1),         \
+                 "enforce %s " #__CMP " %s failed, %s " #__INV_CMP " %s\n%s", \
+                 #__VAL0, #__VAL1, std::to_string(__VAL0),                    \
+                 std::to_string(__VAL1),                                      \
+                 paddle::string::Sprintf("" __VA_ARGS__));
+
+#define __COMPATIBLE_TYPE(__VAL0, __VAL1, __VAL)              \
+  typename paddle::platform::CompatibleType<decltype(__VAL0), \
+                                            decltype(__VAL1)>::type(__VAL)
+
 }  // namespace platform
 }  // namespace paddle

paddle/platform/enforce_test.cc

@@ -34,3 +34,165 @@ TEST(ENFORCE, FAILED) {
   }
   ASSERT_TRUE(in_catch);
 }
+
+TEST(ENFORCE, NO_ARG_OK) {
+  int a = 2;
+  int b = 2;
+  PADDLE_ENFORCE_EQ(a, b);
+  // test enforce with extra message.
+  PADDLE_ENFORCE_EQ(a, b, "some thing wrong %s", "info");
+}
+
+TEST(ENFORCE_EQ, NO_EXTRA_MSG_FAIL) {
+  int a = 2;
+  bool in_catch = false;
+
+  try {
+    PADDLE_ENFORCE_EQ(a, 1 + 3);
+
+  } catch (paddle::platform::EnforceNotMet error) {
+    in_catch = true;
+    const std::string msg = "enforce a == 1 + 3 failed, 2 != 4";
+    const char* what = error.what();
+    for (size_t i = 0; i < msg.length(); ++i) {
+      ASSERT_EQ(what[i], msg[i]);
+    }
+  }
+
+  ASSERT_TRUE(in_catch);
+}
+
+TEST(ENFORCE_EQ, EXTRA_MSG_FAIL) {
+  int a = 2;
+  bool in_catch = false;
+
+  try {
+    PADDLE_ENFORCE_EQ(a, 1 + 3, "%s size not match", "their");
+
+  } catch (paddle::platform::EnforceNotMet error) {
+    in_catch = true;
+    const std::string msg =
+        "enforce a == 1 + 3 failed, 2 != 4\ntheir size not match";
+    const char* what = error.what();
+    for (size_t i = 0; i < msg.length(); ++i) {
+      ASSERT_EQ(what[i], msg[i]);
+    }
+  }
+
+  ASSERT_TRUE(in_catch);
+}
+
+TEST(ENFORCE_NE, OK) {
+  PADDLE_ENFORCE_NE(1, 2);
+  PADDLE_ENFORCE_NE(1.0, 2UL);
+}
+TEST(ENFORCE_NE, FAIL) {
+  bool in_catch = false;
+
+  try {
+    // 2UL here to check data type compatible
+    PADDLE_ENFORCE_NE(1.0, 1UL);
+
+  } catch (paddle::platform::EnforceNotMet error) {
+    in_catch = true;
+    const std::string msg = "enforce 1.0 != 1UL failed, 1.000000 == 1";
+    const char* what = error.what();
+    for (size_t i = 0; i < msg.length(); ++i) {
+      ASSERT_EQ(what[i], msg[i]);
+    }
+  }
+
+  ASSERT_TRUE(in_catch);
+}
+
+TEST(ENFORCE_GT, OK) { PADDLE_ENFORCE_GT(2, 1); }
+TEST(ENFORCE_GT, FAIL) {
+  bool in_catch = false;
+
+  try {
+    // 2UL here to check data type compatible
+    PADDLE_ENFORCE_GT(1, 2UL);
+
+  } catch (paddle::platform::EnforceNotMet error) {
+    in_catch = true;
+    const std::string msg = "enforce 1 > 2UL failed, 1 <= 2";
+    const char* what = error.what();
+    for (size_t i = 0; i < msg.length(); ++i) {
+      ASSERT_EQ(what[i], msg[i]);
+    }
+  }
+
+  ASSERT_TRUE(in_catch);
+}
+
+TEST(ENFORCE_GE, OK) {
+  PADDLE_ENFORCE_GE(2, 2UL);
+  PADDLE_ENFORCE_GE(3, 2UL);
+  PADDLE_ENFORCE_GE(3, 2);
+  PADDLE_ENFORCE_GE(3.21, 2UL);
+}
+TEST(ENFORCE_GE, FAIL) {
+  bool in_catch = false;
+
+  try {
+    PADDLE_ENFORCE_GE(1, 2UL);
+
+  } catch (paddle::platform::EnforceNotMet error) {
+    in_catch = true;
+    const std::string msg = "enforce 1 >= 2UL failed, 1 < 2";
+    const char* what = error.what();
+    for (size_t i = 0; i < msg.length(); ++i) {
+      ASSERT_EQ(what[i], msg[i]);
+    }
+  }
+
+  ASSERT_TRUE(in_catch);
+}
+
+TEST(ENFORCE_LE, OK) {
+  PADDLE_ENFORCE_LE(1, 1);
+  PADDLE_ENFORCE_LE(1, 1UL);
+  PADDLE_ENFORCE_LE(2, 3UL);
+  PADDLE_ENFORCE_LE(2UL, 3);
+  PADDLE_ENFORCE_LE(2UL, 3.2);
+}
+TEST(ENFORCE_LE, FAIL) {
+  bool in_catch = false;
+
+  try {
+    PADDLE_ENFORCE_GT(1, 2UL);
+
+  } catch (paddle::platform::EnforceNotMet error) {
+    in_catch = true;
+    const std::string msg = "enforce 1 > 2UL failed, 1 <= 2";
+    const char* what = error.what();
+    for (size_t i = 0; i < msg.length(); ++i) {
+      ASSERT_EQ(what[i], msg[i]);
+    }
+  }
+
+  ASSERT_TRUE(in_catch);
+}
+
+TEST(ENFORCE_LT, OK) {
+  PADDLE_ENFORCE_LT(3, 10);
+  PADDLE_ENFORCE_LT(2, 3UL);
+  PADDLE_ENFORCE_LT(2UL, 3);
+}
+TEST(ENFORCE_LT, FAIL) {
+  bool in_catch = false;
+
+  try {
+    PADDLE_ENFORCE_LT(1UL, 0.12);
+
+  } catch (paddle::platform::EnforceNotMet error) {
+    in_catch = true;
+    const std::string msg = "enforce 1UL < 0.12 failed, 1 >= 0.12";
+    const char* what = error.what();
+    for (size_t i = 0; i < msg.length(); ++i) {
+      ASSERT_EQ(what[i], msg[i]);
+    }
+  }
+
+  ASSERT_TRUE(in_catch);
+}

paddle/platform/place.h

@@ -32,7 +32,7 @@ struct CPUPlace {
 
 struct GPUPlace {
   GPUPlace() : GPUPlace(0) {}
-  GPUPlace(int d) : device(d) {}
+  explicit GPUPlace(int d) : device(d) {}
 
   // needed for variant equality comparison
   inline bool operator==(const GPUPlace &o) const { return device == o.device; }

paddle/string/piece.h

@@ -39,8 +39,8 @@ public:
   // size_ is 0.
   Piece();
   Piece(const char* d, size_t n);
-  Piece(const char* d);
-  Piece(const std::string& s);
+  Piece(const char* d);         // NOLINT: accept C string into Piece.
+  Piece(const std::string& s);  // NOLINT: accept C++ string into Piece.
 
   const char* data() const { return data_; }
   size_t len() const { return size_; }

paddle/trainer/tests/sample_trainer_config_compare_sparse.conf

+#edit-mode: -*- python -*-
+# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#Todo(luotao02) This config is only used for unitest. It is out of date now, and will be updated later.
+
+# Note: when making change to this file, please make sure
+# sample_trainer_config_rnn.conf is changed accordingly so that the uniitest
+# for comparing these two nets can pass (test_CompareTwoNets)
+
+default_initial_std(0.1)
+default_device(0)
+
+word_dim = 999
+l1 = 0
+l2 = 0
+
+model_type("nn")
+
+sparse_update = get_config_arg("sparse_update", bool, False)
+
+TrainData(ProtoData(        
+            type = "proto_sequence",
+            files = ('trainer/tests/train_sparse.list'), 
+            ))
+
+Settings(
+    algorithm='sgd',
+    batch_size=100,
+    learning_rate=0.0001,
+    learning_rate_decay_a=4e-08,
+    learning_rate_decay_b=0.0,
+    learning_rate_schedule='poly',
+)
+
+
+wordvec_dim = 32
+layer2_dim = 16
+layer3_dim = 16
+hidden_dim = 32
+
+slot_names = ["qb", "qw", "tb", "tw"]
+
+def ltr_network(network_name,
+                word_dim=word_dim,
+                wordvec_dim=wordvec_dim,
+                layer2_dim=layer2_dim,
+                layer3_dim=layer3_dim,
+                hidden_dim=hidden_dim,
+                slot_names=slot_names,
+                l1=l1,
+                l2=l2):
+
+    slotnum = len(slot_names)
+    for i in xrange(slotnum):
+        Inputs(slot_names[i] + network_name)
+    for i in xrange(slotnum):
+        Layer(
+            name = slot_names[i] + network_name,
+            type = "data",
+            size = word_dim,
+            device = -1,
+        )
+        Layer(
+            name = slot_names[i] + "_embedding_" + network_name,
+            type = "mixed",
+            size = wordvec_dim,
+            bias = False,
+            device = -1,
+            inputs = TableProjection(slot_names[i] + network_name,
+                                     parameter_name = "embedding.w0",
+                                     decay_rate_l1=l1,
+                                     sparse_remote_update = True,
+                                     sparse_update = sparse_update,
+                                     ),
+        )
+        Layer(
+            name = slot_names[i] + "_rnn1_" + network_name,
+            type = "recurrent",
+            active_type = "tanh",
+            bias = Bias(initial_std = 0,
+                        parameter_name = "rnn1.bias"),
+            inputs = Input(slot_names[i] + "_embedding_" + network_name,
+                           parameter_name = "rnn1.w0")
+        )
+        Layer(
+            name = slot_names[i] + "_rnnlast_" + network_name,
+            type = "seqlastins",
+            inputs = [
+                slot_names[i] + "_rnn1_" + network_name,
+            ],
+        )
+
+    Layer(
+        name = "layer2_" + network_name,
+        type = "fc",
+        active_type = "tanh",
+        size = layer2_dim,
+        bias = Bias(parameter_name = "layer2.bias"),
+        inputs = [Input(slot_name + "_rnnlast_" + network_name, 
+                        parameter_name = "_layer2_" + slot_name + ".w", 
+                        decay_rate = l2, 
+                        initial_smart = True) for slot_name in slot_names]
+    )
+    Layer(
+        name = "layer3_" + network_name,
+        type = "fc",
+        active_type = "tanh",
+        size = layer3_dim,
+        bias = Bias(parameter_name = "layer3.bias"),
+        inputs = [
+            Input("layer2_" + network_name, 
+                  parameter_name = "_layer3.w", 
+                  decay_rate = l2, 
+                  initial_smart = True),
+        ]
+    )
+    Layer(
+        name = "output_" + network_name,
+        type = "fc",
+        size = 1,
+        bias = False,
+        inputs = [
+                  Input("layer3_" + network_name,
+                       parameter_name = "_layerO.w"),
+                 ],
+        )
+
+
+ltr_network("left")
+ltr_network("right")
+Inputs("label")
+Layer(
+    name = "label",
+    type = "data",
+    size = 1,
+    )
+Outputs("cost", "qb_rnnlast_left")
+Layer(
+    name = "cost",
+    type = "rank-cost",
+    inputs = ["output_left", "output_right", "label"],
+    )

paddle/trainer/tests/test_CompareSparse.cpp

@@ -23,7 +23,7 @@ using namespace paddle;  // NOLINT
 using namespace std;     // NOLINT
 
 static const string& configFile1 =
-    "trainer/tests/sample_trainer_config_qb_rnn.conf";
+    "trainer/tests/sample_trainer_config_compare_sparse.conf";
 
 DECLARE_bool(use_gpu);
 DECLARE_string(config);

paddle/trainer/tests/train_sparse.list

+trainer/tests/compare_sparse_data

python/paddle/v2/dataset/cifar.py

@@ -133,7 +133,7 @@ def convert(path):
     """
     Converts dataset to recordio format
     """
-    paddle.v2.dataset.common.convert(path, train100(), 10, "cifar_train100")
-    paddle.v2.dataset.common.convert(path, test100(), 10, "cifar_test100")
-    paddle.v2.dataset.common.convert(path, train10(), 10, "cifar_train10")
-    paddle.v2.dataset.common.convert(path, test10(), 10, "cifar_test10")
+    paddle.v2.dataset.common.convert(path, train100(), 1000, "cifar_train100")
+    paddle.v2.dataset.common.convert(path, test100(), 1000, "cifar_test100")
+    paddle.v2.dataset.common.convert(path, train10(), 1000, "cifar_train10")
+    paddle.v2.dataset.common.convert(path, test10(), 1000, "cifar_test10")

python/paddle/v2/dataset/common.py

@@ -32,19 +32,24 @@ __all__ = [
 
 DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset')
 
+
 # When running unit tests, there could be multiple processes that
 # trying to create DATA_HOME directory simultaneously, so we cannot
 # use a if condition to check for the existence of the directory;
 # instead, we use the filesystem as the synchronization mechanism by
 # catching returned errors.
-try:
+def must_mkdirs(path):
+    try:
         os.makedirs(DATA_HOME)
-except OSError as exc:
+    except OSError as exc:
         if exc.errno != errno.EEXIST:
             raise
         pass
 
 
+must_mkdirs(DATA_HOME)
+
+
 def md5file(fname):
     hash_md5 = hashlib.md5()
     f = open(fname, "rb")
@@ -93,6 +98,19 @@ def fetch_all():
                 "fetch")()
 
 
+def fetch_all_recordio(path):
+    for module_name in filter(lambda x: not x.startswith("__"),
+                              dir(paddle.v2.dataset)):
+        if "convert" in dir(
+                importlib.import_module("paddle.v2.dataset.%s" % module_name)) and \
+                not module_name == "common":
+            ds_path = os.path.join(path, module_name)
+            must_mkdirs(ds_path)
+            getattr(
+                importlib.import_module("paddle.v2.dataset.%s" % module_name),
+                "convert")(ds_path)
+
+
 def split(reader, line_count, suffix="%05d.pickle", dumper=cPickle.dump):
     """
     you can call the function as:

python/paddle/v2/dataset/conll05.py

@@ -233,5 +233,5 @@ def convert(path):
     """
     Converts dataset to recordio format
     """
-    paddle.v2.dataset.common.convert(path, test(), 10, "conl105_train")
-    paddle.v2.dataset.common.convert(path, test(), 10, "conl105_test")
+    paddle.v2.dataset.common.convert(path, test(), 1000, "conl105_train")
+    paddle.v2.dataset.common.convert(path, test(), 1000, "conl105_test")

python/paddle/v2/dataset/imdb.py

@@ -173,5 +173,5 @@ def convert(path):
     Converts dataset to recordio format
     """
     w = word_dict()
-    paddle.v2.dataset.common.convert(path, lambda: train(w), 10, "imdb_train")
-    paddle.v2.dataset.common.convert(path, lambda: test(w), 10, "imdb_test")
+    paddle.v2.dataset.common.convert(path, lambda: train(w), 1000, "imdb_train")
+    paddle.v2.dataset.common.convert(path, lambda: test(w), 1000, "imdb_test")

python/paddle/v2/dataset/imikolov.py

@@ -155,6 +155,7 @@ def convert(path):
     N = 5
     word_dict = build_dict()
     paddle.v2.dataset.common.convert(path,
-                                     train(word_dict, N), 10, "imikolov_train")
+                                     train(word_dict, N), 1000,
+                                     "imikolov_train")
     paddle.v2.dataset.common.convert(path,
-                                     test(word_dict, N), 10, "imikolov_test")
+                                     test(word_dict, N), 1000, "imikolov_test")

python/paddle/v2/dataset/mnist.py

@@ -119,5 +119,5 @@ def convert(path):
     """
     Converts dataset to recordio format
     """
-    paddle.v2.dataset.common.convert(path, train(), 10, "minist_train")
-    paddle.v2.dataset.common.convert(path, test(), 10, "minist_test")
+    paddle.v2.dataset.common.convert(path, train(), 1000, "minist_train")
+    paddle.v2.dataset.common.convert(path, test(), 1000, "minist_test")

python/paddle/v2/dataset/movielens.py

@@ -254,8 +254,8 @@ def convert(path):
     """
     Converts dataset to recordio format
     """
-    paddle.v2.dataset.common.convert(path, train(), 10, "movielens_train")
-    paddle.v2.dataset.common.convert(path, test(), 10, "movielens_test")
+    paddle.v2.dataset.common.convert(path, train(), 1000, "movielens_train")
+    paddle.v2.dataset.common.convert(path, test(), 1000, "movielens_test")
 
 
 if __name__ == '__main__':

python/paddle/v2/dataset/sentiment.py

@@ -137,5 +137,5 @@ def convert(path):
     """
     Converts dataset to recordio format
     """
-    paddle.v2.dataset.common.convert(path, train, 10, "sentiment_train")
-    paddle.v2.dataset.common.convert(path, test, 10, "sentiment_test")
+    paddle.v2.dataset.common.convert(path, train, 1000, "sentiment_train")
+    paddle.v2.dataset.common.convert(path, test, 1000, "sentiment_test")

python/paddle/v2/dataset/uci_housing.py

@@ -119,5 +119,5 @@ def convert(path):
     """
     Converts dataset to recordio format
     """
-    paddle.v2.dataset.common.convert(path, train(), 10, "uci_housing_train")
-    paddle.v2.dataset.common.convert(path, test(), 10, "uci_houseing_test")
+    paddle.v2.dataset.common.convert(path, train(), 1000, "uci_housing_train")
+    paddle.v2.dataset.common.convert(path, test(), 1000, "uci_houseing_test")

python/paddle/v2/dataset/wmt14.py

@@ -169,5 +169,6 @@ def convert(path):
     Converts dataset to recordio format
     """
     dict_size = 30000
-    paddle.v2.dataset.common.convert(path, train(dict_size), 10, "wmt14_train")
-    paddle.v2.dataset.common.convert(path, test(dict_size), 10, "wmt14_test")
+    paddle.v2.dataset.common.convert(path,
+                                     train(dict_size), 1000, "wmt14_train")
+    paddle.v2.dataset.common.convert(path, test(dict_size), 1000, "wmt14_test")

python/paddle/v2/framework/create_op_creation_methods.py

 import paddle.v2.framework.core as core
 import paddle.v2.framework.proto.op_proto_pb2 as op_proto_pb2
 import paddle.v2.framework.proto.op_desc_pb2 as op_desc_pb2
-import paddle.v2.framework.proto.attr_type_pb2 as attr_type_pb2
+import paddle.v2.framework.proto.attribute_pb2 as attribute_pb2
 import cStringIO
 
 
@@ -57,7 +57,7 @@ class OpDescCreationMethod(object):
             op_desc.attrs.extend([out_format])
         if len(tmp_index) != 0:
             tmp_index_attr = op_desc.attrs.add()
-            tmp_index_attr.type = attr_type_pb2.INTS
+            tmp_index_attr.type = attribute_pb2.INTS
             tmp_index_attr.name = "temporary_index"
             tmp_index_attr.ints.extend(tmp_index)
 
@@ -73,17 +73,17 @@ class OpDescCreationMethod(object):
                 new_attr = op_desc.attrs.add()
                 new_attr.name = attr.name
                 new_attr.type = attr.type
-                if attr.type == attr_type_pb2.INT:
+                if attr.type == attribute_pb2.INT:
                     new_attr.i = user_defined_attr
-                elif attr.type == attr_type_pb2.FLOAT:
+                elif attr.type == attribute_pb2.FLOAT:
                     new_attr.f = user_defined_attr
-                elif attr.type == attr_type_pb2.STRING:
+                elif attr.type == attribute_pb2.STRING:
                     new_attr.s = user_defined_attr
-                elif attr.type == attr_type_pb2.INTS:
+                elif attr.type == attribute_pb2.INTS:
                     new_attr.ints.extend(user_defined_attr)
-                elif attr.type == attr_type_pb2.FLOATS:
+                elif attr.type == attribute_pb2.FLOATS:
                     new_attr.floats.extend(user_defined_attr)
-                elif attr.type == attr_type_pb2.STRINGS:
+                elif attr.type == attribute_pb2.STRINGS:
                     new_attr.strings.extend(user_defined_attr)
                 else:
                     raise NotImplementedError("Not support attribute type " +
@@ -109,7 +109,7 @@ class OpDescCreationMethod(object):
         retv = []
         if multiple:
             var_format = op_desc_pb2.AttrDesc()
-            var_format.type = attr_type_pb2.INTS
+            var_format.type = attribute_pb2.INTS
             var_format.name = "%s_format" % in_out
             var_format.ints.append(0)
 
@@ -185,17 +185,17 @@ def get_docstring_from_op_proto(op_proto):
 
     for attr in op_proto.attrs:
         attr_type = None
-        if attr.type == attr_type_pb2.INT:
+        if attr.type == attribute_pb2.INT:
             attr_type = "int"
-        elif attr.type == attr_type_pb2.FLOAT:
+        elif attr.type == attribute_pb2.FLOAT:
             attr_type = "float"
-        elif attr.type == attr_type_pb2.STRING:
+        elif attr.type == attribute_pb2.STRING:
             attr_type = "basestr"
-        elif attr.type == attr_type_pb2.INTS:
+        elif attr.type == attribute_pb2.INTS:
             attr_type = "list of int"
-        elif attr.type == attr_type_pb2.FLOATS:
+        elif attr.type == attribute_pb2.FLOATS:
             attr_type = "list of float"
-        elif attr.type == attr_type_pb2.STRINGS:
+        elif attr.type == attribute_pb2.STRINGS:
             attr_type = "list of basestr"
 
         if attr_type is None:

python/paddle/v2/framework/tests/CMakeLists.txt

@@ -13,4 +13,5 @@ add_python_test(test_framework
     test_sigmoid_op.py
     test_softmax_op.py
     test_rowwise_add_op.py
-    test_network.py)
+    test_network.py
+    gradient_checker.py)

python/paddle/v2/framework/tests/gradient_checker.py

+import paddle.v2.framework.core as core
+from paddle.v2.framework.create_op_creation_methods import op_creations
+import numpy
+import unittest
+
+__all__ = ['get_numeric_gradient']
+
+
+def get_numeric_gradient(op,
+                         input_values,
+                         output_name,
+                         input_to_check,
+                         delta=1e-2,
+                         local_scope=None):
+    """
+    Get Numeric Gradient for an operator's input.
+    
+    :param op: C++ operator instance, could be an network 
+    :param input_values: The input variables. Should be an dictionary, key is 
+    variable name. Value is numpy array.
+    :param output_name: The final output variable name. 
+    :param input_to_check: The input variable need to get gradient.
+    :param delta: The perturbation value for numeric gradient method. The 
+    smaller delta is, the more accurate result will get. But if that delta is
+     too small, it could occur numerical stability problem.
+    :param local_scope: The local scope used for get_numeric_gradient.
+    :return: The gradient array in numpy format.
+    """
+    if local_scope is None:
+        local_scope = core.Scope()
+
+    # Create all input variable in local_scope
+    for var_name in input_values:
+        var = local_scope.new_var(var_name)
+        tensor = var.get_tensor()
+        tensor.set_dims(input_values[var_name].shape)
+        tensor.alloc_float(core.CPUPlace())
+        tensor.set(input_values[var_name], core.CPUPlace())
+
+    # Create all output variable in local_scope
+    for output in op.outputs():
+        if local_scope.find_var(output) is None:
+            local_scope.new_var(output).get_tensor()
+
+    op.infer_shape(local_scope)
+
+    # allocate output memory
+    for output in op.outputs():
+        local_scope.find_var(output).get_tensor().alloc_float(core.CPUPlace())
+
+    # TODO(yuyang18): Only CPU is support now.
+    cpu_ctx = core.DeviceContext.create(core.CPUPlace())
+
+    def get_output():
+        op.run(local_scope, cpu_ctx)
+        return numpy.array(local_scope.find_var(output_name).get_tensor()).sum()
+
+    def product(dim):
+        return reduce(lambda a, b: a * b, dim, 1)
+
+    tensor_to_check = local_scope.find_var(input_to_check).get_tensor()
+    tensor_size = product(tensor_to_check.get_dims())
+    gradient_flat = numpy.zeros(shape=(tensor_size, ), dtype='float32')
+    for i in xrange(tensor_size):
+        origin = tensor_to_check.get_float_element(i)
+        x_pos = origin + delta
+        tensor_to_check.set_float_element(i, x_pos)
+        y_pos = get_output()
+
+        x_neg = origin - delta
+        tensor_to_check.set_float_element(i, x_neg)
+        y_neg = get_output()
+
+        tensor_to_check.set_float_element(i, origin)  # restore old value
+        gradient_flat[i] = (y_pos - y_neg) / delta / 2
+    return gradient_flat.reshape(tensor_to_check.get_dims())
+
+
+if __name__ == '__main__':
+
+    class GetNumericGradientTest(unittest.TestCase):
+        def test_add_op(self):
+            add_op = op_creations.add_two(X="X", Y="Y", Out="Z")
+            x = numpy.random.random((10, 1)).astype("float32")
+            y = numpy.random.random((10, 1)).astype("float32")
+
+            arr = get_numeric_gradient(add_op, {'X': x, "Y": y}, 'Z', 'X')
+            self.assertAlmostEqual(arr.mean(), 1.0, delta=1e-2)
+
+    unittest.main()

python/paddle/v2/framework/tests/test_op_creation_methods.py

@@ -3,7 +3,7 @@ import paddle.v2.framework.create_op_creation_methods as creation
 import paddle.v2.framework.core as core
 import paddle.v2.framework.proto.op_proto_pb2 as op_proto_pb2
 import paddle.v2.framework.proto.op_desc_pb2 as op_desc_pb2
-import paddle.v2.framework.proto.attr_type_pb2 as attr_type_pb2
+import paddle.v2.framework.proto.attribute_pb2 as attribute_pb2
 
 
 class TestGetAllProtos(unittest.TestCase):
@@ -76,7 +76,7 @@ class TestOpDescCreationMethod(unittest.TestCase):
         expected1.type = 'fc'
         attr = expected1.attrs.add()
         attr.name = 'input_format'
-        attr.type = attr_type_pb2.INTS
+        attr.type = attribute_pb2.INTS
         attr.ints.extend([0, 1, 2, 3])
         self.assertEqual(expected1, generated1)
 
@@ -88,7 +88,7 @@ class TestOpDescCreationMethod(unittest.TestCase):
         expected2.type = 'fc'
         attr = expected2.attrs.add()
         attr.name = 'input_format'
-        attr.type = attr_type_pb2.INTS
+        attr.type = attribute_pb2.INTS
         attr.ints.extend([0, 3, 6, 7])
         self.assertEqual(expected2, generated2)
 
@@ -105,12 +105,12 @@ class TestOpDescCreationMethod(unittest.TestCase):
             attr.comment = ""
             attr.type = type
 
-        __add_attr__("int_attr", attr_type_pb2.INT)
-        __add_attr__("float_attr", attr_type_pb2.FLOAT)
-        __add_attr__("string_attr", attr_type_pb2.STRING)
-        __add_attr__("ints_attr", attr_type_pb2.INTS)
-        __add_attr__("floats_attr", attr_type_pb2.FLOATS)
-        __add_attr__("strings_attr", attr_type_pb2.STRINGS)
+        __add_attr__("int_attr", attribute_pb2.INT)
+        __add_attr__("float_attr", attribute_pb2.FLOAT)
+        __add_attr__("string_attr", attribute_pb2.STRING)
+        __add_attr__("ints_attr", attribute_pb2.INTS)
+        __add_attr__("floats_attr", attribute_pb2.FLOATS)
+        __add_attr__("strings_attr", attribute_pb2.STRINGS)
 
         op.comment = ""
         self.assertTrue(op.IsInitialized())
@@ -131,32 +131,32 @@ class TestOpDescCreationMethod(unittest.TestCase):
         expected.inputs.extend(['a'])
         attr = expected.attrs.add()
         attr.name = "int_attr"
-        attr.type = attr_type_pb2.INT
+        attr.type = attribute_pb2.INT
         attr.i = 10
 
         attr = expected.attrs.add()
         attr.name = "float_attr"
-        attr.type = attr_type_pb2.FLOAT
+        attr.type = attribute_pb2.FLOAT
         attr.f = 3.2
 
         attr = expected.attrs.add()
         attr.name = "string_attr"
-        attr.type = attr_type_pb2.STRING
+        attr.type = attribute_pb2.STRING
         attr.s = "test_str"
 
         attr = expected.attrs.add()
         attr.name = "ints_attr"
-        attr.type = attr_type_pb2.INTS
+        attr.type = attribute_pb2.INTS
         attr.ints.extend([0, 1, 2, 3, 4])
 
         attr = expected.attrs.add()
         attr.name = 'floats_attr'
-        attr.type = attr_type_pb2.FLOATS
+        attr.type = attribute_pb2.FLOATS
         attr.floats.extend([0.2, 3.2, 4.5])
 
         attr = expected.attrs.add()
         attr.name = 'strings_attr'
-        attr.type = attr_type_pb2.STRINGS
+        attr.type = attribute_pb2.STRINGS
         attr.strings.extend(['a', 'b', 'c'])
 
         self.assertEqual(expected, generated)
@@ -185,7 +185,7 @@ class TestOpDescCreationMethod(unittest.TestCase):
         desc.type = "test"
         attr = desc.attrs.add()
         attr.name = "temporary_index"
-        attr.type = attr_type_pb2.INTS
+        attr.type = attribute_pb2.INTS
         attr.ints.append(2)
         self.assertEqual(generated, desc)
 
@@ -219,7 +219,7 @@ This op is used for unit test, not a real op.
 
         test_str = op.attrs.add()
         test_str.name = "str_attr"
-        test_str.type = attr_type_pb2.STRING
+        test_str.type = attribute_pb2.STRING
         test_str.comment = "A string attribute for test op"
 
         actual = creation.get_docstring_from_op_proto(op)

python/paddle/v2/framework/tests/test_protobuf.py

-import paddle.v2.framework.proto.op_proto_pb2
-import paddle.v2.framework.proto.attr_type_pb2
+import paddle.v2.framework.proto.op_proto_pb2 as op_proto_lib
+import paddle.v2.framework.proto.attribute_pb2 as attr_type_lib
 import unittest
 
 
 class TestFrameworkProto(unittest.TestCase):
     def test_all(self):
-        op_proto_lib = paddle.v2.framework.proto.op_proto_pb2
-        attr_type_lib = paddle.v2.framework.proto.attr_type_pb2
         op_proto = op_proto_lib.OpProto()
         ipt0 = op_proto.inputs.add()
         ipt0.name = "a"