Get rid of the calling of inplace op in FCOp.

paddle/operators/fc_op.cc

@@ -26,38 +26,43 @@ class FCOp : public NetOp {
       : NetOp(type, inputs, outputs, attrs) {
     auto x = Inputs("X");
     auto w = Inputs("W");
+    auto mul_out = Outputs("mul_out");
     PADDLE_ENFORCE_EQ(
         x.size(), w.size(),
         "The size of inputs X(%d) should be the same as that of weights W(%d).",
         x.size(), w.size());
+    PADDLE_ENFORCE_EQ(mul_out.size(), x.size(),
+                      "The size of intermediate mul_out(%d) should be the same "
+                      "as that of inputs X(%d).",
+                      mul_out.size(), x.size());
 
     int n = x.size();
     PADDLE_ENFORCE_GE(n, 1,
                       "The size of inputs X(%d) should be no less than 1.", n);
 
-    // mul_out = X[0] * W[0] + ... + X[n-1] * W[n-1]
-    AppendOp(
-        framework::OpRegistry::CreateOp("mul", {{"X", {x[0]}}, {"Y", {w[0]}}},
-                                        {{"Out", {Output("mul_out")}}}, {}));
+    // mul_out[i] = X[i] * W[i]
+    for (int i = 0; i < n; i++) {
+      AppendOp(framework::OpRegistry::CreateOp(
+          "mul", {{"X", {x[i]}}, {"Y", {w[i]}}}, {{"Out", {mul_out[i]}}}, {}));
+    }
 
-    for (int i = 1; i < n; i++) {
-      // mul_out = mul_out + X[i] * W[i]
-      AppendOp(
-          framework::OpRegistry::CreateOp("mul", {{"X", {x[i]}}, {"Y", {w[i]}}},
-                                          {{"Out", {Output("add_out")}}}, {}));
+    // sum_out = X[0] * W[0] + ... + X[n-1] * W[n-1]
+    if (n > 1) {
+      AppendOp(framework::OpRegistry::CreateOp(
+          "sum", {{"X", {mul_out}}}, {{"Out", {Output("sum_out")}}}, {}));
+    } else {
       AppendOp(framework::OpRegistry::CreateOp(
-          "add", {{"X", {Output("mul_out")}}, {"Y", {Output("add_out")}}},
-          {{"Out", {Output("mul_out")}}}, {}));
+          "identity", {{"X", {mul_out[0]}}}, {{"Y", {Output("sum_out")}}}, {}));
     }
 
+    // add_out = sum_out + b
     auto b = Input("b");
-    std::string add_out = "mul_out";
+    std::string add_out = "sum_out";
     if (b != framework::kEmptyVarName) {
-      // add_out = mul_out + b
-      AppendOp(framework::OpRegistry::CreateOp(
-          "rowwise_add", {{"X", {Output("mul_out")}}, {"b", {Input("b")}}},
-          {{"Out", {Output("add_out")}}}, {}));
       add_out = "add_out";
+      AppendOp(framework::OpRegistry::CreateOp(
+          "rowwise_add", {{"X", {Output("sum_out")}}, {"b", {Input("b")}}},
+          {{"Out", {Output(add_out)}}}, {}));
     } else {
       if (Output("add_out") != framework::kEmptyVarName) {
         this->Rename(Output("add_out"), framework::kEmptyVarName);
@@ -68,8 +73,6 @@ class FCOp : public NetOp {
     AppendOp(framework::OpRegistry::CreateOp(
         activation, {{"X", {Output(add_out)}}}, {{"Y", {Output("Y")}}}, {}));
     CompleteAddOp(false);
-
-    std::cout << DebugString() << std::endl;
   }
 };
 
@@ -77,14 +80,24 @@ class FCOpMaker : public framework::OpProtoAndCheckerMaker {
  public:
   FCOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
       : OpProtoAndCheckerMaker(proto, op_checker) {
-    AddInput("X", "The 2-D input matrix of FC operator.").AsDuplicable();
-    AddInput("W", "The 2-D weight matrix of FC operator.").AsDuplicable();
+    AddInput("X", "The inputs of FC operator, a ordered vector of 2-D matrix.")
+        .AsDuplicable();
+    AddInput("W", "The weights of FC operator, a ordered vector of 2-D matrix.")
+        .AsDuplicable();
     AddInput("b", "The 1-D bias vector of FC operator");
 
     AddOutput("Y", "The activated output matrix of FC operator");
-    AddOutput("mul_out", "The non-actived output of FC operator, X * W")
+    AddOutput("mul_out",
+              "The intermediate outputs of FC operator, "
+              "saving the product of X[i] * W[i]")
+        .AsIntermediate()
+        .AsDuplicable();
+    AddOutput("sum_out",
+              "The intermediate output of FC operator, "
+              "saving the sum of products, sum(X[i] * W[i])")
         .AsIntermediate();
-    AddOutput("add_out", "The non-actived output of FC operator, X * W + b")
+    AddOutput("add_out",
+              "The non-actived output of FC operator, saving X * W + b")
         .AsIntermediate();
     AddAttr<std::string>("activation", "The activation type of FC operator.")
         .SetDefault("identity")

python/paddle/v2/framework/tests/test_fc_op.py

@@ -3,33 +3,65 @@ import numpy as np
 from op_test import OpTest
 
 
-class TestFCOp(OpTest):
+class TestFCOp1(OpTest):
     def setUp(self):
-        print "Run"
         self.op_type = "fc"
-        x0 = np.random.random((32, 256)).astype("float32")
-        x1 = np.random.random((32, 256)).astype("float32")
-        w0 = np.random.random((256, 100)).astype("float32")
-        w1 = np.random.random((256, 100)).astype("float32")
-        b = np.random.random(100).astype("float32")
+        x1 = np.random.random((16, 32)).astype("float32")
+        w1 = np.random.random((32, 10)).astype("float32")
+        b = np.random.random(10).astype("float32")
+        self.inputs = {"X": {"X1": x1}, "W": {"W1": w1}, "b": b}
+        mul_out1 = np.dot(x1, w1)
+        sum_out = mul_out1
+        add_out = sum_out + b
+        identity_out = add_out
+        self.outputs = {
+            "mul_out": {
+                "mul_out1": mul_out1,
+            },
+            "sum_out": sum_out,
+            "add_out": add_out,
+            "Y": identity_out
+        }
+
+    def test_check_output(self):
+        self.check_output()
+
+    def test_check_grad(self):
+        self.check_grad(["X1", "W1", "b"], "Y", max_relative_error=0.05)
+
+
+class TestFCOp2(OpTest):
+    def setUp(self):
+        self.op_type = "fc"
+        x1 = np.random.random((16, 32)).astype("float32")
+        x2 = np.random.random((16, 32)).astype("float32")
+        w1 = np.random.random((32, 10)).astype("float32")
+        w2 = np.random.random((32, 10)).astype("float32")
+        b = np.random.random(10).astype("float32")
         self.inputs = {
             "X": {
-                "X0": x0,
-                "X1": x1
+                "X1": x1,
+                "X2": x2
             },
             "W": {
-                "W0": w0,
-                "W1": w1
+                "W1": w1,
+                "W2": w2
             },
             "b": b
         }
         #self.attrs = {"activation": "sigmoid"}
-        mul_out = np.dot(x0, w0) + np.dot(x1, w1)
-        add_out = np.add(mul_out, b)
+        mul_out1 = np.dot(x1, w1)
+        mul_out2 = np.dot(x2, w2)
+        sum_out = mul_out1 + mul_out2
+        add_out = np.add(sum_out, b)
         #sigmoid_out = 1 / (1 + np.exp(-add_out))
         sigmoid_out = add_out
         self.outputs = {
-            "mul_out": mul_out,
+            "mul_out": {
+                "mul_out0": mul_out1,
+                "mul_out1": mul_out2
+            },
+            "sum_out": sum_out,
             "add_out": add_out,
             "Y": sigmoid_out
         }
@@ -37,8 +69,9 @@ class TestFCOp(OpTest):
     def test_check_output(self):
         self.check_output()
 
-    #def test_check_grad(self):
-    #    self.check_grad(["X0", "X1", "W0", "W1", "b"], "Y")
+    def test_check_grad(self):
+        self.check_grad(
+            ["X1", "X2", "W1", "W2", "b"], "Y", max_relative_error=0.05)
 
 
 if __name__ == '__main__':