diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc
index 1677a3ed4c85ef293f0aadc64a4caa809cbd6ced..38194b716dc9675d011dc18fa60aa7c14a65e915 100644
--- a/paddle/framework/backward_test.cc
+++ b/paddle/framework/backward_test.cc
@@ -150,19 +150,20 @@ class AddOpMaker : public OpProtoAndCheckerMaker {
 namespace f = paddle::framework;
 namespace ops = paddle::operators;
 using EnforceNotMet = paddle::platform::EnforceNotMet;
-REGISTER_OP(rowwise_add, f::EmptyOp, f::RowWiseAddOpMaker);
-REGISTER_GRADIENT_OP(rowwise_add, rowwise_add_grad, f::EmptyOp);
-REGISTER_OP(mul, f::EmptyOp, f::MulOpMaker);
-REGISTER_GRADIENT_OP(mul, mul_grad, f::EmptyOp);
-REGISTER_OP(sigmoid, f::EmptyOp, f::SigmoidOpMaker);
-REGISTER_GRADIENT_OP(sigmoid, sigmoid_grad, f::EmptyOp);
-REGISTER_OP(nograd, f::EmptyOp, f::NoGradOpMaker);
-REGISTER_OP(fill_zeros_like, f::EmptyOp, f::FillZeroOpMaker);
-REGISTER_OP(add, f::EmptyOp, f::AddOpMaker);
-REGISTER_GRADIENT_OP(add, add_grad, f::EmptyOp);
-REGISTER_OP(fc, f::FcOp, f::FcOpMaker);
-REGISTER_OP(many_output_op, f::EmptyOp, f::ManyOutputOpMaker);
-REGISTER_GRADIENT_OP(many_output_op, many_output_op_grad, f::EmptyOp);
+REGISTER_OP(rowwise_add, f::EmptyOp, f::RowWiseAddOpMaker, rowwise_add_grad);
+REGISTER_GRADIENT_OP(rowwise_add_grad, f::EmptyOp);
+REGISTER_OP(mul, f::EmptyOp, f::MulOpMaker, mul_grad);
+REGISTER_GRADIENT_OP(mul_grad, f::EmptyOp);
+REGISTER_OP(sigmoid, f::EmptyOp, f::SigmoidOpMaker, sigmoid_grad);
+REGISTER_GRADIENT_OP(sigmoid_grad, f::EmptyOp);
+REGISTER_OP_WITHOUT_GRADIENT(nograd, f::EmptyOp, f::NoGradOpMaker);
+REGISTER_OP_WITHOUT_GRADIENT(fill_zeros_like, f::EmptyOp, f::FillZeroOpMaker);
+REGISTER_OP(add, f::EmptyOp, f::AddOpMaker, add_grad);
+REGISTER_GRADIENT_OP(add_grad, f::EmptyOp);
+REGISTER_OP_WITHOUT_GRADIENT(fc, f::FcOp, f::FcOpMaker);
+REGISTER_OP(many_output_op, f::EmptyOp, f::ManyOutputOpMaker,
+            many_output_op_grad);
+REGISTER_GRADIENT_OP(many_output_op_grad, f::EmptyOp);
 
 TEST(Backward, simple_op_grad) {
   auto fwd = f::OpRegistry::CreateOp("rowwise_add", {"X", "b"}, {"Out"}, {});
diff --git a/paddle/framework/grad_op_builder_test.cc b/paddle/framework/grad_op_builder_test.cc
index f1ebbae52f13d9c0fc9408aec8c4160575ad59c0..ad61b482e05c79105c56a405cbde4332abcddd54 100644
--- a/paddle/framework/grad_op_builder_test.cc
+++ b/paddle/framework/grad_op_builder_test.cc
@@ -61,10 +61,10 @@ TEST(GradOpBuilder, AddTwo) {
   EXPECT_EQ(grad_add_op->Output("Y@GRAD"), "y@GRAD");
 }
 
-REGISTER_OP(mult_io, f::NOP, f::MutiInOutOpMaker);
-REGISTER_GRADIENT_OP(mult_io, mult_io_grad, f::NOP);
-REGISTER_OP(io_ignored, f::NOP, f::IOIgnoredOpMaker);
-REGISTER_GRADIENT_OP(io_ignored, io_ignored_grad, f::NOP);
+REGISTER_OP(mult_io, f::NOP, f::MutiInOutOpMaker, mult_io_grad);
+REGISTER_GRADIENT_OP(mult_io_grad, f::NOP);
+REGISTER_OP(io_ignored, f::NOP, f::IOIgnoredOpMaker, io_ignored_grad);
+REGISTER_GRADIENT_OP(io_ignored_grad, f::NOP);
 
 TEST(GradOpBuilder, MutiInOut) {
   f::AttributeMap attrs{{"input_format", std::vector<int>{0, 1, 4, 5}},
diff --git a/paddle/framework/op_registry_test.cc b/paddle/framework/op_registry_test.cc
index 9894928a7aa19bc6c7ad8b230562fb9a681cfebd..6f21ffc8a4b701b612e4a6f50f1d414bbbfbadb1 100644
--- a/paddle/framework/op_registry_test.cc
+++ b/paddle/framework/op_registry_test.cc
@@ -49,10 +49,10 @@ class MyTestOpProtoAndCheckerMaker : public OpProtoAndCheckerMaker {
 }  // namespace framework
 }  // namespace paddle
 
-REGISTER_OP(cos_sim, paddle::framework::CosineOp,
-            paddle::framework::CosineOpProtoAndCheckerMaker);
-REGISTER_OP(my_test_op, paddle::framework::MyTestOp,
-            paddle::framework::MyTestOpProtoAndCheckerMaker);
+REGISTER_OP_WITHOUT_GRADIENT(cos_sim, paddle::framework::CosineOp,
+                             paddle::framework::CosineOpProtoAndCheckerMaker);
+REGISTER_OP_WITHOUT_GRADIENT(my_test_op, paddle::framework::MyTestOp,
+                             paddle::framework::MyTestOpProtoAndCheckerMaker);
 
 TEST(OpRegistry, CreateOp) {
   paddle::framework::OpDesc op_desc;
diff --git a/paddle/framework/operator_test.cc b/paddle/framework/operator_test.cc
index 387aada749ba62246b44dedc050547c05955caa9..b1976a65149b5373a73f073526ffa1a66faaf206 100644
--- a/paddle/framework/operator_test.cc
+++ b/paddle/framework/operator_test.cc
@@ -54,8 +54,9 @@ class OpeWithoutKernelTestProtoAndCheckerMaker : public OpProtoAndCheckerMaker {
 }  // namespace framework
 }  // namespace paddle
 
-REGISTER_OP(test_operator, paddle::framework::OpWithoutKernelTest,
-            paddle::framework::OpeWithoutKernelTestProtoAndCheckerMaker);
+REGISTER_OP_WITHOUT_GRADIENT(
+    test_operator, paddle::framework::OpWithoutKernelTest,
+    paddle::framework::OpeWithoutKernelTestProtoAndCheckerMaker);
 
 TEST(OperatorBase, all) {
   paddle::framework::OpDesc op_desc;
@@ -212,8 +213,9 @@ TEST(OpKernel, all) {
   ASSERT_EQ(paddle::framework::cpu_kernel_run_num, 1);
 }
 
-REGISTER_OP(op_multi_inputs_with_kernel, paddle::framework::OpWithKernelTest,
-            paddle::framework::OpKernelTestMultiInputsProtoAndCheckerMaker);
+REGISTER_OP_WITHOUT_GRADIENT(
+    op_multi_inputs_with_kernel, paddle::framework::OpWithKernelTest,
+    paddle::framework::OpKernelTestMultiInputsProtoAndCheckerMaker);
 REGISTER_OP_CPU_KERNEL(op_multi_inputs_with_kernel,
                        paddle::framework::CPUKernalMultiInputsTest);
 
diff --git a/paddle/framework/pybind.cc b/paddle/framework/pybind.cc
index 412b416266607b9ea77029ed019d686189b110fd..0416793d3aac1d4872f542a618b80ca3d7e4b073 100644
--- a/paddle/framework/pybind.cc
+++ b/paddle/framework/pybind.cc
@@ -30,9 +30,9 @@ limitations under the License. */
 namespace py = pybind11;
 
 USE_OP(add_two);
-USE_CPU_OP(onehot_cross_entropy);
+USE_CPU_ONLY_OP(onehot_cross_entropy);
 USE_OP_ITSELF(fc);
-USE_NO_GRAD_OP(sgd);
+USE_OP(sgd);
 USE_OP(mul);
 USE_OP(mean);
 USE_OP(sigmoid);
diff --git a/paddle/operators/add_op.cc b/paddle/operators/add_op.cc
index 086245ef62d759ab20a3684ddbc015f6c6258639..e8e26cbe9b4ff4ae301816df46e548cf14948877 100644
--- a/paddle/operators/add_op.cc
+++ b/paddle/operators/add_op.cc
@@ -55,8 +55,8 @@ class AddOpGrad : public framework::OperatorWithKernel {
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-REGISTER_OP(add_two, ops::AddOp, ops::AddOpMaker);
-REGISTER_GRADIENT_OP(add_two, add_two_grad, ops::AddOpGrad);
+REGISTER_OP(add_two, ops::AddOp, ops::AddOpMaker, add_two_grad);
+REGISTER_GRADIENT_OP(add_two_grad, ops::AddOpGrad);
 
 REGISTER_OP_CPU_KERNEL(add_two,
                        ops::AddKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/cross_entropy_op.cc b/paddle/operators/cross_entropy_op.cc
index c813d54e17fa48aa4447ef76b918b7355be52b09..7d0e74e5e4d941b84470537ca8b182e487894ef1 100644
--- a/paddle/operators/cross_entropy_op.cc
+++ b/paddle/operators/cross_entropy_op.cc
@@ -69,11 +69,11 @@ OnehotCrossEntropy Operator.
 
 namespace ops = paddle::operators;
 REGISTER_OP(onehot_cross_entropy, ops::OnehotCrossEntropyOp,
-            ops::OnehotCrossEntropyOpMaker);
+            ops::OnehotCrossEntropyOpMaker, onehot_cross_entropy_grad);
 REGISTER_OP_CPU_KERNEL(
     onehot_cross_entropy,
     ops::OnehotCrossEntropyOpKernel<paddle::platform::CPUPlace, float>);
-REGISTER_GRADIENT_OP(onehot_cross_entropy, onehot_cross_entropy_grad,
+REGISTER_GRADIENT_OP(onehot_cross_entropy_grad,
                      ops::OnehotCrossEntropyGradientOp);
 REGISTER_OP_CPU_KERNEL(
     onehot_cross_entropy_grad,
diff --git a/paddle/operators/fc_op.cc b/paddle/operators/fc_op.cc
index 01a1a81206f160386467b3c789a41206d89576b6..9d32f327bfedefb976a5ed070789e3aa3f481f95 100644
--- a/paddle/operators/fc_op.cc
+++ b/paddle/operators/fc_op.cc
@@ -73,4 +73,5 @@ USE_OP(sigmoid);
 USE_OP(softmax);
 
 namespace ops = paddle::operators;
-REGISTER_OP(fc, ops::FullyConnectedOp, ops::FullyConnectedOpMaker);
+REGISTER_OP_WITHOUT_GRADIENT(fc, ops::FullyConnectedOp,
+                             ops::FullyConnectedOpMaker);
diff --git a/paddle/operators/fill_zeros_like_op.cc b/paddle/operators/fill_zeros_like_op.cc
index 3759a886780e555ccdc6286c4b200a5d14214691..d6fd368b078c8e674a509448703ff77a856e9e20 100644
--- a/paddle/operators/fill_zeros_like_op.cc
+++ b/paddle/operators/fill_zeros_like_op.cc
@@ -51,7 +51,8 @@ The output will have the same size with input.
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-REGISTER_OP(fill_zeros_like, ops::FillZerosLikeOp, ops::FillZerosLikeOpMaker);
+REGISTER_OP_WITHOUT_GRADIENT(fill_zeros_like, ops::FillZerosLikeOp,
+                             ops::FillZerosLikeOpMaker);
 REGISTER_OP_CPU_KERNEL(
     fill_zeros_like,
     ops::FillZerosLikeKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/gaussian_random_op.cc b/paddle/operators/gaussian_random_op.cc
index ef417ae2f06e8a9f10aed80674015e2ee448f4a3..0bbbeaa08a11f05a9eae63d0235ea8b0feb24062 100644
--- a/paddle/operators/gaussian_random_op.cc
+++ b/paddle/operators/gaussian_random_op.cc
@@ -78,5 +78,6 @@ Use to initialize tensor with gaussian random generator.
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-REGISTER_OP(gaussian_random, ops::GaussianRandomOp, ops::GaussianRandomOpMaker);
+REGISTER_OP_WITHOUT_GRADIENT(gaussian_random, ops::GaussianRandomOp,
+                             ops::GaussianRandomOpMaker);
 REGISTER_OP_CPU_KERNEL(gaussian_random, ops::GaussianRandomKernel<float>);
diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc
index 2ea049cb3605f4dedabb992ebc0e8aa276ad5e9a..15e0708c4688ddec6a8e6a65c359b3456834aa8f 100644
--- a/paddle/operators/mean_op.cc
+++ b/paddle/operators/mean_op.cc
@@ -50,9 +50,9 @@ class MeanGradOp : public framework::OperatorWithKernel {
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-REGISTER_OP(mean, ops::MeanOp, ops::MeanOpMaker);
+REGISTER_OP(mean, ops::MeanOp, ops::MeanOpMaker, mean_grad);
 REGISTER_OP_CPU_KERNEL(mean,
                        ops::MeanKernel<paddle::platform::CPUPlace, float>);
-REGISTER_GRADIENT_OP(mean, mean_grad, ops::MeanGradOp);
+REGISTER_GRADIENT_OP(mean_grad, ops::MeanGradOp);
 REGISTER_OP_CPU_KERNEL(mean_grad,
                        ops::MeanGradKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/mul_op.cc b/paddle/operators/mul_op.cc
index db81fd555d1c7bea7c0c3bbd70266b4952ed3724..60550a274254f6c4455d8cd5ed07f4ccc85a6f8b 100644
--- a/paddle/operators/mul_op.cc
+++ b/paddle/operators/mul_op.cc
@@ -65,7 +65,7 @@ class MulOpGrad : public framework::OperatorWithKernel {
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-REGISTER_OP(mul, ops::MulOp, ops::MulOpMaker);
-REGISTER_GRADIENT_OP(mul, mul_grad, ops::MulOpGrad);
+REGISTER_OP(mul, ops::MulOp, ops::MulOpMaker, mul_grad);
+REGISTER_GRADIENT_OP(mul_grad, ops::MulOpGrad);
 
 REGISTER_OP_CPU_KERNEL(mul, ops::MulKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc
index 243837420562634c3d99fd0acf234ebd53539735..91be1ce5193dd5f3fbab6ebc38c18123120c4a85 100644
--- a/paddle/operators/recurrent_op.cc
+++ b/paddle/operators/recurrent_op.cc
@@ -235,5 +235,6 @@ void RecurrentGradientOp::Init() {
 }  // namespace operators
 }  // namespace paddle
 
-REGISTER_OP(recurrent_op, paddle::operators::RecurrentOp,
-            paddle::operators::RecurrentAlgorithmProtoAndCheckerMaker);
+REGISTER_OP_WITHOUT_GRADIENT(
+    recurrent_op, paddle::operators::RecurrentOp,
+    paddle::operators::RecurrentAlgorithmProtoAndCheckerMaker);
diff --git a/paddle/operators/rowwise_add_op.cc b/paddle/operators/rowwise_add_op.cc
index 55ed1c2f4c316656de94b24dd95b053a89d7e74e..262a4127ef98a957547609f1aa1eb4105b0b5e0e 100644
--- a/paddle/operators/rowwise_add_op.cc
+++ b/paddle/operators/rowwise_add_op.cc
@@ -53,6 +53,7 @@ for i in xrange(X.shape[0]):
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-REGISTER_OP(rowwise_add, ops::RowWiseAddOp, ops::RowWiseAddOpMaker);
+REGISTER_OP_WITHOUT_GRADIENT(rowwise_add, ops::RowWiseAddOp,
+                             ops::RowWiseAddOpMaker);
 REGISTER_OP_CPU_KERNEL(
     rowwise_add, ops::RowWiseAddKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/sgd_op.cc b/paddle/operators/sgd_op.cc
index f9a28ff8a6a06c5c239c4e6ec21eacb410cc162f..94d0fe0466a95daf87bc850ce6a356bd327cf30d 100644
--- a/paddle/operators/sgd_op.cc
+++ b/paddle/operators/sgd_op.cc
@@ -52,6 +52,6 @@ param_out = param - learning_rate * grad;
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-REGISTER_OP(sgd, ops::SGDOp, ops::SGDOpMaker);
+REGISTER_OP_WITHOUT_GRADIENT(sgd, ops::SGDOp, ops::SGDOpMaker);
 REGISTER_OP_CPU_KERNEL(sgd,
                        ops::SGDOpKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/sigmoid_op.cc b/paddle/operators/sigmoid_op.cc
index bc5e0bbb183f9bdf0a3fa8a5a02499282fbd6b98..fb27ffbfa1d520e9b7bb5b2d73e91730b44a9d1d 100644
--- a/paddle/operators/sigmoid_op.cc
+++ b/paddle/operators/sigmoid_op.cc
@@ -48,8 +48,8 @@ class SigmoidOpGrad : public framework::OperatorWithKernel {
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-REGISTER_OP(sigmoid, ops::SigmoidOp, ops::SigmoidOpMaker);
-REGISTER_GRADIENT_OP(sigmoid, sigmoid_grad, ops::SigmoidOpGrad);
+REGISTER_OP(sigmoid, ops::SigmoidOp, ops::SigmoidOpMaker, sigmoid_grad);
+REGISTER_GRADIENT_OP(sigmoid_grad, ops::SigmoidOpGrad);
 
 REGISTER_OP_CPU_KERNEL(sigmoid,
                        ops::SigmoidKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/softmax_op.cc b/paddle/operators/softmax_op.cc
index 3dd4e86918a86f408e7867d15b4fdc8f9cbbb5ce..abc21337c528216970ebf435f7297e5893e51829 100644
--- a/paddle/operators/softmax_op.cc
+++ b/paddle/operators/softmax_op.cc
@@ -64,9 +64,9 @@ class SoftmaxOpGrad : public framework::OperatorWithKernel {
 
 namespace ops = paddle::operators;
 
-REGISTER_OP(softmax, ops::SoftmaxOp, ops::SoftmaxOpMaker);
+REGISTER_OP(softmax, ops::SoftmaxOp, ops::SoftmaxOpMaker, softmax_grad);
 REGISTER_OP_CPU_KERNEL(softmax,
                        ops::SoftmaxKernel<paddle::platform::CPUPlace, float>);
-REGISTER_GRADIENT_OP(softmax, softmax_grad, ops::SoftmaxOpGrad);
+REGISTER_GRADIENT_OP(softmax_grad, ops::SoftmaxOpGrad);
 REGISTER_OP_CPU_KERNEL(
     softmax_grad, ops::SoftmaxGradKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/uniform_random_op.cc b/paddle/operators/uniform_random_op.cc
index 405b84b76d2e24db25d2ff16e99495f2f132ef09..37ec7fe427097d808d948737e40be41e310bef0f 100644
--- a/paddle/operators/uniform_random_op.cc
+++ b/paddle/operators/uniform_random_op.cc
@@ -78,7 +78,7 @@ Used to initialize tensor with uniform random generator.
 }  // namespace operators
 }  // namespace paddle
 
-REGISTER_OP(uniform_random, paddle::operators::UniformRandomOp,
-            paddle::operators::UniformRandomOpMaker);
+REGISTER_OP_WITHOUT_GRADIENT(uniform_random, paddle::operators::UniformRandomOp,
+                             paddle::operators::UniformRandomOpMaker);
 REGISTER_OP_CPU_KERNEL(uniform_random,
                        paddle::operators::CPUUniformRandomKernel<float>);