diff --git a/paddle/optimizer/CMakeLists.txt b/paddle/optimizer/CMakeLists.txt
index 0f6c4eb2dd1de3b804825ca11a6192909366a814..746f4a69f84534cdc98e9285eddf559fea786239 100644
--- a/paddle/optimizer/CMakeLists.txt
+++ b/paddle/optimizer/CMakeLists.txt
@@ -9,17 +9,6 @@ set(OPITMIZER_SRCS
     sgd_optmizer.cc
   )
 
-set(OPITMIZER_HEADERS
-    adadelta_optimizer.h
-    adagrad_optimizer.h
-    adam_optimizer.h
-    lr_policy.h
-    optimizer.h
-    parameter_optimizer.h
-    sgd_optimizer.h
-    tensor.h
-  )
-
 add_library(optimizer STATIC ${OPITMIZER_SRCS})
 add_dependencies(optimizer gen_proto_cpp)
 
diff --git a/paddle/optimizer/Tensor.h b/paddle/optimizer/Tensor.h
index a887005cf41d8af24d902acf6b2236daff1cc12c..3dbb3ca05de7514fe6e818207a7c52756c33e41c 100644
--- a/paddle/optimizer/Tensor.h
+++ b/paddle/optimizer/Tensor.h
@@ -1,10 +1,10 @@
-#ifndef PADDLE_OPTIMIZER_TENSOR_H_
-#define PADDLE_OPTIMIZER_TENSOR_H_
+#pragma once
 /**
  * @brief tensor used by optimizer
  */
 
 #include <string.h>
+#include <memory>
 #include "paddle/utils/Common.h"
 #include "paddle/utils/Logging.h"
 
@@ -45,5 +45,3 @@ typedef TensorT<real> Tensor;
 
 }  // namespace optimizer
 }  // namespace paddle
-
-#endif
diff --git a/paddle/optimizer/adadelta_optimizer.h b/paddle/optimizer/adadelta_optimizer.h
index 20801c3794b6c5a67cf64c6d2684cb33bf54273d..d17fae6b70a65b3ec4d81de2063e42a5dafab53c 100644
--- a/paddle/optimizer/adadelta_optimizer.h
+++ b/paddle/optimizer/adadelta_optimizer.h
@@ -1,5 +1,4 @@
-#ifndef PADDLE_ADADELTA_OPTIMIZER_H_
-#define PADDLE_ADADELTA_OPTIMIZER_H_
+#pragma once
 
 #include "parameter_optimizer.h"
 
@@ -31,5 +30,3 @@ private:
 
 }  // namespace optimizer
 }  // namespace paddle
-
-#endif
diff --git a/paddle/optimizer/adagrad_optimizer.h b/paddle/optimizer/adagrad_optimizer.h
index bb64d7d5a778431ec584773a72f539ce32eeae29..6c00665c86c05d5c4c266e1ee8c1672d1224beb1 100644
--- a/paddle/optimizer/adagrad_optimizer.h
+++ b/paddle/optimizer/adagrad_optimizer.h
@@ -1,5 +1,4 @@
-#ifndef PADDLE_ADAGRAD_OPTIMIZER_H_
-#define PADDLE_ADAGRAD_OPTIMIZER_H_
+#pragma once
 
 #include "parameter_optimizer.h"
 
@@ -25,5 +24,3 @@ private:
 
 }  // namespace optimizer
 }  // namespace paddle
-
-#endif
diff --git a/paddle/optimizer/adam_optimizer.h b/paddle/optimizer/adam_optimizer.h
index 89e68346d58a5063eb50f7cc7e0f9544b55ccf6c..718039e451492f90c2be8db5d5def3ec5fdfb7a6 100644
--- a/paddle/optimizer/adam_optimizer.h
+++ b/paddle/optimizer/adam_optimizer.h
@@ -1,5 +1,4 @@
-#ifndef PADDLE_ADAM_OPTIMIZER_H_
-#define PADDLE_ADAM_OPTIMIZER_H_
+#pragma once
 
 #include "parameter_optimizer.h"
 
@@ -34,4 +33,3 @@ private:
 
 }  // namespace optimizer
 }  // namespace paddle
-#endif
diff --git a/paddle/optimizer/lr_policy.h b/paddle/optimizer/lr_policy.h
index b24a17ced01b0851ffd66147b6265cbd63ce2337..51debf675f38688384610d860cc46ec4e80b6dea 100644
--- a/paddle/optimizer/lr_policy.h
+++ b/paddle/optimizer/lr_policy.h
@@ -1,5 +1,4 @@
-#ifndef PADDLE_OPTIMIZER_LR_POLICY_H_
-#define PADDLE_OPTIMIZER_LR_POLICY_H_
+#pragma once
 
 #include <algorithm>
 #include "OptimizerConfig.pb.h"
@@ -41,5 +40,3 @@ private:
 
 }  // namespace optimizer
 }  // namespace paddle
-
-#endif
diff --git a/paddle/optimizer/optimizer.h b/paddle/optimizer/optimizer.h
index c3328331fc38efcfd16f86e46987fee606b01c77..ee017eddabe3b348d96f9ef358b48b7807f38000 100644
--- a/paddle/optimizer/optimizer.h
+++ b/paddle/optimizer/optimizer.h
@@ -1,5 +1,5 @@
-#ifndef PADDLE_LIB_OPTIMIZER_H_
-#define PADDLE_LIB_OPTIMIZER_H_
+#pragma once
+
 #include <stdbool.h>
 #include <stdint.h>
 
@@ -93,4 +93,3 @@ int paddle_optimizer_get_state(paddle_optimizer* o, const char* state);
 #ifdef __cplusplus
 }
 #endif
-#endif
diff --git a/paddle/optimizer/parameter_optimizer.h b/paddle/optimizer/parameter_optimizer.h
index e60c7778205e3a1fdc4394101679a979616629d9..33b3f837b43c2af3fc4372dc6d47b0aaf1894814 100644
--- a/paddle/optimizer/parameter_optimizer.h
+++ b/paddle/optimizer/parameter_optimizer.h
@@ -1,5 +1,4 @@
-#ifndef PADDLE_PARAMETER_OPTIMIZER_H_
-#define PADDLE_PARAMETER_OPTIMIZER_H_
+#pragma once
 
 #include <glog/logging.h>
 #include <functional>
@@ -40,5 +39,3 @@ protected:
 
 }  // namespace optimizer
 }  // namespace paddle
-
-#endif
diff --git a/paddle/optimizer/serialization.h b/paddle/optimizer/serialization.h
index edb90d0a164c4905450f3534bfb496a6381e4318..6caa514cef25eb3fb1a3a7efe972063a3d378416 100644
--- a/paddle/optimizer/serialization.h
+++ b/paddle/optimizer/serialization.h
@@ -1,5 +1,4 @@
-#ifndef PADDLE_OPTIMIZER_SERIALIZARION_H
-#define PADDLE_OPTIMIZER_SERIALIZARION_H
+#pragma once
 
 #include <sstream>
 #include <string>
@@ -33,4 +32,3 @@ static void ProtoToTensor(const TensorProto& proto, Tensor* tensor) {
 
 }  // namespace optimizer
 }  // namespace paddle
-#endif