diff --git a/paddle/fluid/operators/increment_op.cc b/paddle/fluid/operators/increment_op.cc
index 2893ab7127601bc2e787e44859086dc354693e89..ec2e641679fedec776d48716f13445f44375ce3d 100644
--- a/paddle/fluid/operators/increment_op.cc
+++ b/paddle/fluid/operators/increment_op.cc
@@ -33,6 +33,15 @@ class IncrementOp : public framework::OperatorWithKernel {
     ctx->SetOutputDim("Out", ctx->GetInputDim("X"));
     ctx->ShareLoD("X", "Out");
   }
+
+ protected:
+  framework::OpKernelType GetExpectedKernelType(
+      const framework::ExecutionContext &ctx) const override {
+    framework::OpKernelType kt = OperatorWithKernel::GetExpectedKernelType(ctx);
+    // IncrementOp kernel's device type is decided by input tensor place
+    kt.place_ = ctx.Input<framework::LoDTensor>("X")->place();
+    return kt;
+  }
 };
 
 class IncrementOpMaker : public framework::OpProtoAndCheckerMaker {
diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py
index fbfc383d118acca42e16d27a8dc962b0ba0a539b..b9a53eda9144e9e56cf9bc626db40cf4225bd87f 100644
--- a/python/paddle/fluid/layers/control_flow.py
+++ b/python/paddle/fluid/layers/control_flow.py
@@ -1362,7 +1362,8 @@ class DynamicRNN(object):
         self.lod_rank_table = None
         self.max_seq_len = None
         self.step_idx = None
-        self.zero_idx = fill_constant(shape=[1], value=0, dtype='int64')
+        self.zero_idx = fill_constant(
+            shape=[1], value=0, dtype='int64', force_cpu=True)
         self.mem_dict = dict()
         self.output_array = []
         self.outputs = []
@@ -1439,7 +1440,8 @@ class DynamicRNN(object):
     def block(self):
         if self.status != DynamicRNN.BEFORE_RNN:
             raise ValueError("rnn.block() can only be invoke once")
-        self.step_idx = fill_constant(shape=[1], dtype='int64', value=0)
+        self.step_idx = fill_constant(
+            shape=[1], dtype='int64', value=0, force_cpu=True)
         self.step_idx.stop_gradient = False
         self.status = DynamicRNN.IN_RNN
         with self.while_op.block():
diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py
index fdf4185205fd2a29c2426ecf1d81cc347f98afdc..0332556f62c46b187bd79841e4969d9da08b57a5 100644
--- a/python/paddle/fluid/layers/nn.py
+++ b/python/paddle/fluid/layers/nn.py
@@ -3307,7 +3307,8 @@ def autoincreased_step_counter(counter_name=None, begin=1, step=1):
         name=counter_name, dtype='int64', shape=[1], persistable=True)
     if is_new_var:
         helper.set_variable_initializer(
-            counter, initializer=Constant(value=begin - 1))
+            counter, initializer=Constant(
+                value=begin - 1, force_cpu=True))
         helper.main_program.global_block().prepend_op(
             type='increment',
             inputs={'X': [counter]},
diff --git a/python/paddle/fluid/tests/book/test_machine_translation.py b/python/paddle/fluid/tests/book/test_machine_translation.py
index de72a7c3ff815c57fc99bb7ac0719778d8ad2228..3a1a0859ecfd4ac5337e2112f8b22e32d8474f22 100644
--- a/python/paddle/fluid/tests/book/test_machine_translation.py
+++ b/python/paddle/fluid/tests/book/test_machine_translation.py
@@ -83,7 +83,7 @@ def decoder_train(context, is_sparse):
 def decoder_decode(context, is_sparse):
     init_state = context
     array_len = pd.fill_constant(shape=[1], dtype='int64', value=max_length)
-    counter = pd.zeros(shape=[1], dtype='int64')
+    counter = pd.zeros(shape=[1], dtype='int64', force_cpu=True)
 
     # fill the first element with init_state
     state_array = pd.create_array('float32')
diff --git a/python/paddle/fluid/tests/unittests/test_profiler.py b/python/paddle/fluid/tests/unittests/test_profiler.py
index 49ec9c9020595713b79455b5f22a1ce0e6b2de92..cf6fe14a86aa1ab6ea3f60ad15f33d708e9b803a 100644
--- a/python/paddle/fluid/tests/unittests/test_profiler.py
+++ b/python/paddle/fluid/tests/unittests/test_profiler.py
@@ -33,7 +33,8 @@ class TestProfiler(unittest.TestCase):
             image = fluid.layers.data(name='x', shape=[784], dtype='float32')
             hidden1 = fluid.layers.fc(input=image, size=64, act='relu')
             i = layers.zeros(shape=[1], dtype='int64')
-            counter = fluid.layers.zeros(shape=[1], dtype='int64')
+            counter = fluid.layers.zeros(
+                shape=[1], dtype='int64', force_cpu=True)
             until = layers.fill_constant([1], dtype='int64', value=10)
             data_arr = layers.array_write(hidden1, i)
             cond = fluid.layers.less_than(x=counter, y=until)