diff --git a/paddle/fluid/operators/gumbel_softmax_op.cu b/paddle/fluid/operators/gumbel_softmax_op.cu
index bf0ac667411d86ed1a888258fddb518e562f5719..6b6290d4af29f73216311ab41722a17adc237d11 100644
--- a/paddle/fluid/operators/gumbel_softmax_op.cu
+++ b/paddle/fluid/operators/gumbel_softmax_op.cu
@@ -130,7 +130,6 @@ struct GumbleNoiseGenerator<platform::CUDADeviceContext, T> {
     T* random_data =
         random_tensor.mutable_data<T>({size}, platform::CUDAPlace());
     thrust::counting_iterator<unsigned int> index_sequence_begin(0);
-    const unsigned int seed = std::random_device()();
 
     // generate gumbel noise
     int device_id =
@@ -144,6 +143,7 @@ struct GumbleNoiseGenerator<platform::CUDADeviceContext, T> {
           thrust::device_ptr<T>(random_data),
           UniformCUDAGenerator<T>(0.00001, 1, seed_offset.first, gen_offset));
     } else {
+      const unsigned int seed = std::random_device()();
       thrust::transform(index_sequence_begin, index_sequence_begin + size,
                         thrust::device_ptr<T>(random_data),
                         UniformCUDAGenerator<T>(0.00001, 1, seed));
diff --git a/paddle/fluid/operators/gumbel_softmax_op.h b/paddle/fluid/operators/gumbel_softmax_op.h
index c224cc7ca1dd99a64a96d19976727bbe3989d42f..f95a4810f44421e79b263f477baf03df19253a50 100644
--- a/paddle/fluid/operators/gumbel_softmax_op.h
+++ b/paddle/fluid/operators/gumbel_softmax_op.h
@@ -86,8 +86,7 @@ struct GumbleNoiseGenerator<platform::CPUDeviceContext, T> {
     // generate uniform random number
     const int size = size_to_axis * size_from_axis;
     std::uniform_real_distribution<T> dist(0.00001, 1);
-    const int seed = std::random_device()();
-    auto engine = paddle::framework::GetCPURandomEngine(seed);
+    auto engine = paddle::framework::GetCPURandomEngine(0);
     Tensor random_tensor;
     auto* random_data =
         random_tensor.mutable_data<T>({size}, platform::CPUPlace());