diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py
index 0845c9bd88889921c42a71687ca2c9e03f6613eb..2315a2d5ccd7598f0b897722cc35192d477e0d5e 100644
--- a/python/paddle/fluid/layers/nn.py
+++ b/python/paddle/fluid/layers/nn.py
@@ -5921,6 +5921,8 @@ def sampled_softmax_with_cross_entropy(logits,
     sampled_logits \
         = helper.create_variable_for_type_inference(dtype=logits.dtype)
     sampled_label = helper.create_variable_for_type_inference(dtype='int64')
+    sampled_softlabel = helper.create_variable_for_type_inference(
+        dtype=logits.dtype)
 
     helper.append_op(
         type='sample_logits',
@@ -5945,14 +5947,20 @@ def sampled_softmax_with_cross_entropy(logits,
         })
     loss = helper.create_variable_for_type_inference(dtype=logits.dtype)
     softmax = helper.create_variable_for_type_inference(dtype=logits.dtype)
+    helper.append_op(
+        type='one_hot',
+        inputs={'X': sampled_label},
+        attrs={'depth': num_samples + 1},
+        outputs={'Out': sampled_softlabel})
+
     helper.append_op(
         type='softmax_with_cross_entropy',
         inputs={'Logits': sampled_logits,
-                'Label': sampled_label},
+                'Label': sampled_softlabel},
         outputs={'Softmax': softmax,
                  'Loss': loss},
         attrs={
-            'soft_label': False,
+            'soft_label': True,
             'ignore_index': False,
             'numeric_stable_mode': False
         })