diff --git a/paddle/gserver/gradientmachines/ParallelNeuralNetwork.cpp b/paddle/gserver/gradientmachines/ParallelNeuralNetwork.cpp
index 952df60a7d78666c84d5fd9176c3113fdbdacdc9..22698f586701774d884e6eeca943f6bf75fe7a96 100644
--- a/paddle/gserver/gradientmachines/ParallelNeuralNetwork.cpp
+++ b/paddle/gserver/gradientmachines/ParallelNeuralNetwork.cpp
@@ -28,6 +28,12 @@ void ParallelNeuralNetwork::init(
     const std::vector<ParameterType>& parameterTypes, bool useGpu) {
   NeuralNetwork::init(config, callback, parameterTypes, useGpu);
 
+  if (config.type() == "recurrent_nn") {
+    LOG(FATAL)
+      << "You can not add `--parallel_nn=true` on the command line, "
+      << "parallel_nn training mode does not support the recurrent_nn model.";
+  }
+
   useGpu_ = useGpu;
   numDevices_ = 0;
   if (useGpu_) {