diff --git a/paddle/legacy/gserver/gradientmachines/MultiGradientMachine.cpp b/paddle/legacy/gserver/gradientmachines/MultiGradientMachine.cpp
index 637686e443ceb740d52d42524246ae48a85d52f0..3ef0dfbfe2e5842918500a3b0706c1a55024ce46 100644
--- a/paddle/legacy/gserver/gradientmachines/MultiGradientMachine.cpp
+++ b/paddle/legacy/gserver/gradientmachines/MultiGradientMachine.cpp
@@ -532,6 +532,7 @@ void TrainerThread::computeThread() {
         break;
     }
   }
+  hl_fini();
 }
 
 void TrainerThread::prefetch() {
@@ -651,6 +652,7 @@ void TrainerThread::copyGradToBufferThread() {
     }
     partnerThread->notifyGradientCollect(pid);
   }
+  hl_fini();
 }
 
 void TrainerThread::gradCollectThread() {
@@ -693,6 +695,7 @@ void TrainerThread::gradCollectThread() {
       notifyCopyGradToBuffer(pid);
     }
   }
+  hl_fini();
 }
 
 void TrainerThread::doCallback(int pid) {
@@ -741,6 +744,7 @@ void TrainerThread::valueDispatchThread() {
 
     thread->notifyValueReady(pid);
   }
+  hl_fini();
 }
 
 void TrainerThread::notifyValueReady(int paramId) {
diff --git a/paddle/legacy/gserver/gradientmachines/ParallelNeuralNetwork.cpp b/paddle/legacy/gserver/gradientmachines/ParallelNeuralNetwork.cpp
index 450514ca88a319b30ca3ebae669c78502087540a..33d24b5b832fe9011591606860e0f50361367790 100644
--- a/paddle/legacy/gserver/gradientmachines/ParallelNeuralNetwork.cpp
+++ b/paddle/legacy/gserver/gradientmachines/ParallelNeuralNetwork.cpp
@@ -197,6 +197,7 @@ void ParallelThread::computeThread() {
       job_work.layer_->markAllInputGrad();
     }
   }
+  hl_fini();
 }
 
 void ParallelThread::start() {