diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c
index 727a809636d8a63ce93b1271175c3f520ac4455b..ed3dcdea7fe10cb7b521f7ccdb78a1bb5bfc86cc 100644
--- a/drivers/s390/crypto/ap_bus.c
+++ b/drivers/s390/crypto/ap_bus.c
@@ -1145,12 +1145,17 @@ ap_config_timeout(unsigned long ptr)
  */
 static inline void ap_schedule_poll_timer(void)
 {
+	ktime_t hr_time;
 	if (ap_using_interrupts() || ap_suspend_flag)
 		return;
 	if (hrtimer_is_queued(&ap_poll_timer))
 		return;
-	hrtimer_start(&ap_poll_timer, ktime_set(0, poll_timeout),
-		      HRTIMER_MODE_ABS);
+	if (ktime_to_ns(hrtimer_expires_remaining(&ap_poll_timer)) <= 0) {
+		hr_time = ktime_set(0, poll_timeout);
+		hrtimer_forward_now(&ap_poll_timer, hr_time);
+		hrtimer_restart(&ap_poll_timer);
+	}
+	return;
 }
 
 /**