diff --git a/drivers/serial/atmel_serial.c b/drivers/serial/atmel_serial.c
index fad245b064d666a00017aabcc8f6ccb9b4f08a91..d57bf3e708d8fffb187dcbe33e410811cdeb1f66 100644
--- a/drivers/serial/atmel_serial.c
+++ b/drivers/serial/atmel_serial.c
@@ -549,7 +549,7 @@ static irqreturn_t atmel_interrupt(int irq, void *dev_id)
 		atmel_handle_transmit(port, pending);
 	} while (pass_counter++ < ATMEL_ISR_PASS_LIMIT);
 
-	return IRQ_HANDLED;
+	return pass_counter ? IRQ_HANDLED : IRQ_NONE;
 }
 
 /*