diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index 0d5529c382e8ce5d66fceac90f12c3dba7e51440..afa508d92c93fdf6b3a4a96946a223c440514750 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -143,6 +143,11 @@ struct xfrm_state
 	/* Replay detection state at the time we sent the last notification */
 	struct xfrm_replay_state preplay;
 
+	/* internal flag that only holds state for delayed aevent at the
+	 * moment
+	*/
+	u32			xflags;
+
 	/* Replay detection notification settings */
 	u32			replay_maxage;
 	u32			replay_maxdiff;
@@ -168,6 +173,9 @@ struct xfrm_state
 	void			*data;
 };
 
+/* xflags - make enum if more show up */
+#define XFRM_TIME_DEFER	1
+
 enum {
 	XFRM_STATE_VOID,
 	XFRM_STATE_ACQ,
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index a8e14dc1b04ee726e0ccb047dec04d30fa06756d..3dc3e1f3b7aae687346e4f9f8ff8926d658b65d0 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -805,16 +805,22 @@ void xfrm_replay_notify(struct xfrm_state *x, int event)
 	case XFRM_REPLAY_UPDATE:
 		if (x->replay_maxdiff &&
 		    (x->replay.seq - x->preplay.seq < x->replay_maxdiff) &&
-		    (x->replay.oseq - x->preplay.oseq < x->replay_maxdiff))
-			return;
+		    (x->replay.oseq - x->preplay.oseq < x->replay_maxdiff)) {
+			if (x->xflags & XFRM_TIME_DEFER)
+				event = XFRM_REPLAY_TIMEOUT;
+			else
+				return;
+		}
 
 		break;
 
 	case XFRM_REPLAY_TIMEOUT:
 		if ((x->replay.seq == x->preplay.seq) &&
 		    (x->replay.bitmap == x->preplay.bitmap) &&
-		    (x->replay.oseq == x->preplay.oseq))
+		    (x->replay.oseq == x->preplay.oseq)) {
+			x->xflags |= XFRM_TIME_DEFER;
 			return;
+		}
 
 		break;
 	}
@@ -825,8 +831,10 @@ void xfrm_replay_notify(struct xfrm_state *x, int event)
 	km_state_notify(x, &c);
 
 	if (x->replay_maxage &&
-	    !mod_timer(&x->rtimer, jiffies + x->replay_maxage))
+	    !mod_timer(&x->rtimer, jiffies + x->replay_maxage)) {
 		xfrm_state_hold(x);
+		x->xflags &= ~XFRM_TIME_DEFER;
+	}
 }
 EXPORT_SYMBOL(xfrm_replay_notify);
 
@@ -836,10 +844,15 @@ static void xfrm_replay_timer_handler(unsigned long data)
 
 	spin_lock(&x->lock);
 
-	if (xfrm_aevent_is_on() && x->km.state == XFRM_STATE_VALID)
-		xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT);
+	if (x->km.state == XFRM_STATE_VALID) {
+		if (xfrm_aevent_is_on())
+			xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT);
+		else
+			x->xflags |= XFRM_TIME_DEFER;
+	}
 
 	spin_unlock(&x->lock);
+	xfrm_state_put(x);
 }
 
 int xfrm_replay_check(struct xfrm_state *x, u32 seq)