diff --git a/libswresample/resample.c b/libswresample/resample.c
index e65a57a87722b8dd9da9cf4fbe59e3e612e48bba..71dffb9a3ec8d64837ae1152c9d5b87508c8ccb4 100644
--- a/libswresample/resample.c
+++ b/libswresample/resample.c
@@ -449,7 +449,7 @@ static int rebuild_filter_bank_with_compensation(ResampleContext *c)
 static int set_compensation(ResampleContext *c, int sample_delta, int compensation_distance){
     int ret;
 
-    if (compensation_distance) {
+    if (compensation_distance && sample_delta) {
         ret = rebuild_filter_bank_with_compensation(c);
         if (ret < 0)
             return ret;