diff --git a/libavutil/buffer.c b/libavutil/buffer.c
index 668100234594928afd2ffef36bb2639662422123..49a1f2d39558cb355b5a0d776b0f77b58c64bfb2 100644
--- a/libavutil/buffer.c
+++ b/libavutil/buffer.c
@@ -170,7 +170,7 @@ int av_buffer_realloc(AVBufferRef **pbuf, int size)
         return 0;
 
     if (!(buf->buffer->flags & BUFFER_FLAG_REALLOCATABLE) ||
-        !av_buffer_is_writable(buf)) {
+        !av_buffer_is_writable(buf) || buf->data != buf->buffer->data) {
         /* cannot realloc, allocate a new reallocable buffer and copy data */
         AVBufferRef *new = NULL;