diff --git a/arch/avr32/mm/ioremap.c b/arch/avr32/mm/ioremap.c
index 8cfec65e37f76f72ac101aaacc1315affe1da913..3437c82434ac9ec189742862d1ce996223feabf4 100644
--- a/arch/avr32/mm/ioremap.c
+++ b/arch/avr32/mm/ioremap.c
@@ -77,6 +77,8 @@ void __iounmap(void __iomem *addr)
 
 	if ((unsigned long)addr >= P4SEG)
 		return;
+	if (PXSEG(addr) == P2SEG)
+		return;
 
 	p = remove_vm_area((void *)(PAGE_MASK & (unsigned long __force)addr));
 	if (unlikely(!p)) {