diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig
index 9b1f8c3c8cba19006c673629e817b28e59a222b9..7a4c77b39ce043c7766fd8fb767b3dcd62a84a07 100644
--- a/arch/xtensa/Kconfig
+++ b/arch/xtensa/Kconfig
@@ -394,7 +394,7 @@ config XTENSA_PLATFORM_XT2000
 config XTENSA_PLATFORM_XTFPGA
 	bool "XTFPGA"
 	select ETHOC if ETHERNET
-	select PLATFORM_WANT_DEFAULT_MEM
+	select PLATFORM_WANT_DEFAULT_MEM if !MMU
 	select SERIAL_CONSOLE
 	select XTENSA_CALIBRATE_CCOUNT
 	help
@@ -502,11 +502,9 @@ config DEFAULT_MEM_START
 	default 0x00000000 if MMU
 	default 0x60000000 if !MMU
 	help
-	  This is a fallback start address of the default memory area, it is
-	  used when no physical memory size is passed through DTB or through
-	  boot parameter from bootloader.
-
-	  It's also used for TASK_SIZE calculation in noMMU configuration.
+	  This is the base address of the default memory area.
+	  Default memory area has platform-specific meaning, it may be used
+	  for e.g. early cache initialization.
 
 	  If unsure, leave the default value here.
 
@@ -515,11 +513,9 @@ config DEFAULT_MEM_SIZE
 	depends on PLATFORM_WANT_DEFAULT_MEM
 	default 0x04000000
 	help
-	  This is a fallback size of the default memory area, it is used when
-	  no physical memory size is passed through DTB or through boot
-	  parameter from bootloader.
-
-	  It's also used for TASK_SIZE calculation in noMMU configuration.
+	  This is the size of the default memory area.
+	  Default memory area has platform-specific meaning, it may be used
+	  for e.g. early cache initialization.
 
 	  If unsure, leave the default value here.
 
diff --git a/arch/xtensa/include/asm/page.h b/arch/xtensa/include/asm/page.h
index 3b5a49dbf8b20b4c006bcbc10e126dbce11f9f15..976b1d70edbc0a2d77016409fec93ec597df9cad 100644
--- a/arch/xtensa/include/asm/page.h
+++ b/arch/xtensa/include/asm/page.h
@@ -31,10 +31,9 @@
 #define MAX_LOW_PFN	(PHYS_PFN(XCHAL_KSEG_PADDR) + \
 			 PHYS_PFN(XCHAL_KSEG_SIZE))
 #else
-#define PAGE_OFFSET	__XTENSA_UL_CONST(0)
-#define PHYS_OFFSET	__XTENSA_UL_CONST(0)
-#define MAX_LOW_PFN	(PHYS_PFN(PLATFORM_DEFAULT_MEM_START) + \
-			 PHYS_PFN(PLATFORM_DEFAULT_MEM_SIZE))
+#define PAGE_OFFSET	PLATFORM_DEFAULT_MEM_START
+#define PHYS_OFFSET	PLATFORM_DEFAULT_MEM_START
+#define MAX_LOW_PFN	PHYS_PFN(0xfffffffful)
 #endif
 
 #define PGTABLE_START	0x80000000
@@ -163,7 +162,7 @@ void copy_user_highpage(struct page *to, struct page *from,
  * addresses.
  */
 
-#define ARCH_PFN_OFFSET		(PLATFORM_DEFAULT_MEM_START >> PAGE_SHIFT)
+#define ARCH_PFN_OFFSET		(PHYS_OFFSET >> PAGE_SHIFT)
 
 #define __pa(x)	\
 	((unsigned long) (x) - PAGE_OFFSET + PHYS_OFFSET)
diff --git a/arch/xtensa/include/asm/processor.h b/arch/xtensa/include/asm/processor.h
index d2e40d39c6158102de48debffe76a0c5fe4f4cbc..b42d68bfe3cf0a965380bef7aae9afdcdada8050 100644
--- a/arch/xtensa/include/asm/processor.h
+++ b/arch/xtensa/include/asm/processor.h
@@ -37,7 +37,7 @@
 #ifdef CONFIG_MMU
 #define TASK_SIZE	__XTENSA_UL_CONST(0x40000000)
 #else
-#define TASK_SIZE	(PLATFORM_DEFAULT_MEM_START + PLATFORM_DEFAULT_MEM_SIZE)
+#define TASK_SIZE	__XTENSA_UL_CONST(0xffffffff)
 #endif
 
 #define STACK_TOP	TASK_SIZE
diff --git a/arch/xtensa/kernel/setup.c b/arch/xtensa/kernel/setup.c
index 9735691f37f164803ee0a971c3c14c8214b13e69..0e59b8900e6a7d3f53418d94198afd0a0d7cfde6 100644
--- a/arch/xtensa/kernel/setup.c
+++ b/arch/xtensa/kernel/setup.c
@@ -188,7 +188,6 @@ static int __init parse_bootparam(const bp_tag_t* tag)
 }
 
 #ifdef CONFIG_OF
-bool __initdata dt_memory_scan = false;
 
 #if !XCHAL_HAVE_PTP_MMU || XCHAL_HAVE_SPANNING_WAY
 unsigned long xtensa_kio_paddr = XCHAL_KIO_DEFAULT_PADDR;
@@ -228,9 +227,6 @@ static int __init xtensa_dt_io_area(unsigned long node, const char *uname,
 
 void __init early_init_dt_add_memory_arch(u64 base, u64 size)
 {
-	if (!dt_memory_scan)
-		return;
-
 	size &= PAGE_MASK;
 	add_sysmem_bank(base, base + size);
 }
@@ -242,9 +238,6 @@ void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align)
 
 void __init early_init_devtree(void *params)
 {
-	if (sysmem.nr_banks == 0)
-		dt_memory_scan = true;
-
 	early_init_dt_scan(params);
 	of_scan_flat_dt(xtensa_dt_io_area, NULL);
 
@@ -278,12 +271,6 @@ void __init init_arch(bp_tag_t *bp_start)
 	early_init_devtree(dtb_start);
 #endif
 
-	if (sysmem.nr_banks == 0) {
-		add_sysmem_bank(PLATFORM_DEFAULT_MEM_START,
-				PLATFORM_DEFAULT_MEM_START +
-				PLATFORM_DEFAULT_MEM_SIZE);
-	}
-
 #ifdef CONFIG_CMDLINE_BOOL
 	if (!command_line[0])
 		strlcpy(command_line, default_command_line, COMMAND_LINE_SIZE);