mmu.c

/*
 * Copyright (c) 2006-2018, RT-Thread Development Team
 *
 * SPDX-License-Identifier: Apache-2.0
 *
 * Change Logs:
 * Date           Author       Notes
 * 2012-01-10     bernard      porting to AM1808
 */

#include <rtthread.h>
#include <rthw.h>
#include <board.h>

#include "mmu.h"

#ifdef RT_USING_SMART
#include <lwp_mm.h>
#include <page.h>
#endif

#define MMU_LEVEL_MASK 0x1ffUL
#define MMU_LEVEL_SHIFT   9
#define MMU_ADDRESS_BITS  39
#define MMU_ADDRESS_MASK  0x0000fffffffff000UL
#define MMU_ATTRIB_MASK   0xfff0000000000ffcUL

#define MMU_TYPE_MASK     3UL
#define MMU_TYPE_USED     1UL
#define MMU_TYPE_BLOCK    1UL
#define MMU_TYPE_TABLE    3UL
#define MMU_TYPE_PAGE     3UL

#define MMU_TBL_BLOCK_2M_LEVEL 2
#define MMU_TBL_PAGE_4k_LEVEL  3
#define MMU_TBL_LEVEL_NR       4

void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr);

struct page_table
{
    unsigned long page[512];
};

#ifndef RT_USING_SMART
#define MMU_TBL_PAGE_NR_MAX     32

static rt_mutex_t mm_lock = RT_NULL;

void rt_mm_lock(void)
{
    if (rt_thread_self())
    {
        if (!mm_lock)
        {
            mm_lock = rt_mutex_create("mm_lock", RT_IPC_FLAG_FIFO);
        }
        if (mm_lock)
        {
            rt_mutex_take(mm_lock, RT_WAITING_FOREVER);
        }
    }
}

void rt_mm_unlock(void)
{
    if (rt_thread_self())
    {
        if (mm_lock)
        {
            rt_mutex_release(mm_lock);
        }
    }
}

static volatile struct page_table MMUPage[MMU_TBL_PAGE_NR_MAX] __attribute__((aligned(4096)));

#define rt_page_ref_inc(...)

unsigned long rt_pages_alloc(rt_size_t size_bits)
{
    static unsigned long i = 0;

    if (i >= MMU_TBL_PAGE_NR_MAX)
    {
        return RT_NULL;
    }

    ++i;

    return (unsigned long)&MMUPage[i - 1].page;
}
#endif

static struct page_table *__init_page_array;
static unsigned long __page_off = 0UL;
unsigned long get_free_page(void)
{
    if (!__init_page_array)
    {
        unsigned long temp_page_start;
        asm volatile("mov %0, sp":"=r"(temp_page_start));
        __init_page_array = (struct page_table *)(temp_page_start & ~(ARCH_SECTION_MASK));
        __page_off = 2; /* 0, 1 for ttbr0, ttrb1 */
    }
    __page_off++;
    return (unsigned long)(__init_page_array[__page_off - 1].page);
}

void mmu_memset(char *dst, char v,  size_t len)
{
    while (len--)
    {
        *dst++ = v;
    }
}

static int _map_single_page_2M(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long attr)
{
    int level;
    unsigned long *cur_lv_tbl = lv0_tbl;
    unsigned long page;
    unsigned long off;
    int level_shift = MMU_ADDRESS_BITS;

    if (va & ARCH_SECTION_MASK)
    {
        return MMU_MAP_ERROR_VANOTALIGN;
    }
    if (pa & ARCH_SECTION_MASK)
    {
        return MMU_MAP_ERROR_PANOTALIGN;
    }
    for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++)
    {
        off = (va >> level_shift);
        off &= MMU_LEVEL_MASK;
        if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
        {
            page = get_free_page();
            if (!page)
            {
                return MMU_MAP_ERROR_NOPAGE;
            }
            mmu_memset((char *)page, 0, ARCH_PAGE_SIZE);
            cur_lv_tbl[off] = page | MMU_TYPE_TABLE;
        }
        page = cur_lv_tbl[off];
        if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
        {
            /* is block! error! */
            return MMU_MAP_ERROR_CONFLICT;
        }
        cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
        level_shift -= MMU_LEVEL_SHIFT;
    }
    attr &= MMU_ATTRIB_MASK;
    pa |= (attr | MMU_TYPE_BLOCK); /* block */
    off = (va >> ARCH_SECTION_SHIFT);
    off &= MMU_LEVEL_MASK;
    cur_lv_tbl[off] = pa;
    return 0;
}

int armv8_init_map_2M(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long count, unsigned long attr)
{
    unsigned long i;
    int ret;

    if (va & ARCH_SECTION_MASK)
    {
        return -1;
    }
    if (pa & ARCH_SECTION_MASK)
    {
        return -1;
    }
    for (i = 0; i < count; i++)
    {
        ret = _map_single_page_2M(lv0_tbl, va, pa, attr);
        va += ARCH_SECTION_SIZE;
        pa += ARCH_SECTION_SIZE;
        if (ret != 0)
        {
            return ret;
        }
    }
    return 0;
}

static int _kenrel_map_2M(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long attr)
{
    int level;
    unsigned long *cur_lv_tbl = lv0_tbl;
    unsigned long page;
    unsigned long off;
    int level_shift = MMU_ADDRESS_BITS;

    if (va & ARCH_SECTION_MASK)
    {
        return MMU_MAP_ERROR_VANOTALIGN;
    }
    if (pa & ARCH_SECTION_MASK)
    {
        return MMU_MAP_ERROR_PANOTALIGN;
    }
    for (level = 0; level < MMU_TBL_BLOCK_2M_LEVEL; level++)
    {
        off = (va >> level_shift);
        off &= MMU_LEVEL_MASK;
        if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
        {
            page = (unsigned long)rt_pages_alloc(0);
            if (!page)
            {
                return MMU_MAP_ERROR_NOPAGE;
            }
            rt_memset((char *)page, 0, ARCH_PAGE_SIZE);
            rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE);
            cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE;
            rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
        }
        else
        {
            page = cur_lv_tbl[off];
            page &= MMU_ADDRESS_MASK;
            /* page to va */
            page -= PV_OFFSET;
            rt_page_ref_inc((void *)page, 0);
        }
        page = cur_lv_tbl[off];
        if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
        {
            /* is block! error! */
            return MMU_MAP_ERROR_CONFLICT;
        }
        cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
        cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
        level_shift -= MMU_LEVEL_SHIFT;
    }
    attr &= MMU_ATTRIB_MASK;
    pa |= (attr | MMU_TYPE_BLOCK); /* block */
    off = (va >> ARCH_SECTION_SHIFT);
    off &= MMU_LEVEL_MASK;
    cur_lv_tbl[off] = pa;
    rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));

    return 0;
}

struct mmu_level_info
{
    unsigned long *pos;
    void *page;
};

#ifdef RT_USING_LWP
static void _kenrel_unmap_4K(unsigned long *lv0_tbl, void *v_addr)
{
    int level;
    unsigned long va = (unsigned long)v_addr;
    unsigned long *cur_lv_tbl = lv0_tbl;
    unsigned long page;
    unsigned long off;
    struct mmu_level_info level_info[4];
    int ref;
    int level_shift = MMU_ADDRESS_BITS;
    unsigned long *pos;

    rt_memset(level_info, 0, sizeof level_info);
    for (level = 0; level < MMU_TBL_LEVEL_NR; level++)
    {
        off = (va >> level_shift);
        off &= MMU_LEVEL_MASK;
        page = cur_lv_tbl[off];
        if (!(page & MMU_TYPE_USED))
        {
            break;
        }
        if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
        {
            break;
        }
        level_info[level].pos = cur_lv_tbl + off;
        cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
        cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
        level_info[level].page = cur_lv_tbl;
        level_shift -= MMU_LEVEL_SHIFT;
    }

    level = MMU_TBL_PAGE_4k_LEVEL;
    pos = level_info[level].pos;
    if (pos)
    {
        *pos = (unsigned long)RT_NULL;
        rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, pos, sizeof(void *));
    }
    level--;

    while (level >= 0)
    {
        pos = level_info[level].pos;
        if (pos)
        {
            void *cur_page = level_info[level].page;
            ref = rt_page_ref_get(cur_page, 0);
            if (ref == 1)
            {
                *pos = (unsigned long)RT_NULL;
                rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, pos, sizeof(void *));
            }
            rt_pages_free(cur_page, 0);
        }
        level--;
    }
    asm volatile("tlbi vae1, %0\ndsb sy"::"r"(v_addr):"memory");
    return;
}

static int _kenrel_map_4K(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long attr)
{
    int ret = 0;
    int level;
    unsigned long *cur_lv_tbl = lv0_tbl;
    unsigned long page;
    unsigned long off;
    int level_shift = MMU_ADDRESS_BITS;

    if (va & ARCH_PAGE_MASK)
    {
        return MMU_MAP_ERROR_VANOTALIGN;
    }
    if (pa & ARCH_PAGE_MASK)
    {
        return MMU_MAP_ERROR_PANOTALIGN;
    }
    for (level = 0; level < MMU_TBL_PAGE_4k_LEVEL; level++)
    {
        off = (va >> level_shift);
        off &= MMU_LEVEL_MASK;
        if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
        {
            page = (unsigned long)rt_pages_alloc(0);
            if (!page)
            {
                ret = MMU_MAP_ERROR_NOPAGE;
                goto err;
            }
            rt_memset((void *)page, 0, ARCH_PAGE_SIZE);
            rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)page, ARCH_PAGE_SIZE);
            cur_lv_tbl[off] = (page + PV_OFFSET) | MMU_TYPE_TABLE;
            rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
        }
        else
        {
            page = cur_lv_tbl[off];
            page &= MMU_ADDRESS_MASK;
            /* page to va */
            page -= PV_OFFSET;
            rt_page_ref_inc((void *)page, 0);
        }
        page = cur_lv_tbl[off];
        if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
        {
            /* is block! error! */
            ret = MMU_MAP_ERROR_CONFLICT;
            goto err;
        }
        cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
        cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
        level_shift -= MMU_LEVEL_SHIFT;
    }
    /* now is level page */
    attr &= MMU_ATTRIB_MASK;
    pa |= (attr | MMU_TYPE_PAGE); /* page */
    off = (va >> ARCH_PAGE_SHIFT);
    off &= MMU_LEVEL_MASK;
    cur_lv_tbl[off] = pa; /* page */
    rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, cur_lv_tbl + off, sizeof(void *));
    return ret;
err:
    _kenrel_unmap_4K(lv0_tbl, (void *)va);
    return ret;
}
#endif

static int _kernel_map_fixed(unsigned long *lv0_tbl, unsigned long va, unsigned long pa, unsigned long count, unsigned long attr)
{
    unsigned long i;
    int ret;
    unsigned long _attr = MMU_MAP_CUSTOM(MMU_AP_KAUN, attr);

    if (va & ARCH_SECTION_MASK)
    {
        return -1;
    }
    if (pa & ARCH_SECTION_MASK)
    {
        return -1;
    }
    for (i = 0; i < count; i++)
    {
        ret = _kenrel_map_2M(lv0_tbl, va, pa, _attr);
        va += ARCH_SECTION_SIZE;
        pa += ARCH_SECTION_SIZE;
        if (ret != 0)
        {
            return ret;
        }
    }
    return 0;
}

/************ setting el1 mmu register**************
  MAIR_EL1
  index 0 : memory outer writeback, write/read alloc
  index 1 : memory nocache
  index 2 : device nGnRnE
 *****************************************************/
void mmu_tcr_init(void)
{
    unsigned long val64;

    val64 = 0x00447fUL;
    __asm__ volatile("msr MAIR_EL1, %0\n dsb sy\n"::"r"(val64));

    /* TCR_EL1 */
    val64 = (16UL << 0)  /* t0sz 48bit */
            | (0x0UL << 6)   /* reserved */
            | (0x0UL << 7)   /* epd0 */
            | (0x3UL << 8)   /* t0 wb cacheable */
            | (0x3UL << 10)  /* inner shareable */
            | (0x2UL << 12)  /* t0 outer shareable */
            | (0x0UL << 14)  /* t0 4K */
            | (16UL << 16)   /* t1sz 48bit */
            | (0x0UL << 22)  /* define asid use ttbr0.asid */
            | (0x0UL << 23)  /* epd1 */
            | (0x3UL << 24)  /* t1 inner wb cacheable */
            | (0x3UL << 26)  /* t1 outer wb cacheable */
            | (0x2UL << 28)  /* t1 outer shareable */
            | (0x2UL << 30)  /* t1 4k */
            | (0x1UL << 32)  /* 001b 64GB PA */
            | (0x0UL << 35)  /* reserved */
            | (0x1UL << 36)  /* as: 0:8bit 1:16bit */
            | (0x0UL << 37)  /* tbi0 */
            | (0x0UL << 38); /* tbi1 */
    __asm__ volatile("msr TCR_EL1, %0\n"::"r"(val64));
}

/* dump 2nd level page table */
void rt_hw_cpu_dump_page_table_2nd(rt_uint32_t *ptb)
{
}

void rt_hw_cpu_dump_page_table(rt_uint32_t *ptb)
{
}

volatile unsigned long MMUTable[512] __attribute__((aligned(4 * 1024)));
void rt_hw_mmu_setmtt(unsigned long vaddrStart,
                      unsigned long vaddrEnd,
                      unsigned long paddrStart,
                      unsigned long attr)
{
    unsigned long count;

    if (vaddrStart & ARCH_SECTION_MASK)
    {
        while (1);
    }
    if (paddrStart & ARCH_SECTION_MASK)
    {
        while (1);
    }
    if (vaddrStart > vaddrEnd)
    {
        while (1);
    }
    count = vaddrEnd + 1;
    if (count & ARCH_SECTION_MASK)
    {
        while (1);
    }
    count -= vaddrStart;
    if (count == 0)
    {
        while (1);
    }
    count >>= ARCH_SECTION_SHIFT;
    _kernel_map_fixed((unsigned long *)MMUTable, vaddrStart, paddrStart, count, attr);
}

void rt_hw_mmu_ktbl_set(unsigned long tbl)
{
#ifdef RT_USING_SMART
    tbl += PV_OFFSET;
    __asm__ volatile("msr TTBR1_EL1, %0\n dsb sy\nisb"::"r"(tbl):"memory");
#else
    __asm__ volatile("msr TTBR0_EL1, %0\n dsb sy\nisb"::"r"(tbl):"memory");
#endif
    __asm__ volatile("tlbi vmalle1\n dsb sy\nisb":::"memory");
    __asm__ volatile("ic ialluis\n dsb sy\nisb":::"memory");
}

void rt_hw_mmu_setup(struct mem_desc *mdesc, int desc_nr)
{
    /* set page table */
    for (; desc_nr > 0; desc_nr--)
    {
        rt_hw_mmu_setmtt(mdesc->vaddr_start, mdesc->vaddr_end,
                         mdesc->paddr_start, mdesc->attr);
        mdesc++;
    }
    rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, (void *)MMUTable, sizeof MMUTable);
    rt_hw_mmu_ktbl_set((unsigned long)MMUTable);
}

/**
 * This function will initialize rt_mmu_info structure.
 *
 * @param mmu_info   rt_mmu_info structure
 * @param v_address  virtual address
 * @param size       map size
 * @param vtable     mmu table
 * @param pv_off     pv offset in kernel space
 *
 * @return 0 on successful and -1 for fail
 */
int rt_hw_mmu_map_init(rt_mmu_info *mmu_info, void *v_address, size_t size, size_t *vtable, size_t pv_off)
{
    size_t va_s, va_e;

    if (!mmu_info || !vtable)
    {
        return -1;
    }

    va_s = (size_t)v_address;
    va_e = (size_t)v_address + size - 1;

    if (va_e < va_s)
    {
        return -1;
    }

    va_s >>= ARCH_SECTION_SHIFT;
    va_e >>= ARCH_SECTION_SHIFT;

    if (va_s == 0)
    {
        return -1;
    }

    mmu_info->vtable = vtable;
    mmu_info->vstart = va_s;
    mmu_info->vend = va_e;
    mmu_info->pv_off = pv_off;

    return 0;
}

int rt_hw_mmu_ioremap_init(rt_mmu_info *mmu_info, void *v_address, size_t size)
{
    return 0;
}

#ifdef RT_USING_SMART
static size_t find_vaddr(rt_mmu_info *mmu_info, int pages)
{
    size_t loop_pages;
    size_t va;
    size_t find_va = 0;
    int n = 0;
    size_t i;

    if (!pages)
    {
        return 0;
    }

    if (!mmu_info)
    {
        return 0;
    }

    loop_pages = mmu_info->vend - mmu_info->vstart + 1;
    loop_pages <<= (ARCH_SECTION_SHIFT - ARCH_PAGE_SHIFT);
    va = mmu_info->vstart;
    va <<= ARCH_SECTION_SHIFT;
    for (i = 0; i < loop_pages; i++, va += ARCH_PAGE_SIZE)
    {
        if (_rt_hw_mmu_v2p(mmu_info, (void *)va))
        {
            n = 0;
            find_va = 0;
            continue;
        }
        if (!find_va)
        {
            find_va = va;
        }
        n++;
        if (n >= pages)
        {
            return find_va;
        }
    }
    return 0;
}

static int check_vaddr(rt_mmu_info *mmu_info, void *va, int pages)
{
    size_t loop_va;

    if (!pages)
    {
        return -1;
    }

    if (!mmu_info)
    {
        return -1;
    }

    loop_va = ((size_t)va >> ARCH_SECTION_SHIFT);
    if (loop_va < mmu_info->vstart || loop_va > mmu_info->vend)
    {
        return -1;
    }
    loop_va += ((pages << ARCH_PAGE_SHIFT) >> ARCH_SECTION_SHIFT);
    if (loop_va < mmu_info->vstart || loop_va > mmu_info->vend + 1)
    {
        return -1;
    }

    loop_va = (size_t)va & ~ARCH_PAGE_MASK;
    while (pages--)
    {
        if (_rt_hw_mmu_v2p(mmu_info, (void *)loop_va))
        {
            return -1;
        }
        loop_va += ARCH_PAGE_SIZE;
    }
    return 0;
}

static void __rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, size_t npages)
{
    size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK;

    if (!mmu_info || !mmu_info->vtable)
    {
        return;
    }

    while (npages--)
    {
        _kenrel_unmap_4K(mmu_info->vtable, (void *)loop_va);
        loop_va += ARCH_PAGE_SIZE;
    }
}

static int __rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, size_t npages, size_t attr)
{
    int ret = -1;
    size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK;
    size_t loop_pa = (size_t)p_addr & ~ARCH_PAGE_MASK;
    size_t unmap_va = loop_va;

    if (mmu_info)
    {
        while (npages--)
        {
            ret = _kenrel_map_4K(mmu_info->vtable, loop_va, loop_pa, attr);
            if (ret != 0)
            {
                /* error, undo map */
                while (unmap_va != loop_va)
                {
                    _kenrel_unmap_4K(mmu_info->vtable, (void *)unmap_va);
                    unmap_va += ARCH_PAGE_SIZE;
                }
                break;
            }
            loop_va += ARCH_PAGE_SIZE;
            loop_pa += ARCH_PAGE_SIZE;
        }
    }
    return ret;
}
#endif

static void rt_hw_cpu_tlb_invalidate(void)
{
    __asm__ volatile("tlbi vmalle1\n dsb sy\n isb sy\n");
}

#ifdef RT_USING_SMART
void *_rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, size_t size, size_t attr)
{
    size_t pa_s, pa_e;
    size_t vaddr;
    int pages;
    int ret;

    if (!size)
    {
        return 0;
    }
    pa_s = (size_t)p_addr;
    pa_e = (size_t)p_addr + size - 1;
    pa_s >>= ARCH_PAGE_SHIFT;
    pa_e >>= ARCH_PAGE_SHIFT;
    pages = pa_e - pa_s + 1;
    if (v_addr)
    {
        vaddr = (size_t)v_addr;
        pa_s = (size_t)p_addr;
        if ((vaddr & ARCH_PAGE_MASK) != (pa_s & ARCH_PAGE_MASK))
        {
            return 0;
        }
        vaddr &= ~ARCH_PAGE_MASK;
        if (check_vaddr(mmu_info, (void *)vaddr, pages) != 0)
        {
            return 0;
        }
    }
    else
    {
        vaddr = find_vaddr(mmu_info, pages);
    }
    if (vaddr)
    {
        rt_enter_critical();
        ret = __rt_hw_mmu_map(mmu_info, (void *)vaddr, p_addr, pages, attr);
        if (ret == 0)
        {
            rt_hw_cpu_tlb_invalidate();
            rt_exit_critical();
            return (void *)(vaddr + ((size_t)p_addr & ARCH_PAGE_MASK));
        }
        rt_exit_critical();
    }
    return 0;
}
#else
void *_rt_hw_mmu_map(rt_mmu_info *mmu_info, void* p_addr, size_t size, size_t attr)
{
    return p_addr;
}
#endif

#ifdef RT_USING_SMART
static int __rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t npages, size_t attr)
{
    size_t loop_va = (size_t)v_addr & ~ARCH_PAGE_MASK;
    size_t loop_pa;

    if (!mmu_info)
    {
        return -1;
    }

    while (npages--)
    {
        loop_pa = (size_t)rt_pages_alloc(0);
        if (!loop_pa)
        {
            goto err;
        }
        loop_pa += mmu_info->pv_off;
        _kenrel_map_4K(mmu_info->vtable, loop_va, loop_pa, attr);
        loop_va += ARCH_PAGE_SIZE;
    }
    return 0;
err:
    {
        /* error, unmap and quit */
        int i;
        void *va, *pa;

        va = (void *)((size_t)v_addr & ~ARCH_PAGE_MASK);
        for (i = 0; i < npages; i++)
        {
            pa = rt_hw_mmu_v2p(mmu_info, va);
            pa = (void *)((char *)pa - mmu_info->pv_off);
            rt_pages_free(pa, 0);
            va = (void *)((char *)va + ARCH_PAGE_SIZE);
        }

        __rt_hw_mmu_unmap(mmu_info, v_addr, npages);
        return -1;
    }
}

void *_rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr)
{
    size_t vaddr;
    size_t offset;
    int pages;
    int ret;

    if (!size)
    {
        return 0;
    }
    offset = (size_t)v_addr & ARCH_PAGE_MASK;
    size += (offset + ARCH_PAGE_SIZE - 1);
    pages = (size >> ARCH_PAGE_SHIFT);
    if (v_addr)
    {
        vaddr = (size_t)v_addr;
        vaddr &= ~ARCH_PAGE_MASK;
        if (check_vaddr(mmu_info, (void *)vaddr, pages) != 0)
        {
            return 0;
        }
    }
    else
    {
        vaddr = find_vaddr(mmu_info, pages);
    }
    if (vaddr)
    {
        rt_enter_critical();
        ret = __rt_hw_mmu_map_auto(mmu_info, (void *)vaddr, pages, attr);
        if (ret == 0)
        {
            rt_hw_cpu_tlb_invalidate();
            rt_exit_critical();
            return (void *)((char *)vaddr + offset);
        }
        rt_exit_critical();
    }
    return 0;
}

void _rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, size_t size)
{
    size_t va_s, va_e;
    int pages;

    va_s = (size_t)v_addr;
    va_e = (size_t)v_addr + size - 1;
    va_s >>= ARCH_PAGE_SHIFT;
    va_e >>= ARCH_PAGE_SHIFT;
    pages = va_e - va_s + 1;
    rt_enter_critical();
    __rt_hw_mmu_unmap(mmu_info, v_addr, pages);
    rt_hw_cpu_tlb_invalidate();
    rt_exit_critical();
}

void *rt_hw_mmu_map(rt_mmu_info *mmu_info, void *v_addr, void *p_addr, size_t size, size_t attr)
{
    void *ret;

    rt_mm_lock();
    ret = _rt_hw_mmu_map(mmu_info, v_addr, p_addr, size, attr);
    rt_mm_unlock();
    return ret;
}

void *rt_hw_mmu_map_auto(rt_mmu_info *mmu_info, void *v_addr, size_t size, size_t attr)
{
    void *ret;

    rt_mm_lock();
    ret = _rt_hw_mmu_map_auto(mmu_info, v_addr, size, attr);
    rt_mm_unlock();
    return ret;
}

void rt_hw_mmu_unmap(rt_mmu_info *mmu_info, void *v_addr, size_t size)
{
    rt_mm_lock();
    _rt_hw_mmu_unmap(mmu_info, v_addr, size);
    rt_mm_unlock();
}
#else

#include <cache.h>

void rt_hw_init_mmu_table(struct mem_desc *mdesc, rt_size_t desc_nr)
{
    rt_memset((void *)MMUTable, 0, sizeof(MMUTable));
    rt_memset((void *)MMUPage, 0, sizeof(MMUPage));

    /* set page table */
    for (; desc_nr > 0; --desc_nr)
    {
        rt_hw_mmu_setmtt(mdesc->vaddr_start, mdesc->vaddr_end, mdesc->paddr_start, mdesc->attr);
        ++mdesc;
    }

    rt_hw_cpu_dcache_clean((void *)MMUTable, sizeof(MMUTable));
}


void rt_hw_mmu_init(void)
{
    unsigned long reg_val;

    reg_val = 0x00447fUL;
    __asm__ volatile("msr mair_el1, %0"::"r"(reg_val));

    rt_hw_isb();

    reg_val = (16UL << 0)   /* t0sz 48bit */
            | (0UL  << 6)   /* reserved */
            | (0UL  << 7)   /* epd0 */
            | (3UL  << 8)   /* t0 wb cacheable */
            | (3UL  << 10)  /* inner shareable */
            | (2UL  << 12)  /* t0 outer shareable */
            | (0UL  << 14)  /* t0 4K */
            | (16UL << 16)  /* t1sz 48bit */
            | (0UL  << 22)  /* define asid use ttbr0.asid */
            | (0UL  << 23)  /* epd1 */
            | (3UL  << 24)  /* t1 inner wb cacheable */
            | (3UL  << 26)  /* t1 outer wb cacheable */
            | (2UL  << 28)  /* t1 outer shareable */
            | (2UL  << 30)  /* t1 4k */
            | (1UL  << 32)  /* 001b 64GB PA */
            | (0UL  << 35)  /* reserved */
            | (1UL  << 36)  /* as: 0:8bit 1:16bit */
            | (0UL  << 37)  /* tbi0 */
            | (0UL  << 38); /* tbi1 */
    __asm__ volatile("msr tcr_el1, %0"::"r"(reg_val));

    rt_hw_isb();

    __asm__ volatile ("mrs %0, sctlr_el1":"=r"(reg_val));

    reg_val |= 1 << 2;  /* enable dcache */
    reg_val |= 1 << 0;  /* enable mmu */

    __asm__ volatile (
        "msr ttbr0_el1, %0\n\r"
        "msr sctlr_el1, %1\n\r"
        "dsb sy\n\r"
        "isb sy\n\r"
        ::"r"(MMUTable), "r"(reg_val) :"memory");

    rt_hw_cpu_tlb_invalidate();
}
#endif

void *_rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr)
{
    int level;
    unsigned long va = (unsigned long)v_addr;
    unsigned long pa;
    unsigned long *cur_lv_tbl;
    unsigned long page;
    unsigned long off;
    unsigned long off_addr;
    int level_shift = MMU_ADDRESS_BITS;

    if (!mmu_info)
    {
        return (void *)0;
    }
    cur_lv_tbl = mmu_info->vtable;
    for (level = 0; level < MMU_TBL_PAGE_4k_LEVEL; level++)
    {
        off = (va >> level_shift);
        off &= MMU_LEVEL_MASK;
        if (!(cur_lv_tbl[off] & MMU_TYPE_USED))
        {
            return (void *)0;
        }
        page = cur_lv_tbl[off];
        if ((page & MMU_TYPE_MASK) == MMU_TYPE_BLOCK)
        {
            off_addr = va & ((1UL << level_shift) - 1);
            pa = (page & MMU_ADDRESS_MASK);
            pa += off_addr;
            return (void *)pa;
        }
        cur_lv_tbl = (unsigned long *)(page & MMU_ADDRESS_MASK);
        cur_lv_tbl = (unsigned long *)((unsigned long)cur_lv_tbl - PV_OFFSET);
        level_shift -= MMU_LEVEL_SHIFT;
    }
    /* now is level MMU_TBL_PAGE_4k_LEVEL */
    off = (va >> ARCH_PAGE_SHIFT);
    off &= MMU_LEVEL_MASK;
    page = cur_lv_tbl[off];
    if (!(page & MMU_TYPE_USED))
    {
        return (void *)0;
    }
    pa = (page & MMU_ADDRESS_MASK);
    pa += (va & ARCH_PAGE_MASK);
    return (void *)pa;
}

void *rt_hw_mmu_v2p(rt_mmu_info *mmu_info, void *v_addr)
{
    void *ret;

    rt_mm_lock();
    ret = _rt_hw_mmu_v2p(mmu_info, v_addr);
    rt_mm_unlock();
    return ret;
}


void rt_hw_mmu_setup_early(unsigned long *tbl0, unsigned long *tbl1, unsigned long size, unsigned long pv_off)
{
    int ret;
    unsigned long va = KERNEL_VADDR_START;
    unsigned long count = (size + ARCH_SECTION_MASK) >> ARCH_SECTION_SHIFT;
    unsigned long normal_attr = MMU_MAP_CUSTOM(MMU_AP_KAUN, NORMAL_MEM);

    /* clean the first two pages */
    mmu_memset((char *)tbl0, 0, ARCH_PAGE_SIZE);
    mmu_memset((char *)tbl1, 0, ARCH_PAGE_SIZE);

    ret = armv8_init_map_2M(tbl1, va, va + pv_off, count, normal_attr);
    if (ret != 0)
    {
        while (1);
    }
    ret = armv8_init_map_2M(tbl0, va + pv_off, va + pv_off, count, normal_attr);
    if (ret != 0)
    {
        while (1);
    }
}