exec.c 122.7 KB
Newer Older
B
bellard 已提交
1
/*
B
bellard 已提交
2
 *  virtual page mapping and translated block handling
3
 *
B
bellard 已提交
4 5 6 7 8 9 10 11 12 13 14 15 16
 *  Copyright (c) 2003 Fabrice Bellard
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
17
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
B
bellard 已提交
18
 */
B
bellard 已提交
19
#include "config.h"
B
bellard 已提交
20 21 22
#ifdef _WIN32
#include <windows.h>
#else
B
bellard 已提交
23
#include <sys/types.h>
B
bellard 已提交
24 25
#include <sys/mman.h>
#endif
B
bellard 已提交
26

27
#include "qemu-common.h"
B
bellard 已提交
28 29
#include "cpu.h"
#include "exec-all.h"
B
bellard 已提交
30
#include "tcg.h"
31
#include "hw/hw.h"
32
#include "hw/qdev.h"
A
aliguori 已提交
33
#include "osdep.h"
A
aliguori 已提交
34
#include "kvm.h"
B
Blue Swirl 已提交
35
#include "qemu-timer.h"
36 37
#if defined(CONFIG_USER_ONLY)
#include <qemu.h>
38
#include <signal.h>
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
#include <sys/param.h>
#if __FreeBSD_version >= 700104
#define HAVE_KINFO_GETVMMAP
#define sigqueue sigqueue_freebsd  /* avoid redefinition */
#include <sys/time.h>
#include <sys/proc.h>
#include <machine/profile.h>
#define _KERNEL
#include <sys/user.h>
#undef _KERNEL
#undef sigqueue
#include <libutil.h>
#endif
#endif
54
#endif
B
bellard 已提交
55

B
bellard 已提交
56
//#define DEBUG_TB_INVALIDATE
B
bellard 已提交
57
//#define DEBUG_FLUSH
58
//#define DEBUG_TLB
P
pbrook 已提交
59
//#define DEBUG_UNASSIGNED
B
bellard 已提交
60 61

/* make various TB consistency checks */
62 63
//#define DEBUG_TB_CHECK
//#define DEBUG_TLB_CHECK
B
bellard 已提交
64

T
ths 已提交
65
//#define DEBUG_IOPORT
66
//#define DEBUG_SUBPAGE
T
ths 已提交
67

68 69 70 71 72
#if !defined(CONFIG_USER_ONLY)
/* TB consistency checks only implemented for usermode emulation.  */
#undef DEBUG_TB_CHECK
#endif

73 74
#define SMC_BITMAP_USE_THRESHOLD 10

B
blueswir1 已提交
75
static TranslationBlock *tbs;
76
static int code_gen_max_blocks;
77
TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
B
blueswir1 已提交
78
static int nb_tbs;
B
bellard 已提交
79
/* any access to the tbs or the page table must use this lock */
A
Anthony Liguori 已提交
80
spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
B
bellard 已提交
81

B
blueswir1 已提交
82 83 84
#if defined(__arm__) || defined(__sparc_v9__)
/* The prologue must be reachable with a direct jump. ARM and Sparc64
 have limited branch ranges (possibly also PPC) so place it in a
85 86 87 88
 section close to code segment. */
#define code_gen_section                                \
    __attribute__((__section__(".gen_code")))           \
    __attribute__((aligned (32)))
89 90 91 92
#elif defined(_WIN32)
/* Maximum alignment for Win32 is 16. */
#define code_gen_section                                \
    __attribute__((aligned (16)))
93 94 95 96 97 98
#else
#define code_gen_section                                \
    __attribute__((aligned (32)))
#endif

uint8_t code_gen_prologue[1024] code_gen_section;
B
blueswir1 已提交
99 100
static uint8_t *code_gen_buffer;
static unsigned long code_gen_buffer_size;
101
/* threshold to flush the translated code buffer */
B
blueswir1 已提交
102
static unsigned long code_gen_buffer_max_size;
103
static uint8_t *code_gen_ptr;
B
bellard 已提交
104

105
#if !defined(CONFIG_USER_ONLY)
106
int phys_ram_fd;
A
aliguori 已提交
107
static int in_migration;
P
pbrook 已提交
108

A
Alex Williamson 已提交
109
RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list) };
110
#endif
111

B
bellard 已提交
112 113 114
CPUState *first_cpu;
/* current CPU in the current thread. It is only valid inside
   cpu_exec() */
115
CPUState *cpu_single_env;
P
pbrook 已提交
116
/* 0 = Do not count executed instructions.
T
ths 已提交
117
   1 = Precise instruction counting.
P
pbrook 已提交
118 119 120 121 122
   2 = Adaptive rate instruction counting.  */
int use_icount = 0;
/* Current instruction counter.  While executing translated code this may
   include some instructions that have not yet been executed.  */
int64_t qemu_icount;
B
bellard 已提交
123

B
bellard 已提交
124
typedef struct PageDesc {
B
bellard 已提交
125
    /* list of TBs intersecting this ram page */
B
bellard 已提交
126
    TranslationBlock *first_tb;
127 128 129 130 131 132 133
    /* in order to optimize self modifying code, we count the number
       of lookups we do to a given page to use a bitmap */
    unsigned int code_write_count;
    uint8_t *code_bitmap;
#if defined(CONFIG_USER_ONLY)
    unsigned long flags;
#endif
B
bellard 已提交
134 135
} PageDesc;

P
Paul Brook 已提交
136
/* In system mode we want L1_MAP to be based on ram offsets,
137 138
   while in user mode we want it to be based on virtual addresses.  */
#if !defined(CONFIG_USER_ONLY)
P
Paul Brook 已提交
139 140 141
#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
# define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
#else
142
# define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
P
Paul Brook 已提交
143
#endif
144
#else
145
# define L1_MAP_ADDR_SPACE_BITS  TARGET_VIRT_ADDR_SPACE_BITS
146
#endif
B
bellard 已提交
147

148 149
/* Size of the L2 (and L3, etc) page tables.  */
#define L2_BITS 10
B
bellard 已提交
150 151
#define L2_SIZE (1 << L2_BITS)

152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
/* The bits remaining after N lower levels of page tables.  */
#define P_L1_BITS_REM \
    ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
#define V_L1_BITS_REM \
    ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)

/* Size of the L1 page table.  Avoid silly small sizes.  */
#if P_L1_BITS_REM < 4
#define P_L1_BITS  (P_L1_BITS_REM + L2_BITS)
#else
#define P_L1_BITS  P_L1_BITS_REM
#endif

#if V_L1_BITS_REM < 4
#define V_L1_BITS  (V_L1_BITS_REM + L2_BITS)
#else
#define V_L1_BITS  V_L1_BITS_REM
#endif

#define P_L1_SIZE  ((target_phys_addr_t)1 << P_L1_BITS)
#define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)

#define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)

177 178 179 180
unsigned long qemu_real_host_page_size;
unsigned long qemu_host_page_bits;
unsigned long qemu_host_page_size;
unsigned long qemu_host_page_mask;
B
bellard 已提交
181

182 183 184
/* This is a multi-level map on the virtual address space.
   The bottom level has pointers to PageDesc.  */
static void *l1_map[V_L1_SIZE];
B
bellard 已提交
185

186
#if !defined(CONFIG_USER_ONLY)
P
Paul Brook 已提交
187 188 189 190 191 192
typedef struct PhysPageDesc {
    /* offset in host memory of the page + io_index in the low bits */
    ram_addr_t phys_offset;
    ram_addr_t region_offset;
} PhysPageDesc;

193 194 195
/* This is a multi-level map on the physical address space.
   The bottom level has pointers to PhysPageDesc.  */
static void *l1_phys_map[P_L1_SIZE];
196

197 198
static void io_mem_init(void);

199 200 201
/* io memory support */
CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
B
bellard 已提交
202
void *io_mem_opaque[IO_MEM_NB_ENTRIES];
203
static char io_mem_used[IO_MEM_NB_ENTRIES];
204 205
static int io_mem_watch;
#endif
206

207
/* log support */
J
Juha Riihimäki 已提交
208 209 210
#ifdef WIN32
static const char *logfilename = "qemu.log";
#else
211
static const char *logfilename = "/tmp/qemu.log";
J
Juha Riihimäki 已提交
212
#endif
213 214
FILE *logfile;
int loglevel;
P
pbrook 已提交
215
static int log_append = 0;
216

B
bellard 已提交
217
/* statistics */
218
#if !defined(CONFIG_USER_ONLY)
B
bellard 已提交
219
static int tlb_flush_count;
220
#endif
B
bellard 已提交
221 222 223
static int tb_flush_count;
static int tb_phys_invalidate_count;

224 225 226 227 228 229 230 231 232 233 234
#ifdef _WIN32
static void map_exec(void *addr, long size)
{
    DWORD old_protect;
    VirtualProtect(addr, size,
                   PAGE_EXECUTE_READWRITE, &old_protect);
    
}
#else
static void map_exec(void *addr, long size)
{
235
    unsigned long start, end, page_size;
236
    
237
    page_size = getpagesize();
238
    start = (unsigned long)addr;
239
    start &= ~(page_size - 1);
240 241
    
    end = (unsigned long)addr + size;
242 243
    end += page_size - 1;
    end &= ~(page_size - 1);
244 245 246 247 248 249
    
    mprotect((void *)start, end - start,
             PROT_READ | PROT_WRITE | PROT_EXEC);
}
#endif

B
bellard 已提交
250
static void page_init(void)
B
bellard 已提交
251
{
252
    /* NOTE: we can always suppose that qemu_host_page_size >=
B
bellard 已提交
253
       TARGET_PAGE_SIZE */
254 255 256 257 258 259 260 261 262 263
#ifdef _WIN32
    {
        SYSTEM_INFO system_info;

        GetSystemInfo(&system_info);
        qemu_real_host_page_size = system_info.dwPageSize;
    }
#else
    qemu_real_host_page_size = getpagesize();
#endif
264 265 266 267 268 269 270 271
    if (qemu_host_page_size == 0)
        qemu_host_page_size = qemu_real_host_page_size;
    if (qemu_host_page_size < TARGET_PAGE_SIZE)
        qemu_host_page_size = TARGET_PAGE_SIZE;
    qemu_host_page_bits = 0;
    while ((1 << qemu_host_page_bits) < qemu_host_page_size)
        qemu_host_page_bits++;
    qemu_host_page_mask = ~(qemu_host_page_size - 1);
272

P
Paul Brook 已提交
273
#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
274
    {
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291
#ifdef HAVE_KINFO_GETVMMAP
        struct kinfo_vmentry *freep;
        int i, cnt;

        freep = kinfo_getvmmap(getpid(), &cnt);
        if (freep) {
            mmap_lock();
            for (i = 0; i < cnt; i++) {
                unsigned long startaddr, endaddr;

                startaddr = freep[i].kve_start;
                endaddr = freep[i].kve_end;
                if (h2g_valid(startaddr)) {
                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;

                    if (h2g_valid(endaddr)) {
                        endaddr = h2g(endaddr);
292
                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
293 294 295
                    } else {
#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
                        endaddr = ~0ul;
296
                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
297 298 299 300 301 302 303 304
#endif
                    }
                }
            }
            free(freep);
            mmap_unlock();
        }
#else
305 306
        FILE *f;

P
pbrook 已提交
307
        last_brk = (unsigned long)sbrk(0);
308

309
        f = fopen("/compat/linux/proc/self/maps", "r");
310
        if (f) {
311 312
            mmap_lock();

313
            do {
314 315 316 317 318 319 320 321 322 323 324 325 326 327
                unsigned long startaddr, endaddr;
                int n;

                n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);

                if (n == 2 && h2g_valid(startaddr)) {
                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;

                    if (h2g_valid(endaddr)) {
                        endaddr = h2g(endaddr);
                    } else {
                        endaddr = ~0ul;
                    }
                    page_set_flags(startaddr, endaddr, PAGE_RESERVED);
328 329
                }
            } while (!feof(f));
330

331
            fclose(f);
332
            mmap_unlock();
333
        }
334
#endif
335 336
    }
#endif
B
bellard 已提交
337 338
}

P
Paul Brook 已提交
339
static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
B
bellard 已提交
340
{
P
Paul Brook 已提交
341 342 343 344
    PageDesc *pd;
    void **lp;
    int i;

345
#if defined(CONFIG_USER_ONLY)
P
Paul Brook 已提交
346
    /* We can't use qemu_malloc because it may recurse into a locked mutex. */
347 348 349 350 351 352 353 354
# define ALLOC(P, SIZE)                                 \
    do {                                                \
        P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,    \
                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);   \
    } while (0)
#else
# define ALLOC(P, SIZE) \
    do { P = qemu_mallocz(SIZE); } while (0)
355
#endif
356

357 358 359 360 361 362 363 364 365 366 367 368 369
    /* Level 1.  Always allocated.  */
    lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));

    /* Level 2..N-1.  */
    for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
        void **p = *lp;

        if (p == NULL) {
            if (!alloc) {
                return NULL;
            }
            ALLOC(p, sizeof(void *) * L2_SIZE);
            *lp = p;
370
        }
371 372 373 374 375 376 377 378 379 380 381

        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
    }

    pd = *lp;
    if (pd == NULL) {
        if (!alloc) {
            return NULL;
        }
        ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
        *lp = pd;
B
bellard 已提交
382
    }
383 384 385 386

#undef ALLOC

    return pd + (index & (L2_SIZE - 1));
B
bellard 已提交
387 388
}

P
Paul Brook 已提交
389
static inline PageDesc *page_find(tb_page_addr_t index)
B
bellard 已提交
390
{
391
    return page_find_alloc(index, 0);
B
bellard 已提交
392 393
}

394
#if !defined(CONFIG_USER_ONLY)
A
Anthony Liguori 已提交
395
static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
B
bellard 已提交
396
{
397
    PhysPageDesc *pd;
398 399
    void **lp;
    int i;
B
bellard 已提交
400

401 402
    /* Level 1.  Always allocated.  */
    lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
403

404 405 406 407 408 409 410 411 412 413
    /* Level 2..N-1.  */
    for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
        void **p = *lp;
        if (p == NULL) {
            if (!alloc) {
                return NULL;
            }
            *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
        }
        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
414
    }
415

416
    pd = *lp;
417
    if (pd == NULL) {
418
        int i;
419 420

        if (!alloc) {
421
            return NULL;
422 423 424 425
        }

        *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);

P
pbrook 已提交
426
        for (i = 0; i < L2_SIZE; i++) {
427 428
            pd[i].phys_offset = IO_MEM_UNASSIGNED;
            pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
P
pbrook 已提交
429
        }
B
bellard 已提交
430
    }
431 432

    return pd + (index & (L2_SIZE - 1));
B
bellard 已提交
433 434
}

A
Anthony Liguori 已提交
435
static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
B
bellard 已提交
436
{
437
    return phys_page_find_alloc(index, 0);
B
bellard 已提交
438 439
}

A
Anthony Liguori 已提交
440 441
static void tlb_protect_code(ram_addr_t ram_addr);
static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
442
                                    target_ulong vaddr);
P
pbrook 已提交
443 444
#define mmap_lock() do { } while(0)
#define mmap_unlock() do { } while(0)
445
#endif
B
bellard 已提交
446

447 448 449
#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)

#if defined(CONFIG_USER_ONLY)
S
Stuart Brady 已提交
450
/* Currently it is not recommended to allocate big chunks of data in
451 452 453 454 455
   user mode. It will change when a dedicated libc will be used */
#define USE_STATIC_CODE_GEN_BUFFER
#endif

#ifdef USE_STATIC_CODE_GEN_BUFFER
456 457
static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
               __attribute__((aligned (CODE_GEN_ALIGN)));
458 459
#endif

460
static void code_gen_alloc(unsigned long tb_size)
461
{
462 463 464 465 466
#ifdef USE_STATIC_CODE_GEN_BUFFER
    code_gen_buffer = static_code_gen_buffer;
    code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
    map_exec(code_gen_buffer, code_gen_buffer_size);
#else
467 468
    code_gen_buffer_size = tb_size;
    if (code_gen_buffer_size == 0) {
469 470 471 472
#if defined(CONFIG_USER_ONLY)
        /* in user mode, phys_ram_size is not meaningful */
        code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
#else
S
Stuart Brady 已提交
473
        /* XXX: needs adjustments */
P
pbrook 已提交
474
        code_gen_buffer_size = (unsigned long)(ram_size / 4);
475
#endif
476 477 478 479 480 481 482 483
    }
    if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
        code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
    /* The code gen buffer location may have constraints depending on
       the host cpu and OS */
#if defined(__linux__) 
    {
        int flags;
B
blueswir1 已提交
484 485
        void *start = NULL;

486 487 488 489 490 491
        flags = MAP_PRIVATE | MAP_ANONYMOUS;
#if defined(__x86_64__)
        flags |= MAP_32BIT;
        /* Cannot map more than that */
        if (code_gen_buffer_size > (800 * 1024 * 1024))
            code_gen_buffer_size = (800 * 1024 * 1024);
B
blueswir1 已提交
492 493 494 495 496 497
#elif defined(__sparc_v9__)
        // Map the buffer below 2G, so we can use direct calls and branches
        flags |= MAP_FIXED;
        start = (void *) 0x60000000UL;
        if (code_gen_buffer_size > (512 * 1024 * 1024))
            code_gen_buffer_size = (512 * 1024 * 1024);
498
#elif defined(__arm__)
B
balrog 已提交
499
        /* Map the buffer below 32M, so we can use direct calls and branches */
500 501 502 503
        flags |= MAP_FIXED;
        start = (void *) 0x01000000UL;
        if (code_gen_buffer_size > 16 * 1024 * 1024)
            code_gen_buffer_size = 16 * 1024 * 1024;
504 505 506 507 508 509 510
#elif defined(__s390x__)
        /* Map the buffer so that we can use direct calls and branches.  */
        /* We have a +- 4GB range on the branches; leave some slop.  */
        if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
            code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
        }
        start = (void *)0x90000000UL;
511
#endif
B
blueswir1 已提交
512 513
        code_gen_buffer = mmap(start, code_gen_buffer_size,
                               PROT_WRITE | PROT_READ | PROT_EXEC,
514 515 516 517 518 519
                               flags, -1, 0);
        if (code_gen_buffer == MAP_FAILED) {
            fprintf(stderr, "Could not allocate dynamic translator buffer\n");
            exit(1);
        }
    }
A
Aurelien Jarno 已提交
520
#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541
    {
        int flags;
        void *addr = NULL;
        flags = MAP_PRIVATE | MAP_ANONYMOUS;
#if defined(__x86_64__)
        /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
         * 0x40000000 is free */
        flags |= MAP_FIXED;
        addr = (void *)0x40000000;
        /* Cannot map more than that */
        if (code_gen_buffer_size > (800 * 1024 * 1024))
            code_gen_buffer_size = (800 * 1024 * 1024);
#endif
        code_gen_buffer = mmap(addr, code_gen_buffer_size,
                               PROT_WRITE | PROT_READ | PROT_EXEC, 
                               flags, -1, 0);
        if (code_gen_buffer == MAP_FAILED) {
            fprintf(stderr, "Could not allocate dynamic translator buffer\n");
            exit(1);
        }
    }
542 543 544 545
#else
    code_gen_buffer = qemu_malloc(code_gen_buffer_size);
    map_exec(code_gen_buffer, code_gen_buffer_size);
#endif
546
#endif /* !USE_STATIC_CODE_GEN_BUFFER */
547 548
    map_exec(code_gen_prologue, sizeof(code_gen_prologue));
    code_gen_buffer_max_size = code_gen_buffer_size - 
549
        (TCG_MAX_OP_SIZE * OPC_MAX_SIZE);
550 551 552 553 554 555 556 557 558 559 560 561
    code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
    tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
}

/* Must be called before using the QEMU cpus. 'tb_size' is the size
   (in bytes) allocated to the translation buffer. Zero means default
   size. */
void cpu_exec_init_all(unsigned long tb_size)
{
    cpu_gen_init();
    code_gen_alloc(tb_size);
    code_gen_ptr = code_gen_buffer;
562
    page_init();
563
#if !defined(CONFIG_USER_ONLY)
564
    io_mem_init();
565
#endif
566 567 568 569 570
#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
    /* There's no guest base to take into account, so go ahead and
       initialize the prologue now.  */
    tcg_prologue_init(&tcg_ctx);
#endif
571 572
}

573 574
#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)

575
static int cpu_common_post_load(void *opaque, int version_id)
J
Juan Quintela 已提交
576 577
{
    CPUState *env = opaque;
578

579 580 581
    /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
       version_id is increased. */
    env->interrupt_request &= ~0x01;
582 583 584 585
    tlb_flush(env, 1);

    return 0;
}
J
Juan Quintela 已提交
586 587 588 589 590 591 592 593 594 595 596 597 598

static const VMStateDescription vmstate_cpu_common = {
    .name = "cpu_common",
    .version_id = 1,
    .minimum_version_id = 1,
    .minimum_version_id_old = 1,
    .post_load = cpu_common_post_load,
    .fields      = (VMStateField []) {
        VMSTATE_UINT32(halted, CPUState),
        VMSTATE_UINT32(interrupt_request, CPUState),
        VMSTATE_END_OF_LIST()
    }
};
599 600
#endif

G
Glauber Costa 已提交
601 602 603 604 605 606 607 608 609 610 611 612 613
CPUState *qemu_get_cpu(int cpu)
{
    CPUState *env = first_cpu;

    while (env) {
        if (env->cpu_index == cpu)
            break;
        env = env->next_cpu;
    }

    return env;
}

B
bellard 已提交
614
void cpu_exec_init(CPUState *env)
B
bellard 已提交
615
{
B
bellard 已提交
616 617 618
    CPUState **penv;
    int cpu_index;

619 620 621
#if defined(CONFIG_USER_ONLY)
    cpu_list_lock();
#endif
B
bellard 已提交
622 623 624 625
    env->next_cpu = NULL;
    penv = &first_cpu;
    cpu_index = 0;
    while (*penv != NULL) {
626
        penv = &(*penv)->next_cpu;
B
bellard 已提交
627 628 629
        cpu_index++;
    }
    env->cpu_index = cpu_index;
630
    env->numa_node = 0;
B
Blue Swirl 已提交
631 632
    QTAILQ_INIT(&env->breakpoints);
    QTAILQ_INIT(&env->watchpoints);
B
bellard 已提交
633
    *penv = env;
634 635 636
#if defined(CONFIG_USER_ONLY)
    cpu_list_unlock();
#endif
637
#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
A
Alex Williamson 已提交
638 639
    vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
    register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
640 641
                    cpu_save, cpu_load, env);
#endif
B
bellard 已提交
642 643
}

644 645 646
static inline void invalidate_page_bitmap(PageDesc *p)
{
    if (p->code_bitmap) {
647
        qemu_free(p->code_bitmap);
648 649 650 651 652
        p->code_bitmap = NULL;
    }
    p->code_write_count = 0;
}

653 654 655
/* Set to NULL all the 'first_tb' fields in all PageDescs. */

static void page_flush_tb_1 (int level, void **lp)
B
bellard 已提交
656
{
657
    int i;
B
bellard 已提交
658

659 660 661 662 663
    if (*lp == NULL) {
        return;
    }
    if (level == 0) {
        PageDesc *pd = *lp;
P
Paul Brook 已提交
664
        for (i = 0; i < L2_SIZE; ++i) {
665 666
            pd[i].first_tb = NULL;
            invalidate_page_bitmap(pd + i);
B
bellard 已提交
667
        }
668 669
    } else {
        void **pp = *lp;
P
Paul Brook 已提交
670
        for (i = 0; i < L2_SIZE; ++i) {
671 672 673 674 675 676 677 678 679 680
            page_flush_tb_1 (level - 1, pp + i);
        }
    }
}

static void page_flush_tb(void)
{
    int i;
    for (i = 0; i < V_L1_SIZE; i++) {
        page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
B
bellard 已提交
681 682 683 684
    }
}

/* flush all the translation blocks */
B
bellard 已提交
685
/* XXX: tb_flush is currently not thread safe */
B
bellard 已提交
686
void tb_flush(CPUState *env1)
B
bellard 已提交
687
{
B
bellard 已提交
688
    CPUState *env;
689
#if defined(DEBUG_FLUSH)
B
blueswir1 已提交
690 691 692 693
    printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
           (unsigned long)(code_gen_ptr - code_gen_buffer),
           nb_tbs, nb_tbs > 0 ?
           ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
B
bellard 已提交
694
#endif
695
    if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
P
pbrook 已提交
696 697
        cpu_abort(env1, "Internal error: code buffer overflow\n");

B
bellard 已提交
698
    nb_tbs = 0;
699

B
bellard 已提交
700 701 702
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
        memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
    }
703

B
bellard 已提交
704
    memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
B
bellard 已提交
705
    page_flush_tb();
706

B
bellard 已提交
707
    code_gen_ptr = code_gen_buffer;
B
bellard 已提交
708 709
    /* XXX: flush processor icache at this point if cache flush is
       expensive */
B
bellard 已提交
710
    tb_flush_count++;
B
bellard 已提交
711 712 713 714
}

#ifdef DEBUG_TB_CHECK

J
j_mayer 已提交
715
static void tb_invalidate_check(target_ulong address)
B
bellard 已提交
716 717 718 719
{
    TranslationBlock *tb;
    int i;
    address &= TARGET_PAGE_MASK;
720 721
    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
B
bellard 已提交
722 723
            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
                  address >= tb->pc + tb->size)) {
724 725
                printf("ERROR invalidate: address=" TARGET_FMT_lx
                       " PC=%08lx size=%04x\n",
726
                       address, (long)tb->pc, tb->size);
B
bellard 已提交
727 728 729 730 731 732 733 734 735 736
            }
        }
    }
}

/* verify that all the pages have correct rights for code */
static void tb_page_check(void)
{
    TranslationBlock *tb;
    int i, flags1, flags2;
737

738 739
    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
B
bellard 已提交
740 741 742 743
            flags1 = page_get_flags(tb->pc);
            flags2 = page_get_flags(tb->pc + tb->size - 1);
            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
744
                       (long)tb->pc, tb->size, flags1, flags2);
B
bellard 已提交
745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766
            }
        }
    }
}

#endif

/* invalidate one TB */
static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
                             int next_offset)
{
    TranslationBlock *tb1;
    for(;;) {
        tb1 = *ptb;
        if (tb1 == tb) {
            *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
            break;
        }
        ptb = (TranslationBlock **)((char *)tb1 + next_offset);
    }
}

767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783
static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
{
    TranslationBlock *tb1;
    unsigned int n1;

    for(;;) {
        tb1 = *ptb;
        n1 = (long)tb1 & 3;
        tb1 = (TranslationBlock *)((long)tb1 & ~3);
        if (tb1 == tb) {
            *ptb = tb1->page_next[n1];
            break;
        }
        ptb = &tb1->page_next[n1];
    }
}

B
bellard 已提交
784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818
static inline void tb_jmp_remove(TranslationBlock *tb, int n)
{
    TranslationBlock *tb1, **ptb;
    unsigned int n1;

    ptb = &tb->jmp_next[n];
    tb1 = *ptb;
    if (tb1) {
        /* find tb(n) in circular list */
        for(;;) {
            tb1 = *ptb;
            n1 = (long)tb1 & 3;
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
            if (n1 == n && tb1 == tb)
                break;
            if (n1 == 2) {
                ptb = &tb1->jmp_first;
            } else {
                ptb = &tb1->jmp_next[n1];
            }
        }
        /* now we can suppress tb(n) from the list */
        *ptb = tb->jmp_next[n];

        tb->jmp_next[n] = NULL;
    }
}

/* reset the jump entry 'n' of a TB so that it is not chained to
   another TB */
static inline void tb_reset_jump(TranslationBlock *tb, int n)
{
    tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
}

P
Paul Brook 已提交
819
void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
B
bellard 已提交
820
{
B
bellard 已提交
821
    CPUState *env;
822
    PageDesc *p;
B
bellard 已提交
823
    unsigned int h, n1;
P
Paul Brook 已提交
824
    tb_page_addr_t phys_pc;
825
    TranslationBlock *tb1, *tb2;
826

827 828 829
    /* remove the TB from the hash list */
    phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
    h = tb_phys_hash_func(phys_pc);
830
    tb_remove(&tb_phys_hash[h], tb,
831 832 833 834 835 836 837 838 839 840 841 842 843 844
              offsetof(TranslationBlock, phys_hash_next));

    /* remove the TB from the page list */
    if (tb->page_addr[0] != page_addr) {
        p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
        tb_page_remove(&p->first_tb, tb);
        invalidate_page_bitmap(p);
    }
    if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
        p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
        tb_page_remove(&p->first_tb, tb);
        invalidate_page_bitmap(p);
    }

845
    tb_invalidated_flag = 1;
846

B
bellard 已提交
847
    /* remove the TB from the hash list */
848
    h = tb_jmp_cache_hash_func(tb->pc);
B
bellard 已提交
849 850 851 852
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
        if (env->tb_jmp_cache[h] == tb)
            env->tb_jmp_cache[h] = NULL;
    }
B
bellard 已提交
853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870

    /* suppress this TB from the two jump lists */
    tb_jmp_remove(tb, 0);
    tb_jmp_remove(tb, 1);

    /* suppress any remaining jumps to this TB */
    tb1 = tb->jmp_first;
    for(;;) {
        n1 = (long)tb1 & 3;
        if (n1 == 2)
            break;
        tb1 = (TranslationBlock *)((long)tb1 & ~3);
        tb2 = tb1->jmp_next[n1];
        tb_reset_jump(tb1, n1);
        tb1->jmp_next[n1] = NULL;
        tb1 = tb2;
    }
    tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
871

B
bellard 已提交
872
    tb_phys_invalidate_count++;
873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905
}

static inline void set_bits(uint8_t *tab, int start, int len)
{
    int end, mask, end1;

    end = start + len;
    tab += start >> 3;
    mask = 0xff << (start & 7);
    if ((start & ~7) == (end & ~7)) {
        if (start < end) {
            mask &= ~(0xff << (end & 7));
            *tab |= mask;
        }
    } else {
        *tab++ |= mask;
        start = (start + 8) & ~7;
        end1 = end & ~7;
        while (start < end1) {
            *tab++ = 0xff;
            start += 8;
        }
        if (start < end) {
            mask = ~(0xff << (end & 7));
            *tab |= mask;
        }
    }
}

static void build_page_bitmap(PageDesc *p)
{
    int n, tb_start, tb_end;
    TranslationBlock *tb;
906

P
pbrook 已提交
907
    p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929

    tb = p->first_tb;
    while (tb != NULL) {
        n = (long)tb & 3;
        tb = (TranslationBlock *)((long)tb & ~3);
        /* NOTE: this is subtle as a TB may span two physical pages */
        if (n == 0) {
            /* NOTE: tb_end may be after the end of the page, but
               it is not a problem */
            tb_start = tb->pc & ~TARGET_PAGE_MASK;
            tb_end = tb_start + tb->size;
            if (tb_end > TARGET_PAGE_SIZE)
                tb_end = TARGET_PAGE_SIZE;
        } else {
            tb_start = 0;
            tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
        }
        set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
        tb = tb->page_next[n];
    }
}

P
pbrook 已提交
930 931 932
TranslationBlock *tb_gen_code(CPUState *env,
                              target_ulong pc, target_ulong cs_base,
                              int flags, int cflags)
B
bellard 已提交
933 934 935
{
    TranslationBlock *tb;
    uint8_t *tc_ptr;
P
Paul Brook 已提交
936 937
    tb_page_addr_t phys_pc, phys_page2;
    target_ulong virt_page2;
B
bellard 已提交
938 939
    int code_gen_size;

P
Paul Brook 已提交
940
    phys_pc = get_page_addr_code(env, pc);
B
bellard 已提交
941
    tb = tb_alloc(pc);
B
bellard 已提交
942 943 944 945
    if (!tb) {
        /* flush must be done */
        tb_flush(env);
        /* cannot fail at this point */
B
bellard 已提交
946
        tb = tb_alloc(pc);
P
pbrook 已提交
947 948
        /* Don't forget to invalidate previous TB info.  */
        tb_invalidated_flag = 1;
B
bellard 已提交
949 950 951 952 953 954
    }
    tc_ptr = code_gen_ptr;
    tb->tc_ptr = tc_ptr;
    tb->cs_base = cs_base;
    tb->flags = flags;
    tb->cflags = cflags;
955
    cpu_gen_code(env, tb, &code_gen_size);
B
bellard 已提交
956
    code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
957

B
bellard 已提交
958
    /* check next page if needed */
B
bellard 已提交
959
    virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
B
bellard 已提交
960
    phys_page2 = -1;
B
bellard 已提交
961
    if ((pc & TARGET_PAGE_MASK) != virt_page2) {
P
Paul Brook 已提交
962
        phys_page2 = get_page_addr_code(env, virt_page2);
B
bellard 已提交
963
    }
P
Paul Brook 已提交
964
    tb_link_page(tb, phys_pc, phys_page2);
P
pbrook 已提交
965
    return tb;
B
bellard 已提交
966
}
967

968 969
/* invalidate all TBs which intersect with the target physical page
   starting in range [start;end[. NOTE: start and end must refer to
B
bellard 已提交
970 971 972
   the same physical page. 'is_cpu_write_access' should be true if called
   from a real cpu write access: the virtual CPU will exit the current
   TB if code is modified inside this TB. */
P
Paul Brook 已提交
973
void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
B
bellard 已提交
974 975
                                   int is_cpu_write_access)
{
976
    TranslationBlock *tb, *tb_next, *saved_tb;
B
bellard 已提交
977
    CPUState *env = cpu_single_env;
P
Paul Brook 已提交
978
    tb_page_addr_t tb_start, tb_end;
979 980 981 982 983 984 985 986 987 988
    PageDesc *p;
    int n;
#ifdef TARGET_HAS_PRECISE_SMC
    int current_tb_not_found = is_cpu_write_access;
    TranslationBlock *current_tb = NULL;
    int current_tb_modified = 0;
    target_ulong current_pc = 0;
    target_ulong current_cs_base = 0;
    int current_flags = 0;
#endif /* TARGET_HAS_PRECISE_SMC */
989 990

    p = page_find(start >> TARGET_PAGE_BITS);
991
    if (!p)
992
        return;
993
    if (!p->code_bitmap &&
B
bellard 已提交
994 995
        ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
        is_cpu_write_access) {
996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017
        /* build code bitmap */
        build_page_bitmap(p);
    }

    /* we remove all the TBs in the range [start, end[ */
    /* XXX: see if in some cases it could be faster to invalidate all the code */
    tb = p->first_tb;
    while (tb != NULL) {
        n = (long)tb & 3;
        tb = (TranslationBlock *)((long)tb & ~3);
        tb_next = tb->page_next[n];
        /* NOTE: this is subtle as a TB may span two physical pages */
        if (n == 0) {
            /* NOTE: tb_end may be after the end of the page, but
               it is not a problem */
            tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
            tb_end = tb_start + tb->size;
        } else {
            tb_start = tb->page_addr[1];
            tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
        }
        if (!(tb_end <= start || tb_start >= end)) {
B
bellard 已提交
1018 1019 1020 1021
#ifdef TARGET_HAS_PRECISE_SMC
            if (current_tb_not_found) {
                current_tb_not_found = 0;
                current_tb = NULL;
P
pbrook 已提交
1022
                if (env->mem_io_pc) {
B
bellard 已提交
1023
                    /* now we have a real cpu fault */
P
pbrook 已提交
1024
                    current_tb = tb_find_pc(env->mem_io_pc);
B
bellard 已提交
1025 1026 1027
                }
            }
            if (current_tb == tb &&
P
pbrook 已提交
1028
                (current_tb->cflags & CF_COUNT_MASK) != 1) {
B
bellard 已提交
1029 1030 1031 1032 1033
                /* If we are modifying the current TB, we must stop
                its execution. We could be more precise by checking
                that the modification is after the current PC, but it
                would require a specialized function to partially
                restore the CPU state */
1034

B
bellard 已提交
1035
                current_tb_modified = 1;
1036
                cpu_restore_state(current_tb, env,
P
pbrook 已提交
1037
                                  env->mem_io_pc, NULL);
1038 1039
                cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
                                     &current_flags);
B
bellard 已提交
1040 1041
            }
#endif /* TARGET_HAS_PRECISE_SMC */
1042 1043 1044 1045 1046 1047 1048
            /* we need to do that to handle the case where a signal
               occurs while doing tb_phys_invalidate() */
            saved_tb = NULL;
            if (env) {
                saved_tb = env->current_tb;
                env->current_tb = NULL;
            }
1049
            tb_phys_invalidate(tb, -1);
1050 1051 1052 1053 1054
            if (env) {
                env->current_tb = saved_tb;
                if (env->interrupt_request && env->current_tb)
                    cpu_interrupt(env, env->interrupt_request);
            }
1055 1056 1057 1058 1059 1060 1061
        }
        tb = tb_next;
    }
#if !defined(CONFIG_USER_ONLY)
    /* if no code remaining, no need to continue to use slow writes */
    if (!p->first_tb) {
        invalidate_page_bitmap(p);
B
bellard 已提交
1062
        if (is_cpu_write_access) {
P
pbrook 已提交
1063
            tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
B
bellard 已提交
1064 1065 1066 1067 1068 1069 1070 1071
        }
    }
#endif
#ifdef TARGET_HAS_PRECISE_SMC
    if (current_tb_modified) {
        /* we generate a block containing just the instruction
           modifying the memory. It will ensure that it cannot modify
           itself */
1072
        env->current_tb = NULL;
P
pbrook 已提交
1073
        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
B
bellard 已提交
1074
        cpu_resume_from_signal(env, NULL);
1075
    }
B
bellard 已提交
1076
#endif
1077
}
B
bellard 已提交
1078

1079
/* len must be <= 8 and start must be a multiple of len */
P
Paul Brook 已提交
1080
static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1081 1082 1083
{
    PageDesc *p;
    int offset, b;
1084
#if 0
B
bellard 已提交
1085
    if (1) {
1086 1087 1088 1089
        qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
                  cpu_single_env->mem_io_vaddr, len,
                  cpu_single_env->eip,
                  cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1090 1091
    }
#endif
1092
    p = page_find(start >> TARGET_PAGE_BITS);
1093
    if (!p)
1094 1095 1096 1097 1098 1099 1100 1101
        return;
    if (p->code_bitmap) {
        offset = start & ~TARGET_PAGE_MASK;
        b = p->code_bitmap[offset >> 3] >> (offset & 7);
        if (b & ((1 << len) - 1))
            goto do_invalidate;
    } else {
    do_invalidate:
B
bellard 已提交
1102
        tb_invalidate_phys_page_range(start, start + len, 1);
1103 1104 1105 1106
    }
}

#if !defined(CONFIG_SOFTMMU)
P
Paul Brook 已提交
1107
static void tb_invalidate_phys_page(tb_page_addr_t addr,
B
bellard 已提交
1108
                                    unsigned long pc, void *puc)
1109
{
1110
    TranslationBlock *tb;
1111
    PageDesc *p;
1112
    int n;
B
bellard 已提交
1113
#ifdef TARGET_HAS_PRECISE_SMC
1114
    TranslationBlock *current_tb = NULL;
B
bellard 已提交
1115
    CPUState *env = cpu_single_env;
1116 1117 1118 1119
    int current_tb_modified = 0;
    target_ulong current_pc = 0;
    target_ulong current_cs_base = 0;
    int current_flags = 0;
B
bellard 已提交
1120
#endif
1121 1122 1123

    addr &= TARGET_PAGE_MASK;
    p = page_find(addr >> TARGET_PAGE_BITS);
1124
    if (!p)
1125 1126
        return;
    tb = p->first_tb;
B
bellard 已提交
1127 1128 1129 1130 1131
#ifdef TARGET_HAS_PRECISE_SMC
    if (tb && pc != 0) {
        current_tb = tb_find_pc(pc);
    }
#endif
1132 1133 1134
    while (tb != NULL) {
        n = (long)tb & 3;
        tb = (TranslationBlock *)((long)tb & ~3);
B
bellard 已提交
1135 1136
#ifdef TARGET_HAS_PRECISE_SMC
        if (current_tb == tb &&
P
pbrook 已提交
1137
            (current_tb->cflags & CF_COUNT_MASK) != 1) {
B
bellard 已提交
1138 1139 1140 1141 1142
                /* If we are modifying the current TB, we must stop
                   its execution. We could be more precise by checking
                   that the modification is after the current PC, but it
                   would require a specialized function to partially
                   restore the CPU state */
1143

B
bellard 已提交
1144 1145
            current_tb_modified = 1;
            cpu_restore_state(current_tb, env, pc, puc);
1146 1147
            cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
                                 &current_flags);
B
bellard 已提交
1148 1149
        }
#endif /* TARGET_HAS_PRECISE_SMC */
1150 1151 1152
        tb_phys_invalidate(tb, addr);
        tb = tb->page_next[n];
    }
B
bellard 已提交
1153
    p->first_tb = NULL;
B
bellard 已提交
1154 1155 1156 1157 1158
#ifdef TARGET_HAS_PRECISE_SMC
    if (current_tb_modified) {
        /* we generate a block containing just the instruction
           modifying the memory. It will ensure that it cannot modify
           itself */
1159
        env->current_tb = NULL;
P
pbrook 已提交
1160
        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
B
bellard 已提交
1161 1162 1163
        cpu_resume_from_signal(env, puc);
    }
#endif
B
bellard 已提交
1164
}
1165
#endif
B
bellard 已提交
1166 1167

/* add the tb in the target page and protect it if necessary */
1168
static inline void tb_alloc_page(TranslationBlock *tb,
P
Paul Brook 已提交
1169
                                 unsigned int n, tb_page_addr_t page_addr)
B
bellard 已提交
1170 1171
{
    PageDesc *p;
1172 1173 1174
    TranslationBlock *last_first_tb;

    tb->page_addr[n] = page_addr;
1175
    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1176 1177 1178 1179
    tb->page_next[n] = p->first_tb;
    last_first_tb = p->first_tb;
    p->first_tb = (TranslationBlock *)((long)tb | n);
    invalidate_page_bitmap(p);
B
bellard 已提交
1180

1181
#if defined(TARGET_HAS_SMC) || 1
B
bellard 已提交
1182

1183
#if defined(CONFIG_USER_ONLY)
B
bellard 已提交
1184
    if (p->flags & PAGE_WRITE) {
1185 1186
        target_ulong addr;
        PageDesc *p2;
1187 1188
        int prot;

B
bellard 已提交
1189 1190
        /* force the host page as non writable (writes will have a
           page fault + mprotect overhead) */
1191
        page_addr &= qemu_host_page_mask;
B
bellard 已提交
1192
        prot = 0;
1193 1194 1195 1196 1197 1198 1199 1200 1201
        for(addr = page_addr; addr < page_addr + qemu_host_page_size;
            addr += TARGET_PAGE_SIZE) {

            p2 = page_find (addr >> TARGET_PAGE_BITS);
            if (!p2)
                continue;
            prot |= p2->flags;
            p2->flags &= ~PAGE_WRITE;
          }
1202
        mprotect(g2h(page_addr), qemu_host_page_size,
B
bellard 已提交
1203 1204
                 (prot & PAGE_BITS) & ~PAGE_WRITE);
#ifdef DEBUG_TB_INVALIDATE
B
blueswir1 已提交
1205
        printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1206
               page_addr);
B
bellard 已提交
1207 1208
#endif
    }
1209 1210 1211 1212 1213
#else
    /* if some code is already present, then the pages are already
       protected. So we handle the case where only the first TB is
       allocated in a physical page */
    if (!last_first_tb) {
B
bellard 已提交
1214
        tlb_protect_code(page_addr);
1215 1216
    }
#endif
B
bellard 已提交
1217 1218

#endif /* TARGET_HAS_SMC */
B
bellard 已提交
1219 1220 1221 1222
}

/* Allocate a new translation block. Flush the translation buffer if
   too many translation blocks or too much generated code. */
B
bellard 已提交
1223
TranslationBlock *tb_alloc(target_ulong pc)
B
bellard 已提交
1224 1225 1226
{
    TranslationBlock *tb;

1227 1228
    if (nb_tbs >= code_gen_max_blocks ||
        (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
B
bellard 已提交
1229
        return NULL;
B
bellard 已提交
1230 1231
    tb = &tbs[nb_tbs++];
    tb->pc = pc;
1232
    tb->cflags = 0;
B
bellard 已提交
1233 1234 1235
    return tb;
}

P
pbrook 已提交
1236 1237
void tb_free(TranslationBlock *tb)
{
T
ths 已提交
1238
    /* In practice this is mostly used for single use temporary TB
P
pbrook 已提交
1239 1240 1241 1242 1243 1244 1245 1246
       Ignore the hard cases and just back up if this TB happens to
       be the last one generated.  */
    if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
        code_gen_ptr = tb->tc_ptr;
        nb_tbs--;
    }
}

1247 1248
/* add a new TB and link it to the physical page tables. phys_page2 is
   (-1) to indicate that only one page contains the TB. */
P
Paul Brook 已提交
1249 1250
void tb_link_page(TranslationBlock *tb,
                  tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
B
bellard 已提交
1251
{
1252 1253 1254
    unsigned int h;
    TranslationBlock **ptb;

P
pbrook 已提交
1255 1256 1257
    /* Grab the mmap lock to stop another thread invalidating this TB
       before we are done.  */
    mmap_lock();
1258 1259 1260 1261 1262
    /* add in the physical hash table */
    h = tb_phys_hash_func(phys_pc);
    ptb = &tb_phys_hash[h];
    tb->phys_hash_next = *ptb;
    *ptb = tb;
B
bellard 已提交
1263 1264

    /* add in the page list */
1265 1266 1267 1268 1269 1270
    tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
    if (phys_page2 != -1)
        tb_alloc_page(tb, 1, phys_page2);
    else
        tb->page_addr[1] = -1;

B
bellard 已提交
1271 1272 1273 1274 1275 1276 1277 1278 1279
    tb->jmp_first = (TranslationBlock *)((long)tb | 2);
    tb->jmp_next[0] = NULL;
    tb->jmp_next[1] = NULL;

    /* init original jump addresses */
    if (tb->tb_next_offset[0] != 0xffff)
        tb_reset_jump(tb, 0);
    if (tb->tb_next_offset[1] != 0xffff)
        tb_reset_jump(tb, 1);
1280 1281 1282 1283

#ifdef DEBUG_TB_CHECK
    tb_page_check();
#endif
P
pbrook 已提交
1284
    mmap_unlock();
B
bellard 已提交
1285 1286
}

1287 1288 1289
/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
   tb[1].tc_ptr. Return NULL if not found */
TranslationBlock *tb_find_pc(unsigned long tc_ptr)
B
bellard 已提交
1290
{
1291 1292 1293
    int m_min, m_max, m;
    unsigned long v;
    TranslationBlock *tb;
B
bellard 已提交
1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313

    if (nb_tbs <= 0)
        return NULL;
    if (tc_ptr < (unsigned long)code_gen_buffer ||
        tc_ptr >= (unsigned long)code_gen_ptr)
        return NULL;
    /* binary search (cf Knuth) */
    m_min = 0;
    m_max = nb_tbs - 1;
    while (m_min <= m_max) {
        m = (m_min + m_max) >> 1;
        tb = &tbs[m];
        v = (unsigned long)tb->tc_ptr;
        if (v == tc_ptr)
            return tb;
        else if (tc_ptr < v) {
            m_max = m - 1;
        } else {
            m_min = m + 1;
        }
1314
    }
B
bellard 已提交
1315 1316
    return &tbs[m_max];
}
B
bellard 已提交
1317

B
bellard 已提交
1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349
static void tb_reset_jump_recursive(TranslationBlock *tb);

static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
{
    TranslationBlock *tb1, *tb_next, **ptb;
    unsigned int n1;

    tb1 = tb->jmp_next[n];
    if (tb1 != NULL) {
        /* find head of list */
        for(;;) {
            n1 = (long)tb1 & 3;
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
            if (n1 == 2)
                break;
            tb1 = tb1->jmp_next[n1];
        }
        /* we are now sure now that tb jumps to tb1 */
        tb_next = tb1;

        /* remove tb from the jmp_first list */
        ptb = &tb_next->jmp_first;
        for(;;) {
            tb1 = *ptb;
            n1 = (long)tb1 & 3;
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
            if (n1 == n && tb1 == tb)
                break;
            ptb = &tb1->jmp_next[n1];
        }
        *ptb = tb->jmp_next[n];
        tb->jmp_next[n] = NULL;
1350

B
bellard 已提交
1351 1352 1353
        /* suppress the jump to next tb in generated code */
        tb_reset_jump(tb, n);

1354
        /* suppress jumps in the tb on which we could have jumped */
B
bellard 已提交
1355 1356 1357 1358 1359 1360 1361 1362 1363 1364
        tb_reset_jump_recursive(tb_next);
    }
}

static void tb_reset_jump_recursive(TranslationBlock *tb)
{
    tb_reset_jump_recursive2(tb, 0);
    tb_reset_jump_recursive2(tb, 1);
}

B
bellard 已提交
1365
#if defined(TARGET_HAS_ICE)
1366 1367 1368 1369 1370 1371
#if defined(CONFIG_USER_ONLY)
static void breakpoint_invalidate(CPUState *env, target_ulong pc)
{
    tb_invalidate_phys_page_range(pc, pc + 1, 0);
}
#else
B
bellard 已提交
1372 1373
static void breakpoint_invalidate(CPUState *env, target_ulong pc)
{
A
Anthony Liguori 已提交
1374
    target_phys_addr_t addr;
1375
    target_ulong pd;
A
Anthony Liguori 已提交
1376
    ram_addr_t ram_addr;
P
pbrook 已提交
1377
    PhysPageDesc *p;
B
bellard 已提交
1378

P
pbrook 已提交
1379 1380 1381 1382 1383 1384 1385 1386
    addr = cpu_get_phys_page_debug(env, pc);
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
    ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
P
pbrook 已提交
1387
    tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
B
bellard 已提交
1388
}
B
bellard 已提交
1389
#endif
1390
#endif /* TARGET_HAS_ICE */
B
bellard 已提交
1391

1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403
#if defined(CONFIG_USER_ONLY)
void cpu_watchpoint_remove_all(CPUState *env, int mask)

{
}

int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
                          int flags, CPUWatchpoint **watchpoint)
{
    return -ENOSYS;
}
#else
1404
/* Add a watchpoint.  */
1405 1406
int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
                          int flags, CPUWatchpoint **watchpoint)
1407
{
1408
    target_ulong len_mask = ~(len - 1);
1409
    CPUWatchpoint *wp;
1410

1411 1412 1413 1414 1415 1416
    /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
    if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
        fprintf(stderr, "qemu: tried to set invalid watchpoint at "
                TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
        return -EINVAL;
    }
1417 1418 1419
    wp = qemu_malloc(sizeof(*wp));

    wp->vaddr = addr;
1420
    wp->len_mask = len_mask;
1421 1422
    wp->flags = flags;

1423
    /* keep all GDB-injected watchpoints in front */
1424
    if (flags & BP_GDB)
B
Blue Swirl 已提交
1425
        QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1426
    else
B
Blue Swirl 已提交
1427
        QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1428 1429

    tlb_flush_page(env, addr);
1430 1431 1432 1433

    if (watchpoint)
        *watchpoint = wp;
    return 0;
1434 1435
}

1436 1437 1438
/* Remove a specific watchpoint.  */
int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
                          int flags)
1439
{
1440
    target_ulong len_mask = ~(len - 1);
1441
    CPUWatchpoint *wp;
1442

B
Blue Swirl 已提交
1443
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1444
        if (addr == wp->vaddr && len_mask == wp->len_mask
1445
                && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1446
            cpu_watchpoint_remove_by_ref(env, wp);
1447 1448 1449
            return 0;
        }
    }
1450
    return -ENOENT;
1451 1452
}

1453 1454 1455
/* Remove a specific watchpoint by reference.  */
void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
{
B
Blue Swirl 已提交
1456
    QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1457

1458 1459 1460 1461 1462 1463 1464 1465
    tlb_flush_page(env, watchpoint->vaddr);

    qemu_free(watchpoint);
}

/* Remove all matching watchpoints.  */
void cpu_watchpoint_remove_all(CPUState *env, int mask)
{
1466
    CPUWatchpoint *wp, *next;
1467

B
Blue Swirl 已提交
1468
    QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1469 1470
        if (wp->flags & mask)
            cpu_watchpoint_remove_by_ref(env, wp);
1471
    }
1472
}
1473
#endif
1474

1475 1476 1477
/* Add a breakpoint.  */
int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
                          CPUBreakpoint **breakpoint)
B
bellard 已提交
1478
{
B
bellard 已提交
1479
#if defined(TARGET_HAS_ICE)
1480
    CPUBreakpoint *bp;
1481

1482
    bp = qemu_malloc(sizeof(*bp));
B
bellard 已提交
1483

1484 1485 1486
    bp->pc = pc;
    bp->flags = flags;

1487
    /* keep all GDB-injected breakpoints in front */
1488
    if (flags & BP_GDB)
B
Blue Swirl 已提交
1489
        QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1490
    else
B
Blue Swirl 已提交
1491
        QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1492

B
bellard 已提交
1493
    breakpoint_invalidate(env, pc);
1494 1495 1496

    if (breakpoint)
        *breakpoint = bp;
B
bellard 已提交
1497 1498
    return 0;
#else
1499
    return -ENOSYS;
B
bellard 已提交
1500 1501 1502
#endif
}

1503 1504 1505
/* Remove a specific breakpoint.  */
int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
{
1506
#if defined(TARGET_HAS_ICE)
1507 1508
    CPUBreakpoint *bp;

B
Blue Swirl 已提交
1509
    QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1510 1511 1512 1513
        if (bp->pc == pc && bp->flags == flags) {
            cpu_breakpoint_remove_by_ref(env, bp);
            return 0;
        }
1514
    }
1515 1516 1517
    return -ENOENT;
#else
    return -ENOSYS;
1518 1519 1520
#endif
}

1521 1522
/* Remove a specific breakpoint by reference.  */
void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
B
bellard 已提交
1523
{
B
bellard 已提交
1524
#if defined(TARGET_HAS_ICE)
B
Blue Swirl 已提交
1525
    QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
B
bellard 已提交
1526

1527 1528 1529 1530 1531 1532 1533 1534 1535 1536
    breakpoint_invalidate(env, breakpoint->pc);

    qemu_free(breakpoint);
#endif
}

/* Remove all matching breakpoints. */
void cpu_breakpoint_remove_all(CPUState *env, int mask)
{
#if defined(TARGET_HAS_ICE)
1537
    CPUBreakpoint *bp, *next;
1538

B
Blue Swirl 已提交
1539
    QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1540 1541
        if (bp->flags & mask)
            cpu_breakpoint_remove_by_ref(env, bp);
1542
    }
B
bellard 已提交
1543 1544 1545
#endif
}

B
bellard 已提交
1546 1547 1548 1549
/* enable or disable single step mode. EXCP_DEBUG is returned by the
   CPU loop after each instruction */
void cpu_single_step(CPUState *env, int enabled)
{
B
bellard 已提交
1550
#if defined(TARGET_HAS_ICE)
B
bellard 已提交
1551 1552
    if (env->singlestep_enabled != enabled) {
        env->singlestep_enabled = enabled;
1553 1554 1555
        if (kvm_enabled())
            kvm_update_guest_debug(env, 0);
        else {
S
Stuart Brady 已提交
1556
            /* must flush all the translated code to avoid inconsistencies */
1557 1558 1559
            /* XXX: only flush what is necessary */
            tb_flush(env);
        }
B
bellard 已提交
1560 1561 1562 1563
    }
#endif
}

1564 1565 1566 1567 1568
/* enable or disable low levels log */
void cpu_set_log(int log_flags)
{
    loglevel = log_flags;
    if (loglevel && !logfile) {
P
pbrook 已提交
1569
        logfile = fopen(logfilename, log_append ? "a" : "w");
1570 1571 1572 1573
        if (!logfile) {
            perror(logfilename);
            _exit(1);
        }
1574 1575 1576
#if !defined(CONFIG_SOFTMMU)
        /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
        {
1577
            static char logfile_buf[4096];
1578 1579
            setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
        }
1580 1581
#elif !defined(_WIN32)
        /* Win32 doesn't support line-buffering and requires size >= 2 */
1582
        setvbuf(logfile, NULL, _IOLBF, 0);
1583
#endif
P
pbrook 已提交
1584 1585 1586 1587 1588
        log_append = 1;
    }
    if (!loglevel && logfile) {
        fclose(logfile);
        logfile = NULL;
1589 1590 1591 1592 1593 1594
    }
}

void cpu_set_log_filename(const char *filename)
{
    logfilename = strdup(filename);
P
pbrook 已提交
1595 1596 1597 1598 1599
    if (logfile) {
        fclose(logfile);
        logfile = NULL;
    }
    cpu_set_log(loglevel);
1600
}
B
bellard 已提交
1601

1602
static void cpu_unlink_tb(CPUState *env)
B
bellard 已提交
1603
{
1604 1605 1606 1607
    /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
       problem and hope the cpu will stop of its own accord.  For userspace
       emulation this often isn't actually as bad as it sounds.  Often
       signals are used primarily to interrupt blocking syscalls.  */
B
bellard 已提交
1608
    TranslationBlock *tb;
A
Anthony Liguori 已提交
1609
    static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1610

R
Riku Voipio 已提交
1611
    spin_lock(&interrupt_lock);
1612 1613 1614
    tb = env->current_tb;
    /* if the cpu is currently executing code, we must unlink it and
       all the potentially executing TB */
1615
    if (tb) {
1616 1617
        env->current_tb = NULL;
        tb_reset_jump_recursive(tb);
1618
    }
R
Riku Voipio 已提交
1619
    spin_unlock(&interrupt_lock);
1620 1621 1622 1623 1624 1625
}

/* mask must never be zero, except for A20 change call */
void cpu_interrupt(CPUState *env, int mask)
{
    int old_mask;
1626

P
pbrook 已提交
1627
    old_mask = env->interrupt_request;
B
bellard 已提交
1628
    env->interrupt_request |= mask;
1629

1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640
#ifndef CONFIG_USER_ONLY
    /*
     * If called from iothread context, wake the target cpu in
     * case its halted.
     */
    if (!qemu_cpu_self(env)) {
        qemu_cpu_kick(env);
        return;
    }
#endif

P
pbrook 已提交
1641
    if (use_icount) {
P
pbrook 已提交
1642
        env->icount_decr.u16.high = 0xffff;
P
pbrook 已提交
1643 1644
#ifndef CONFIG_USER_ONLY
        if (!can_do_io(env)
1645
            && (mask & ~old_mask) != 0) {
P
pbrook 已提交
1646 1647 1648 1649
            cpu_abort(env, "Raised interrupt while not in I/O function");
        }
#endif
    } else {
1650
        cpu_unlink_tb(env);
B
bellard 已提交
1651 1652 1653
    }
}

1654 1655 1656 1657 1658
void cpu_reset_interrupt(CPUState *env, int mask)
{
    env->interrupt_request &= ~mask;
}

1659 1660 1661 1662 1663 1664
void cpu_exit(CPUState *env)
{
    env->exit_request = 1;
    cpu_unlink_tb(env);
}

B
blueswir1 已提交
1665
const CPULogItem cpu_log_items[] = {
1666
    { CPU_LOG_TB_OUT_ASM, "out_asm",
1667 1668 1669
      "show generated host assembly code for each compiled TB" },
    { CPU_LOG_TB_IN_ASM, "in_asm",
      "show target assembly code for each compiled TB" },
1670
    { CPU_LOG_TB_OP, "op",
B
bellard 已提交
1671
      "show micro ops for each compiled TB" },
1672
    { CPU_LOG_TB_OP_OPT, "op_opt",
B
blueswir1 已提交
1673 1674 1675
      "show micro ops "
#ifdef TARGET_I386
      "before eflags optimization and "
1676
#endif
B
blueswir1 已提交
1677
      "after liveness analysis" },
1678 1679 1680 1681
    { CPU_LOG_INT, "int",
      "show interrupts/exceptions in short format" },
    { CPU_LOG_EXEC, "exec",
      "show trace before each executed TB (lots of logs)" },
1682
    { CPU_LOG_TB_CPU, "cpu",
T
ths 已提交
1683
      "show CPU state before block translation" },
1684 1685 1686
#ifdef TARGET_I386
    { CPU_LOG_PCALL, "pcall",
      "show protected mode far calls/returns/exceptions" },
A
aliguori 已提交
1687 1688
    { CPU_LOG_RESET, "cpu_reset",
      "show CPU state before CPU resets" },
1689
#endif
B
bellard 已提交
1690
#ifdef DEBUG_IOPORT
1691 1692
    { CPU_LOG_IOPORT, "ioport",
      "show all i/o ports accesses" },
B
bellard 已提交
1693
#endif
1694 1695 1696
    { 0, NULL, NULL },
};

M
Michael S. Tsirkin 已提交
1697 1698 1699 1700 1701
#ifndef CONFIG_USER_ONLY
static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
    = QLIST_HEAD_INITIALIZER(memory_client_list);

static void cpu_notify_set_memory(target_phys_addr_t start_addr,
Y
Yoshiaki Tamura 已提交
1702 1703
                                  ram_addr_t size,
                                  ram_addr_t phys_offset)
M
Michael S. Tsirkin 已提交
1704 1705 1706 1707 1708 1709 1710 1711
{
    CPUPhysMemoryClient *client;
    QLIST_FOREACH(client, &memory_client_list, list) {
        client->set_memory(client, start_addr, size, phys_offset);
    }
}

static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
Y
Yoshiaki Tamura 已提交
1712
                                        target_phys_addr_t end)
M
Michael S. Tsirkin 已提交
1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733
{
    CPUPhysMemoryClient *client;
    QLIST_FOREACH(client, &memory_client_list, list) {
        int r = client->sync_dirty_bitmap(client, start, end);
        if (r < 0)
            return r;
    }
    return 0;
}

static int cpu_notify_migration_log(int enable)
{
    CPUPhysMemoryClient *client;
    QLIST_FOREACH(client, &memory_client_list, list) {
        int r = client->migration_log(client, enable);
        if (r < 0)
            return r;
    }
    return 0;
}

1734 1735
static void phys_page_for_each_1(CPUPhysMemoryClient *client,
                                 int level, void **lp)
M
Michael S. Tsirkin 已提交
1736
{
1737
    int i;
M
Michael S. Tsirkin 已提交
1738

1739 1740 1741 1742 1743
    if (*lp == NULL) {
        return;
    }
    if (level == 0) {
        PhysPageDesc *pd = *lp;
P
Paul Brook 已提交
1744
        for (i = 0; i < L2_SIZE; ++i) {
1745 1746 1747
            if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
                client->set_memory(client, pd[i].region_offset,
                                   TARGET_PAGE_SIZE, pd[i].phys_offset);
M
Michael S. Tsirkin 已提交
1748
            }
1749 1750 1751
        }
    } else {
        void **pp = *lp;
P
Paul Brook 已提交
1752
        for (i = 0; i < L2_SIZE; ++i) {
1753
            phys_page_for_each_1(client, level - 1, pp + i);
M
Michael S. Tsirkin 已提交
1754 1755 1756 1757 1758 1759
        }
    }
}

static void phys_page_for_each(CPUPhysMemoryClient *client)
{
1760 1761 1762 1763
    int i;
    for (i = 0; i < P_L1_SIZE; ++i) {
        phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
                             l1_phys_map + 1);
M
Michael S. Tsirkin 已提交
1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778
    }
}

void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
{
    QLIST_INSERT_HEAD(&memory_client_list, client, list);
    phys_page_for_each(client);
}

void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
{
    QLIST_REMOVE(client, list);
}
#endif

1779 1780 1781 1782 1783 1784
static int cmp1(const char *s1, int n, const char *s2)
{
    if (strlen(s2) != n)
        return 0;
    return memcmp(s1, s2, n) == 0;
}
1785

1786 1787 1788
/* takes a comma separated list of log masks. Return 0 if error. */
int cpu_str_to_log_mask(const char *str)
{
B
blueswir1 已提交
1789
    const CPULogItem *item;
1790 1791 1792 1793 1794 1795 1796 1797 1798
    int mask;
    const char *p, *p1;

    p = str;
    mask = 0;
    for(;;) {
        p1 = strchr(p, ',');
        if (!p1)
            p1 = p + strlen(p);
Y
Yoshiaki Tamura 已提交
1799 1800 1801 1802 1803 1804 1805 1806 1807 1808
        if(cmp1(p,p1-p,"all")) {
            for(item = cpu_log_items; item->mask != 0; item++) {
                mask |= item->mask;
            }
        } else {
            for(item = cpu_log_items; item->mask != 0; item++) {
                if (cmp1(p, p1 - p, item->name))
                    goto found;
            }
            return 0;
1809 1810 1811 1812 1813 1814 1815 1816 1817
        }
    found:
        mask |= item->mask;
        if (*p1 != ',')
            break;
        p = p1 + 1;
    }
    return mask;
}
B
bellard 已提交
1818

B
bellard 已提交
1819 1820 1821
void cpu_abort(CPUState *env, const char *fmt, ...)
{
    va_list ap;
P
pbrook 已提交
1822
    va_list ap2;
B
bellard 已提交
1823 1824

    va_start(ap, fmt);
P
pbrook 已提交
1825
    va_copy(ap2, ap);
B
bellard 已提交
1826 1827 1828 1829
    fprintf(stderr, "qemu: fatal: ");
    vfprintf(stderr, fmt, ap);
    fprintf(stderr, "\n");
#ifdef TARGET_I386
B
bellard 已提交
1830 1831 1832
    cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
#else
    cpu_dump_state(env, stderr, fprintf, 0);
B
bellard 已提交
1833
#endif
1834 1835 1836 1837
    if (qemu_log_enabled()) {
        qemu_log("qemu: fatal: ");
        qemu_log_vprintf(fmt, ap2);
        qemu_log("\n");
1838
#ifdef TARGET_I386
1839
        log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1840
#else
1841
        log_cpu_state(env, 0);
1842
#endif
1843
        qemu_log_flush();
1844
        qemu_log_close();
1845
    }
P
pbrook 已提交
1846
    va_end(ap2);
1847
    va_end(ap);
1848 1849 1850 1851 1852 1853 1854 1855
#if defined(CONFIG_USER_ONLY)
    {
        struct sigaction act;
        sigfillset(&act.sa_mask);
        act.sa_handler = SIG_DFL;
        sigaction(SIGABRT, &act, NULL);
    }
#endif
B
bellard 已提交
1856 1857 1858
    abort();
}

1859 1860
CPUState *cpu_copy(CPUState *env)
{
1861
    CPUState *new_env = cpu_init(env->cpu_model_str);
1862 1863
    CPUState *next_cpu = new_env->next_cpu;
    int cpu_index = new_env->cpu_index;
1864 1865 1866 1867 1868
#if defined(TARGET_HAS_ICE)
    CPUBreakpoint *bp;
    CPUWatchpoint *wp;
#endif

1869
    memcpy(new_env, env, sizeof(CPUState));
1870 1871

    /* Preserve chaining and index. */
1872 1873
    new_env->next_cpu = next_cpu;
    new_env->cpu_index = cpu_index;
1874 1875 1876 1877

    /* Clone all break/watchpoints.
       Note: Once we support ptrace with hw-debug register access, make sure
       BP_CPU break/watchpoints are handled correctly on clone. */
B
Blue Swirl 已提交
1878 1879
    QTAILQ_INIT(&env->breakpoints);
    QTAILQ_INIT(&env->watchpoints);
1880
#if defined(TARGET_HAS_ICE)
B
Blue Swirl 已提交
1881
    QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1882 1883
        cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
    }
B
Blue Swirl 已提交
1884
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1885 1886 1887 1888 1889
        cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
                              wp->flags, NULL);
    }
#endif

1890 1891 1892
    return new_env;
}

1893 1894
#if !defined(CONFIG_USER_ONLY)

1895 1896 1897 1898 1899 1900 1901 1902
static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
{
    unsigned int i;

    /* Discard jump cache entries for any tb which might potentially
       overlap the flushed page.  */
    i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
    memset (&env->tb_jmp_cache[i], 0, 
Y
Yoshiaki Tamura 已提交
1903
            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1904 1905 1906

    i = tb_jmp_cache_hash_page(addr);
    memset (&env->tb_jmp_cache[i], 0, 
Y
Yoshiaki Tamura 已提交
1907
            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1908 1909
}

I
Igor Kovalenko 已提交
1910 1911 1912 1913 1914 1915 1916
static CPUTLBEntry s_cputlb_empty_entry = {
    .addr_read  = -1,
    .addr_write = -1,
    .addr_code  = -1,
    .addend     = -1,
};

1917 1918 1919
/* NOTE: if flush_global is true, also flush global entries (not
   implemented yet) */
void tlb_flush(CPUState *env, int flush_global)
1920 1921
{
    int i;
1922

1923 1924 1925
#if defined(DEBUG_TLB)
    printf("tlb_flush:\n");
#endif
1926 1927 1928 1929
    /* must reset current TB so that interrupts cannot modify the
       links while we are modifying them */
    env->current_tb = NULL;

1930
    for(i = 0; i < CPU_TLB_SIZE; i++) {
1931 1932
        int mmu_idx;
        for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
I
Igor Kovalenko 已提交
1933
            env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1934
        }
1935
    }
1936

1937
    memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1938

P
Paul Brook 已提交
1939 1940
    env->tlb_flush_addr = -1;
    env->tlb_flush_mask = 0;
B
bellard 已提交
1941
    tlb_flush_count++;
1942 1943
}

B
bellard 已提交
1944
static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
B
bellard 已提交
1945
{
1946
    if (addr == (tlb_entry->addr_read &
B
bellard 已提交
1947
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1948
        addr == (tlb_entry->addr_write &
B
bellard 已提交
1949
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1950
        addr == (tlb_entry->addr_code &
B
bellard 已提交
1951
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
I
Igor Kovalenko 已提交
1952
        *tlb_entry = s_cputlb_empty_entry;
B
bellard 已提交
1953
    }
B
bellard 已提交
1954 1955
}

1956
void tlb_flush_page(CPUState *env, target_ulong addr)
1957
{
1958
    int i;
1959
    int mmu_idx;
1960

1961
#if defined(DEBUG_TLB)
1962
    printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1963
#endif
P
Paul Brook 已提交
1964 1965 1966 1967 1968 1969 1970 1971 1972 1973
    /* Check if we need to flush due to large pages.  */
    if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
#if defined(DEBUG_TLB)
        printf("tlb_flush_page: forced full flush ("
               TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
               env->tlb_flush_addr, env->tlb_flush_mask);
#endif
        tlb_flush(env, 1);
        return;
    }
1974 1975 1976
    /* must reset current TB so that interrupts cannot modify the
       links while we are modifying them */
    env->current_tb = NULL;
B
bellard 已提交
1977 1978 1979

    addr &= TARGET_PAGE_MASK;
    i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1980 1981
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
        tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
1982

1983
    tlb_flush_jmp_cache(env, addr);
1984 1985 1986 1987
}

/* update the TLBs so that writes to code in the virtual page 'addr'
   can be detected */
A
Anthony Liguori 已提交
1988
static void tlb_protect_code(ram_addr_t ram_addr)
1989
{
1990
    cpu_physical_memory_reset_dirty(ram_addr,
B
bellard 已提交
1991 1992
                                    ram_addr + TARGET_PAGE_SIZE,
                                    CODE_DIRTY_FLAG);
1993 1994 1995
}

/* update the TLB so that writes in physical page 'phys_addr' are no longer
1996
   tested for self modifying code */
A
Anthony Liguori 已提交
1997
static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1998
                                    target_ulong vaddr)
1999
{
2000
    cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
2001 2002
}

2003
static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
2004 2005 2006
                                         unsigned long start, unsigned long length)
{
    unsigned long addr;
B
bellard 已提交
2007 2008
    if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
        addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2009
        if ((addr - start) < length) {
P
pbrook 已提交
2010
            tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
2011 2012 2013 2014
        }
    }
}

P
pbrook 已提交
2015
/* Note: start and end must be within the same ram block.  */
A
Anthony Liguori 已提交
2016
void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
B
bellard 已提交
2017
                                     int dirty_flags)
2018 2019
{
    CPUState *env;
B
bellard 已提交
2020
    unsigned long length, start1;
2021
    int i;
2022 2023 2024 2025 2026 2027 2028

    start &= TARGET_PAGE_MASK;
    end = TARGET_PAGE_ALIGN(end);

    length = end - start;
    if (length == 0)
        return;
2029
    cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
B
bellard 已提交
2030

2031 2032
    /* we modify the TLB cache so that the dirty bit will be set again
       when accessing the range */
2033
    start1 = (unsigned long)qemu_safe_ram_ptr(start);
P
pbrook 已提交
2034 2035
    /* Chek that we don't span multiple blocks - this breaks the
       address comparisons below.  */
2036
    if ((unsigned long)qemu_safe_ram_ptr(end - 1) - start1
P
pbrook 已提交
2037 2038 2039 2040
            != (end - 1) - start) {
        abort();
    }

B
bellard 已提交
2041
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
2042 2043 2044 2045 2046 2047
        int mmu_idx;
        for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
            for(i = 0; i < CPU_TLB_SIZE; i++)
                tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
                                      start1, length);
        }
B
bellard 已提交
2048
    }
2049 2050
}

A
aliguori 已提交
2051 2052
int cpu_physical_memory_set_dirty_tracking(int enable)
{
M
Michael S. Tsirkin 已提交
2053
    int ret = 0;
A
aliguori 已提交
2054
    in_migration = enable;
M
Michael S. Tsirkin 已提交
2055 2056
    ret = cpu_notify_migration_log(!!enable);
    return ret;
A
aliguori 已提交
2057 2058 2059 2060 2061 2062 2063
}

int cpu_physical_memory_get_dirty_tracking(void)
{
    return in_migration;
}

A
Anthony Liguori 已提交
2064 2065
int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
                                   target_phys_addr_t end_addr)
A
aliguori 已提交
2066
{
2067
    int ret;
2068

M
Michael S. Tsirkin 已提交
2069
    ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
2070
    return ret;
A
aliguori 已提交
2071 2072
}

2073 2074
static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
{
A
Anthony Liguori 已提交
2075
    ram_addr_t ram_addr;
P
pbrook 已提交
2076
    void *p;
2077

B
bellard 已提交
2078
    if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
P
pbrook 已提交
2079 2080
        p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
            + tlb_entry->addend);
M
Marcelo Tosatti 已提交
2081
        ram_addr = qemu_ram_addr_from_host_nofail(p);
2082
        if (!cpu_physical_memory_is_dirty(ram_addr)) {
P
pbrook 已提交
2083
            tlb_entry->addr_write |= TLB_NOTDIRTY;
2084 2085 2086 2087 2088 2089 2090 2091
        }
    }
}

/* update the TLB according to the current state of the dirty bits */
void cpu_tlb_update_dirty(CPUState *env)
{
    int i;
2092 2093 2094 2095 2096
    int mmu_idx;
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
        for(i = 0; i < CPU_TLB_SIZE; i++)
            tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
    }
2097 2098
}

P
pbrook 已提交
2099
static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2100
{
P
pbrook 已提交
2101 2102
    if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
        tlb_entry->addr_write = vaddr;
2103 2104
}

P
pbrook 已提交
2105 2106 2107
/* update the TLB corresponding to virtual page vaddr
   so that it is no longer dirty */
static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2108 2109
{
    int i;
2110
    int mmu_idx;
2111

P
pbrook 已提交
2112
    vaddr &= TARGET_PAGE_MASK;
2113
    i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2114 2115
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
        tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2116 2117
}

P
Paul Brook 已提交
2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146
/* Our TLB does not support large pages, so remember the area covered by
   large pages and trigger a full TLB flush if these are invalidated.  */
static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
                               target_ulong size)
{
    target_ulong mask = ~(size - 1);

    if (env->tlb_flush_addr == (target_ulong)-1) {
        env->tlb_flush_addr = vaddr & mask;
        env->tlb_flush_mask = mask;
        return;
    }
    /* Extend the existing region to include the new page.
       This is a compromise between unnecessary flushes and the cost
       of maintaining a full variable size TLB.  */
    mask &= env->tlb_flush_mask;
    while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
        mask <<= 1;
    }
    env->tlb_flush_addr &= mask;
    env->tlb_flush_mask = mask;
}

/* Add a new TLB entry. At most one entry for a given virtual address
   is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
   supplied size is only used by tlb_flush_page.  */
void tlb_set_page(CPUState *env, target_ulong vaddr,
                  target_phys_addr_t paddr, int prot,
                  int mmu_idx, target_ulong size)
2147
{
B
bellard 已提交
2148
    PhysPageDesc *p;
B
bellard 已提交
2149
    unsigned long pd;
2150
    unsigned int index;
B
bellard 已提交
2151
    target_ulong address;
P
pbrook 已提交
2152
    target_ulong code_address;
2153
    unsigned long addend;
B
bellard 已提交
2154
    CPUTLBEntry *te;
2155
    CPUWatchpoint *wp;
A
Anthony Liguori 已提交
2156
    target_phys_addr_t iotlb;
2157

P
Paul Brook 已提交
2158 2159 2160 2161
    assert(size >= TARGET_PAGE_SIZE);
    if (size != TARGET_PAGE_SIZE) {
        tlb_add_large_page(env, vaddr, size);
    }
B
bellard 已提交
2162
    p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2163 2164 2165 2166 2167 2168
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
#if defined(DEBUG_TLB)
2169 2170 2171
    printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
           " prot=%x idx=%d pd=0x%08lx\n",
           vaddr, paddr, prot, mmu_idx, pd);
2172 2173
#endif

P
pbrook 已提交
2174 2175 2176 2177 2178
    address = vaddr;
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
        /* IO memory case (romd handled later) */
        address |= TLB_MMIO;
    }
P
pbrook 已提交
2179
    addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
P
pbrook 已提交
2180 2181 2182 2183 2184 2185 2186 2187
    if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
        /* Normal RAM.  */
        iotlb = pd & TARGET_PAGE_MASK;
        if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
            iotlb |= IO_MEM_NOTDIRTY;
        else
            iotlb |= IO_MEM_ROM;
    } else {
S
Stuart Brady 已提交
2188
        /* IO handlers are currently passed a physical address.
P
pbrook 已提交
2189 2190 2191 2192 2193
           It would be nice to pass an offset from the base address
           of that region.  This would avoid having to special case RAM,
           and avoid full address decoding in every device.
           We can't use the high bits of pd for this because
           IO_MEM_ROMD uses these as a ram address.  */
2194 2195 2196 2197 2198 2199
        iotlb = (pd & ~TARGET_PAGE_MASK);
        if (p) {
            iotlb += p->region_offset;
        } else {
            iotlb += paddr;
        }
P
pbrook 已提交
2200 2201 2202 2203 2204
    }

    code_address = address;
    /* Make accesses to pages with watchpoints go via the
       watchpoint trap routines.  */
B
Blue Swirl 已提交
2205
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2206
        if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
J
Jun Koi 已提交
2207 2208 2209 2210 2211 2212
            /* Avoid trapping reads of pages with a write breakpoint. */
            if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
                iotlb = io_mem_watch + paddr;
                address |= TLB_MMIO;
                break;
            }
2213
        }
P
pbrook 已提交
2214
    }
2215

P
pbrook 已提交
2216 2217 2218 2219 2220 2221 2222 2223 2224
    index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    env->iotlb[mmu_idx][index] = iotlb - vaddr;
    te = &env->tlb_table[mmu_idx][index];
    te->addend = addend - vaddr;
    if (prot & PAGE_READ) {
        te->addr_read = address;
    } else {
        te->addr_read = -1;
    }
2225

P
pbrook 已提交
2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238
    if (prot & PAGE_EXEC) {
        te->addr_code = code_address;
    } else {
        te->addr_code = -1;
    }
    if (prot & PAGE_WRITE) {
        if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
            (pd & IO_MEM_ROMD)) {
            /* Write access calls the I/O callback.  */
            te->addr_write = address | TLB_MMIO;
        } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
                   !cpu_physical_memory_is_dirty(pd)) {
            te->addr_write = address | TLB_NOTDIRTY;
2239
        } else {
P
pbrook 已提交
2240
            te->addr_write = address;
2241
        }
P
pbrook 已提交
2242 2243
    } else {
        te->addr_write = -1;
2244 2245 2246
    }
}

2247 2248
#else

2249
void tlb_flush(CPUState *env, int flush_global)
2250 2251 2252
{
}

2253
void tlb_flush_page(CPUState *env, target_ulong addr)
2254 2255 2256
{
}

2257 2258 2259 2260
/*
 * Walks guest process memory "regions" one by one
 * and calls callback function 'fn' for each region.
 */
2261 2262 2263 2264 2265 2266 2267 2268 2269 2270

struct walk_memory_regions_data
{
    walk_memory_regions_fn fn;
    void *priv;
    unsigned long start;
    int prot;
};

static int walk_memory_regions_end(struct walk_memory_regions_data *data,
P
Paul Brook 已提交
2271
                                   abi_ulong end, int new_prot)
2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286
{
    if (data->start != -1ul) {
        int rc = data->fn(data->priv, data->start, end, data->prot);
        if (rc != 0) {
            return rc;
        }
    }

    data->start = (new_prot ? end : -1ul);
    data->prot = new_prot;

    return 0;
}

static int walk_memory_regions_1(struct walk_memory_regions_data *data,
P
Paul Brook 已提交
2287
                                 abi_ulong base, int level, void **lp)
2288
{
P
Paul Brook 已提交
2289
    abi_ulong pa;
2290 2291 2292 2293 2294 2295 2296 2297
    int i, rc;

    if (*lp == NULL) {
        return walk_memory_regions_end(data, base, 0);
    }

    if (level == 0) {
        PageDesc *pd = *lp;
P
Paul Brook 已提交
2298
        for (i = 0; i < L2_SIZE; ++i) {
2299 2300 2301 2302 2303 2304 2305
            int prot = pd[i].flags;

            pa = base | (i << TARGET_PAGE_BITS);
            if (prot != data->prot) {
                rc = walk_memory_regions_end(data, pa, prot);
                if (rc != 0) {
                    return rc;
2306 2307
                }
            }
2308 2309 2310
        }
    } else {
        void **pp = *lp;
P
Paul Brook 已提交
2311
        for (i = 0; i < L2_SIZE; ++i) {
P
Paul Brook 已提交
2312 2313
            pa = base | ((abi_ulong)i <<
                (TARGET_PAGE_BITS + L2_BITS * level));
2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334
            rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
            if (rc != 0) {
                return rc;
            }
        }
    }

    return 0;
}

int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
{
    struct walk_memory_regions_data data;
    unsigned long i;

    data.fn = fn;
    data.priv = priv;
    data.start = -1ul;
    data.prot = 0;

    for (i = 0; i < V_L1_SIZE; i++) {
P
Paul Brook 已提交
2335
        int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
2336 2337 2338
                                       V_L1_SHIFT / L2_BITS - 1, l1_map + i);
        if (rc != 0) {
            return rc;
2339
        }
2340
    }
2341 2342

    return walk_memory_regions_end(&data, 0, 0);
2343 2344
}

P
Paul Brook 已提交
2345 2346
static int dump_region(void *priv, abi_ulong start,
    abi_ulong end, unsigned long prot)
2347 2348 2349
{
    FILE *f = (FILE *)priv;

P
Paul Brook 已提交
2350 2351
    (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
        " "TARGET_ABI_FMT_lx" %c%c%c\n",
2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365
        start, end, end - start,
        ((prot & PAGE_READ) ? 'r' : '-'),
        ((prot & PAGE_WRITE) ? 'w' : '-'),
        ((prot & PAGE_EXEC) ? 'x' : '-'));

    return (0);
}

/* dump memory mappings */
void page_dump(FILE *f)
{
    (void) fprintf(f, "%-8s %-8s %-8s %s\n",
            "start", "end", "size", "prot");
    walk_memory_regions(f, dump_region);
2366 2367
}

2368
int page_get_flags(target_ulong address)
2369
{
2370 2371 2372
    PageDesc *p;

    p = page_find(address >> TARGET_PAGE_BITS);
2373
    if (!p)
2374 2375 2376 2377
        return 0;
    return p->flags;
}

2378 2379 2380
/* Modify the flags of a page and invalidate the code if necessary.
   The flag PAGE_WRITE_ORG is positioned automatically depending
   on PAGE_WRITE.  The mmap_lock should already be held.  */
2381
void page_set_flags(target_ulong start, target_ulong end, int flags)
2382
{
2383 2384 2385 2386 2387
    target_ulong addr, len;

    /* This function should never be called with addresses outside the
       guest address space.  If this assert fires, it probably indicates
       a missing call to h2g_valid.  */
P
Paul Brook 已提交
2388 2389
#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
    assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2390 2391
#endif
    assert(start < end);
2392 2393 2394

    start = start & TARGET_PAGE_MASK;
    end = TARGET_PAGE_ALIGN(end);
2395 2396

    if (flags & PAGE_WRITE) {
2397
        flags |= PAGE_WRITE_ORG;
2398 2399 2400 2401 2402 2403 2404 2405 2406
    }

    for (addr = start, len = end - start;
         len != 0;
         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
        PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);

        /* If the write protection bit is set, then we invalidate
           the code inside.  */
2407
        if (!(p->flags & PAGE_WRITE) &&
2408 2409
            (flags & PAGE_WRITE) &&
            p->first_tb) {
B
bellard 已提交
2410
            tb_invalidate_phys_page(addr, 0, NULL);
2411 2412 2413
        }
        p->flags = flags;
    }
2414 2415
}

2416 2417 2418 2419 2420 2421
int page_check_range(target_ulong start, target_ulong len, int flags)
{
    PageDesc *p;
    target_ulong end;
    target_ulong addr;

2422 2423 2424
    /* This function should never be called with addresses outside the
       guest address space.  If this assert fires, it probably indicates
       a missing call to h2g_valid.  */
2425 2426
#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
    assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2427 2428
#endif

R
Richard Henderson 已提交
2429 2430 2431
    if (len == 0) {
        return 0;
    }
2432 2433
    if (start + len - 1 < start) {
        /* We've wrapped around.  */
2434
        return -1;
2435
    }
2436

2437 2438 2439
    end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
    start = start & TARGET_PAGE_MASK;

2440 2441 2442
    for (addr = start, len = end - start;
         len != 0;
         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2443 2444 2445 2446 2447 2448
        p = page_find(addr >> TARGET_PAGE_BITS);
        if( !p )
            return -1;
        if( !(p->flags & PAGE_VALID) )
            return -1;

2449
        if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2450
            return -1;
2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461
        if (flags & PAGE_WRITE) {
            if (!(p->flags & PAGE_WRITE_ORG))
                return -1;
            /* unprotect the page if it was put read-only because it
               contains translated code */
            if (!(p->flags & PAGE_WRITE)) {
                if (!page_unprotect(addr, 0, NULL))
                    return -1;
            }
            return 0;
        }
2462 2463 2464 2465
    }
    return 0;
}

2466
/* called from signal handler: invalidate the code and unprotect the
S
Stuart Brady 已提交
2467
   page. Return TRUE if the fault was successfully handled. */
2468
int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2469
{
2470 2471
    unsigned int prot;
    PageDesc *p;
2472
    target_ulong host_start, host_end, addr;
2473

P
pbrook 已提交
2474 2475 2476 2477 2478
    /* Technically this isn't safe inside a signal handler.  However we
       know this only ever happens in a synchronous SEGV handler, so in
       practice it seems to be ok.  */
    mmap_lock();

2479 2480
    p = page_find(address >> TARGET_PAGE_BITS);
    if (!p) {
P
pbrook 已提交
2481
        mmap_unlock();
2482
        return 0;
P
pbrook 已提交
2483
    }
2484

2485 2486
    /* if the page was really writable, then we change its
       protection back to writable */
2487 2488 2489 2490 2491 2492 2493 2494 2495 2496
    if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
        host_start = address & qemu_host_page_mask;
        host_end = host_start + qemu_host_page_size;

        prot = 0;
        for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
            p = page_find(addr >> TARGET_PAGE_BITS);
            p->flags |= PAGE_WRITE;
            prot |= p->flags;

2497 2498
            /* and since the content will be modified, we must invalidate
               the corresponding translated code. */
2499
            tb_invalidate_phys_page(addr, pc, puc);
2500
#ifdef DEBUG_TB_CHECK
2501
            tb_invalidate_check(addr);
2502 2503
#endif
        }
2504 2505 2506 2507 2508
        mprotect((void *)g2h(host_start), qemu_host_page_size,
                 prot & PAGE_BITS);

        mmap_unlock();
        return 1;
2509
    }
P
pbrook 已提交
2510
    mmap_unlock();
2511 2512 2513
    return 0;
}

B
bellard 已提交
2514 2515
static inline void tlb_set_dirty(CPUState *env,
                                 unsigned long addr, target_ulong vaddr)
2516 2517
{
}
2518 2519
#endif /* defined(CONFIG_USER_ONLY) */

2520
#if !defined(CONFIG_USER_ONLY)
2521

P
Paul Brook 已提交
2522 2523 2524
#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
typedef struct subpage_t {
    target_phys_addr_t base;
R
Richard Henderson 已提交
2525 2526
    ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
    ram_addr_t region_offset[TARGET_PAGE_SIZE];
P
Paul Brook 已提交
2527 2528
} subpage_t;

A
Anthony Liguori 已提交
2529 2530
static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
                             ram_addr_t memory, ram_addr_t region_offset);
R
Richard Henderson 已提交
2531 2532 2533
static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
                                ram_addr_t orig_memory,
                                ram_addr_t region_offset);
2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544
#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
                      need_subpage)                                     \
    do {                                                                \
        if (addr > start_addr)                                          \
            start_addr2 = 0;                                            \
        else {                                                          \
            start_addr2 = start_addr & ~TARGET_PAGE_MASK;               \
            if (start_addr2 > 0)                                        \
                need_subpage = 1;                                       \
        }                                                               \
                                                                        \
2545
        if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE)        \
2546 2547 2548 2549 2550 2551 2552 2553
            end_addr2 = TARGET_PAGE_SIZE - 1;                           \
        else {                                                          \
            end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
            if (end_addr2 < TARGET_PAGE_SIZE - 1)                       \
                need_subpage = 1;                                       \
        }                                                               \
    } while (0)

2554 2555 2556
/* register physical memory.
   For RAM, 'size' must be a multiple of the target page size.
   If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2557 2558
   io memory page.  The address used when calling the IO function is
   the offset from the start of the region, plus region_offset.  Both
S
Stuart Brady 已提交
2559
   start_addr and region_offset are rounded down to a page boundary
2560 2561
   before calculating this offset.  This should not be a problem unless
   the low bits of start_addr and region_offset differ.  */
A
Anthony Liguori 已提交
2562 2563 2564 2565
void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
                                         ram_addr_t size,
                                         ram_addr_t phys_offset,
                                         ram_addr_t region_offset)
2566
{
A
Anthony Liguori 已提交
2567
    target_phys_addr_t addr, end_addr;
B
bellard 已提交
2568
    PhysPageDesc *p;
2569
    CPUState *env;
A
Anthony Liguori 已提交
2570
    ram_addr_t orig_size = size;
R
Richard Henderson 已提交
2571
    subpage_t *subpage;
2572

M
Michael S. Tsirkin 已提交
2573 2574
    cpu_notify_set_memory(start_addr, size, phys_offset);

P
pbrook 已提交
2575 2576 2577
    if (phys_offset == IO_MEM_UNASSIGNED) {
        region_offset = start_addr;
    }
2578
    region_offset &= TARGET_PAGE_MASK;
B
bellard 已提交
2579
    size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
A
Anthony Liguori 已提交
2580
    end_addr = start_addr + (target_phys_addr_t)size;
2581
    for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2582 2583
        p = phys_page_find(addr >> TARGET_PAGE_BITS);
        if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
A
Anthony Liguori 已提交
2584 2585
            ram_addr_t orig_memory = p->phys_offset;
            target_phys_addr_t start_addr2, end_addr2;
2586 2587 2588 2589
            int need_subpage = 0;

            CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
                          need_subpage);
R
Richard Henderson 已提交
2590
            if (need_subpage) {
2591 2592
                if (!(orig_memory & IO_MEM_SUBPAGE)) {
                    subpage = subpage_init((addr & TARGET_PAGE_MASK),
2593 2594
                                           &p->phys_offset, orig_memory,
                                           p->region_offset);
2595 2596 2597 2598
                } else {
                    subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
                                            >> IO_MEM_SHIFT];
                }
2599 2600 2601
                subpage_register(subpage, start_addr2, end_addr2, phys_offset,
                                 region_offset);
                p->region_offset = 0;
2602 2603 2604 2605 2606 2607 2608 2609 2610
            } else {
                p->phys_offset = phys_offset;
                if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
                    (phys_offset & IO_MEM_ROMD))
                    phys_offset += TARGET_PAGE_SIZE;
            }
        } else {
            p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
            p->phys_offset = phys_offset;
2611
            p->region_offset = region_offset;
2612
            if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2613
                (phys_offset & IO_MEM_ROMD)) {
2614
                phys_offset += TARGET_PAGE_SIZE;
P
pbrook 已提交
2615
            } else {
A
Anthony Liguori 已提交
2616
                target_phys_addr_t start_addr2, end_addr2;
2617 2618 2619 2620 2621
                int need_subpage = 0;

                CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
                              end_addr2, need_subpage);

R
Richard Henderson 已提交
2622
                if (need_subpage) {
2623
                    subpage = subpage_init((addr & TARGET_PAGE_MASK),
2624
                                           &p->phys_offset, IO_MEM_UNASSIGNED,
P
pbrook 已提交
2625
                                           addr & TARGET_PAGE_MASK);
2626
                    subpage_register(subpage, start_addr2, end_addr2,
2627 2628
                                     phys_offset, region_offset);
                    p->region_offset = 0;
2629 2630 2631
                }
            }
        }
2632
        region_offset += TARGET_PAGE_SIZE;
2633
    }
2634

2635 2636 2637 2638 2639 2640
    /* since each CPU stores ram addresses in its TLB cache, we must
       reset the modified entries */
    /* XXX: slow ! */
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
        tlb_flush(env, 1);
    }
2641 2642
}

B
bellard 已提交
2643
/* XXX: temporary until new memory mapping API */
A
Anthony Liguori 已提交
2644
ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
B
bellard 已提交
2645 2646 2647 2648 2649 2650 2651 2652 2653
{
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p)
        return IO_MEM_UNASSIGNED;
    return p->phys_offset;
}

A
Anthony Liguori 已提交
2654
void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
A
aliguori 已提交
2655 2656 2657 2658 2659
{
    if (kvm_enabled())
        kvm_coalesce_mmio_region(addr, size);
}

A
Anthony Liguori 已提交
2660
void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
A
aliguori 已提交
2661 2662 2663 2664 2665
{
    if (kvm_enabled())
        kvm_uncoalesce_mmio_region(addr, size);
}

2666 2667 2668 2669 2670 2671
void qemu_flush_coalesced_mmio_buffer(void)
{
    if (kvm_enabled())
        kvm_flush_coalesced_mmio_buffer();
}

2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683
#if defined(__linux__) && !defined(TARGET_S390X)

#include <sys/vfs.h>

#define HUGETLBFS_MAGIC       0x958458f6

static long gethugepagesize(const char *path)
{
    struct statfs fs;
    int ret;

    do {
Y
Yoshiaki Tamura 已提交
2684
        ret = statfs(path, &fs);
2685 2686 2687
    } while (ret != 0 && errno == EINTR);

    if (ret != 0) {
Y
Yoshiaki Tamura 已提交
2688 2689
        perror(path);
        return 0;
2690 2691 2692
    }

    if (fs.f_type != HUGETLBFS_MAGIC)
Y
Yoshiaki Tamura 已提交
2693
        fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
2694 2695 2696 2697

    return fs.f_bsize;
}

A
Alex Williamson 已提交
2698 2699 2700
static void *file_ram_alloc(RAMBlock *block,
                            ram_addr_t memory,
                            const char *path)
2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711
{
    char *filename;
    void *area;
    int fd;
#ifdef MAP_POPULATE
    int flags;
#endif
    unsigned long hpagesize;

    hpagesize = gethugepagesize(path);
    if (!hpagesize) {
Y
Yoshiaki Tamura 已提交
2712
        return NULL;
2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724
    }

    if (memory < hpagesize) {
        return NULL;
    }

    if (kvm_enabled() && !kvm_has_sync_mmu()) {
        fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
        return NULL;
    }

    if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
Y
Yoshiaki Tamura 已提交
2725
        return NULL;
2726 2727 2728 2729
    }

    fd = mkstemp(filename);
    if (fd < 0) {
Y
Yoshiaki Tamura 已提交
2730 2731 2732
        perror("unable to create backing store for hugepages");
        free(filename);
        return NULL;
2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745
    }
    unlink(filename);
    free(filename);

    memory = (memory+hpagesize-1) & ~(hpagesize-1);

    /*
     * ftruncate is not supported by hugetlbfs in older
     * hosts, so don't bother bailing out on errors.
     * If anything goes wrong with it under other filesystems,
     * mmap will fail.
     */
    if (ftruncate(fd, memory))
Y
Yoshiaki Tamura 已提交
2746
        perror("ftruncate");
2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758

#ifdef MAP_POPULATE
    /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
     * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
     * to sidestep this quirk.
     */
    flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
    area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
#else
    area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
#endif
    if (area == MAP_FAILED) {
Y
Yoshiaki Tamura 已提交
2759 2760 2761
        perror("file_ram_alloc: can't mmap RAM pages");
        close(fd);
        return (NULL);
2762
    }
A
Alex Williamson 已提交
2763
    block->fd = fd;
2764 2765 2766 2767
    return area;
}
#endif

2768
static ram_addr_t find_ram_offset(ram_addr_t size)
A
Alex Williamson 已提交
2769 2770
{
    RAMBlock *block, *next_block;
2771
    ram_addr_t offset = 0, mingap = ULONG_MAX;
A
Alex Williamson 已提交
2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794

    if (QLIST_EMPTY(&ram_list.blocks))
        return 0;

    QLIST_FOREACH(block, &ram_list.blocks, next) {
        ram_addr_t end, next = ULONG_MAX;

        end = block->offset + block->length;

        QLIST_FOREACH(next_block, &ram_list.blocks, next) {
            if (next_block->offset >= end) {
                next = MIN(next, next_block->offset);
            }
        }
        if (next - end >= size && next - end < mingap) {
            offset =  end;
            mingap = next - end;
        }
    }
    return offset;
}

static ram_addr_t last_ram_offset(void)
2795 2796 2797 2798 2799 2800 2801 2802 2803 2804
{
    RAMBlock *block;
    ram_addr_t last = 0;

    QLIST_FOREACH(block, &ram_list.blocks, next)
        last = MAX(last, block->offset + block->length);

    return last;
}

2805
ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
2806
                                   ram_addr_t size, void *host)
2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829
{
    RAMBlock *new_block, *block;

    size = TARGET_PAGE_ALIGN(size);
    new_block = qemu_mallocz(sizeof(*new_block));

    if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
        char *id = dev->parent_bus->info->get_dev_path(dev);
        if (id) {
            snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
            qemu_free(id);
        }
    }
    pstrcat(new_block->idstr, sizeof(new_block->idstr), name);

    QLIST_FOREACH(block, &ram_list.blocks, next) {
        if (!strcmp(block->idstr, new_block->idstr)) {
            fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
                    new_block->idstr);
            abort();
        }
    }

2830 2831 2832 2833
    if (host) {
        new_block->host = host;
    } else {
        if (mem_path) {
2834
#if defined (__linux__) && !defined(TARGET_S390X)
2835 2836 2837
            new_block->host = file_ram_alloc(new_block, size, mem_path);
            if (!new_block->host) {
                new_block->host = qemu_vmalloc(size);
A
Andreas Färber 已提交
2838
                qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2839
            }
2840
#else
2841 2842
            fprintf(stderr, "-mem-path option unsupported\n");
            exit(1);
2843
#endif
2844
        } else {
2845
#if defined(TARGET_S390X) && defined(CONFIG_KVM)
2846 2847 2848 2849
            /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
            new_block->host = mmap((void*)0x1000000, size,
                                   PROT_EXEC|PROT_READ|PROT_WRITE,
                                   MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2850
#else
2851
            new_block->host = qemu_vmalloc(size);
2852
#endif
A
Andreas Färber 已提交
2853
            qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2854
        }
2855
    }
2856

2857
    new_block->offset = find_ram_offset(size);
P
pbrook 已提交
2858 2859
    new_block->length = size;

A
Alex Williamson 已提交
2860
    QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
2861
    fprintf(stderr, "alloc ram %s len 0x%x\n", new_block->idstr, (int)new_block->length);
P
pbrook 已提交
2862

A
Alex Williamson 已提交
2863
    ram_list.phys_dirty = qemu_realloc(ram_list.phys_dirty,
A
Alex Williamson 已提交
2864
                                       last_ram_offset() >> TARGET_PAGE_BITS);
2865
    memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
P
pbrook 已提交
2866 2867
           0xff, size >> TARGET_PAGE_BITS);

2868 2869 2870
    if (kvm_enabled())
        kvm_setup_guest_memory(new_block->host, size);

P
pbrook 已提交
2871 2872
    return new_block->offset;
}
B
bellard 已提交
2873

2874 2875 2876 2877 2878
ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size)
{
    return qemu_ram_alloc_from_ptr(dev, name, size, NULL);
}

A
Anthony Liguori 已提交
2879
void qemu_ram_free(ram_addr_t addr)
B
bellard 已提交
2880
{
A
Alex Williamson 已提交
2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906
    RAMBlock *block;

    QLIST_FOREACH(block, &ram_list.blocks, next) {
        if (addr == block->offset) {
            QLIST_REMOVE(block, next);
            if (mem_path) {
#if defined (__linux__) && !defined(TARGET_S390X)
                if (block->fd) {
                    munmap(block->host, block->length);
                    close(block->fd);
                } else {
                    qemu_vfree(block->host);
                }
#endif
            } else {
#if defined(TARGET_S390X) && defined(CONFIG_KVM)
                munmap(block->host, block->length);
#else
                qemu_vfree(block->host);
#endif
            }
            qemu_free(block);
            return;
        }
    }

B
bellard 已提交
2907 2908
}

2909
/* Return a host pointer to ram allocated with qemu_ram_alloc.
P
pbrook 已提交
2910 2911 2912 2913 2914 2915 2916
   With the exception of the softmmu code in this file, this should
   only be used for local memory (e.g. video ram) that the device owns,
   and knows it isn't going to access beyond the end of the block.

   It should not be used for general purpose DMA.
   Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
 */
A
Anthony Liguori 已提交
2917
void *qemu_get_ram_ptr(ram_addr_t addr)
2918
{
P
pbrook 已提交
2919 2920
    RAMBlock *block;

A
Alex Williamson 已提交
2921 2922 2923 2924 2925 2926
    QLIST_FOREACH(block, &ram_list.blocks, next) {
        if (addr - block->offset < block->length) {
            QLIST_REMOVE(block, next);
            QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
            return block->host + (addr - block->offset);
        }
P
pbrook 已提交
2927
    }
A
Alex Williamson 已提交
2928 2929 2930 2931 2932

    fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
    abort();

    return NULL;
2933 2934
}

2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953
/* Return a host pointer to ram allocated with qemu_ram_alloc.
 * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
 */
void *qemu_safe_ram_ptr(ram_addr_t addr)
{
    RAMBlock *block;

    QLIST_FOREACH(block, &ram_list.blocks, next) {
        if (addr - block->offset < block->length) {
            return block->host + (addr - block->offset);
        }
    }

    fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
    abort();

    return NULL;
}

M
Marcelo Tosatti 已提交
2954
int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
P
pbrook 已提交
2955
{
P
pbrook 已提交
2956 2957 2958
    RAMBlock *block;
    uint8_t *host = ptr;

A
Alex Williamson 已提交
2959 2960
    QLIST_FOREACH(block, &ram_list.blocks, next) {
        if (host - block->host < block->length) {
M
Marcelo Tosatti 已提交
2961 2962
            *ram_addr = block->offset + (host - block->host);
            return 0;
A
Alex Williamson 已提交
2963
        }
P
pbrook 已提交
2964
    }
M
Marcelo Tosatti 已提交
2965 2966
    return -1;
}
A
Alex Williamson 已提交
2967

M
Marcelo Tosatti 已提交
2968 2969 2970 2971 2972
/* Some of the softmmu routines need to translate from a host pointer
   (typically a TLB entry) back to a ram offset.  */
ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
{
    ram_addr_t ram_addr;
A
Alex Williamson 已提交
2973

M
Marcelo Tosatti 已提交
2974 2975 2976 2977 2978
    if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
        fprintf(stderr, "Bad ram pointer %p\n", ptr);
        abort();
    }
    return ram_addr;
P
pbrook 已提交
2979 2980
}

A
Anthony Liguori 已提交
2981
static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2982
{
P
pbrook 已提交
2983
#ifdef DEBUG_UNASSIGNED
B
blueswir1 已提交
2984
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2985
#endif
2986
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2987 2988 2989 2990 2991
    do_unassigned_access(addr, 0, 0, 0, 1);
#endif
    return 0;
}

A
Anthony Liguori 已提交
2992
static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2993 2994 2995 2996
{
#ifdef DEBUG_UNASSIGNED
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
#endif
2997
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2998 2999 3000 3001 3002
    do_unassigned_access(addr, 0, 0, 0, 2);
#endif
    return 0;
}

A
Anthony Liguori 已提交
3003
static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
3004 3005 3006 3007
{
#ifdef DEBUG_UNASSIGNED
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
#endif
3008
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3009
    do_unassigned_access(addr, 0, 0, 0, 4);
P
pbrook 已提交
3010
#endif
3011 3012 3013
    return 0;
}

A
Anthony Liguori 已提交
3014
static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
3015
{
P
pbrook 已提交
3016
#ifdef DEBUG_UNASSIGNED
B
blueswir1 已提交
3017
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
P
pbrook 已提交
3018
#endif
3019
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3020 3021 3022 3023
    do_unassigned_access(addr, 1, 0, 0, 1);
#endif
}

A
Anthony Liguori 已提交
3024
static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
3025 3026 3027 3028
{
#ifdef DEBUG_UNASSIGNED
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
#endif
3029
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3030 3031 3032 3033
    do_unassigned_access(addr, 1, 0, 0, 2);
#endif
}

A
Anthony Liguori 已提交
3034
static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
3035 3036 3037 3038
{
#ifdef DEBUG_UNASSIGNED
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
#endif
3039
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3040
    do_unassigned_access(addr, 1, 0, 0, 4);
3041
#endif
3042 3043
}

3044
static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
3045
    unassigned_mem_readb,
3046 3047
    unassigned_mem_readw,
    unassigned_mem_readl,
3048 3049
};

3050
static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
3051
    unassigned_mem_writeb,
3052 3053
    unassigned_mem_writew,
    unassigned_mem_writel,
3054 3055
};

A
Anthony Liguori 已提交
3056
static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
P
pbrook 已提交
3057
                                uint32_t val)
3058
{
3059
    int dirty_flags;
3060
    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3061
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3062
#if !defined(CONFIG_USER_ONLY)
3063
        tb_invalidate_phys_page_fast(ram_addr, 1);
3064
        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3065
#endif
3066
    }
P
pbrook 已提交
3067
    stb_p(qemu_get_ram_ptr(ram_addr), val);
B
bellard 已提交
3068
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3069
    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
B
bellard 已提交
3070 3071 3072
    /* we remove the notdirty callback only if the code has been
       flushed */
    if (dirty_flags == 0xff)
P
pbrook 已提交
3073
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3074 3075
}

A
Anthony Liguori 已提交
3076
static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
P
pbrook 已提交
3077
                                uint32_t val)
3078
{
3079
    int dirty_flags;
3080
    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3081
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3082
#if !defined(CONFIG_USER_ONLY)
3083
        tb_invalidate_phys_page_fast(ram_addr, 2);
3084
        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3085
#endif
3086
    }
P
pbrook 已提交
3087
    stw_p(qemu_get_ram_ptr(ram_addr), val);
B
bellard 已提交
3088
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3089
    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
B
bellard 已提交
3090 3091 3092
    /* we remove the notdirty callback only if the code has been
       flushed */
    if (dirty_flags == 0xff)
P
pbrook 已提交
3093
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3094 3095
}

A
Anthony Liguori 已提交
3096
static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
P
pbrook 已提交
3097
                                uint32_t val)
3098
{
3099
    int dirty_flags;
3100
    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3101
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3102
#if !defined(CONFIG_USER_ONLY)
3103
        tb_invalidate_phys_page_fast(ram_addr, 4);
3104
        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3105
#endif
3106
    }
P
pbrook 已提交
3107
    stl_p(qemu_get_ram_ptr(ram_addr), val);
B
bellard 已提交
3108
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3109
    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
B
bellard 已提交
3110 3111 3112
    /* we remove the notdirty callback only if the code has been
       flushed */
    if (dirty_flags == 0xff)
P
pbrook 已提交
3113
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3114 3115
}

3116
static CPUReadMemoryFunc * const error_mem_read[3] = {
3117 3118 3119 3120 3121
    NULL, /* never used */
    NULL, /* never used */
    NULL, /* never used */
};

3122
static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
3123 3124 3125 3126 3127
    notdirty_mem_writeb,
    notdirty_mem_writew,
    notdirty_mem_writel,
};

P
pbrook 已提交
3128
/* Generate a debug exception if a watchpoint has been hit.  */
3129
static void check_watchpoint(int offset, int len_mask, int flags)
P
pbrook 已提交
3130 3131
{
    CPUState *env = cpu_single_env;
3132 3133
    target_ulong pc, cs_base;
    TranslationBlock *tb;
P
pbrook 已提交
3134
    target_ulong vaddr;
3135
    CPUWatchpoint *wp;
3136
    int cpu_flags;
P
pbrook 已提交
3137

3138 3139 3140 3141 3142 3143 3144
    if (env->watchpoint_hit) {
        /* We re-entered the check after replacing the TB. Now raise
         * the debug interrupt so that is will trigger after the
         * current instruction. */
        cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
        return;
    }
P
pbrook 已提交
3145
    vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
B
Blue Swirl 已提交
3146
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
3147 3148
        if ((vaddr == (wp->vaddr & len_mask) ||
             (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165
            wp->flags |= BP_WATCHPOINT_HIT;
            if (!env->watchpoint_hit) {
                env->watchpoint_hit = wp;
                tb = tb_find_pc(env->mem_io_pc);
                if (!tb) {
                    cpu_abort(env, "check_watchpoint: could not find TB for "
                              "pc=%p", (void *)env->mem_io_pc);
                }
                cpu_restore_state(tb, env, env->mem_io_pc, NULL);
                tb_phys_invalidate(tb, -1);
                if (wp->flags & BP_STOP_BEFORE_ACCESS) {
                    env->exception_index = EXCP_DEBUG;
                } else {
                    cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
                    tb_gen_code(env, pc, cs_base, cpu_flags, 1);
                }
                cpu_resume_from_signal(env, NULL);
3166
            }
3167 3168
        } else {
            wp->flags &= ~BP_WATCHPOINT_HIT;
P
pbrook 已提交
3169 3170 3171 3172
        }
    }
}

3173 3174 3175
/* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
   so these check for a hit then pass through to the normal out-of-line
   phys routines.  */
A
Anthony Liguori 已提交
3176
static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
3177
{
3178
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
3179 3180 3181
    return ldub_phys(addr);
}

A
Anthony Liguori 已提交
3182
static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
3183
{
3184
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
3185 3186 3187
    return lduw_phys(addr);
}

A
Anthony Liguori 已提交
3188
static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
3189
{
3190
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
3191 3192 3193
    return ldl_phys(addr);
}

A
Anthony Liguori 已提交
3194
static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
3195 3196
                             uint32_t val)
{
3197
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
3198 3199 3200
    stb_phys(addr, val);
}

A
Anthony Liguori 已提交
3201
static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
3202 3203
                             uint32_t val)
{
3204
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
3205 3206 3207
    stw_phys(addr, val);
}

A
Anthony Liguori 已提交
3208
static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
3209 3210
                             uint32_t val)
{
3211
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
3212 3213 3214
    stl_phys(addr, val);
}

3215
static CPUReadMemoryFunc * const watch_mem_read[3] = {
3216 3217 3218 3219 3220
    watch_mem_readb,
    watch_mem_readw,
    watch_mem_readl,
};

3221
static CPUWriteMemoryFunc * const watch_mem_write[3] = {
3222 3223 3224 3225 3226
    watch_mem_writeb,
    watch_mem_writew,
    watch_mem_writel,
};

R
Richard Henderson 已提交
3227 3228 3229
static inline uint32_t subpage_readlen (subpage_t *mmio,
                                        target_phys_addr_t addr,
                                        unsigned int len)
3230
{
R
Richard Henderson 已提交
3231
    unsigned int idx = SUBPAGE_IDX(addr);
3232 3233 3234 3235 3236
#if defined(DEBUG_SUBPAGE)
    printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
           mmio, len, addr, idx);
#endif

R
Richard Henderson 已提交
3237 3238 3239
    addr += mmio->region_offset[idx];
    idx = mmio->sub_io_index[idx];
    return io_mem_read[idx][len](io_mem_opaque[idx], addr);
3240 3241
}

A
Anthony Liguori 已提交
3242
static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
R
Richard Henderson 已提交
3243
                                     uint32_t value, unsigned int len)
3244
{
R
Richard Henderson 已提交
3245
    unsigned int idx = SUBPAGE_IDX(addr);
3246
#if defined(DEBUG_SUBPAGE)
R
Richard Henderson 已提交
3247 3248
    printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n",
           __func__, mmio, len, addr, idx, value);
3249
#endif
R
Richard Henderson 已提交
3250 3251 3252 3253

    addr += mmio->region_offset[idx];
    idx = mmio->sub_io_index[idx];
    io_mem_write[idx][len](io_mem_opaque[idx], addr, value);
3254 3255
}

A
Anthony Liguori 已提交
3256
static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
3257 3258 3259 3260
{
    return subpage_readlen(opaque, addr, 0);
}

A
Anthony Liguori 已提交
3261
static void subpage_writeb (void *opaque, target_phys_addr_t addr,
3262 3263 3264 3265 3266
                            uint32_t value)
{
    subpage_writelen(opaque, addr, value, 0);
}

A
Anthony Liguori 已提交
3267
static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
3268 3269 3270 3271
{
    return subpage_readlen(opaque, addr, 1);
}

A
Anthony Liguori 已提交
3272
static void subpage_writew (void *opaque, target_phys_addr_t addr,
3273 3274 3275 3276 3277
                            uint32_t value)
{
    subpage_writelen(opaque, addr, value, 1);
}

A
Anthony Liguori 已提交
3278
static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
3279 3280 3281 3282
{
    return subpage_readlen(opaque, addr, 2);
}

R
Richard Henderson 已提交
3283 3284
static void subpage_writel (void *opaque, target_phys_addr_t addr,
                            uint32_t value)
3285 3286 3287 3288
{
    subpage_writelen(opaque, addr, value, 2);
}

3289
static CPUReadMemoryFunc * const subpage_read[] = {
3290 3291 3292 3293 3294
    &subpage_readb,
    &subpage_readw,
    &subpage_readl,
};

3295
static CPUWriteMemoryFunc * const subpage_write[] = {
3296 3297 3298 3299 3300
    &subpage_writeb,
    &subpage_writew,
    &subpage_writel,
};

A
Anthony Liguori 已提交
3301 3302
static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
                             ram_addr_t memory, ram_addr_t region_offset)
3303 3304 3305 3306 3307 3308 3309 3310
{
    int idx, eidx;

    if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
        return -1;
    idx = SUBPAGE_IDX(start);
    eidx = SUBPAGE_IDX(end);
#if defined(DEBUG_SUBPAGE)
3311
    printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
3312 3313
           mmio, start, end, idx, eidx, memory);
#endif
3314 3315
    if ((memory & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
        memory = IO_MEM_UNASSIGNED;
R
Richard Henderson 已提交
3316
    memory = (memory >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3317
    for (; idx <= eidx; idx++) {
R
Richard Henderson 已提交
3318 3319
        mmio->sub_io_index[idx] = memory;
        mmio->region_offset[idx] = region_offset;
3320 3321 3322 3323 3324
    }

    return 0;
}

R
Richard Henderson 已提交
3325 3326 3327
static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
                                ram_addr_t orig_memory,
                                ram_addr_t region_offset)
3328
{
A
Anthony Liguori 已提交
3329
    subpage_t *mmio;
3330 3331
    int subpage_memory;

A
Anthony Liguori 已提交
3332
    mmio = qemu_mallocz(sizeof(subpage_t));
3333 3334

    mmio->base = base;
3335
    subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio);
3336
#if defined(DEBUG_SUBPAGE)
3337 3338
    printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
           mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3339
#endif
3340
    *phys = subpage_memory | IO_MEM_SUBPAGE;
R
Richard Henderson 已提交
3341
    subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
3342 3343 3344 3345

    return mmio;
}

3346 3347 3348 3349 3350 3351 3352 3353 3354
static int get_free_io_mem_idx(void)
{
    int i;

    for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
        if (!io_mem_used[i]) {
            io_mem_used[i] = 1;
            return i;
        }
3355
    fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
3356 3357 3358
    return -1;
}

3359 3360
/* mem_read and mem_write are arrays of functions containing the
   function to access byte (index 0), word (index 1) and dword (index
3361
   2). Functions can be omitted with a NULL function pointer.
3362
   If io_index is non zero, the corresponding io zone is
3363 3364 3365
   modified. If it is zero, a new io zone is allocated. The return
   value can be used with cpu_register_physical_memory(). (-1) is
   returned if error. */
3366
static int cpu_register_io_memory_fixed(int io_index,
3367 3368
                                        CPUReadMemoryFunc * const *mem_read,
                                        CPUWriteMemoryFunc * const *mem_write,
3369
                                        void *opaque)
3370
{
3371 3372
    int i;

3373
    if (io_index <= 0) {
3374 3375 3376
        io_index = get_free_io_mem_idx();
        if (io_index == -1)
            return io_index;
3377
    } else {
3378
        io_index >>= IO_MEM_SHIFT;
3379 3380 3381
        if (io_index >= IO_MEM_NB_ENTRIES)
            return -1;
    }
B
bellard 已提交
3382

3383 3384 3385 3386 3387 3388 3389 3390
    for (i = 0; i < 3; ++i) {
        io_mem_read[io_index][i]
            = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
    }
    for (i = 0; i < 3; ++i) {
        io_mem_write[io_index][i]
            = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
    }
B
bellard 已提交
3391
    io_mem_opaque[io_index] = opaque;
R
Richard Henderson 已提交
3392 3393

    return (io_index << IO_MEM_SHIFT);
3394
}
B
bellard 已提交
3395

3396 3397
int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
                           CPUWriteMemoryFunc * const *mem_write,
3398 3399 3400 3401 3402
                           void *opaque)
{
    return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque);
}

3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415
void cpu_unregister_io_memory(int io_table_address)
{
    int i;
    int io_index = io_table_address >> IO_MEM_SHIFT;

    for (i=0;i < 3; i++) {
        io_mem_read[io_index][i] = unassigned_mem_read[i];
        io_mem_write[io_index][i] = unassigned_mem_write[i];
    }
    io_mem_opaque[io_index] = NULL;
    io_mem_used[io_index] = 0;
}

A
Avi Kivity 已提交
3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429
static void io_mem_init(void)
{
    int i;

    cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read, unassigned_mem_write, NULL);
    cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read, unassigned_mem_write, NULL);
    cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read, notdirty_mem_write, NULL);
    for (i=0; i<5; i++)
        io_mem_used[i] = 1;

    io_mem_watch = cpu_register_io_memory(watch_mem_read,
                                          watch_mem_write, NULL);
}

3430 3431
#endif /* !defined(CONFIG_USER_ONLY) */

B
bellard 已提交
3432 3433
/* physical memory access (slow version, mainly for debug) */
#if defined(CONFIG_USER_ONLY)
P
Paul Brook 已提交
3434 3435
int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
                        uint8_t *buf, int len, int is_write)
B
bellard 已提交
3436 3437 3438
{
    int l, flags;
    target_ulong page;
3439
    void * p;
B
bellard 已提交
3440 3441 3442 3443 3444 3445 3446 3447

    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
        flags = page_get_flags(page);
        if (!(flags & PAGE_VALID))
P
Paul Brook 已提交
3448
            return -1;
B
bellard 已提交
3449 3450
        if (is_write) {
            if (!(flags & PAGE_WRITE))
P
Paul Brook 已提交
3451
                return -1;
3452
            /* XXX: this code should not depend on lock_user */
A
aurel32 已提交
3453
            if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
P
Paul Brook 已提交
3454
                return -1;
A
aurel32 已提交
3455 3456
            memcpy(p, buf, l);
            unlock_user(p, addr, l);
B
bellard 已提交
3457 3458
        } else {
            if (!(flags & PAGE_READ))
P
Paul Brook 已提交
3459
                return -1;
3460
            /* XXX: this code should not depend on lock_user */
A
aurel32 已提交
3461
            if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
P
Paul Brook 已提交
3462
                return -1;
A
aurel32 已提交
3463
            memcpy(buf, p, l);
A
aurel32 已提交
3464
            unlock_user(p, addr, 0);
B
bellard 已提交
3465 3466 3467 3468 3469
        }
        len -= l;
        buf += l;
        addr += l;
    }
P
Paul Brook 已提交
3470
    return 0;
B
bellard 已提交
3471
}
B
bellard 已提交
3472

B
bellard 已提交
3473
#else
A
Anthony Liguori 已提交
3474
void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
B
bellard 已提交
3475 3476 3477 3478 3479
                            int len, int is_write)
{
    int l, io_index;
    uint8_t *ptr;
    uint32_t val;
A
Anthony Liguori 已提交
3480
    target_phys_addr_t page;
3481
    unsigned long pd;
B
bellard 已提交
3482
    PhysPageDesc *p;
3483

B
bellard 已提交
3484 3485 3486 3487 3488
    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
B
bellard 已提交
3489
        p = phys_page_find(page >> TARGET_PAGE_BITS);
B
bellard 已提交
3490 3491 3492 3493 3494
        if (!p) {
            pd = IO_MEM_UNASSIGNED;
        } else {
            pd = p->phys_offset;
        }
3495

B
bellard 已提交
3496
        if (is_write) {
3497
            if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
A
Anthony Liguori 已提交
3498
                target_phys_addr_t addr1 = addr;
B
bellard 已提交
3499
                io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3500
                if (p)
3501
                    addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
3502 3503
                /* XXX: could force cpu_single_env to NULL to avoid
                   potential bugs */
3504
                if (l >= 4 && ((addr1 & 3) == 0)) {
B
bellard 已提交
3505
                    /* 32 bit write access */
B
bellard 已提交
3506
                    val = ldl_p(buf);
3507
                    io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
B
bellard 已提交
3508
                    l = 4;
3509
                } else if (l >= 2 && ((addr1 & 1) == 0)) {
B
bellard 已提交
3510
                    /* 16 bit write access */
B
bellard 已提交
3511
                    val = lduw_p(buf);
3512
                    io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
B
bellard 已提交
3513 3514
                    l = 2;
                } else {
B
bellard 已提交
3515
                    /* 8 bit write access */
B
bellard 已提交
3516
                    val = ldub_p(buf);
3517
                    io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
B
bellard 已提交
3518 3519 3520
                    l = 1;
                }
            } else {
3521 3522
                unsigned long addr1;
                addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
B
bellard 已提交
3523
                /* RAM case */
P
pbrook 已提交
3524
                ptr = qemu_get_ram_ptr(addr1);
B
bellard 已提交
3525
                memcpy(ptr, buf, l);
3526 3527 3528 3529
                if (!cpu_physical_memory_is_dirty(addr1)) {
                    /* invalidate code */
                    tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
                    /* set dirty bit */
3530 3531
                    cpu_physical_memory_set_dirty_flags(
                        addr1, (0xff & ~CODE_DIRTY_FLAG));
3532
                }
B
bellard 已提交
3533 3534
            }
        } else {
3535
            if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3536
                !(pd & IO_MEM_ROMD)) {
A
Anthony Liguori 已提交
3537
                target_phys_addr_t addr1 = addr;
B
bellard 已提交
3538 3539
                /* I/O case */
                io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3540
                if (p)
3541 3542
                    addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
                if (l >= 4 && ((addr1 & 3) == 0)) {
B
bellard 已提交
3543
                    /* 32 bit read access */
3544
                    val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
B
bellard 已提交
3545
                    stl_p(buf, val);
B
bellard 已提交
3546
                    l = 4;
3547
                } else if (l >= 2 && ((addr1 & 1) == 0)) {
B
bellard 已提交
3548
                    /* 16 bit read access */
3549
                    val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
B
bellard 已提交
3550
                    stw_p(buf, val);
B
bellard 已提交
3551 3552
                    l = 2;
                } else {
B
bellard 已提交
3553
                    /* 8 bit read access */
3554
                    val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
B
bellard 已提交
3555
                    stb_p(buf, val);
B
bellard 已提交
3556 3557 3558 3559
                    l = 1;
                }
            } else {
                /* RAM case */
P
pbrook 已提交
3560
                ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
B
bellard 已提交
3561 3562 3563 3564 3565 3566 3567 3568 3569
                    (addr & ~TARGET_PAGE_MASK);
                memcpy(buf, ptr, l);
            }
        }
        len -= l;
        buf += l;
        addr += l;
    }
}
B
bellard 已提交
3570

B
bellard 已提交
3571
/* used for ROM loading : can write in RAM and ROM */
A
Anthony Liguori 已提交
3572
void cpu_physical_memory_write_rom(target_phys_addr_t addr,
B
bellard 已提交
3573 3574 3575 3576
                                   const uint8_t *buf, int len)
{
    int l;
    uint8_t *ptr;
A
Anthony Liguori 已提交
3577
    target_phys_addr_t page;
B
bellard 已提交
3578 3579
    unsigned long pd;
    PhysPageDesc *p;
3580

B
bellard 已提交
3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591
    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
        p = phys_page_find(page >> TARGET_PAGE_BITS);
        if (!p) {
            pd = IO_MEM_UNASSIGNED;
        } else {
            pd = p->phys_offset;
        }
3592

B
bellard 已提交
3593
        if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3594 3595
            (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
            !(pd & IO_MEM_ROMD)) {
B
bellard 已提交
3596 3597 3598 3599 3600
            /* do nothing */
        } else {
            unsigned long addr1;
            addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
            /* ROM/RAM case */
P
pbrook 已提交
3601
            ptr = qemu_get_ram_ptr(addr1);
B
bellard 已提交
3602 3603 3604 3605 3606 3607 3608 3609
            memcpy(ptr, buf, l);
        }
        len -= l;
        buf += l;
        addr += l;
    }
}

3610 3611
typedef struct {
    void *buffer;
A
Anthony Liguori 已提交
3612 3613
    target_phys_addr_t addr;
    target_phys_addr_t len;
3614 3615 3616 3617
} BounceBuffer;

static BounceBuffer bounce;

3618 3619 3620
typedef struct MapClient {
    void *opaque;
    void (*callback)(void *opaque);
B
Blue Swirl 已提交
3621
    QLIST_ENTRY(MapClient) link;
3622 3623
} MapClient;

B
Blue Swirl 已提交
3624 3625
static QLIST_HEAD(map_client_list, MapClient) map_client_list
    = QLIST_HEAD_INITIALIZER(map_client_list);
3626 3627 3628 3629 3630 3631 3632

void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
{
    MapClient *client = qemu_malloc(sizeof(*client));

    client->opaque = opaque;
    client->callback = callback;
B
Blue Swirl 已提交
3633
    QLIST_INSERT_HEAD(&map_client_list, client, link);
3634 3635 3636 3637 3638 3639 3640
    return client;
}

void cpu_unregister_map_client(void *_client)
{
    MapClient *client = (MapClient *)_client;

B
Blue Swirl 已提交
3641
    QLIST_REMOVE(client, link);
3642
    qemu_free(client);
3643 3644 3645 3646 3647 3648
}

static void cpu_notify_map_clients(void)
{
    MapClient *client;

B
Blue Swirl 已提交
3649 3650
    while (!QLIST_EMPTY(&map_client_list)) {
        client = QLIST_FIRST(&map_client_list);
3651
        client->callback(client->opaque);
3652
        cpu_unregister_map_client(client);
3653 3654 3655
    }
}

3656 3657 3658 3659
/* Map a physical memory region into a host virtual address.
 * May map a subset of the requested range, given by and returned in *plen.
 * May return NULL if resources needed to perform the mapping are exhausted.
 * Use only for reads OR writes - not for read-modify-write operations.
3660 3661
 * Use cpu_register_map_client() to know when retrying the map operation is
 * likely to succeed.
3662
 */
A
Anthony Liguori 已提交
3663 3664
void *cpu_physical_memory_map(target_phys_addr_t addr,
                              target_phys_addr_t *plen,
3665 3666
                              int is_write)
{
A
Anthony Liguori 已提交
3667 3668
    target_phys_addr_t len = *plen;
    target_phys_addr_t done = 0;
3669 3670 3671
    int l;
    uint8_t *ret = NULL;
    uint8_t *ptr;
A
Anthony Liguori 已提交
3672
    target_phys_addr_t page;
3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701
    unsigned long pd;
    PhysPageDesc *p;
    unsigned long addr1;

    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
        p = phys_page_find(page >> TARGET_PAGE_BITS);
        if (!p) {
            pd = IO_MEM_UNASSIGNED;
        } else {
            pd = p->phys_offset;
        }

        if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
            if (done || bounce.buffer) {
                break;
            }
            bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
            bounce.addr = addr;
            bounce.len = l;
            if (!is_write) {
                cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
            }
            ptr = bounce.buffer;
        } else {
            addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
P
pbrook 已提交
3702
            ptr = qemu_get_ram_ptr(addr1);
3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721
        }
        if (!done) {
            ret = ptr;
        } else if (ret + done != ptr) {
            break;
        }

        len -= l;
        addr += l;
        done += l;
    }
    *plen = done;
    return ret;
}

/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
 * Will also mark the memory as dirty if is_write == 1.  access_len gives
 * the amount of memory that was actually read or written by the caller.
 */
A
Anthony Liguori 已提交
3722 3723
void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
                               int is_write, target_phys_addr_t access_len)
3724 3725 3726
{
    if (buffer != bounce.buffer) {
        if (is_write) {
M
Marcelo Tosatti 已提交
3727
            ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
3728 3729 3730 3731 3732 3733 3734 3735 3736
            while (access_len) {
                unsigned l;
                l = TARGET_PAGE_SIZE;
                if (l > access_len)
                    l = access_len;
                if (!cpu_physical_memory_is_dirty(addr1)) {
                    /* invalidate code */
                    tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
                    /* set dirty bit */
3737 3738
                    cpu_physical_memory_set_dirty_flags(
                        addr1, (0xff & ~CODE_DIRTY_FLAG));
3739 3740 3741 3742 3743 3744 3745 3746 3747 3748
                }
                addr1 += l;
                access_len -= l;
            }
        }
        return;
    }
    if (is_write) {
        cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
    }
3749
    qemu_vfree(bounce.buffer);
3750
    bounce.buffer = NULL;
3751
    cpu_notify_map_clients();
3752
}
B
bellard 已提交
3753

B
bellard 已提交
3754
/* warning: addr must be aligned */
A
Anthony Liguori 已提交
3755
uint32_t ldl_phys(target_phys_addr_t addr)
B
bellard 已提交
3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768
{
    int io_index;
    uint8_t *ptr;
    uint32_t val;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3769

3770
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3771
        !(pd & IO_MEM_ROMD)) {
B
bellard 已提交
3772 3773
        /* I/O case */
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3774 3775
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
3776 3777 3778
        val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
    } else {
        /* RAM case */
P
pbrook 已提交
3779
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
B
bellard 已提交
3780 3781 3782 3783 3784 3785
            (addr & ~TARGET_PAGE_MASK);
        val = ldl_p(ptr);
    }
    return val;
}

B
bellard 已提交
3786
/* warning: addr must be aligned */
A
Anthony Liguori 已提交
3787
uint64_t ldq_phys(target_phys_addr_t addr)
B
bellard 已提交
3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800
{
    int io_index;
    uint8_t *ptr;
    uint64_t val;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3801

3802 3803
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
        !(pd & IO_MEM_ROMD)) {
B
bellard 已提交
3804 3805
        /* I/O case */
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3806 3807
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
3808 3809 3810 3811 3812 3813 3814 3815 3816
#ifdef TARGET_WORDS_BIGENDIAN
        val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
        val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
#else
        val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
        val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
#endif
    } else {
        /* RAM case */
P
pbrook 已提交
3817
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
B
bellard 已提交
3818 3819 3820 3821 3822 3823
            (addr & ~TARGET_PAGE_MASK);
        val = ldq_p(ptr);
    }
    return val;
}

B
bellard 已提交
3824
/* XXX: optimize */
A
Anthony Liguori 已提交
3825
uint32_t ldub_phys(target_phys_addr_t addr)
B
bellard 已提交
3826 3827 3828 3829 3830 3831
{
    uint8_t val;
    cpu_physical_memory_read(addr, &val, 1);
    return val;
}

3832
/* warning: addr must be aligned */
A
Anthony Liguori 已提交
3833
uint32_t lduw_phys(target_phys_addr_t addr)
B
bellard 已提交
3834
{
3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861
    int io_index;
    uint8_t *ptr;
    uint64_t val;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }

    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
        !(pd & IO_MEM_ROMD)) {
        /* I/O case */
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
        val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
    } else {
        /* RAM case */
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
            (addr & ~TARGET_PAGE_MASK);
        val = lduw_p(ptr);
    }
    return val;
B
bellard 已提交
3862 3863
}

B
bellard 已提交
3864 3865 3866
/* warning: addr must be aligned. The ram page is not masked as dirty
   and the code inside is not invalidated. It is useful if the dirty
   bits are used to track modified PTEs */
A
Anthony Liguori 已提交
3867
void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
B
bellard 已提交
3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879
{
    int io_index;
    uint8_t *ptr;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3880

3881
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
B
bellard 已提交
3882
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3883 3884
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
3885 3886
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
    } else {
A
aliguori 已提交
3887
        unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
P
pbrook 已提交
3888
        ptr = qemu_get_ram_ptr(addr1);
B
bellard 已提交
3889
        stl_p(ptr, val);
A
aliguori 已提交
3890 3891 3892 3893 3894 3895

        if (unlikely(in_migration)) {
            if (!cpu_physical_memory_is_dirty(addr1)) {
                /* invalidate code */
                tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
                /* set dirty bit */
3896 3897
                cpu_physical_memory_set_dirty_flags(
                    addr1, (0xff & ~CODE_DIRTY_FLAG));
A
aliguori 已提交
3898 3899
            }
        }
B
bellard 已提交
3900 3901 3902
    }
}

A
Anthony Liguori 已提交
3903
void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
J
j_mayer 已提交
3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915
{
    int io_index;
    uint8_t *ptr;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3916

J
j_mayer 已提交
3917 3918
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3919 3920
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
J
j_mayer 已提交
3921 3922 3923 3924 3925 3926 3927 3928
#ifdef TARGET_WORDS_BIGENDIAN
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
#else
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
#endif
    } else {
P
pbrook 已提交
3929
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
J
j_mayer 已提交
3930 3931 3932 3933 3934
            (addr & ~TARGET_PAGE_MASK);
        stq_p(ptr, val);
    }
}

B
bellard 已提交
3935
/* warning: addr must be aligned */
A
Anthony Liguori 已提交
3936
void stl_phys(target_phys_addr_t addr, uint32_t val)
B
bellard 已提交
3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948
{
    int io_index;
    uint8_t *ptr;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3949

3950
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
B
bellard 已提交
3951
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3952 3953
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
3954 3955 3956 3957 3958
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
    } else {
        unsigned long addr1;
        addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
        /* RAM case */
P
pbrook 已提交
3959
        ptr = qemu_get_ram_ptr(addr1);
B
bellard 已提交
3960
        stl_p(ptr, val);
3961 3962 3963 3964
        if (!cpu_physical_memory_is_dirty(addr1)) {
            /* invalidate code */
            tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
            /* set dirty bit */
3965 3966
            cpu_physical_memory_set_dirty_flags(addr1,
                (0xff & ~CODE_DIRTY_FLAG));
3967
        }
B
bellard 已提交
3968 3969 3970
    }
}

B
bellard 已提交
3971
/* XXX: optimize */
A
Anthony Liguori 已提交
3972
void stb_phys(target_phys_addr_t addr, uint32_t val)
B
bellard 已提交
3973 3974 3975 3976 3977
{
    uint8_t v = val;
    cpu_physical_memory_write(addr, &v, 1);
}

3978
/* warning: addr must be aligned */
A
Anthony Liguori 已提交
3979
void stw_phys(target_phys_addr_t addr, uint32_t val)
B
bellard 已提交
3980
{
3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011
    int io_index;
    uint8_t *ptr;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }

    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
        io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
    } else {
        unsigned long addr1;
        addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
        /* RAM case */
        ptr = qemu_get_ram_ptr(addr1);
        stw_p(ptr, val);
        if (!cpu_physical_memory_is_dirty(addr1)) {
            /* invalidate code */
            tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
            /* set dirty bit */
            cpu_physical_memory_set_dirty_flags(addr1,
                (0xff & ~CODE_DIRTY_FLAG));
        }
    }
B
bellard 已提交
4012 4013 4014
}

/* XXX: optimize */
A
Anthony Liguori 已提交
4015
void stq_phys(target_phys_addr_t addr, uint64_t val)
B
bellard 已提交
4016 4017 4018 4019 4020
{
    val = tswap64(val);
    cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
}

4021
/* virtual memory access for debug (includes writing to ROM) */
4022
int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
4023
                        uint8_t *buf, int len, int is_write)
B
bellard 已提交
4024 4025
{
    int l;
A
Anthony Liguori 已提交
4026
    target_phys_addr_t phys_addr;
4027
    target_ulong page;
B
bellard 已提交
4028 4029 4030 4031 4032 4033 4034 4035 4036 4037

    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        phys_addr = cpu_get_phys_page_debug(env, page);
        /* if no physical page mapped, return an error */
        if (phys_addr == -1)
            return -1;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
4038 4039 4040 4041 4042
        phys_addr += (addr & ~TARGET_PAGE_MASK);
        if (is_write)
            cpu_physical_memory_write_rom(phys_addr, buf, l);
        else
            cpu_physical_memory_rw(phys_addr, buf, l, is_write);
B
bellard 已提交
4043 4044 4045 4046 4047 4048
        len -= l;
        buf += l;
        addr += l;
    }
    return 0;
}
P
Paul Brook 已提交
4049
#endif
B
bellard 已提交
4050

P
pbrook 已提交
4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067
/* in deterministic execution mode, instructions doing device I/Os
   must be at the end of the TB */
void cpu_io_recompile(CPUState *env, void *retaddr)
{
    TranslationBlock *tb;
    uint32_t n, cflags;
    target_ulong pc, cs_base;
    uint64_t flags;

    tb = tb_find_pc((unsigned long)retaddr);
    if (!tb) {
        cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p", 
                  retaddr);
    }
    n = env->icount_decr.u16.low + tb->icount;
    cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
    /* Calculate how many instructions had been executed before the fault
T
ths 已提交
4068
       occurred.  */
P
pbrook 已提交
4069 4070 4071 4072 4073
    n = n - env->icount_decr.u16.low;
    /* Generate a new TB ending on the I/O insn.  */
    n++;
    /* On MIPS and SH, delay slot instructions can only be restarted if
       they were already the first instruction in the TB.  If this is not
T
ths 已提交
4074
       the first instruction in a TB then re-execute the preceding
P
pbrook 已提交
4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101
       branch.  */
#if defined(TARGET_MIPS)
    if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
        env->active_tc.PC -= 4;
        env->icount_decr.u16.low++;
        env->hflags &= ~MIPS_HFLAG_BMASK;
    }
#elif defined(TARGET_SH4)
    if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
            && n > 1) {
        env->pc -= 2;
        env->icount_decr.u16.low++;
        env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
    }
#endif
    /* This should never happen.  */
    if (n > CF_COUNT_MASK)
        cpu_abort(env, "TB too big during recompile");

    cflags = n | CF_LAST_IO;
    pc = tb->pc;
    cs_base = tb->cs_base;
    flags = tb->flags;
    tb_phys_invalidate(tb, -1);
    /* FIXME: In theory this could raise an exception.  In practice
       we have already translated the block once so it's probably ok.  */
    tb_gen_code(env, pc, cs_base, flags, cflags);
T
ths 已提交
4102
    /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
P
pbrook 已提交
4103 4104 4105 4106 4107 4108 4109
       the first in the TB) then we end up generating a whole new TB and
       repeating the fault, which is horribly inefficient.
       Better would be to execute just this insn uncached, or generate a
       second new TB.  */
    cpu_resume_from_signal(env, NULL);
}

4110 4111
#if !defined(CONFIG_USER_ONLY)

4112
void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
B
bellard 已提交
4113 4114 4115 4116
{
    int i, target_code_size, max_target_code_size;
    int direct_jmp_count, direct_jmp2_count, cross_page;
    TranslationBlock *tb;
4117

B
bellard 已提交
4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137
    target_code_size = 0;
    max_target_code_size = 0;
    cross_page = 0;
    direct_jmp_count = 0;
    direct_jmp2_count = 0;
    for(i = 0; i < nb_tbs; i++) {
        tb = &tbs[i];
        target_code_size += tb->size;
        if (tb->size > max_target_code_size)
            max_target_code_size = tb->size;
        if (tb->page_addr[1] != -1)
            cross_page++;
        if (tb->tb_next_offset[0] != 0xffff) {
            direct_jmp_count++;
            if (tb->tb_next_offset[1] != 0xffff) {
                direct_jmp2_count++;
            }
        }
    }
    /* XXX: avoid using doubles ? */
B
bellard 已提交
4138
    cpu_fprintf(f, "Translation buffer state:\n");
4139
    cpu_fprintf(f, "gen code size       %td/%ld\n",
4140 4141 4142
                code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
    cpu_fprintf(f, "TB count            %d/%d\n", 
                nb_tbs, code_gen_max_blocks);
4143
    cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
B
bellard 已提交
4144 4145
                nb_tbs ? target_code_size / nb_tbs : 0,
                max_target_code_size);
4146
    cpu_fprintf(f, "TB avg host size    %td bytes (expansion ratio: %0.1f)\n",
B
bellard 已提交
4147 4148
                nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
                target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
4149 4150
    cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
            cross_page,
B
bellard 已提交
4151 4152
            nb_tbs ? (cross_page * 100) / nb_tbs : 0);
    cpu_fprintf(f, "direct jump count   %d (%d%%) (2 jumps=%d %d%%)\n",
4153
                direct_jmp_count,
B
bellard 已提交
4154 4155 4156
                nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
                direct_jmp2_count,
                nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
B
bellard 已提交
4157
    cpu_fprintf(f, "\nStatistics:\n");
B
bellard 已提交
4158 4159 4160
    cpu_fprintf(f, "TB flush count      %d\n", tb_flush_count);
    cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
    cpu_fprintf(f, "TLB flush count     %d\n", tlb_flush_count);
B
bellard 已提交
4161
    tcg_dump_info(f, cpu_fprintf);
B
bellard 已提交
4162 4163
}

B
bellard 已提交
4164 4165 4166
#define MMUSUFFIX _cmmu
#define GETPC() NULL
#define env cpu_single_env
B
bellard 已提交
4167
#define SOFTMMU_CODE_ACCESS
B
bellard 已提交
4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183

#define SHIFT 0
#include "softmmu_template.h"

#define SHIFT 1
#include "softmmu_template.h"

#define SHIFT 2
#include "softmmu_template.h"

#define SHIFT 3
#include "softmmu_template.h"

#undef env

#endif