exec.c 122.1 KB
Newer Older
B
bellard 已提交
1
/*
B
bellard 已提交
2
 *  virtual page mapping and translated block handling
3
 *
B
bellard 已提交
4 5 6 7 8 9 10 11 12 13 14 15 16
 *  Copyright (c) 2003 Fabrice Bellard
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
17
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
B
bellard 已提交
18
 */
B
bellard 已提交
19
#include "config.h"
B
bellard 已提交
20 21 22
#ifdef _WIN32
#include <windows.h>
#else
B
bellard 已提交
23
#include <sys/types.h>
B
bellard 已提交
24 25
#include <sys/mman.h>
#endif
B
bellard 已提交
26 27 28 29 30 31 32 33
#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <inttypes.h>

B
bellard 已提交
34 35
#include "cpu.h"
#include "exec-all.h"
36
#include "qemu-common.h"
B
bellard 已提交
37
#include "tcg.h"
38
#include "hw/hw.h"
39
#include "hw/qdev.h"
A
aliguori 已提交
40
#include "osdep.h"
A
aliguori 已提交
41
#include "kvm.h"
B
Blue Swirl 已提交
42
#include "qemu-timer.h"
43 44
#if defined(CONFIG_USER_ONLY)
#include <qemu.h>
45
#include <signal.h>
46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
#include <sys/param.h>
#if __FreeBSD_version >= 700104
#define HAVE_KINFO_GETVMMAP
#define sigqueue sigqueue_freebsd  /* avoid redefinition */
#include <sys/time.h>
#include <sys/proc.h>
#include <machine/profile.h>
#define _KERNEL
#include <sys/user.h>
#undef _KERNEL
#undef sigqueue
#include <libutil.h>
#endif
#endif
61
#endif
B
bellard 已提交
62

B
bellard 已提交
63
//#define DEBUG_TB_INVALIDATE
B
bellard 已提交
64
//#define DEBUG_FLUSH
65
//#define DEBUG_TLB
P
pbrook 已提交
66
//#define DEBUG_UNASSIGNED
B
bellard 已提交
67 68

/* make various TB consistency checks */
69 70
//#define DEBUG_TB_CHECK
//#define DEBUG_TLB_CHECK
B
bellard 已提交
71

T
ths 已提交
72
//#define DEBUG_IOPORT
73
//#define DEBUG_SUBPAGE
T
ths 已提交
74

75 76 77 78 79
#if !defined(CONFIG_USER_ONLY)
/* TB consistency checks only implemented for usermode emulation.  */
#undef DEBUG_TB_CHECK
#endif

80 81
#define SMC_BITMAP_USE_THRESHOLD 10

B
blueswir1 已提交
82
static TranslationBlock *tbs;
83
static int code_gen_max_blocks;
84
TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
B
blueswir1 已提交
85
static int nb_tbs;
B
bellard 已提交
86
/* any access to the tbs or the page table must use this lock */
A
Anthony Liguori 已提交
87
spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
B
bellard 已提交
88

B
blueswir1 已提交
89 90 91
#if defined(__arm__) || defined(__sparc_v9__)
/* The prologue must be reachable with a direct jump. ARM and Sparc64
 have limited branch ranges (possibly also PPC) so place it in a
92 93 94 95
 section close to code segment. */
#define code_gen_section                                \
    __attribute__((__section__(".gen_code")))           \
    __attribute__((aligned (32)))
96 97 98 99
#elif defined(_WIN32)
/* Maximum alignment for Win32 is 16. */
#define code_gen_section                                \
    __attribute__((aligned (16)))
100 101 102 103 104 105
#else
#define code_gen_section                                \
    __attribute__((aligned (32)))
#endif

uint8_t code_gen_prologue[1024] code_gen_section;
B
blueswir1 已提交
106 107
static uint8_t *code_gen_buffer;
static unsigned long code_gen_buffer_size;
108
/* threshold to flush the translated code buffer */
B
blueswir1 已提交
109
static unsigned long code_gen_buffer_max_size;
110
static uint8_t *code_gen_ptr;
B
bellard 已提交
111

112
#if !defined(CONFIG_USER_ONLY)
113
int phys_ram_fd;
A
aliguori 已提交
114
static int in_migration;
P
pbrook 已提交
115

A
Alex Williamson 已提交
116
RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list) };
117
#endif
118

B
bellard 已提交
119 120 121
CPUState *first_cpu;
/* current CPU in the current thread. It is only valid inside
   cpu_exec() */
122
CPUState *cpu_single_env;
P
pbrook 已提交
123
/* 0 = Do not count executed instructions.
T
ths 已提交
124
   1 = Precise instruction counting.
P
pbrook 已提交
125 126 127 128 129
   2 = Adaptive rate instruction counting.  */
int use_icount = 0;
/* Current instruction counter.  While executing translated code this may
   include some instructions that have not yet been executed.  */
int64_t qemu_icount;
B
bellard 已提交
130

B
bellard 已提交
131
typedef struct PageDesc {
B
bellard 已提交
132
    /* list of TBs intersecting this ram page */
B
bellard 已提交
133
    TranslationBlock *first_tb;
134 135 136 137 138 139 140
    /* in order to optimize self modifying code, we count the number
       of lookups we do to a given page to use a bitmap */
    unsigned int code_write_count;
    uint8_t *code_bitmap;
#if defined(CONFIG_USER_ONLY)
    unsigned long flags;
#endif
B
bellard 已提交
141 142
} PageDesc;

P
Paul Brook 已提交
143
/* In system mode we want L1_MAP to be based on ram offsets,
144 145
   while in user mode we want it to be based on virtual addresses.  */
#if !defined(CONFIG_USER_ONLY)
P
Paul Brook 已提交
146 147 148
#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
# define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
#else
149
# define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
P
Paul Brook 已提交
150
#endif
151
#else
152
# define L1_MAP_ADDR_SPACE_BITS  TARGET_VIRT_ADDR_SPACE_BITS
153
#endif
B
bellard 已提交
154

155 156
/* Size of the L2 (and L3, etc) page tables.  */
#define L2_BITS 10
B
bellard 已提交
157 158
#define L2_SIZE (1 << L2_BITS)

159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183
/* The bits remaining after N lower levels of page tables.  */
#define P_L1_BITS_REM \
    ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
#define V_L1_BITS_REM \
    ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)

/* Size of the L1 page table.  Avoid silly small sizes.  */
#if P_L1_BITS_REM < 4
#define P_L1_BITS  (P_L1_BITS_REM + L2_BITS)
#else
#define P_L1_BITS  P_L1_BITS_REM
#endif

#if V_L1_BITS_REM < 4
#define V_L1_BITS  (V_L1_BITS_REM + L2_BITS)
#else
#define V_L1_BITS  V_L1_BITS_REM
#endif

#define P_L1_SIZE  ((target_phys_addr_t)1 << P_L1_BITS)
#define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)

#define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)

184 185 186 187
unsigned long qemu_real_host_page_size;
unsigned long qemu_host_page_bits;
unsigned long qemu_host_page_size;
unsigned long qemu_host_page_mask;
B
bellard 已提交
188

189 190 191
/* This is a multi-level map on the virtual address space.
   The bottom level has pointers to PageDesc.  */
static void *l1_map[V_L1_SIZE];
B
bellard 已提交
192

193
#if !defined(CONFIG_USER_ONLY)
P
Paul Brook 已提交
194 195 196 197 198 199
typedef struct PhysPageDesc {
    /* offset in host memory of the page + io_index in the low bits */
    ram_addr_t phys_offset;
    ram_addr_t region_offset;
} PhysPageDesc;

200 201 202
/* This is a multi-level map on the physical address space.
   The bottom level has pointers to PhysPageDesc.  */
static void *l1_phys_map[P_L1_SIZE];
203

204 205
static void io_mem_init(void);

206 207 208
/* io memory support */
CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
B
bellard 已提交
209
void *io_mem_opaque[IO_MEM_NB_ENTRIES];
210
static char io_mem_used[IO_MEM_NB_ENTRIES];
211 212
static int io_mem_watch;
#endif
213

214
/* log support */
J
Juha Riihimäki 已提交
215 216 217
#ifdef WIN32
static const char *logfilename = "qemu.log";
#else
218
static const char *logfilename = "/tmp/qemu.log";
J
Juha Riihimäki 已提交
219
#endif
220 221
FILE *logfile;
int loglevel;
P
pbrook 已提交
222
static int log_append = 0;
223

B
bellard 已提交
224
/* statistics */
225
#if !defined(CONFIG_USER_ONLY)
B
bellard 已提交
226
static int tlb_flush_count;
227
#endif
B
bellard 已提交
228 229 230
static int tb_flush_count;
static int tb_phys_invalidate_count;

231 232 233 234 235 236 237 238 239 240 241
#ifdef _WIN32
static void map_exec(void *addr, long size)
{
    DWORD old_protect;
    VirtualProtect(addr, size,
                   PAGE_EXECUTE_READWRITE, &old_protect);
    
}
#else
static void map_exec(void *addr, long size)
{
242
    unsigned long start, end, page_size;
243
    
244
    page_size = getpagesize();
245
    start = (unsigned long)addr;
246
    start &= ~(page_size - 1);
247 248
    
    end = (unsigned long)addr + size;
249 250
    end += page_size - 1;
    end &= ~(page_size - 1);
251 252 253 254 255 256
    
    mprotect((void *)start, end - start,
             PROT_READ | PROT_WRITE | PROT_EXEC);
}
#endif

B
bellard 已提交
257
static void page_init(void)
B
bellard 已提交
258
{
259
    /* NOTE: we can always suppose that qemu_host_page_size >=
B
bellard 已提交
260
       TARGET_PAGE_SIZE */
261 262 263 264 265 266 267 268 269 270
#ifdef _WIN32
    {
        SYSTEM_INFO system_info;

        GetSystemInfo(&system_info);
        qemu_real_host_page_size = system_info.dwPageSize;
    }
#else
    qemu_real_host_page_size = getpagesize();
#endif
271 272 273 274 275 276 277 278
    if (qemu_host_page_size == 0)
        qemu_host_page_size = qemu_real_host_page_size;
    if (qemu_host_page_size < TARGET_PAGE_SIZE)
        qemu_host_page_size = TARGET_PAGE_SIZE;
    qemu_host_page_bits = 0;
    while ((1 << qemu_host_page_bits) < qemu_host_page_size)
        qemu_host_page_bits++;
    qemu_host_page_mask = ~(qemu_host_page_size - 1);
279

P
Paul Brook 已提交
280
#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
281
    {
282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
#ifdef HAVE_KINFO_GETVMMAP
        struct kinfo_vmentry *freep;
        int i, cnt;

        freep = kinfo_getvmmap(getpid(), &cnt);
        if (freep) {
            mmap_lock();
            for (i = 0; i < cnt; i++) {
                unsigned long startaddr, endaddr;

                startaddr = freep[i].kve_start;
                endaddr = freep[i].kve_end;
                if (h2g_valid(startaddr)) {
                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;

                    if (h2g_valid(endaddr)) {
                        endaddr = h2g(endaddr);
299
                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
300 301 302
                    } else {
#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
                        endaddr = ~0ul;
303
                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
304 305 306 307 308 309 310 311
#endif
                    }
                }
            }
            free(freep);
            mmap_unlock();
        }
#else
312 313
        FILE *f;

P
pbrook 已提交
314
        last_brk = (unsigned long)sbrk(0);
315

316
        f = fopen("/compat/linux/proc/self/maps", "r");
317
        if (f) {
318 319
            mmap_lock();

320
            do {
321 322 323 324 325 326 327 328 329 330 331 332 333 334
                unsigned long startaddr, endaddr;
                int n;

                n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);

                if (n == 2 && h2g_valid(startaddr)) {
                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;

                    if (h2g_valid(endaddr)) {
                        endaddr = h2g(endaddr);
                    } else {
                        endaddr = ~0ul;
                    }
                    page_set_flags(startaddr, endaddr, PAGE_RESERVED);
335 336
                }
            } while (!feof(f));
337

338
            fclose(f);
339
            mmap_unlock();
340
        }
341
#endif
342 343
    }
#endif
B
bellard 已提交
344 345
}

P
Paul Brook 已提交
346
static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
B
bellard 已提交
347
{
P
Paul Brook 已提交
348 349 350 351
    PageDesc *pd;
    void **lp;
    int i;

352
#if defined(CONFIG_USER_ONLY)
P
Paul Brook 已提交
353
    /* We can't use qemu_malloc because it may recurse into a locked mutex. */
354 355 356 357 358 359 360 361
# define ALLOC(P, SIZE)                                 \
    do {                                                \
        P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,    \
                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);   \
    } while (0)
#else
# define ALLOC(P, SIZE) \
    do { P = qemu_mallocz(SIZE); } while (0)
362
#endif
363

364 365 366 367 368 369 370 371 372 373 374 375 376
    /* Level 1.  Always allocated.  */
    lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));

    /* Level 2..N-1.  */
    for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
        void **p = *lp;

        if (p == NULL) {
            if (!alloc) {
                return NULL;
            }
            ALLOC(p, sizeof(void *) * L2_SIZE);
            *lp = p;
377
        }
378 379 380 381 382 383 384 385 386 387 388

        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
    }

    pd = *lp;
    if (pd == NULL) {
        if (!alloc) {
            return NULL;
        }
        ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
        *lp = pd;
B
bellard 已提交
389
    }
390 391 392 393

#undef ALLOC

    return pd + (index & (L2_SIZE - 1));
B
bellard 已提交
394 395
}

P
Paul Brook 已提交
396
static inline PageDesc *page_find(tb_page_addr_t index)
B
bellard 已提交
397
{
398
    return page_find_alloc(index, 0);
B
bellard 已提交
399 400
}

401
#if !defined(CONFIG_USER_ONLY)
A
Anthony Liguori 已提交
402
static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
B
bellard 已提交
403
{
404
    PhysPageDesc *pd;
405 406
    void **lp;
    int i;
B
bellard 已提交
407

408 409
    /* Level 1.  Always allocated.  */
    lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
410

411 412 413 414 415 416 417 418 419 420
    /* Level 2..N-1.  */
    for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
        void **p = *lp;
        if (p == NULL) {
            if (!alloc) {
                return NULL;
            }
            *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
        }
        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
421
    }
422

423
    pd = *lp;
424
    if (pd == NULL) {
425
        int i;
426 427

        if (!alloc) {
428
            return NULL;
429 430 431 432
        }

        *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);

P
pbrook 已提交
433
        for (i = 0; i < L2_SIZE; i++) {
434 435
            pd[i].phys_offset = IO_MEM_UNASSIGNED;
            pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
P
pbrook 已提交
436
        }
B
bellard 已提交
437
    }
438 439

    return pd + (index & (L2_SIZE - 1));
B
bellard 已提交
440 441
}

A
Anthony Liguori 已提交
442
static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
B
bellard 已提交
443
{
444
    return phys_page_find_alloc(index, 0);
B
bellard 已提交
445 446
}

A
Anthony Liguori 已提交
447 448
static void tlb_protect_code(ram_addr_t ram_addr);
static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
449
                                    target_ulong vaddr);
P
pbrook 已提交
450 451
#define mmap_lock() do { } while(0)
#define mmap_unlock() do { } while(0)
452
#endif
B
bellard 已提交
453

454 455 456
#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)

#if defined(CONFIG_USER_ONLY)
S
Stuart Brady 已提交
457
/* Currently it is not recommended to allocate big chunks of data in
458 459 460 461 462
   user mode. It will change when a dedicated libc will be used */
#define USE_STATIC_CODE_GEN_BUFFER
#endif

#ifdef USE_STATIC_CODE_GEN_BUFFER
463 464
static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
               __attribute__((aligned (CODE_GEN_ALIGN)));
465 466
#endif

467
static void code_gen_alloc(unsigned long tb_size)
468
{
469 470 471 472 473
#ifdef USE_STATIC_CODE_GEN_BUFFER
    code_gen_buffer = static_code_gen_buffer;
    code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
    map_exec(code_gen_buffer, code_gen_buffer_size);
#else
474 475
    code_gen_buffer_size = tb_size;
    if (code_gen_buffer_size == 0) {
476 477 478 479
#if defined(CONFIG_USER_ONLY)
        /* in user mode, phys_ram_size is not meaningful */
        code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
#else
S
Stuart Brady 已提交
480
        /* XXX: needs adjustments */
P
pbrook 已提交
481
        code_gen_buffer_size = (unsigned long)(ram_size / 4);
482
#endif
483 484 485 486 487 488 489 490
    }
    if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
        code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
    /* The code gen buffer location may have constraints depending on
       the host cpu and OS */
#if defined(__linux__) 
    {
        int flags;
B
blueswir1 已提交
491 492
        void *start = NULL;

493 494 495 496 497 498
        flags = MAP_PRIVATE | MAP_ANONYMOUS;
#if defined(__x86_64__)
        flags |= MAP_32BIT;
        /* Cannot map more than that */
        if (code_gen_buffer_size > (800 * 1024 * 1024))
            code_gen_buffer_size = (800 * 1024 * 1024);
B
blueswir1 已提交
499 500 501 502 503 504
#elif defined(__sparc_v9__)
        // Map the buffer below 2G, so we can use direct calls and branches
        flags |= MAP_FIXED;
        start = (void *) 0x60000000UL;
        if (code_gen_buffer_size > (512 * 1024 * 1024))
            code_gen_buffer_size = (512 * 1024 * 1024);
505
#elif defined(__arm__)
B
balrog 已提交
506
        /* Map the buffer below 32M, so we can use direct calls and branches */
507 508 509 510
        flags |= MAP_FIXED;
        start = (void *) 0x01000000UL;
        if (code_gen_buffer_size > 16 * 1024 * 1024)
            code_gen_buffer_size = 16 * 1024 * 1024;
511 512 513 514 515 516 517
#elif defined(__s390x__)
        /* Map the buffer so that we can use direct calls and branches.  */
        /* We have a +- 4GB range on the branches; leave some slop.  */
        if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
            code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
        }
        start = (void *)0x90000000UL;
518
#endif
B
blueswir1 已提交
519 520
        code_gen_buffer = mmap(start, code_gen_buffer_size,
                               PROT_WRITE | PROT_READ | PROT_EXEC,
521 522 523 524 525 526
                               flags, -1, 0);
        if (code_gen_buffer == MAP_FAILED) {
            fprintf(stderr, "Could not allocate dynamic translator buffer\n");
            exit(1);
        }
    }
A
Aurelien Jarno 已提交
527
#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548
    {
        int flags;
        void *addr = NULL;
        flags = MAP_PRIVATE | MAP_ANONYMOUS;
#if defined(__x86_64__)
        /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
         * 0x40000000 is free */
        flags |= MAP_FIXED;
        addr = (void *)0x40000000;
        /* Cannot map more than that */
        if (code_gen_buffer_size > (800 * 1024 * 1024))
            code_gen_buffer_size = (800 * 1024 * 1024);
#endif
        code_gen_buffer = mmap(addr, code_gen_buffer_size,
                               PROT_WRITE | PROT_READ | PROT_EXEC, 
                               flags, -1, 0);
        if (code_gen_buffer == MAP_FAILED) {
            fprintf(stderr, "Could not allocate dynamic translator buffer\n");
            exit(1);
        }
    }
549 550 551 552
#else
    code_gen_buffer = qemu_malloc(code_gen_buffer_size);
    map_exec(code_gen_buffer, code_gen_buffer_size);
#endif
553
#endif /* !USE_STATIC_CODE_GEN_BUFFER */
554 555
    map_exec(code_gen_prologue, sizeof(code_gen_prologue));
    code_gen_buffer_max_size = code_gen_buffer_size - 
556
        (TCG_MAX_OP_SIZE * OPC_MAX_SIZE);
557 558 559 560 561 562 563 564 565 566 567 568
    code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
    tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
}

/* Must be called before using the QEMU cpus. 'tb_size' is the size
   (in bytes) allocated to the translation buffer. Zero means default
   size. */
void cpu_exec_init_all(unsigned long tb_size)
{
    cpu_gen_init();
    code_gen_alloc(tb_size);
    code_gen_ptr = code_gen_buffer;
569
    page_init();
570
#if !defined(CONFIG_USER_ONLY)
571
    io_mem_init();
572
#endif
573 574 575 576 577
#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
    /* There's no guest base to take into account, so go ahead and
       initialize the prologue now.  */
    tcg_prologue_init(&tcg_ctx);
#endif
578 579
}

580 581
#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)

582
static int cpu_common_post_load(void *opaque, int version_id)
J
Juan Quintela 已提交
583 584
{
    CPUState *env = opaque;
585

586 587 588
    /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
       version_id is increased. */
    env->interrupt_request &= ~0x01;
589 590 591 592
    tlb_flush(env, 1);

    return 0;
}
J
Juan Quintela 已提交
593 594 595 596 597 598 599 600 601 602 603 604 605

static const VMStateDescription vmstate_cpu_common = {
    .name = "cpu_common",
    .version_id = 1,
    .minimum_version_id = 1,
    .minimum_version_id_old = 1,
    .post_load = cpu_common_post_load,
    .fields      = (VMStateField []) {
        VMSTATE_UINT32(halted, CPUState),
        VMSTATE_UINT32(interrupt_request, CPUState),
        VMSTATE_END_OF_LIST()
    }
};
606 607
#endif

G
Glauber Costa 已提交
608 609 610 611 612 613 614 615 616 617 618 619 620
CPUState *qemu_get_cpu(int cpu)
{
    CPUState *env = first_cpu;

    while (env) {
        if (env->cpu_index == cpu)
            break;
        env = env->next_cpu;
    }

    return env;
}

B
bellard 已提交
621
void cpu_exec_init(CPUState *env)
B
bellard 已提交
622
{
B
bellard 已提交
623 624 625
    CPUState **penv;
    int cpu_index;

626 627 628
#if defined(CONFIG_USER_ONLY)
    cpu_list_lock();
#endif
B
bellard 已提交
629 630 631 632
    env->next_cpu = NULL;
    penv = &first_cpu;
    cpu_index = 0;
    while (*penv != NULL) {
633
        penv = &(*penv)->next_cpu;
B
bellard 已提交
634 635 636
        cpu_index++;
    }
    env->cpu_index = cpu_index;
637
    env->numa_node = 0;
B
Blue Swirl 已提交
638 639
    QTAILQ_INIT(&env->breakpoints);
    QTAILQ_INIT(&env->watchpoints);
B
bellard 已提交
640
    *penv = env;
641 642 643
#if defined(CONFIG_USER_ONLY)
    cpu_list_unlock();
#endif
644
#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
A
Alex Williamson 已提交
645 646
    vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
    register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
647 648
                    cpu_save, cpu_load, env);
#endif
B
bellard 已提交
649 650
}

651 652 653
static inline void invalidate_page_bitmap(PageDesc *p)
{
    if (p->code_bitmap) {
654
        qemu_free(p->code_bitmap);
655 656 657 658 659
        p->code_bitmap = NULL;
    }
    p->code_write_count = 0;
}

660 661 662
/* Set to NULL all the 'first_tb' fields in all PageDescs. */

static void page_flush_tb_1 (int level, void **lp)
B
bellard 已提交
663
{
664
    int i;
B
bellard 已提交
665

666 667 668 669 670
    if (*lp == NULL) {
        return;
    }
    if (level == 0) {
        PageDesc *pd = *lp;
P
Paul Brook 已提交
671
        for (i = 0; i < L2_SIZE; ++i) {
672 673
            pd[i].first_tb = NULL;
            invalidate_page_bitmap(pd + i);
B
bellard 已提交
674
        }
675 676
    } else {
        void **pp = *lp;
P
Paul Brook 已提交
677
        for (i = 0; i < L2_SIZE; ++i) {
678 679 680 681 682 683 684 685 686 687
            page_flush_tb_1 (level - 1, pp + i);
        }
    }
}

static void page_flush_tb(void)
{
    int i;
    for (i = 0; i < V_L1_SIZE; i++) {
        page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
B
bellard 已提交
688 689 690 691
    }
}

/* flush all the translation blocks */
B
bellard 已提交
692
/* XXX: tb_flush is currently not thread safe */
B
bellard 已提交
693
void tb_flush(CPUState *env1)
B
bellard 已提交
694
{
B
bellard 已提交
695
    CPUState *env;
696
#if defined(DEBUG_FLUSH)
B
blueswir1 已提交
697 698 699 700
    printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
           (unsigned long)(code_gen_ptr - code_gen_buffer),
           nb_tbs, nb_tbs > 0 ?
           ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
B
bellard 已提交
701
#endif
702
    if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
P
pbrook 已提交
703 704
        cpu_abort(env1, "Internal error: code buffer overflow\n");

B
bellard 已提交
705
    nb_tbs = 0;
706

B
bellard 已提交
707 708 709
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
        memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
    }
710

B
bellard 已提交
711
    memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
B
bellard 已提交
712
    page_flush_tb();
713

B
bellard 已提交
714
    code_gen_ptr = code_gen_buffer;
B
bellard 已提交
715 716
    /* XXX: flush processor icache at this point if cache flush is
       expensive */
B
bellard 已提交
717
    tb_flush_count++;
B
bellard 已提交
718 719 720 721
}

#ifdef DEBUG_TB_CHECK

J
j_mayer 已提交
722
static void tb_invalidate_check(target_ulong address)
B
bellard 已提交
723 724 725 726
{
    TranslationBlock *tb;
    int i;
    address &= TARGET_PAGE_MASK;
727 728
    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
B
bellard 已提交
729 730
            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
                  address >= tb->pc + tb->size)) {
731 732
                printf("ERROR invalidate: address=" TARGET_FMT_lx
                       " PC=%08lx size=%04x\n",
733
                       address, (long)tb->pc, tb->size);
B
bellard 已提交
734 735 736 737 738 739 740 741 742 743
            }
        }
    }
}

/* verify that all the pages have correct rights for code */
static void tb_page_check(void)
{
    TranslationBlock *tb;
    int i, flags1, flags2;
744

745 746
    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
B
bellard 已提交
747 748 749 750
            flags1 = page_get_flags(tb->pc);
            flags2 = page_get_flags(tb->pc + tb->size - 1);
            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
751
                       (long)tb->pc, tb->size, flags1, flags2);
B
bellard 已提交
752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773
            }
        }
    }
}

#endif

/* invalidate one TB */
static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
                             int next_offset)
{
    TranslationBlock *tb1;
    for(;;) {
        tb1 = *ptb;
        if (tb1 == tb) {
            *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
            break;
        }
        ptb = (TranslationBlock **)((char *)tb1 + next_offset);
    }
}

774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790
static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
{
    TranslationBlock *tb1;
    unsigned int n1;

    for(;;) {
        tb1 = *ptb;
        n1 = (long)tb1 & 3;
        tb1 = (TranslationBlock *)((long)tb1 & ~3);
        if (tb1 == tb) {
            *ptb = tb1->page_next[n1];
            break;
        }
        ptb = &tb1->page_next[n1];
    }
}

B
bellard 已提交
791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825
static inline void tb_jmp_remove(TranslationBlock *tb, int n)
{
    TranslationBlock *tb1, **ptb;
    unsigned int n1;

    ptb = &tb->jmp_next[n];
    tb1 = *ptb;
    if (tb1) {
        /* find tb(n) in circular list */
        for(;;) {
            tb1 = *ptb;
            n1 = (long)tb1 & 3;
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
            if (n1 == n && tb1 == tb)
                break;
            if (n1 == 2) {
                ptb = &tb1->jmp_first;
            } else {
                ptb = &tb1->jmp_next[n1];
            }
        }
        /* now we can suppress tb(n) from the list */
        *ptb = tb->jmp_next[n];

        tb->jmp_next[n] = NULL;
    }
}

/* reset the jump entry 'n' of a TB so that it is not chained to
   another TB */
static inline void tb_reset_jump(TranslationBlock *tb, int n)
{
    tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
}

P
Paul Brook 已提交
826
void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
B
bellard 已提交
827
{
B
bellard 已提交
828
    CPUState *env;
829
    PageDesc *p;
B
bellard 已提交
830
    unsigned int h, n1;
P
Paul Brook 已提交
831
    tb_page_addr_t phys_pc;
832
    TranslationBlock *tb1, *tb2;
833

834 835 836
    /* remove the TB from the hash list */
    phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
    h = tb_phys_hash_func(phys_pc);
837
    tb_remove(&tb_phys_hash[h], tb,
838 839 840 841 842 843 844 845 846 847 848 849 850 851
              offsetof(TranslationBlock, phys_hash_next));

    /* remove the TB from the page list */
    if (tb->page_addr[0] != page_addr) {
        p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
        tb_page_remove(&p->first_tb, tb);
        invalidate_page_bitmap(p);
    }
    if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
        p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
        tb_page_remove(&p->first_tb, tb);
        invalidate_page_bitmap(p);
    }

852
    tb_invalidated_flag = 1;
853

B
bellard 已提交
854
    /* remove the TB from the hash list */
855
    h = tb_jmp_cache_hash_func(tb->pc);
B
bellard 已提交
856 857 858 859
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
        if (env->tb_jmp_cache[h] == tb)
            env->tb_jmp_cache[h] = NULL;
    }
B
bellard 已提交
860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877

    /* suppress this TB from the two jump lists */
    tb_jmp_remove(tb, 0);
    tb_jmp_remove(tb, 1);

    /* suppress any remaining jumps to this TB */
    tb1 = tb->jmp_first;
    for(;;) {
        n1 = (long)tb1 & 3;
        if (n1 == 2)
            break;
        tb1 = (TranslationBlock *)((long)tb1 & ~3);
        tb2 = tb1->jmp_next[n1];
        tb_reset_jump(tb1, n1);
        tb1->jmp_next[n1] = NULL;
        tb1 = tb2;
    }
    tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
878

B
bellard 已提交
879
    tb_phys_invalidate_count++;
880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912
}

static inline void set_bits(uint8_t *tab, int start, int len)
{
    int end, mask, end1;

    end = start + len;
    tab += start >> 3;
    mask = 0xff << (start & 7);
    if ((start & ~7) == (end & ~7)) {
        if (start < end) {
            mask &= ~(0xff << (end & 7));
            *tab |= mask;
        }
    } else {
        *tab++ |= mask;
        start = (start + 8) & ~7;
        end1 = end & ~7;
        while (start < end1) {
            *tab++ = 0xff;
            start += 8;
        }
        if (start < end) {
            mask = ~(0xff << (end & 7));
            *tab |= mask;
        }
    }
}

static void build_page_bitmap(PageDesc *p)
{
    int n, tb_start, tb_end;
    TranslationBlock *tb;
913

P
pbrook 已提交
914
    p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936

    tb = p->first_tb;
    while (tb != NULL) {
        n = (long)tb & 3;
        tb = (TranslationBlock *)((long)tb & ~3);
        /* NOTE: this is subtle as a TB may span two physical pages */
        if (n == 0) {
            /* NOTE: tb_end may be after the end of the page, but
               it is not a problem */
            tb_start = tb->pc & ~TARGET_PAGE_MASK;
            tb_end = tb_start + tb->size;
            if (tb_end > TARGET_PAGE_SIZE)
                tb_end = TARGET_PAGE_SIZE;
        } else {
            tb_start = 0;
            tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
        }
        set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
        tb = tb->page_next[n];
    }
}

P
pbrook 已提交
937 938 939
TranslationBlock *tb_gen_code(CPUState *env,
                              target_ulong pc, target_ulong cs_base,
                              int flags, int cflags)
B
bellard 已提交
940 941 942
{
    TranslationBlock *tb;
    uint8_t *tc_ptr;
P
Paul Brook 已提交
943 944
    tb_page_addr_t phys_pc, phys_page2;
    target_ulong virt_page2;
B
bellard 已提交
945 946
    int code_gen_size;

P
Paul Brook 已提交
947
    phys_pc = get_page_addr_code(env, pc);
B
bellard 已提交
948
    tb = tb_alloc(pc);
B
bellard 已提交
949 950 951 952
    if (!tb) {
        /* flush must be done */
        tb_flush(env);
        /* cannot fail at this point */
B
bellard 已提交
953
        tb = tb_alloc(pc);
P
pbrook 已提交
954 955
        /* Don't forget to invalidate previous TB info.  */
        tb_invalidated_flag = 1;
B
bellard 已提交
956 957 958 959 960 961
    }
    tc_ptr = code_gen_ptr;
    tb->tc_ptr = tc_ptr;
    tb->cs_base = cs_base;
    tb->flags = flags;
    tb->cflags = cflags;
962
    cpu_gen_code(env, tb, &code_gen_size);
B
bellard 已提交
963
    code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
964

B
bellard 已提交
965
    /* check next page if needed */
B
bellard 已提交
966
    virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
B
bellard 已提交
967
    phys_page2 = -1;
B
bellard 已提交
968
    if ((pc & TARGET_PAGE_MASK) != virt_page2) {
P
Paul Brook 已提交
969
        phys_page2 = get_page_addr_code(env, virt_page2);
B
bellard 已提交
970
    }
P
Paul Brook 已提交
971
    tb_link_page(tb, phys_pc, phys_page2);
P
pbrook 已提交
972
    return tb;
B
bellard 已提交
973
}
974

975 976
/* invalidate all TBs which intersect with the target physical page
   starting in range [start;end[. NOTE: start and end must refer to
B
bellard 已提交
977 978 979
   the same physical page. 'is_cpu_write_access' should be true if called
   from a real cpu write access: the virtual CPU will exit the current
   TB if code is modified inside this TB. */
P
Paul Brook 已提交
980
void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
B
bellard 已提交
981 982
                                   int is_cpu_write_access)
{
983
    TranslationBlock *tb, *tb_next, *saved_tb;
B
bellard 已提交
984
    CPUState *env = cpu_single_env;
P
Paul Brook 已提交
985
    tb_page_addr_t tb_start, tb_end;
986 987 988 989 990 991 992 993 994 995
    PageDesc *p;
    int n;
#ifdef TARGET_HAS_PRECISE_SMC
    int current_tb_not_found = is_cpu_write_access;
    TranslationBlock *current_tb = NULL;
    int current_tb_modified = 0;
    target_ulong current_pc = 0;
    target_ulong current_cs_base = 0;
    int current_flags = 0;
#endif /* TARGET_HAS_PRECISE_SMC */
996 997

    p = page_find(start >> TARGET_PAGE_BITS);
998
    if (!p)
999
        return;
1000
    if (!p->code_bitmap &&
B
bellard 已提交
1001 1002
        ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
        is_cpu_write_access) {
1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024
        /* build code bitmap */
        build_page_bitmap(p);
    }

    /* we remove all the TBs in the range [start, end[ */
    /* XXX: see if in some cases it could be faster to invalidate all the code */
    tb = p->first_tb;
    while (tb != NULL) {
        n = (long)tb & 3;
        tb = (TranslationBlock *)((long)tb & ~3);
        tb_next = tb->page_next[n];
        /* NOTE: this is subtle as a TB may span two physical pages */
        if (n == 0) {
            /* NOTE: tb_end may be after the end of the page, but
               it is not a problem */
            tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
            tb_end = tb_start + tb->size;
        } else {
            tb_start = tb->page_addr[1];
            tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
        }
        if (!(tb_end <= start || tb_start >= end)) {
B
bellard 已提交
1025 1026 1027 1028
#ifdef TARGET_HAS_PRECISE_SMC
            if (current_tb_not_found) {
                current_tb_not_found = 0;
                current_tb = NULL;
P
pbrook 已提交
1029
                if (env->mem_io_pc) {
B
bellard 已提交
1030
                    /* now we have a real cpu fault */
P
pbrook 已提交
1031
                    current_tb = tb_find_pc(env->mem_io_pc);
B
bellard 已提交
1032 1033 1034
                }
            }
            if (current_tb == tb &&
P
pbrook 已提交
1035
                (current_tb->cflags & CF_COUNT_MASK) != 1) {
B
bellard 已提交
1036 1037 1038 1039 1040
                /* If we are modifying the current TB, we must stop
                its execution. We could be more precise by checking
                that the modification is after the current PC, but it
                would require a specialized function to partially
                restore the CPU state */
1041

B
bellard 已提交
1042
                current_tb_modified = 1;
1043
                cpu_restore_state(current_tb, env,
P
pbrook 已提交
1044
                                  env->mem_io_pc, NULL);
1045 1046
                cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
                                     &current_flags);
B
bellard 已提交
1047 1048
            }
#endif /* TARGET_HAS_PRECISE_SMC */
1049 1050 1051 1052 1053 1054 1055
            /* we need to do that to handle the case where a signal
               occurs while doing tb_phys_invalidate() */
            saved_tb = NULL;
            if (env) {
                saved_tb = env->current_tb;
                env->current_tb = NULL;
            }
1056
            tb_phys_invalidate(tb, -1);
1057 1058 1059 1060 1061
            if (env) {
                env->current_tb = saved_tb;
                if (env->interrupt_request && env->current_tb)
                    cpu_interrupt(env, env->interrupt_request);
            }
1062 1063 1064 1065 1066 1067 1068
        }
        tb = tb_next;
    }
#if !defined(CONFIG_USER_ONLY)
    /* if no code remaining, no need to continue to use slow writes */
    if (!p->first_tb) {
        invalidate_page_bitmap(p);
B
bellard 已提交
1069
        if (is_cpu_write_access) {
P
pbrook 已提交
1070
            tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
B
bellard 已提交
1071 1072 1073 1074 1075 1076 1077 1078
        }
    }
#endif
#ifdef TARGET_HAS_PRECISE_SMC
    if (current_tb_modified) {
        /* we generate a block containing just the instruction
           modifying the memory. It will ensure that it cannot modify
           itself */
1079
        env->current_tb = NULL;
P
pbrook 已提交
1080
        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
B
bellard 已提交
1081
        cpu_resume_from_signal(env, NULL);
1082
    }
B
bellard 已提交
1083
#endif
1084
}
B
bellard 已提交
1085

1086
/* len must be <= 8 and start must be a multiple of len */
P
Paul Brook 已提交
1087
static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1088 1089 1090
{
    PageDesc *p;
    int offset, b;
1091
#if 0
B
bellard 已提交
1092
    if (1) {
1093 1094 1095 1096
        qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
                  cpu_single_env->mem_io_vaddr, len,
                  cpu_single_env->eip,
                  cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1097 1098
    }
#endif
1099
    p = page_find(start >> TARGET_PAGE_BITS);
1100
    if (!p)
1101 1102 1103 1104 1105 1106 1107 1108
        return;
    if (p->code_bitmap) {
        offset = start & ~TARGET_PAGE_MASK;
        b = p->code_bitmap[offset >> 3] >> (offset & 7);
        if (b & ((1 << len) - 1))
            goto do_invalidate;
    } else {
    do_invalidate:
B
bellard 已提交
1109
        tb_invalidate_phys_page_range(start, start + len, 1);
1110 1111 1112 1113
    }
}

#if !defined(CONFIG_SOFTMMU)
P
Paul Brook 已提交
1114
static void tb_invalidate_phys_page(tb_page_addr_t addr,
B
bellard 已提交
1115
                                    unsigned long pc, void *puc)
1116
{
1117
    TranslationBlock *tb;
1118
    PageDesc *p;
1119
    int n;
B
bellard 已提交
1120
#ifdef TARGET_HAS_PRECISE_SMC
1121
    TranslationBlock *current_tb = NULL;
B
bellard 已提交
1122
    CPUState *env = cpu_single_env;
1123 1124 1125 1126
    int current_tb_modified = 0;
    target_ulong current_pc = 0;
    target_ulong current_cs_base = 0;
    int current_flags = 0;
B
bellard 已提交
1127
#endif
1128 1129 1130

    addr &= TARGET_PAGE_MASK;
    p = page_find(addr >> TARGET_PAGE_BITS);
1131
    if (!p)
1132 1133
        return;
    tb = p->first_tb;
B
bellard 已提交
1134 1135 1136 1137 1138
#ifdef TARGET_HAS_PRECISE_SMC
    if (tb && pc != 0) {
        current_tb = tb_find_pc(pc);
    }
#endif
1139 1140 1141
    while (tb != NULL) {
        n = (long)tb & 3;
        tb = (TranslationBlock *)((long)tb & ~3);
B
bellard 已提交
1142 1143
#ifdef TARGET_HAS_PRECISE_SMC
        if (current_tb == tb &&
P
pbrook 已提交
1144
            (current_tb->cflags & CF_COUNT_MASK) != 1) {
B
bellard 已提交
1145 1146 1147 1148 1149
                /* If we are modifying the current TB, we must stop
                   its execution. We could be more precise by checking
                   that the modification is after the current PC, but it
                   would require a specialized function to partially
                   restore the CPU state */
1150

B
bellard 已提交
1151 1152
            current_tb_modified = 1;
            cpu_restore_state(current_tb, env, pc, puc);
1153 1154
            cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
                                 &current_flags);
B
bellard 已提交
1155 1156
        }
#endif /* TARGET_HAS_PRECISE_SMC */
1157 1158 1159
        tb_phys_invalidate(tb, addr);
        tb = tb->page_next[n];
    }
B
bellard 已提交
1160
    p->first_tb = NULL;
B
bellard 已提交
1161 1162 1163 1164 1165
#ifdef TARGET_HAS_PRECISE_SMC
    if (current_tb_modified) {
        /* we generate a block containing just the instruction
           modifying the memory. It will ensure that it cannot modify
           itself */
1166
        env->current_tb = NULL;
P
pbrook 已提交
1167
        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
B
bellard 已提交
1168 1169 1170
        cpu_resume_from_signal(env, puc);
    }
#endif
B
bellard 已提交
1171
}
1172
#endif
B
bellard 已提交
1173 1174

/* add the tb in the target page and protect it if necessary */
1175
static inline void tb_alloc_page(TranslationBlock *tb,
P
Paul Brook 已提交
1176
                                 unsigned int n, tb_page_addr_t page_addr)
B
bellard 已提交
1177 1178
{
    PageDesc *p;
1179 1180 1181
    TranslationBlock *last_first_tb;

    tb->page_addr[n] = page_addr;
1182
    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1183 1184 1185 1186
    tb->page_next[n] = p->first_tb;
    last_first_tb = p->first_tb;
    p->first_tb = (TranslationBlock *)((long)tb | n);
    invalidate_page_bitmap(p);
B
bellard 已提交
1187

1188
#if defined(TARGET_HAS_SMC) || 1
B
bellard 已提交
1189

1190
#if defined(CONFIG_USER_ONLY)
B
bellard 已提交
1191
    if (p->flags & PAGE_WRITE) {
1192 1193
        target_ulong addr;
        PageDesc *p2;
1194 1195
        int prot;

B
bellard 已提交
1196 1197
        /* force the host page as non writable (writes will have a
           page fault + mprotect overhead) */
1198
        page_addr &= qemu_host_page_mask;
B
bellard 已提交
1199
        prot = 0;
1200 1201 1202 1203 1204 1205 1206 1207 1208
        for(addr = page_addr; addr < page_addr + qemu_host_page_size;
            addr += TARGET_PAGE_SIZE) {

            p2 = page_find (addr >> TARGET_PAGE_BITS);
            if (!p2)
                continue;
            prot |= p2->flags;
            p2->flags &= ~PAGE_WRITE;
          }
1209
        mprotect(g2h(page_addr), qemu_host_page_size,
B
bellard 已提交
1210 1211
                 (prot & PAGE_BITS) & ~PAGE_WRITE);
#ifdef DEBUG_TB_INVALIDATE
B
blueswir1 已提交
1212
        printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1213
               page_addr);
B
bellard 已提交
1214 1215
#endif
    }
1216 1217 1218 1219 1220
#else
    /* if some code is already present, then the pages are already
       protected. So we handle the case where only the first TB is
       allocated in a physical page */
    if (!last_first_tb) {
B
bellard 已提交
1221
        tlb_protect_code(page_addr);
1222 1223
    }
#endif
B
bellard 已提交
1224 1225

#endif /* TARGET_HAS_SMC */
B
bellard 已提交
1226 1227 1228 1229
}

/* Allocate a new translation block. Flush the translation buffer if
   too many translation blocks or too much generated code. */
B
bellard 已提交
1230
TranslationBlock *tb_alloc(target_ulong pc)
B
bellard 已提交
1231 1232 1233
{
    TranslationBlock *tb;

1234 1235
    if (nb_tbs >= code_gen_max_blocks ||
        (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
B
bellard 已提交
1236
        return NULL;
B
bellard 已提交
1237 1238
    tb = &tbs[nb_tbs++];
    tb->pc = pc;
1239
    tb->cflags = 0;
B
bellard 已提交
1240 1241 1242
    return tb;
}

P
pbrook 已提交
1243 1244
void tb_free(TranslationBlock *tb)
{
T
ths 已提交
1245
    /* In practice this is mostly used for single use temporary TB
P
pbrook 已提交
1246 1247 1248 1249 1250 1251 1252 1253
       Ignore the hard cases and just back up if this TB happens to
       be the last one generated.  */
    if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
        code_gen_ptr = tb->tc_ptr;
        nb_tbs--;
    }
}

1254 1255
/* add a new TB and link it to the physical page tables. phys_page2 is
   (-1) to indicate that only one page contains the TB. */
P
Paul Brook 已提交
1256 1257
void tb_link_page(TranslationBlock *tb,
                  tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
B
bellard 已提交
1258
{
1259 1260 1261
    unsigned int h;
    TranslationBlock **ptb;

P
pbrook 已提交
1262 1263 1264
    /* Grab the mmap lock to stop another thread invalidating this TB
       before we are done.  */
    mmap_lock();
1265 1266 1267 1268 1269
    /* add in the physical hash table */
    h = tb_phys_hash_func(phys_pc);
    ptb = &tb_phys_hash[h];
    tb->phys_hash_next = *ptb;
    *ptb = tb;
B
bellard 已提交
1270 1271

    /* add in the page list */
1272 1273 1274 1275 1276 1277
    tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
    if (phys_page2 != -1)
        tb_alloc_page(tb, 1, phys_page2);
    else
        tb->page_addr[1] = -1;

B
bellard 已提交
1278 1279 1280 1281 1282 1283 1284 1285 1286
    tb->jmp_first = (TranslationBlock *)((long)tb | 2);
    tb->jmp_next[0] = NULL;
    tb->jmp_next[1] = NULL;

    /* init original jump addresses */
    if (tb->tb_next_offset[0] != 0xffff)
        tb_reset_jump(tb, 0);
    if (tb->tb_next_offset[1] != 0xffff)
        tb_reset_jump(tb, 1);
1287 1288 1289 1290

#ifdef DEBUG_TB_CHECK
    tb_page_check();
#endif
P
pbrook 已提交
1291
    mmap_unlock();
B
bellard 已提交
1292 1293
}

1294 1295 1296
/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
   tb[1].tc_ptr. Return NULL if not found */
TranslationBlock *tb_find_pc(unsigned long tc_ptr)
B
bellard 已提交
1297
{
1298 1299 1300
    int m_min, m_max, m;
    unsigned long v;
    TranslationBlock *tb;
B
bellard 已提交
1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320

    if (nb_tbs <= 0)
        return NULL;
    if (tc_ptr < (unsigned long)code_gen_buffer ||
        tc_ptr >= (unsigned long)code_gen_ptr)
        return NULL;
    /* binary search (cf Knuth) */
    m_min = 0;
    m_max = nb_tbs - 1;
    while (m_min <= m_max) {
        m = (m_min + m_max) >> 1;
        tb = &tbs[m];
        v = (unsigned long)tb->tc_ptr;
        if (v == tc_ptr)
            return tb;
        else if (tc_ptr < v) {
            m_max = m - 1;
        } else {
            m_min = m + 1;
        }
1321
    }
B
bellard 已提交
1322 1323
    return &tbs[m_max];
}
B
bellard 已提交
1324

B
bellard 已提交
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356
static void tb_reset_jump_recursive(TranslationBlock *tb);

static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
{
    TranslationBlock *tb1, *tb_next, **ptb;
    unsigned int n1;

    tb1 = tb->jmp_next[n];
    if (tb1 != NULL) {
        /* find head of list */
        for(;;) {
            n1 = (long)tb1 & 3;
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
            if (n1 == 2)
                break;
            tb1 = tb1->jmp_next[n1];
        }
        /* we are now sure now that tb jumps to tb1 */
        tb_next = tb1;

        /* remove tb from the jmp_first list */
        ptb = &tb_next->jmp_first;
        for(;;) {
            tb1 = *ptb;
            n1 = (long)tb1 & 3;
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
            if (n1 == n && tb1 == tb)
                break;
            ptb = &tb1->jmp_next[n1];
        }
        *ptb = tb->jmp_next[n];
        tb->jmp_next[n] = NULL;
1357

B
bellard 已提交
1358 1359 1360
        /* suppress the jump to next tb in generated code */
        tb_reset_jump(tb, n);

1361
        /* suppress jumps in the tb on which we could have jumped */
B
bellard 已提交
1362 1363 1364 1365 1366 1367 1368 1369 1370 1371
        tb_reset_jump_recursive(tb_next);
    }
}

static void tb_reset_jump_recursive(TranslationBlock *tb)
{
    tb_reset_jump_recursive2(tb, 0);
    tb_reset_jump_recursive2(tb, 1);
}

B
bellard 已提交
1372
#if defined(TARGET_HAS_ICE)
1373 1374 1375 1376 1377 1378
#if defined(CONFIG_USER_ONLY)
static void breakpoint_invalidate(CPUState *env, target_ulong pc)
{
    tb_invalidate_phys_page_range(pc, pc + 1, 0);
}
#else
B
bellard 已提交
1379 1380
static void breakpoint_invalidate(CPUState *env, target_ulong pc)
{
A
Anthony Liguori 已提交
1381
    target_phys_addr_t addr;
1382
    target_ulong pd;
A
Anthony Liguori 已提交
1383
    ram_addr_t ram_addr;
P
pbrook 已提交
1384
    PhysPageDesc *p;
B
bellard 已提交
1385

P
pbrook 已提交
1386 1387 1388 1389 1390 1391 1392 1393
    addr = cpu_get_phys_page_debug(env, pc);
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
    ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
P
pbrook 已提交
1394
    tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
B
bellard 已提交
1395
}
B
bellard 已提交
1396
#endif
1397
#endif /* TARGET_HAS_ICE */
B
bellard 已提交
1398

1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410
#if defined(CONFIG_USER_ONLY)
void cpu_watchpoint_remove_all(CPUState *env, int mask)

{
}

int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
                          int flags, CPUWatchpoint **watchpoint)
{
    return -ENOSYS;
}
#else
1411
/* Add a watchpoint.  */
1412 1413
int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
                          int flags, CPUWatchpoint **watchpoint)
1414
{
1415
    target_ulong len_mask = ~(len - 1);
1416
    CPUWatchpoint *wp;
1417

1418 1419 1420 1421 1422 1423
    /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
    if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
        fprintf(stderr, "qemu: tried to set invalid watchpoint at "
                TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
        return -EINVAL;
    }
1424 1425 1426
    wp = qemu_malloc(sizeof(*wp));

    wp->vaddr = addr;
1427
    wp->len_mask = len_mask;
1428 1429
    wp->flags = flags;

1430
    /* keep all GDB-injected watchpoints in front */
1431
    if (flags & BP_GDB)
B
Blue Swirl 已提交
1432
        QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1433
    else
B
Blue Swirl 已提交
1434
        QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1435 1436

    tlb_flush_page(env, addr);
1437 1438 1439 1440

    if (watchpoint)
        *watchpoint = wp;
    return 0;
1441 1442
}

1443 1444 1445
/* Remove a specific watchpoint.  */
int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
                          int flags)
1446
{
1447
    target_ulong len_mask = ~(len - 1);
1448
    CPUWatchpoint *wp;
1449

B
Blue Swirl 已提交
1450
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1451
        if (addr == wp->vaddr && len_mask == wp->len_mask
1452
                && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1453
            cpu_watchpoint_remove_by_ref(env, wp);
1454 1455 1456
            return 0;
        }
    }
1457
    return -ENOENT;
1458 1459
}

1460 1461 1462
/* Remove a specific watchpoint by reference.  */
void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
{
B
Blue Swirl 已提交
1463
    QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1464

1465 1466 1467 1468 1469 1470 1471 1472
    tlb_flush_page(env, watchpoint->vaddr);

    qemu_free(watchpoint);
}

/* Remove all matching watchpoints.  */
void cpu_watchpoint_remove_all(CPUState *env, int mask)
{
1473
    CPUWatchpoint *wp, *next;
1474

B
Blue Swirl 已提交
1475
    QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1476 1477
        if (wp->flags & mask)
            cpu_watchpoint_remove_by_ref(env, wp);
1478
    }
1479
}
1480
#endif
1481

1482 1483 1484
/* Add a breakpoint.  */
int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
                          CPUBreakpoint **breakpoint)
B
bellard 已提交
1485
{
B
bellard 已提交
1486
#if defined(TARGET_HAS_ICE)
1487
    CPUBreakpoint *bp;
1488

1489
    bp = qemu_malloc(sizeof(*bp));
B
bellard 已提交
1490

1491 1492 1493
    bp->pc = pc;
    bp->flags = flags;

1494
    /* keep all GDB-injected breakpoints in front */
1495
    if (flags & BP_GDB)
B
Blue Swirl 已提交
1496
        QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1497
    else
B
Blue Swirl 已提交
1498
        QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1499

B
bellard 已提交
1500
    breakpoint_invalidate(env, pc);
1501 1502 1503

    if (breakpoint)
        *breakpoint = bp;
B
bellard 已提交
1504 1505
    return 0;
#else
1506
    return -ENOSYS;
B
bellard 已提交
1507 1508 1509
#endif
}

1510 1511 1512
/* Remove a specific breakpoint.  */
int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
{
1513
#if defined(TARGET_HAS_ICE)
1514 1515
    CPUBreakpoint *bp;

B
Blue Swirl 已提交
1516
    QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1517 1518 1519 1520
        if (bp->pc == pc && bp->flags == flags) {
            cpu_breakpoint_remove_by_ref(env, bp);
            return 0;
        }
1521
    }
1522 1523 1524
    return -ENOENT;
#else
    return -ENOSYS;
1525 1526 1527
#endif
}

1528 1529
/* Remove a specific breakpoint by reference.  */
void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
B
bellard 已提交
1530
{
B
bellard 已提交
1531
#if defined(TARGET_HAS_ICE)
B
Blue Swirl 已提交
1532
    QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
B
bellard 已提交
1533

1534 1535 1536 1537 1538 1539 1540 1541 1542 1543
    breakpoint_invalidate(env, breakpoint->pc);

    qemu_free(breakpoint);
#endif
}

/* Remove all matching breakpoints. */
void cpu_breakpoint_remove_all(CPUState *env, int mask)
{
#if defined(TARGET_HAS_ICE)
1544
    CPUBreakpoint *bp, *next;
1545

B
Blue Swirl 已提交
1546
    QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1547 1548
        if (bp->flags & mask)
            cpu_breakpoint_remove_by_ref(env, bp);
1549
    }
B
bellard 已提交
1550 1551 1552
#endif
}

B
bellard 已提交
1553 1554 1555 1556
/* enable or disable single step mode. EXCP_DEBUG is returned by the
   CPU loop after each instruction */
void cpu_single_step(CPUState *env, int enabled)
{
B
bellard 已提交
1557
#if defined(TARGET_HAS_ICE)
B
bellard 已提交
1558 1559
    if (env->singlestep_enabled != enabled) {
        env->singlestep_enabled = enabled;
1560 1561 1562
        if (kvm_enabled())
            kvm_update_guest_debug(env, 0);
        else {
S
Stuart Brady 已提交
1563
            /* must flush all the translated code to avoid inconsistencies */
1564 1565 1566
            /* XXX: only flush what is necessary */
            tb_flush(env);
        }
B
bellard 已提交
1567 1568 1569 1570
    }
#endif
}

1571 1572 1573 1574 1575
/* enable or disable low levels log */
void cpu_set_log(int log_flags)
{
    loglevel = log_flags;
    if (loglevel && !logfile) {
P
pbrook 已提交
1576
        logfile = fopen(logfilename, log_append ? "a" : "w");
1577 1578 1579 1580
        if (!logfile) {
            perror(logfilename);
            _exit(1);
        }
1581 1582 1583
#if !defined(CONFIG_SOFTMMU)
        /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
        {
1584
            static char logfile_buf[4096];
1585 1586
            setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
        }
1587 1588
#elif !defined(_WIN32)
        /* Win32 doesn't support line-buffering and requires size >= 2 */
1589
        setvbuf(logfile, NULL, _IOLBF, 0);
1590
#endif
P
pbrook 已提交
1591 1592 1593 1594 1595
        log_append = 1;
    }
    if (!loglevel && logfile) {
        fclose(logfile);
        logfile = NULL;
1596 1597 1598 1599 1600 1601
    }
}

void cpu_set_log_filename(const char *filename)
{
    logfilename = strdup(filename);
P
pbrook 已提交
1602 1603 1604 1605 1606
    if (logfile) {
        fclose(logfile);
        logfile = NULL;
    }
    cpu_set_log(loglevel);
1607
}
B
bellard 已提交
1608

1609
static void cpu_unlink_tb(CPUState *env)
B
bellard 已提交
1610
{
1611 1612 1613 1614
    /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
       problem and hope the cpu will stop of its own accord.  For userspace
       emulation this often isn't actually as bad as it sounds.  Often
       signals are used primarily to interrupt blocking syscalls.  */
B
bellard 已提交
1615
    TranslationBlock *tb;
A
Anthony Liguori 已提交
1616
    static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1617

R
Riku Voipio 已提交
1618
    spin_lock(&interrupt_lock);
1619 1620 1621
    tb = env->current_tb;
    /* if the cpu is currently executing code, we must unlink it and
       all the potentially executing TB */
1622
    if (tb) {
1623 1624
        env->current_tb = NULL;
        tb_reset_jump_recursive(tb);
1625
    }
R
Riku Voipio 已提交
1626
    spin_unlock(&interrupt_lock);
1627 1628 1629 1630 1631 1632
}

/* mask must never be zero, except for A20 change call */
void cpu_interrupt(CPUState *env, int mask)
{
    int old_mask;
1633

P
pbrook 已提交
1634
    old_mask = env->interrupt_request;
B
bellard 已提交
1635
    env->interrupt_request |= mask;
1636

1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647
#ifndef CONFIG_USER_ONLY
    /*
     * If called from iothread context, wake the target cpu in
     * case its halted.
     */
    if (!qemu_cpu_self(env)) {
        qemu_cpu_kick(env);
        return;
    }
#endif

P
pbrook 已提交
1648
    if (use_icount) {
P
pbrook 已提交
1649
        env->icount_decr.u16.high = 0xffff;
P
pbrook 已提交
1650 1651
#ifndef CONFIG_USER_ONLY
        if (!can_do_io(env)
1652
            && (mask & ~old_mask) != 0) {
P
pbrook 已提交
1653 1654 1655 1656
            cpu_abort(env, "Raised interrupt while not in I/O function");
        }
#endif
    } else {
1657
        cpu_unlink_tb(env);
B
bellard 已提交
1658 1659 1660
    }
}

1661 1662 1663 1664 1665
void cpu_reset_interrupt(CPUState *env, int mask)
{
    env->interrupt_request &= ~mask;
}

1666 1667 1668 1669 1670 1671
void cpu_exit(CPUState *env)
{
    env->exit_request = 1;
    cpu_unlink_tb(env);
}

B
blueswir1 已提交
1672
const CPULogItem cpu_log_items[] = {
1673
    { CPU_LOG_TB_OUT_ASM, "out_asm",
1674 1675 1676
      "show generated host assembly code for each compiled TB" },
    { CPU_LOG_TB_IN_ASM, "in_asm",
      "show target assembly code for each compiled TB" },
1677
    { CPU_LOG_TB_OP, "op",
B
bellard 已提交
1678
      "show micro ops for each compiled TB" },
1679
    { CPU_LOG_TB_OP_OPT, "op_opt",
B
blueswir1 已提交
1680 1681 1682
      "show micro ops "
#ifdef TARGET_I386
      "before eflags optimization and "
1683
#endif
B
blueswir1 已提交
1684
      "after liveness analysis" },
1685 1686 1687 1688
    { CPU_LOG_INT, "int",
      "show interrupts/exceptions in short format" },
    { CPU_LOG_EXEC, "exec",
      "show trace before each executed TB (lots of logs)" },
1689
    { CPU_LOG_TB_CPU, "cpu",
T
ths 已提交
1690
      "show CPU state before block translation" },
1691 1692 1693
#ifdef TARGET_I386
    { CPU_LOG_PCALL, "pcall",
      "show protected mode far calls/returns/exceptions" },
A
aliguori 已提交
1694 1695
    { CPU_LOG_RESET, "cpu_reset",
      "show CPU state before CPU resets" },
1696
#endif
B
bellard 已提交
1697
#ifdef DEBUG_IOPORT
1698 1699
    { CPU_LOG_IOPORT, "ioport",
      "show all i/o ports accesses" },
B
bellard 已提交
1700
#endif
1701 1702 1703
    { 0, NULL, NULL },
};

M
Michael S. Tsirkin 已提交
1704 1705 1706 1707 1708
#ifndef CONFIG_USER_ONLY
static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
    = QLIST_HEAD_INITIALIZER(memory_client_list);

static void cpu_notify_set_memory(target_phys_addr_t start_addr,
Y
Yoshiaki Tamura 已提交
1709 1710
                                  ram_addr_t size,
                                  ram_addr_t phys_offset)
M
Michael S. Tsirkin 已提交
1711 1712 1713 1714 1715 1716 1717 1718
{
    CPUPhysMemoryClient *client;
    QLIST_FOREACH(client, &memory_client_list, list) {
        client->set_memory(client, start_addr, size, phys_offset);
    }
}

static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
Y
Yoshiaki Tamura 已提交
1719
                                        target_phys_addr_t end)
M
Michael S. Tsirkin 已提交
1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740
{
    CPUPhysMemoryClient *client;
    QLIST_FOREACH(client, &memory_client_list, list) {
        int r = client->sync_dirty_bitmap(client, start, end);
        if (r < 0)
            return r;
    }
    return 0;
}

static int cpu_notify_migration_log(int enable)
{
    CPUPhysMemoryClient *client;
    QLIST_FOREACH(client, &memory_client_list, list) {
        int r = client->migration_log(client, enable);
        if (r < 0)
            return r;
    }
    return 0;
}

1741 1742
static void phys_page_for_each_1(CPUPhysMemoryClient *client,
                                 int level, void **lp)
M
Michael S. Tsirkin 已提交
1743
{
1744
    int i;
M
Michael S. Tsirkin 已提交
1745

1746 1747 1748 1749 1750
    if (*lp == NULL) {
        return;
    }
    if (level == 0) {
        PhysPageDesc *pd = *lp;
P
Paul Brook 已提交
1751
        for (i = 0; i < L2_SIZE; ++i) {
1752 1753 1754
            if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
                client->set_memory(client, pd[i].region_offset,
                                   TARGET_PAGE_SIZE, pd[i].phys_offset);
M
Michael S. Tsirkin 已提交
1755
            }
1756 1757 1758
        }
    } else {
        void **pp = *lp;
P
Paul Brook 已提交
1759
        for (i = 0; i < L2_SIZE; ++i) {
1760
            phys_page_for_each_1(client, level - 1, pp + i);
M
Michael S. Tsirkin 已提交
1761 1762 1763 1764 1765 1766
        }
    }
}

static void phys_page_for_each(CPUPhysMemoryClient *client)
{
1767 1768 1769 1770
    int i;
    for (i = 0; i < P_L1_SIZE; ++i) {
        phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
                             l1_phys_map + 1);
M
Michael S. Tsirkin 已提交
1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785
    }
}

void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
{
    QLIST_INSERT_HEAD(&memory_client_list, client, list);
    phys_page_for_each(client);
}

void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
{
    QLIST_REMOVE(client, list);
}
#endif

1786 1787 1788 1789 1790 1791
static int cmp1(const char *s1, int n, const char *s2)
{
    if (strlen(s2) != n)
        return 0;
    return memcmp(s1, s2, n) == 0;
}
1792

1793 1794 1795
/* takes a comma separated list of log masks. Return 0 if error. */
int cpu_str_to_log_mask(const char *str)
{
B
blueswir1 已提交
1796
    const CPULogItem *item;
1797 1798 1799 1800 1801 1802 1803 1804 1805
    int mask;
    const char *p, *p1;

    p = str;
    mask = 0;
    for(;;) {
        p1 = strchr(p, ',');
        if (!p1)
            p1 = p + strlen(p);
Y
Yoshiaki Tamura 已提交
1806 1807 1808 1809 1810 1811 1812 1813 1814 1815
        if(cmp1(p,p1-p,"all")) {
            for(item = cpu_log_items; item->mask != 0; item++) {
                mask |= item->mask;
            }
        } else {
            for(item = cpu_log_items; item->mask != 0; item++) {
                if (cmp1(p, p1 - p, item->name))
                    goto found;
            }
            return 0;
1816 1817 1818 1819 1820 1821 1822 1823 1824
        }
    found:
        mask |= item->mask;
        if (*p1 != ',')
            break;
        p = p1 + 1;
    }
    return mask;
}
B
bellard 已提交
1825

B
bellard 已提交
1826 1827 1828
void cpu_abort(CPUState *env, const char *fmt, ...)
{
    va_list ap;
P
pbrook 已提交
1829
    va_list ap2;
B
bellard 已提交
1830 1831

    va_start(ap, fmt);
P
pbrook 已提交
1832
    va_copy(ap2, ap);
B
bellard 已提交
1833 1834 1835 1836
    fprintf(stderr, "qemu: fatal: ");
    vfprintf(stderr, fmt, ap);
    fprintf(stderr, "\n");
#ifdef TARGET_I386
B
bellard 已提交
1837 1838 1839
    cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
#else
    cpu_dump_state(env, stderr, fprintf, 0);
B
bellard 已提交
1840
#endif
1841 1842 1843 1844
    if (qemu_log_enabled()) {
        qemu_log("qemu: fatal: ");
        qemu_log_vprintf(fmt, ap2);
        qemu_log("\n");
1845
#ifdef TARGET_I386
1846
        log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1847
#else
1848
        log_cpu_state(env, 0);
1849
#endif
1850
        qemu_log_flush();
1851
        qemu_log_close();
1852
    }
P
pbrook 已提交
1853
    va_end(ap2);
1854
    va_end(ap);
1855 1856 1857 1858 1859 1860 1861 1862
#if defined(CONFIG_USER_ONLY)
    {
        struct sigaction act;
        sigfillset(&act.sa_mask);
        act.sa_handler = SIG_DFL;
        sigaction(SIGABRT, &act, NULL);
    }
#endif
B
bellard 已提交
1863 1864 1865
    abort();
}

1866 1867
CPUState *cpu_copy(CPUState *env)
{
1868
    CPUState *new_env = cpu_init(env->cpu_model_str);
1869 1870
    CPUState *next_cpu = new_env->next_cpu;
    int cpu_index = new_env->cpu_index;
1871 1872 1873 1874 1875
#if defined(TARGET_HAS_ICE)
    CPUBreakpoint *bp;
    CPUWatchpoint *wp;
#endif

1876
    memcpy(new_env, env, sizeof(CPUState));
1877 1878

    /* Preserve chaining and index. */
1879 1880
    new_env->next_cpu = next_cpu;
    new_env->cpu_index = cpu_index;
1881 1882 1883 1884

    /* Clone all break/watchpoints.
       Note: Once we support ptrace with hw-debug register access, make sure
       BP_CPU break/watchpoints are handled correctly on clone. */
B
Blue Swirl 已提交
1885 1886
    QTAILQ_INIT(&env->breakpoints);
    QTAILQ_INIT(&env->watchpoints);
1887
#if defined(TARGET_HAS_ICE)
B
Blue Swirl 已提交
1888
    QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1889 1890
        cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
    }
B
Blue Swirl 已提交
1891
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1892 1893 1894 1895 1896
        cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
                              wp->flags, NULL);
    }
#endif

1897 1898 1899
    return new_env;
}

1900 1901
#if !defined(CONFIG_USER_ONLY)

1902 1903 1904 1905 1906 1907 1908 1909
static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
{
    unsigned int i;

    /* Discard jump cache entries for any tb which might potentially
       overlap the flushed page.  */
    i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
    memset (&env->tb_jmp_cache[i], 0, 
Y
Yoshiaki Tamura 已提交
1910
            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1911 1912 1913

    i = tb_jmp_cache_hash_page(addr);
    memset (&env->tb_jmp_cache[i], 0, 
Y
Yoshiaki Tamura 已提交
1914
            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1915 1916
}

I
Igor Kovalenko 已提交
1917 1918 1919 1920 1921 1922 1923
static CPUTLBEntry s_cputlb_empty_entry = {
    .addr_read  = -1,
    .addr_write = -1,
    .addr_code  = -1,
    .addend     = -1,
};

1924 1925 1926
/* NOTE: if flush_global is true, also flush global entries (not
   implemented yet) */
void tlb_flush(CPUState *env, int flush_global)
1927 1928
{
    int i;
1929

1930 1931 1932
#if defined(DEBUG_TLB)
    printf("tlb_flush:\n");
#endif
1933 1934 1935 1936
    /* must reset current TB so that interrupts cannot modify the
       links while we are modifying them */
    env->current_tb = NULL;

1937
    for(i = 0; i < CPU_TLB_SIZE; i++) {
1938 1939
        int mmu_idx;
        for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
I
Igor Kovalenko 已提交
1940
            env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1941
        }
1942
    }
1943

1944
    memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1945

P
Paul Brook 已提交
1946 1947
    env->tlb_flush_addr = -1;
    env->tlb_flush_mask = 0;
B
bellard 已提交
1948
    tlb_flush_count++;
1949 1950
}

B
bellard 已提交
1951
static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
B
bellard 已提交
1952
{
1953
    if (addr == (tlb_entry->addr_read &
B
bellard 已提交
1954
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1955
        addr == (tlb_entry->addr_write &
B
bellard 已提交
1956
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1957
        addr == (tlb_entry->addr_code &
B
bellard 已提交
1958
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
I
Igor Kovalenko 已提交
1959
        *tlb_entry = s_cputlb_empty_entry;
B
bellard 已提交
1960
    }
B
bellard 已提交
1961 1962
}

1963
void tlb_flush_page(CPUState *env, target_ulong addr)
1964
{
1965
    int i;
1966
    int mmu_idx;
1967

1968
#if defined(DEBUG_TLB)
1969
    printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1970
#endif
P
Paul Brook 已提交
1971 1972 1973 1974 1975 1976 1977 1978 1979 1980
    /* Check if we need to flush due to large pages.  */
    if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
#if defined(DEBUG_TLB)
        printf("tlb_flush_page: forced full flush ("
               TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
               env->tlb_flush_addr, env->tlb_flush_mask);
#endif
        tlb_flush(env, 1);
        return;
    }
1981 1982 1983
    /* must reset current TB so that interrupts cannot modify the
       links while we are modifying them */
    env->current_tb = NULL;
B
bellard 已提交
1984 1985 1986

    addr &= TARGET_PAGE_MASK;
    i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1987 1988
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
        tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
1989

1990
    tlb_flush_jmp_cache(env, addr);
1991 1992 1993 1994
}

/* update the TLBs so that writes to code in the virtual page 'addr'
   can be detected */
A
Anthony Liguori 已提交
1995
static void tlb_protect_code(ram_addr_t ram_addr)
1996
{
1997
    cpu_physical_memory_reset_dirty(ram_addr,
B
bellard 已提交
1998 1999
                                    ram_addr + TARGET_PAGE_SIZE,
                                    CODE_DIRTY_FLAG);
2000 2001 2002
}

/* update the TLB so that writes in physical page 'phys_addr' are no longer
2003
   tested for self modifying code */
A
Anthony Liguori 已提交
2004
static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
2005
                                    target_ulong vaddr)
2006
{
2007
    cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
2008 2009
}

2010
static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
2011 2012 2013
                                         unsigned long start, unsigned long length)
{
    unsigned long addr;
B
bellard 已提交
2014 2015
    if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
        addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2016
        if ((addr - start) < length) {
P
pbrook 已提交
2017
            tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
2018 2019 2020 2021
        }
    }
}

P
pbrook 已提交
2022
/* Note: start and end must be within the same ram block.  */
A
Anthony Liguori 已提交
2023
void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
B
bellard 已提交
2024
                                     int dirty_flags)
2025 2026
{
    CPUState *env;
B
bellard 已提交
2027
    unsigned long length, start1;
2028
    int i;
2029 2030 2031 2032 2033 2034 2035

    start &= TARGET_PAGE_MASK;
    end = TARGET_PAGE_ALIGN(end);

    length = end - start;
    if (length == 0)
        return;
2036
    cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
B
bellard 已提交
2037

2038 2039
    /* we modify the TLB cache so that the dirty bit will be set again
       when accessing the range */
P
pbrook 已提交
2040 2041 2042 2043 2044 2045 2046 2047
    start1 = (unsigned long)qemu_get_ram_ptr(start);
    /* Chek that we don't span multiple blocks - this breaks the
       address comparisons below.  */
    if ((unsigned long)qemu_get_ram_ptr(end - 1) - start1
            != (end - 1) - start) {
        abort();
    }

B
bellard 已提交
2048
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
2049 2050 2051 2052 2053 2054
        int mmu_idx;
        for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
            for(i = 0; i < CPU_TLB_SIZE; i++)
                tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
                                      start1, length);
        }
B
bellard 已提交
2055
    }
2056 2057
}

A
aliguori 已提交
2058 2059
int cpu_physical_memory_set_dirty_tracking(int enable)
{
M
Michael S. Tsirkin 已提交
2060
    int ret = 0;
A
aliguori 已提交
2061
    in_migration = enable;
M
Michael S. Tsirkin 已提交
2062 2063
    ret = cpu_notify_migration_log(!!enable);
    return ret;
A
aliguori 已提交
2064 2065 2066 2067 2068 2069 2070
}

int cpu_physical_memory_get_dirty_tracking(void)
{
    return in_migration;
}

A
Anthony Liguori 已提交
2071 2072
int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
                                   target_phys_addr_t end_addr)
A
aliguori 已提交
2073
{
2074
    int ret;
2075

M
Michael S. Tsirkin 已提交
2076
    ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
2077
    return ret;
A
aliguori 已提交
2078 2079
}

2080 2081
static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
{
A
Anthony Liguori 已提交
2082
    ram_addr_t ram_addr;
P
pbrook 已提交
2083
    void *p;
2084

B
bellard 已提交
2085
    if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
P
pbrook 已提交
2086 2087 2088
        p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
            + tlb_entry->addend);
        ram_addr = qemu_ram_addr_from_host(p);
2089
        if (!cpu_physical_memory_is_dirty(ram_addr)) {
P
pbrook 已提交
2090
            tlb_entry->addr_write |= TLB_NOTDIRTY;
2091 2092 2093 2094 2095 2096 2097 2098
        }
    }
}

/* update the TLB according to the current state of the dirty bits */
void cpu_tlb_update_dirty(CPUState *env)
{
    int i;
2099 2100 2101 2102 2103
    int mmu_idx;
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
        for(i = 0; i < CPU_TLB_SIZE; i++)
            tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
    }
2104 2105
}

P
pbrook 已提交
2106
static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2107
{
P
pbrook 已提交
2108 2109
    if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
        tlb_entry->addr_write = vaddr;
2110 2111
}

P
pbrook 已提交
2112 2113 2114
/* update the TLB corresponding to virtual page vaddr
   so that it is no longer dirty */
static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2115 2116
{
    int i;
2117
    int mmu_idx;
2118

P
pbrook 已提交
2119
    vaddr &= TARGET_PAGE_MASK;
2120
    i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2121 2122
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
        tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2123 2124
}

P
Paul Brook 已提交
2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153
/* Our TLB does not support large pages, so remember the area covered by
   large pages and trigger a full TLB flush if these are invalidated.  */
static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
                               target_ulong size)
{
    target_ulong mask = ~(size - 1);

    if (env->tlb_flush_addr == (target_ulong)-1) {
        env->tlb_flush_addr = vaddr & mask;
        env->tlb_flush_mask = mask;
        return;
    }
    /* Extend the existing region to include the new page.
       This is a compromise between unnecessary flushes and the cost
       of maintaining a full variable size TLB.  */
    mask &= env->tlb_flush_mask;
    while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
        mask <<= 1;
    }
    env->tlb_flush_addr &= mask;
    env->tlb_flush_mask = mask;
}

/* Add a new TLB entry. At most one entry for a given virtual address
   is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
   supplied size is only used by tlb_flush_page.  */
void tlb_set_page(CPUState *env, target_ulong vaddr,
                  target_phys_addr_t paddr, int prot,
                  int mmu_idx, target_ulong size)
2154
{
B
bellard 已提交
2155
    PhysPageDesc *p;
B
bellard 已提交
2156
    unsigned long pd;
2157
    unsigned int index;
B
bellard 已提交
2158
    target_ulong address;
P
pbrook 已提交
2159
    target_ulong code_address;
2160
    unsigned long addend;
B
bellard 已提交
2161
    CPUTLBEntry *te;
2162
    CPUWatchpoint *wp;
A
Anthony Liguori 已提交
2163
    target_phys_addr_t iotlb;
2164

P
Paul Brook 已提交
2165 2166 2167 2168
    assert(size >= TARGET_PAGE_SIZE);
    if (size != TARGET_PAGE_SIZE) {
        tlb_add_large_page(env, vaddr, size);
    }
B
bellard 已提交
2169
    p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2170 2171 2172 2173 2174 2175
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
#if defined(DEBUG_TLB)
2176 2177
    printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
           vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
2178 2179
#endif

P
pbrook 已提交
2180 2181 2182 2183 2184
    address = vaddr;
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
        /* IO memory case (romd handled later) */
        address |= TLB_MMIO;
    }
P
pbrook 已提交
2185
    addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
P
pbrook 已提交
2186 2187 2188 2189 2190 2191 2192 2193
    if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
        /* Normal RAM.  */
        iotlb = pd & TARGET_PAGE_MASK;
        if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
            iotlb |= IO_MEM_NOTDIRTY;
        else
            iotlb |= IO_MEM_ROM;
    } else {
S
Stuart Brady 已提交
2194
        /* IO handlers are currently passed a physical address.
P
pbrook 已提交
2195 2196 2197 2198 2199
           It would be nice to pass an offset from the base address
           of that region.  This would avoid having to special case RAM,
           and avoid full address decoding in every device.
           We can't use the high bits of pd for this because
           IO_MEM_ROMD uses these as a ram address.  */
2200 2201 2202 2203 2204 2205
        iotlb = (pd & ~TARGET_PAGE_MASK);
        if (p) {
            iotlb += p->region_offset;
        } else {
            iotlb += paddr;
        }
P
pbrook 已提交
2206 2207 2208 2209 2210
    }

    code_address = address;
    /* Make accesses to pages with watchpoints go via the
       watchpoint trap routines.  */
B
Blue Swirl 已提交
2211
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2212
        if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
J
Jun Koi 已提交
2213 2214 2215 2216 2217 2218
            /* Avoid trapping reads of pages with a write breakpoint. */
            if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
                iotlb = io_mem_watch + paddr;
                address |= TLB_MMIO;
                break;
            }
2219
        }
P
pbrook 已提交
2220
    }
2221

P
pbrook 已提交
2222 2223 2224 2225 2226 2227 2228 2229 2230
    index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    env->iotlb[mmu_idx][index] = iotlb - vaddr;
    te = &env->tlb_table[mmu_idx][index];
    te->addend = addend - vaddr;
    if (prot & PAGE_READ) {
        te->addr_read = address;
    } else {
        te->addr_read = -1;
    }
2231

P
pbrook 已提交
2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244
    if (prot & PAGE_EXEC) {
        te->addr_code = code_address;
    } else {
        te->addr_code = -1;
    }
    if (prot & PAGE_WRITE) {
        if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
            (pd & IO_MEM_ROMD)) {
            /* Write access calls the I/O callback.  */
            te->addr_write = address | TLB_MMIO;
        } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
                   !cpu_physical_memory_is_dirty(pd)) {
            te->addr_write = address | TLB_NOTDIRTY;
2245
        } else {
P
pbrook 已提交
2246
            te->addr_write = address;
2247
        }
P
pbrook 已提交
2248 2249
    } else {
        te->addr_write = -1;
2250 2251 2252
    }
}

2253 2254
#else

2255
void tlb_flush(CPUState *env, int flush_global)
2256 2257 2258
{
}

2259
void tlb_flush_page(CPUState *env, target_ulong addr)
2260 2261 2262
{
}

2263 2264 2265 2266
/*
 * Walks guest process memory "regions" one by one
 * and calls callback function 'fn' for each region.
 */
2267 2268 2269 2270 2271 2272 2273 2274 2275 2276

struct walk_memory_regions_data
{
    walk_memory_regions_fn fn;
    void *priv;
    unsigned long start;
    int prot;
};

static int walk_memory_regions_end(struct walk_memory_regions_data *data,
P
Paul Brook 已提交
2277
                                   abi_ulong end, int new_prot)
2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292
{
    if (data->start != -1ul) {
        int rc = data->fn(data->priv, data->start, end, data->prot);
        if (rc != 0) {
            return rc;
        }
    }

    data->start = (new_prot ? end : -1ul);
    data->prot = new_prot;

    return 0;
}

static int walk_memory_regions_1(struct walk_memory_regions_data *data,
P
Paul Brook 已提交
2293
                                 abi_ulong base, int level, void **lp)
2294
{
P
Paul Brook 已提交
2295
    abi_ulong pa;
2296 2297 2298 2299 2300 2301 2302 2303
    int i, rc;

    if (*lp == NULL) {
        return walk_memory_regions_end(data, base, 0);
    }

    if (level == 0) {
        PageDesc *pd = *lp;
P
Paul Brook 已提交
2304
        for (i = 0; i < L2_SIZE; ++i) {
2305 2306 2307 2308 2309 2310 2311
            int prot = pd[i].flags;

            pa = base | (i << TARGET_PAGE_BITS);
            if (prot != data->prot) {
                rc = walk_memory_regions_end(data, pa, prot);
                if (rc != 0) {
                    return rc;
2312 2313
                }
            }
2314 2315 2316
        }
    } else {
        void **pp = *lp;
P
Paul Brook 已提交
2317
        for (i = 0; i < L2_SIZE; ++i) {
P
Paul Brook 已提交
2318 2319
            pa = base | ((abi_ulong)i <<
                (TARGET_PAGE_BITS + L2_BITS * level));
2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340
            rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
            if (rc != 0) {
                return rc;
            }
        }
    }

    return 0;
}

int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
{
    struct walk_memory_regions_data data;
    unsigned long i;

    data.fn = fn;
    data.priv = priv;
    data.start = -1ul;
    data.prot = 0;

    for (i = 0; i < V_L1_SIZE; i++) {
P
Paul Brook 已提交
2341
        int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
2342 2343 2344
                                       V_L1_SHIFT / L2_BITS - 1, l1_map + i);
        if (rc != 0) {
            return rc;
2345
        }
2346
    }
2347 2348

    return walk_memory_regions_end(&data, 0, 0);
2349 2350
}

P
Paul Brook 已提交
2351 2352
static int dump_region(void *priv, abi_ulong start,
    abi_ulong end, unsigned long prot)
2353 2354 2355
{
    FILE *f = (FILE *)priv;

P
Paul Brook 已提交
2356 2357
    (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
        " "TARGET_ABI_FMT_lx" %c%c%c\n",
2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371
        start, end, end - start,
        ((prot & PAGE_READ) ? 'r' : '-'),
        ((prot & PAGE_WRITE) ? 'w' : '-'),
        ((prot & PAGE_EXEC) ? 'x' : '-'));

    return (0);
}

/* dump memory mappings */
void page_dump(FILE *f)
{
    (void) fprintf(f, "%-8s %-8s %-8s %s\n",
            "start", "end", "size", "prot");
    walk_memory_regions(f, dump_region);
2372 2373
}

2374
int page_get_flags(target_ulong address)
2375
{
2376 2377 2378
    PageDesc *p;

    p = page_find(address >> TARGET_PAGE_BITS);
2379
    if (!p)
2380 2381 2382 2383
        return 0;
    return p->flags;
}

2384 2385 2386
/* Modify the flags of a page and invalidate the code if necessary.
   The flag PAGE_WRITE_ORG is positioned automatically depending
   on PAGE_WRITE.  The mmap_lock should already be held.  */
2387
void page_set_flags(target_ulong start, target_ulong end, int flags)
2388
{
2389 2390 2391 2392 2393
    target_ulong addr, len;

    /* This function should never be called with addresses outside the
       guest address space.  If this assert fires, it probably indicates
       a missing call to h2g_valid.  */
P
Paul Brook 已提交
2394 2395
#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
    assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2396 2397
#endif
    assert(start < end);
2398 2399 2400

    start = start & TARGET_PAGE_MASK;
    end = TARGET_PAGE_ALIGN(end);
2401 2402

    if (flags & PAGE_WRITE) {
2403
        flags |= PAGE_WRITE_ORG;
2404 2405 2406 2407 2408 2409 2410 2411 2412
    }

    for (addr = start, len = end - start;
         len != 0;
         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
        PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);

        /* If the write protection bit is set, then we invalidate
           the code inside.  */
2413
        if (!(p->flags & PAGE_WRITE) &&
2414 2415
            (flags & PAGE_WRITE) &&
            p->first_tb) {
B
bellard 已提交
2416
            tb_invalidate_phys_page(addr, 0, NULL);
2417 2418 2419
        }
        p->flags = flags;
    }
2420 2421
}

2422 2423 2424 2425 2426 2427
int page_check_range(target_ulong start, target_ulong len, int flags)
{
    PageDesc *p;
    target_ulong end;
    target_ulong addr;

2428 2429 2430
    /* This function should never be called with addresses outside the
       guest address space.  If this assert fires, it probably indicates
       a missing call to h2g_valid.  */
2431 2432
#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
    assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2433 2434
#endif

R
Richard Henderson 已提交
2435 2436 2437
    if (len == 0) {
        return 0;
    }
2438 2439
    if (start + len - 1 < start) {
        /* We've wrapped around.  */
2440
        return -1;
2441
    }
2442

2443 2444 2445
    end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
    start = start & TARGET_PAGE_MASK;

2446 2447 2448
    for (addr = start, len = end - start;
         len != 0;
         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2449 2450 2451 2452 2453 2454
        p = page_find(addr >> TARGET_PAGE_BITS);
        if( !p )
            return -1;
        if( !(p->flags & PAGE_VALID) )
            return -1;

2455
        if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2456
            return -1;
2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467
        if (flags & PAGE_WRITE) {
            if (!(p->flags & PAGE_WRITE_ORG))
                return -1;
            /* unprotect the page if it was put read-only because it
               contains translated code */
            if (!(p->flags & PAGE_WRITE)) {
                if (!page_unprotect(addr, 0, NULL))
                    return -1;
            }
            return 0;
        }
2468 2469 2470 2471
    }
    return 0;
}

2472
/* called from signal handler: invalidate the code and unprotect the
S
Stuart Brady 已提交
2473
   page. Return TRUE if the fault was successfully handled. */
2474
int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2475
{
2476 2477
    unsigned int prot;
    PageDesc *p;
2478
    target_ulong host_start, host_end, addr;
2479

P
pbrook 已提交
2480 2481 2482 2483 2484
    /* Technically this isn't safe inside a signal handler.  However we
       know this only ever happens in a synchronous SEGV handler, so in
       practice it seems to be ok.  */
    mmap_lock();

2485 2486
    p = page_find(address >> TARGET_PAGE_BITS);
    if (!p) {
P
pbrook 已提交
2487
        mmap_unlock();
2488
        return 0;
P
pbrook 已提交
2489
    }
2490

2491 2492
    /* if the page was really writable, then we change its
       protection back to writable */
2493 2494 2495 2496 2497 2498 2499 2500 2501 2502
    if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
        host_start = address & qemu_host_page_mask;
        host_end = host_start + qemu_host_page_size;

        prot = 0;
        for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
            p = page_find(addr >> TARGET_PAGE_BITS);
            p->flags |= PAGE_WRITE;
            prot |= p->flags;

2503 2504
            /* and since the content will be modified, we must invalidate
               the corresponding translated code. */
2505
            tb_invalidate_phys_page(addr, pc, puc);
2506
#ifdef DEBUG_TB_CHECK
2507
            tb_invalidate_check(addr);
2508 2509
#endif
        }
2510 2511 2512 2513 2514
        mprotect((void *)g2h(host_start), qemu_host_page_size,
                 prot & PAGE_BITS);

        mmap_unlock();
        return 1;
2515
    }
P
pbrook 已提交
2516
    mmap_unlock();
2517 2518 2519
    return 0;
}

B
bellard 已提交
2520 2521
static inline void tlb_set_dirty(CPUState *env,
                                 unsigned long addr, target_ulong vaddr)
2522 2523
{
}
2524 2525
#endif /* defined(CONFIG_USER_ONLY) */

2526
#if !defined(CONFIG_USER_ONLY)
2527

P
Paul Brook 已提交
2528 2529 2530
#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
typedef struct subpage_t {
    target_phys_addr_t base;
R
Richard Henderson 已提交
2531 2532
    ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
    ram_addr_t region_offset[TARGET_PAGE_SIZE];
P
Paul Brook 已提交
2533 2534
} subpage_t;

A
Anthony Liguori 已提交
2535 2536
static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
                             ram_addr_t memory, ram_addr_t region_offset);
R
Richard Henderson 已提交
2537 2538 2539
static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
                                ram_addr_t orig_memory,
                                ram_addr_t region_offset);
2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550
#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
                      need_subpage)                                     \
    do {                                                                \
        if (addr > start_addr)                                          \
            start_addr2 = 0;                                            \
        else {                                                          \
            start_addr2 = start_addr & ~TARGET_PAGE_MASK;               \
            if (start_addr2 > 0)                                        \
                need_subpage = 1;                                       \
        }                                                               \
                                                                        \
2551
        if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE)        \
2552 2553 2554 2555 2556 2557 2558 2559
            end_addr2 = TARGET_PAGE_SIZE - 1;                           \
        else {                                                          \
            end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
            if (end_addr2 < TARGET_PAGE_SIZE - 1)                       \
                need_subpage = 1;                                       \
        }                                                               \
    } while (0)

2560 2561 2562
/* register physical memory.
   For RAM, 'size' must be a multiple of the target page size.
   If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2563 2564
   io memory page.  The address used when calling the IO function is
   the offset from the start of the region, plus region_offset.  Both
S
Stuart Brady 已提交
2565
   start_addr and region_offset are rounded down to a page boundary
2566 2567
   before calculating this offset.  This should not be a problem unless
   the low bits of start_addr and region_offset differ.  */
A
Anthony Liguori 已提交
2568 2569 2570 2571
void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
                                         ram_addr_t size,
                                         ram_addr_t phys_offset,
                                         ram_addr_t region_offset)
2572
{
A
Anthony Liguori 已提交
2573
    target_phys_addr_t addr, end_addr;
B
bellard 已提交
2574
    PhysPageDesc *p;
2575
    CPUState *env;
A
Anthony Liguori 已提交
2576
    ram_addr_t orig_size = size;
R
Richard Henderson 已提交
2577
    subpage_t *subpage;
2578

M
Michael S. Tsirkin 已提交
2579 2580
    cpu_notify_set_memory(start_addr, size, phys_offset);

P
pbrook 已提交
2581 2582 2583
    if (phys_offset == IO_MEM_UNASSIGNED) {
        region_offset = start_addr;
    }
2584
    region_offset &= TARGET_PAGE_MASK;
B
bellard 已提交
2585
    size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
A
Anthony Liguori 已提交
2586
    end_addr = start_addr + (target_phys_addr_t)size;
2587
    for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2588 2589
        p = phys_page_find(addr >> TARGET_PAGE_BITS);
        if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
A
Anthony Liguori 已提交
2590 2591
            ram_addr_t orig_memory = p->phys_offset;
            target_phys_addr_t start_addr2, end_addr2;
2592 2593 2594 2595
            int need_subpage = 0;

            CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
                          need_subpage);
R
Richard Henderson 已提交
2596
            if (need_subpage) {
2597 2598
                if (!(orig_memory & IO_MEM_SUBPAGE)) {
                    subpage = subpage_init((addr & TARGET_PAGE_MASK),
2599 2600
                                           &p->phys_offset, orig_memory,
                                           p->region_offset);
2601 2602 2603 2604
                } else {
                    subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
                                            >> IO_MEM_SHIFT];
                }
2605 2606 2607
                subpage_register(subpage, start_addr2, end_addr2, phys_offset,
                                 region_offset);
                p->region_offset = 0;
2608 2609 2610 2611 2612 2613 2614 2615 2616
            } else {
                p->phys_offset = phys_offset;
                if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
                    (phys_offset & IO_MEM_ROMD))
                    phys_offset += TARGET_PAGE_SIZE;
            }
        } else {
            p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
            p->phys_offset = phys_offset;
2617
            p->region_offset = region_offset;
2618
            if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2619
                (phys_offset & IO_MEM_ROMD)) {
2620
                phys_offset += TARGET_PAGE_SIZE;
P
pbrook 已提交
2621
            } else {
A
Anthony Liguori 已提交
2622
                target_phys_addr_t start_addr2, end_addr2;
2623 2624 2625 2626 2627
                int need_subpage = 0;

                CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
                              end_addr2, need_subpage);

R
Richard Henderson 已提交
2628
                if (need_subpage) {
2629
                    subpage = subpage_init((addr & TARGET_PAGE_MASK),
2630
                                           &p->phys_offset, IO_MEM_UNASSIGNED,
P
pbrook 已提交
2631
                                           addr & TARGET_PAGE_MASK);
2632
                    subpage_register(subpage, start_addr2, end_addr2,
2633 2634
                                     phys_offset, region_offset);
                    p->region_offset = 0;
2635 2636 2637
                }
            }
        }
2638
        region_offset += TARGET_PAGE_SIZE;
2639
    }
2640

2641 2642 2643 2644 2645 2646
    /* since each CPU stores ram addresses in its TLB cache, we must
       reset the modified entries */
    /* XXX: slow ! */
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
        tlb_flush(env, 1);
    }
2647 2648
}

B
bellard 已提交
2649
/* XXX: temporary until new memory mapping API */
A
Anthony Liguori 已提交
2650
ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
B
bellard 已提交
2651 2652 2653 2654 2655 2656 2657 2658 2659
{
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p)
        return IO_MEM_UNASSIGNED;
    return p->phys_offset;
}

A
Anthony Liguori 已提交
2660
void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
A
aliguori 已提交
2661 2662 2663 2664 2665
{
    if (kvm_enabled())
        kvm_coalesce_mmio_region(addr, size);
}

A
Anthony Liguori 已提交
2666
void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
A
aliguori 已提交
2667 2668 2669 2670 2671
{
    if (kvm_enabled())
        kvm_uncoalesce_mmio_region(addr, size);
}

2672 2673 2674 2675 2676 2677
void qemu_flush_coalesced_mmio_buffer(void)
{
    if (kvm_enabled())
        kvm_flush_coalesced_mmio_buffer();
}

2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689
#if defined(__linux__) && !defined(TARGET_S390X)

#include <sys/vfs.h>

#define HUGETLBFS_MAGIC       0x958458f6

static long gethugepagesize(const char *path)
{
    struct statfs fs;
    int ret;

    do {
Y
Yoshiaki Tamura 已提交
2690
        ret = statfs(path, &fs);
2691 2692 2693
    } while (ret != 0 && errno == EINTR);

    if (ret != 0) {
Y
Yoshiaki Tamura 已提交
2694 2695
        perror(path);
        return 0;
2696 2697 2698
    }

    if (fs.f_type != HUGETLBFS_MAGIC)
Y
Yoshiaki Tamura 已提交
2699
        fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
2700 2701 2702 2703

    return fs.f_bsize;
}

A
Alex Williamson 已提交
2704 2705 2706
static void *file_ram_alloc(RAMBlock *block,
                            ram_addr_t memory,
                            const char *path)
2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717
{
    char *filename;
    void *area;
    int fd;
#ifdef MAP_POPULATE
    int flags;
#endif
    unsigned long hpagesize;

    hpagesize = gethugepagesize(path);
    if (!hpagesize) {
Y
Yoshiaki Tamura 已提交
2718
        return NULL;
2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730
    }

    if (memory < hpagesize) {
        return NULL;
    }

    if (kvm_enabled() && !kvm_has_sync_mmu()) {
        fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
        return NULL;
    }

    if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
Y
Yoshiaki Tamura 已提交
2731
        return NULL;
2732 2733 2734 2735
    }

    fd = mkstemp(filename);
    if (fd < 0) {
Y
Yoshiaki Tamura 已提交
2736 2737 2738
        perror("unable to create backing store for hugepages");
        free(filename);
        return NULL;
2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751
    }
    unlink(filename);
    free(filename);

    memory = (memory+hpagesize-1) & ~(hpagesize-1);

    /*
     * ftruncate is not supported by hugetlbfs in older
     * hosts, so don't bother bailing out on errors.
     * If anything goes wrong with it under other filesystems,
     * mmap will fail.
     */
    if (ftruncate(fd, memory))
Y
Yoshiaki Tamura 已提交
2752
        perror("ftruncate");
2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764

#ifdef MAP_POPULATE
    /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
     * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
     * to sidestep this quirk.
     */
    flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
    area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
#else
    area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
#endif
    if (area == MAP_FAILED) {
Y
Yoshiaki Tamura 已提交
2765 2766 2767
        perror("file_ram_alloc: can't mmap RAM pages");
        close(fd);
        return (NULL);
2768
    }
A
Alex Williamson 已提交
2769
    block->fd = fd;
2770 2771 2772 2773
    return area;
}
#endif

2774
static ram_addr_t find_ram_offset(ram_addr_t size)
A
Alex Williamson 已提交
2775 2776
{
    RAMBlock *block, *next_block;
2777
    ram_addr_t offset = 0, mingap = ULONG_MAX;
A
Alex Williamson 已提交
2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800

    if (QLIST_EMPTY(&ram_list.blocks))
        return 0;

    QLIST_FOREACH(block, &ram_list.blocks, next) {
        ram_addr_t end, next = ULONG_MAX;

        end = block->offset + block->length;

        QLIST_FOREACH(next_block, &ram_list.blocks, next) {
            if (next_block->offset >= end) {
                next = MIN(next, next_block->offset);
            }
        }
        if (next - end >= size && next - end < mingap) {
            offset =  end;
            mingap = next - end;
        }
    }
    return offset;
}

static ram_addr_t last_ram_offset(void)
2801 2802 2803 2804 2805 2806 2807 2808 2809 2810
{
    RAMBlock *block;
    ram_addr_t last = 0;

    QLIST_FOREACH(block, &ram_list.blocks, next)
        last = MAX(last, block->offset + block->length);

    return last;
}

2811
ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
2812
                                   ram_addr_t size, void *host)
2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835
{
    RAMBlock *new_block, *block;

    size = TARGET_PAGE_ALIGN(size);
    new_block = qemu_mallocz(sizeof(*new_block));

    if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
        char *id = dev->parent_bus->info->get_dev_path(dev);
        if (id) {
            snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
            qemu_free(id);
        }
    }
    pstrcat(new_block->idstr, sizeof(new_block->idstr), name);

    QLIST_FOREACH(block, &ram_list.blocks, next) {
        if (!strcmp(block->idstr, new_block->idstr)) {
            fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
                    new_block->idstr);
            abort();
        }
    }

2836 2837 2838 2839
    if (host) {
        new_block->host = host;
    } else {
        if (mem_path) {
2840
#if defined (__linux__) && !defined(TARGET_S390X)
2841 2842 2843
            new_block->host = file_ram_alloc(new_block, size, mem_path);
            if (!new_block->host) {
                new_block->host = qemu_vmalloc(size);
A
Andreas Färber 已提交
2844
                qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2845
            }
2846
#else
2847 2848
            fprintf(stderr, "-mem-path option unsupported\n");
            exit(1);
2849
#endif
2850
        } else {
2851
#if defined(TARGET_S390X) && defined(CONFIG_KVM)
2852 2853 2854 2855
            /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
            new_block->host = mmap((void*)0x1000000, size,
                                   PROT_EXEC|PROT_READ|PROT_WRITE,
                                   MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2856
#else
2857
            new_block->host = qemu_vmalloc(size);
2858
#endif
A
Andreas Färber 已提交
2859
            qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2860
        }
2861
    }
2862

2863
    new_block->offset = find_ram_offset(size);
P
pbrook 已提交
2864 2865
    new_block->length = size;

A
Alex Williamson 已提交
2866
    QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
P
pbrook 已提交
2867

A
Alex Williamson 已提交
2868
    ram_list.phys_dirty = qemu_realloc(ram_list.phys_dirty,
A
Alex Williamson 已提交
2869
                                       last_ram_offset() >> TARGET_PAGE_BITS);
2870
    memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
P
pbrook 已提交
2871 2872
           0xff, size >> TARGET_PAGE_BITS);

2873 2874 2875
    if (kvm_enabled())
        kvm_setup_guest_memory(new_block->host, size);

P
pbrook 已提交
2876 2877
    return new_block->offset;
}
B
bellard 已提交
2878

2879 2880 2881 2882 2883
ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size)
{
    return qemu_ram_alloc_from_ptr(dev, name, size, NULL);
}

A
Anthony Liguori 已提交
2884
void qemu_ram_free(ram_addr_t addr)
B
bellard 已提交
2885
{
A
Alex Williamson 已提交
2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911
    RAMBlock *block;

    QLIST_FOREACH(block, &ram_list.blocks, next) {
        if (addr == block->offset) {
            QLIST_REMOVE(block, next);
            if (mem_path) {
#if defined (__linux__) && !defined(TARGET_S390X)
                if (block->fd) {
                    munmap(block->host, block->length);
                    close(block->fd);
                } else {
                    qemu_vfree(block->host);
                }
#endif
            } else {
#if defined(TARGET_S390X) && defined(CONFIG_KVM)
                munmap(block->host, block->length);
#else
                qemu_vfree(block->host);
#endif
            }
            qemu_free(block);
            return;
        }
    }

B
bellard 已提交
2912 2913
}

2914
/* Return a host pointer to ram allocated with qemu_ram_alloc.
P
pbrook 已提交
2915 2916 2917 2918 2919 2920 2921
   With the exception of the softmmu code in this file, this should
   only be used for local memory (e.g. video ram) that the device owns,
   and knows it isn't going to access beyond the end of the block.

   It should not be used for general purpose DMA.
   Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
 */
A
Anthony Liguori 已提交
2922
void *qemu_get_ram_ptr(ram_addr_t addr)
2923
{
P
pbrook 已提交
2924 2925
    RAMBlock *block;

A
Alex Williamson 已提交
2926 2927 2928 2929 2930 2931
    QLIST_FOREACH(block, &ram_list.blocks, next) {
        if (addr - block->offset < block->length) {
            QLIST_REMOVE(block, next);
            QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
            return block->host + (addr - block->offset);
        }
P
pbrook 已提交
2932
    }
A
Alex Williamson 已提交
2933 2934 2935 2936 2937

    fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
    abort();

    return NULL;
2938 2939
}

P
pbrook 已提交
2940 2941
/* Some of the softmmu routines need to translate from a host pointer
   (typically a TLB entry) back to a ram offset.  */
A
Anthony Liguori 已提交
2942
ram_addr_t qemu_ram_addr_from_host(void *ptr)
P
pbrook 已提交
2943
{
P
pbrook 已提交
2944 2945 2946
    RAMBlock *block;
    uint8_t *host = ptr;

A
Alex Williamson 已提交
2947 2948 2949 2950
    QLIST_FOREACH(block, &ram_list.blocks, next) {
        if (host - block->host < block->length) {
            return block->offset + (host - block->host);
        }
P
pbrook 已提交
2951
    }
A
Alex Williamson 已提交
2952 2953 2954 2955 2956

    fprintf(stderr, "Bad ram pointer %p\n", ptr);
    abort();

    return 0;
P
pbrook 已提交
2957 2958
}

A
Anthony Liguori 已提交
2959
static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2960
{
P
pbrook 已提交
2961
#ifdef DEBUG_UNASSIGNED
B
blueswir1 已提交
2962
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2963
#endif
2964
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2965 2966 2967 2968 2969
    do_unassigned_access(addr, 0, 0, 0, 1);
#endif
    return 0;
}

A
Anthony Liguori 已提交
2970
static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2971 2972 2973 2974
{
#ifdef DEBUG_UNASSIGNED
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
#endif
2975
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2976 2977 2978 2979 2980
    do_unassigned_access(addr, 0, 0, 0, 2);
#endif
    return 0;
}

A
Anthony Liguori 已提交
2981
static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2982 2983 2984 2985
{
#ifdef DEBUG_UNASSIGNED
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
#endif
2986
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2987
    do_unassigned_access(addr, 0, 0, 0, 4);
P
pbrook 已提交
2988
#endif
2989 2990 2991
    return 0;
}

A
Anthony Liguori 已提交
2992
static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2993
{
P
pbrook 已提交
2994
#ifdef DEBUG_UNASSIGNED
B
blueswir1 已提交
2995
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
P
pbrook 已提交
2996
#endif
2997
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2998 2999 3000 3001
    do_unassigned_access(addr, 1, 0, 0, 1);
#endif
}

A
Anthony Liguori 已提交
3002
static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
3003 3004 3005 3006
{
#ifdef DEBUG_UNASSIGNED
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
#endif
3007
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3008 3009 3010 3011
    do_unassigned_access(addr, 1, 0, 0, 2);
#endif
}

A
Anthony Liguori 已提交
3012
static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
3013 3014 3015 3016
{
#ifdef DEBUG_UNASSIGNED
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
#endif
3017
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3018
    do_unassigned_access(addr, 1, 0, 0, 4);
3019
#endif
3020 3021
}

3022
static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
3023
    unassigned_mem_readb,
3024 3025
    unassigned_mem_readw,
    unassigned_mem_readl,
3026 3027
};

3028
static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
3029
    unassigned_mem_writeb,
3030 3031
    unassigned_mem_writew,
    unassigned_mem_writel,
3032 3033
};

A
Anthony Liguori 已提交
3034
static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
P
pbrook 已提交
3035
                                uint32_t val)
3036
{
3037
    int dirty_flags;
3038
    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3039
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3040
#if !defined(CONFIG_USER_ONLY)
3041
        tb_invalidate_phys_page_fast(ram_addr, 1);
3042
        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3043
#endif
3044
    }
P
pbrook 已提交
3045
    stb_p(qemu_get_ram_ptr(ram_addr), val);
B
bellard 已提交
3046
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3047
    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
B
bellard 已提交
3048 3049 3050
    /* we remove the notdirty callback only if the code has been
       flushed */
    if (dirty_flags == 0xff)
P
pbrook 已提交
3051
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3052 3053
}

A
Anthony Liguori 已提交
3054
static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
P
pbrook 已提交
3055
                                uint32_t val)
3056
{
3057
    int dirty_flags;
3058
    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3059
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3060
#if !defined(CONFIG_USER_ONLY)
3061
        tb_invalidate_phys_page_fast(ram_addr, 2);
3062
        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3063
#endif
3064
    }
P
pbrook 已提交
3065
    stw_p(qemu_get_ram_ptr(ram_addr), val);
B
bellard 已提交
3066
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3067
    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
B
bellard 已提交
3068 3069 3070
    /* we remove the notdirty callback only if the code has been
       flushed */
    if (dirty_flags == 0xff)
P
pbrook 已提交
3071
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3072 3073
}

A
Anthony Liguori 已提交
3074
static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
P
pbrook 已提交
3075
                                uint32_t val)
3076
{
3077
    int dirty_flags;
3078
    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3079
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3080
#if !defined(CONFIG_USER_ONLY)
3081
        tb_invalidate_phys_page_fast(ram_addr, 4);
3082
        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3083
#endif
3084
    }
P
pbrook 已提交
3085
    stl_p(qemu_get_ram_ptr(ram_addr), val);
B
bellard 已提交
3086
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3087
    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
B
bellard 已提交
3088 3089 3090
    /* we remove the notdirty callback only if the code has been
       flushed */
    if (dirty_flags == 0xff)
P
pbrook 已提交
3091
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3092 3093
}

3094
static CPUReadMemoryFunc * const error_mem_read[3] = {
3095 3096 3097 3098 3099
    NULL, /* never used */
    NULL, /* never used */
    NULL, /* never used */
};

3100
static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
3101 3102 3103 3104 3105
    notdirty_mem_writeb,
    notdirty_mem_writew,
    notdirty_mem_writel,
};

P
pbrook 已提交
3106
/* Generate a debug exception if a watchpoint has been hit.  */
3107
static void check_watchpoint(int offset, int len_mask, int flags)
P
pbrook 已提交
3108 3109
{
    CPUState *env = cpu_single_env;
3110 3111
    target_ulong pc, cs_base;
    TranslationBlock *tb;
P
pbrook 已提交
3112
    target_ulong vaddr;
3113
    CPUWatchpoint *wp;
3114
    int cpu_flags;
P
pbrook 已提交
3115

3116 3117 3118 3119 3120 3121 3122
    if (env->watchpoint_hit) {
        /* We re-entered the check after replacing the TB. Now raise
         * the debug interrupt so that is will trigger after the
         * current instruction. */
        cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
        return;
    }
P
pbrook 已提交
3123
    vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
B
Blue Swirl 已提交
3124
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
3125 3126
        if ((vaddr == (wp->vaddr & len_mask) ||
             (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143
            wp->flags |= BP_WATCHPOINT_HIT;
            if (!env->watchpoint_hit) {
                env->watchpoint_hit = wp;
                tb = tb_find_pc(env->mem_io_pc);
                if (!tb) {
                    cpu_abort(env, "check_watchpoint: could not find TB for "
                              "pc=%p", (void *)env->mem_io_pc);
                }
                cpu_restore_state(tb, env, env->mem_io_pc, NULL);
                tb_phys_invalidate(tb, -1);
                if (wp->flags & BP_STOP_BEFORE_ACCESS) {
                    env->exception_index = EXCP_DEBUG;
                } else {
                    cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
                    tb_gen_code(env, pc, cs_base, cpu_flags, 1);
                }
                cpu_resume_from_signal(env, NULL);
3144
            }
3145 3146
        } else {
            wp->flags &= ~BP_WATCHPOINT_HIT;
P
pbrook 已提交
3147 3148 3149 3150
        }
    }
}

3151 3152 3153
/* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
   so these check for a hit then pass through to the normal out-of-line
   phys routines.  */
A
Anthony Liguori 已提交
3154
static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
3155
{
3156
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
3157 3158 3159
    return ldub_phys(addr);
}

A
Anthony Liguori 已提交
3160
static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
3161
{
3162
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
3163 3164 3165
    return lduw_phys(addr);
}

A
Anthony Liguori 已提交
3166
static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
3167
{
3168
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
3169 3170 3171
    return ldl_phys(addr);
}

A
Anthony Liguori 已提交
3172
static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
3173 3174
                             uint32_t val)
{
3175
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
3176 3177 3178
    stb_phys(addr, val);
}

A
Anthony Liguori 已提交
3179
static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
3180 3181
                             uint32_t val)
{
3182
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
3183 3184 3185
    stw_phys(addr, val);
}

A
Anthony Liguori 已提交
3186
static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
3187 3188
                             uint32_t val)
{
3189
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
3190 3191 3192
    stl_phys(addr, val);
}

3193
static CPUReadMemoryFunc * const watch_mem_read[3] = {
3194 3195 3196 3197 3198
    watch_mem_readb,
    watch_mem_readw,
    watch_mem_readl,
};

3199
static CPUWriteMemoryFunc * const watch_mem_write[3] = {
3200 3201 3202 3203 3204
    watch_mem_writeb,
    watch_mem_writew,
    watch_mem_writel,
};

R
Richard Henderson 已提交
3205 3206 3207
static inline uint32_t subpage_readlen (subpage_t *mmio,
                                        target_phys_addr_t addr,
                                        unsigned int len)
3208
{
R
Richard Henderson 已提交
3209
    unsigned int idx = SUBPAGE_IDX(addr);
3210 3211 3212 3213 3214
#if defined(DEBUG_SUBPAGE)
    printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
           mmio, len, addr, idx);
#endif

R
Richard Henderson 已提交
3215 3216 3217
    addr += mmio->region_offset[idx];
    idx = mmio->sub_io_index[idx];
    return io_mem_read[idx][len](io_mem_opaque[idx], addr);
3218 3219
}

A
Anthony Liguori 已提交
3220
static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
R
Richard Henderson 已提交
3221
                                     uint32_t value, unsigned int len)
3222
{
R
Richard Henderson 已提交
3223
    unsigned int idx = SUBPAGE_IDX(addr);
3224
#if defined(DEBUG_SUBPAGE)
R
Richard Henderson 已提交
3225 3226
    printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n",
           __func__, mmio, len, addr, idx, value);
3227
#endif
R
Richard Henderson 已提交
3228 3229 3230 3231

    addr += mmio->region_offset[idx];
    idx = mmio->sub_io_index[idx];
    io_mem_write[idx][len](io_mem_opaque[idx], addr, value);
3232 3233
}

A
Anthony Liguori 已提交
3234
static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
3235 3236 3237 3238
{
    return subpage_readlen(opaque, addr, 0);
}

A
Anthony Liguori 已提交
3239
static void subpage_writeb (void *opaque, target_phys_addr_t addr,
3240 3241 3242 3243 3244
                            uint32_t value)
{
    subpage_writelen(opaque, addr, value, 0);
}

A
Anthony Liguori 已提交
3245
static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
3246 3247 3248 3249
{
    return subpage_readlen(opaque, addr, 1);
}

A
Anthony Liguori 已提交
3250
static void subpage_writew (void *opaque, target_phys_addr_t addr,
3251 3252 3253 3254 3255
                            uint32_t value)
{
    subpage_writelen(opaque, addr, value, 1);
}

A
Anthony Liguori 已提交
3256
static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
3257 3258 3259 3260
{
    return subpage_readlen(opaque, addr, 2);
}

R
Richard Henderson 已提交
3261 3262
static void subpage_writel (void *opaque, target_phys_addr_t addr,
                            uint32_t value)
3263 3264 3265 3266
{
    subpage_writelen(opaque, addr, value, 2);
}

3267
static CPUReadMemoryFunc * const subpage_read[] = {
3268 3269 3270 3271 3272
    &subpage_readb,
    &subpage_readw,
    &subpage_readl,
};

3273
static CPUWriteMemoryFunc * const subpage_write[] = {
3274 3275 3276 3277 3278
    &subpage_writeb,
    &subpage_writew,
    &subpage_writel,
};

A
Anthony Liguori 已提交
3279 3280
static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
                             ram_addr_t memory, ram_addr_t region_offset)
3281 3282 3283 3284 3285 3286 3287 3288
{
    int idx, eidx;

    if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
        return -1;
    idx = SUBPAGE_IDX(start);
    eidx = SUBPAGE_IDX(end);
#if defined(DEBUG_SUBPAGE)
3289
    printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
3290 3291
           mmio, start, end, idx, eidx, memory);
#endif
3292 3293
    if ((memory & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
        memory = IO_MEM_UNASSIGNED;
R
Richard Henderson 已提交
3294
    memory = (memory >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3295
    for (; idx <= eidx; idx++) {
R
Richard Henderson 已提交
3296 3297
        mmio->sub_io_index[idx] = memory;
        mmio->region_offset[idx] = region_offset;
3298 3299 3300 3301 3302
    }

    return 0;
}

R
Richard Henderson 已提交
3303 3304 3305
static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
                                ram_addr_t orig_memory,
                                ram_addr_t region_offset)
3306
{
A
Anthony Liguori 已提交
3307
    subpage_t *mmio;
3308 3309
    int subpage_memory;

A
Anthony Liguori 已提交
3310
    mmio = qemu_mallocz(sizeof(subpage_t));
3311 3312

    mmio->base = base;
3313
    subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio);
3314
#if defined(DEBUG_SUBPAGE)
3315 3316
    printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
           mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3317
#endif
3318
    *phys = subpage_memory | IO_MEM_SUBPAGE;
R
Richard Henderson 已提交
3319
    subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
3320 3321 3322 3323

    return mmio;
}

3324 3325 3326 3327 3328 3329 3330 3331 3332
static int get_free_io_mem_idx(void)
{
    int i;

    for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
        if (!io_mem_used[i]) {
            io_mem_used[i] = 1;
            return i;
        }
3333
    fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
3334 3335 3336
    return -1;
}

3337 3338
/* mem_read and mem_write are arrays of functions containing the
   function to access byte (index 0), word (index 1) and dword (index
3339
   2). Functions can be omitted with a NULL function pointer.
3340
   If io_index is non zero, the corresponding io zone is
3341 3342 3343
   modified. If it is zero, a new io zone is allocated. The return
   value can be used with cpu_register_physical_memory(). (-1) is
   returned if error. */
3344
static int cpu_register_io_memory_fixed(int io_index,
3345 3346
                                        CPUReadMemoryFunc * const *mem_read,
                                        CPUWriteMemoryFunc * const *mem_write,
3347
                                        void *opaque)
3348
{
3349 3350
    int i;

3351
    if (io_index <= 0) {
3352 3353 3354
        io_index = get_free_io_mem_idx();
        if (io_index == -1)
            return io_index;
3355
    } else {
3356
        io_index >>= IO_MEM_SHIFT;
3357 3358 3359
        if (io_index >= IO_MEM_NB_ENTRIES)
            return -1;
    }
B
bellard 已提交
3360

3361 3362 3363 3364 3365 3366 3367 3368
    for (i = 0; i < 3; ++i) {
        io_mem_read[io_index][i]
            = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
    }
    for (i = 0; i < 3; ++i) {
        io_mem_write[io_index][i]
            = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
    }
B
bellard 已提交
3369
    io_mem_opaque[io_index] = opaque;
R
Richard Henderson 已提交
3370 3371

    return (io_index << IO_MEM_SHIFT);
3372
}
B
bellard 已提交
3373

3374 3375
int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
                           CPUWriteMemoryFunc * const *mem_write,
3376 3377 3378 3379 3380
                           void *opaque)
{
    return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque);
}

3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393
void cpu_unregister_io_memory(int io_table_address)
{
    int i;
    int io_index = io_table_address >> IO_MEM_SHIFT;

    for (i=0;i < 3; i++) {
        io_mem_read[io_index][i] = unassigned_mem_read[i];
        io_mem_write[io_index][i] = unassigned_mem_write[i];
    }
    io_mem_opaque[io_index] = NULL;
    io_mem_used[io_index] = 0;
}

A
Avi Kivity 已提交
3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407
static void io_mem_init(void)
{
    int i;

    cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read, unassigned_mem_write, NULL);
    cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read, unassigned_mem_write, NULL);
    cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read, notdirty_mem_write, NULL);
    for (i=0; i<5; i++)
        io_mem_used[i] = 1;

    io_mem_watch = cpu_register_io_memory(watch_mem_read,
                                          watch_mem_write, NULL);
}

3408 3409
#endif /* !defined(CONFIG_USER_ONLY) */

B
bellard 已提交
3410 3411
/* physical memory access (slow version, mainly for debug) */
#if defined(CONFIG_USER_ONLY)
P
Paul Brook 已提交
3412 3413
int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
                        uint8_t *buf, int len, int is_write)
B
bellard 已提交
3414 3415 3416
{
    int l, flags;
    target_ulong page;
3417
    void * p;
B
bellard 已提交
3418 3419 3420 3421 3422 3423 3424 3425

    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
        flags = page_get_flags(page);
        if (!(flags & PAGE_VALID))
P
Paul Brook 已提交
3426
            return -1;
B
bellard 已提交
3427 3428
        if (is_write) {
            if (!(flags & PAGE_WRITE))
P
Paul Brook 已提交
3429
                return -1;
3430
            /* XXX: this code should not depend on lock_user */
A
aurel32 已提交
3431
            if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
P
Paul Brook 已提交
3432
                return -1;
A
aurel32 已提交
3433 3434
            memcpy(p, buf, l);
            unlock_user(p, addr, l);
B
bellard 已提交
3435 3436
        } else {
            if (!(flags & PAGE_READ))
P
Paul Brook 已提交
3437
                return -1;
3438
            /* XXX: this code should not depend on lock_user */
A
aurel32 已提交
3439
            if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
P
Paul Brook 已提交
3440
                return -1;
A
aurel32 已提交
3441
            memcpy(buf, p, l);
A
aurel32 已提交
3442
            unlock_user(p, addr, 0);
B
bellard 已提交
3443 3444 3445 3446 3447
        }
        len -= l;
        buf += l;
        addr += l;
    }
P
Paul Brook 已提交
3448
    return 0;
B
bellard 已提交
3449
}
B
bellard 已提交
3450

B
bellard 已提交
3451
#else
A
Anthony Liguori 已提交
3452
void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
B
bellard 已提交
3453 3454 3455 3456 3457
                            int len, int is_write)
{
    int l, io_index;
    uint8_t *ptr;
    uint32_t val;
A
Anthony Liguori 已提交
3458
    target_phys_addr_t page;
3459
    unsigned long pd;
B
bellard 已提交
3460
    PhysPageDesc *p;
3461

B
bellard 已提交
3462 3463 3464 3465 3466
    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
B
bellard 已提交
3467
        p = phys_page_find(page >> TARGET_PAGE_BITS);
B
bellard 已提交
3468 3469 3470 3471 3472
        if (!p) {
            pd = IO_MEM_UNASSIGNED;
        } else {
            pd = p->phys_offset;
        }
3473

B
bellard 已提交
3474
        if (is_write) {
3475
            if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
A
Anthony Liguori 已提交
3476
                target_phys_addr_t addr1 = addr;
B
bellard 已提交
3477
                io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3478
                if (p)
3479
                    addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
3480 3481
                /* XXX: could force cpu_single_env to NULL to avoid
                   potential bugs */
3482
                if (l >= 4 && ((addr1 & 3) == 0)) {
B
bellard 已提交
3483
                    /* 32 bit write access */
B
bellard 已提交
3484
                    val = ldl_p(buf);
3485
                    io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
B
bellard 已提交
3486
                    l = 4;
3487
                } else if (l >= 2 && ((addr1 & 1) == 0)) {
B
bellard 已提交
3488
                    /* 16 bit write access */
B
bellard 已提交
3489
                    val = lduw_p(buf);
3490
                    io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
B
bellard 已提交
3491 3492
                    l = 2;
                } else {
B
bellard 已提交
3493
                    /* 8 bit write access */
B
bellard 已提交
3494
                    val = ldub_p(buf);
3495
                    io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
B
bellard 已提交
3496 3497 3498
                    l = 1;
                }
            } else {
3499 3500
                unsigned long addr1;
                addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
B
bellard 已提交
3501
                /* RAM case */
P
pbrook 已提交
3502
                ptr = qemu_get_ram_ptr(addr1);
B
bellard 已提交
3503
                memcpy(ptr, buf, l);
3504 3505 3506 3507
                if (!cpu_physical_memory_is_dirty(addr1)) {
                    /* invalidate code */
                    tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
                    /* set dirty bit */
3508 3509
                    cpu_physical_memory_set_dirty_flags(
                        addr1, (0xff & ~CODE_DIRTY_FLAG));
3510
                }
B
bellard 已提交
3511 3512
            }
        } else {
3513
            if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3514
                !(pd & IO_MEM_ROMD)) {
A
Anthony Liguori 已提交
3515
                target_phys_addr_t addr1 = addr;
B
bellard 已提交
3516 3517
                /* I/O case */
                io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3518
                if (p)
3519 3520
                    addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
                if (l >= 4 && ((addr1 & 3) == 0)) {
B
bellard 已提交
3521
                    /* 32 bit read access */
3522
                    val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
B
bellard 已提交
3523
                    stl_p(buf, val);
B
bellard 已提交
3524
                    l = 4;
3525
                } else if (l >= 2 && ((addr1 & 1) == 0)) {
B
bellard 已提交
3526
                    /* 16 bit read access */
3527
                    val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
B
bellard 已提交
3528
                    stw_p(buf, val);
B
bellard 已提交
3529 3530
                    l = 2;
                } else {
B
bellard 已提交
3531
                    /* 8 bit read access */
3532
                    val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
B
bellard 已提交
3533
                    stb_p(buf, val);
B
bellard 已提交
3534 3535 3536 3537
                    l = 1;
                }
            } else {
                /* RAM case */
P
pbrook 已提交
3538
                ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
B
bellard 已提交
3539 3540 3541 3542 3543 3544 3545 3546 3547
                    (addr & ~TARGET_PAGE_MASK);
                memcpy(buf, ptr, l);
            }
        }
        len -= l;
        buf += l;
        addr += l;
    }
}
B
bellard 已提交
3548

B
bellard 已提交
3549
/* used for ROM loading : can write in RAM and ROM */
A
Anthony Liguori 已提交
3550
void cpu_physical_memory_write_rom(target_phys_addr_t addr,
B
bellard 已提交
3551 3552 3553 3554
                                   const uint8_t *buf, int len)
{
    int l;
    uint8_t *ptr;
A
Anthony Liguori 已提交
3555
    target_phys_addr_t page;
B
bellard 已提交
3556 3557
    unsigned long pd;
    PhysPageDesc *p;
3558

B
bellard 已提交
3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569
    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
        p = phys_page_find(page >> TARGET_PAGE_BITS);
        if (!p) {
            pd = IO_MEM_UNASSIGNED;
        } else {
            pd = p->phys_offset;
        }
3570

B
bellard 已提交
3571
        if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3572 3573
            (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
            !(pd & IO_MEM_ROMD)) {
B
bellard 已提交
3574 3575 3576 3577 3578
            /* do nothing */
        } else {
            unsigned long addr1;
            addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
            /* ROM/RAM case */
P
pbrook 已提交
3579
            ptr = qemu_get_ram_ptr(addr1);
B
bellard 已提交
3580 3581 3582 3583 3584 3585 3586 3587
            memcpy(ptr, buf, l);
        }
        len -= l;
        buf += l;
        addr += l;
    }
}

3588 3589
typedef struct {
    void *buffer;
A
Anthony Liguori 已提交
3590 3591
    target_phys_addr_t addr;
    target_phys_addr_t len;
3592 3593 3594 3595
} BounceBuffer;

static BounceBuffer bounce;

3596 3597 3598
typedef struct MapClient {
    void *opaque;
    void (*callback)(void *opaque);
B
Blue Swirl 已提交
3599
    QLIST_ENTRY(MapClient) link;
3600 3601
} MapClient;

B
Blue Swirl 已提交
3602 3603
static QLIST_HEAD(map_client_list, MapClient) map_client_list
    = QLIST_HEAD_INITIALIZER(map_client_list);
3604 3605 3606 3607 3608 3609 3610

void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
{
    MapClient *client = qemu_malloc(sizeof(*client));

    client->opaque = opaque;
    client->callback = callback;
B
Blue Swirl 已提交
3611
    QLIST_INSERT_HEAD(&map_client_list, client, link);
3612 3613 3614 3615 3616 3617 3618
    return client;
}

void cpu_unregister_map_client(void *_client)
{
    MapClient *client = (MapClient *)_client;

B
Blue Swirl 已提交
3619
    QLIST_REMOVE(client, link);
3620
    qemu_free(client);
3621 3622 3623 3624 3625 3626
}

static void cpu_notify_map_clients(void)
{
    MapClient *client;

B
Blue Swirl 已提交
3627 3628
    while (!QLIST_EMPTY(&map_client_list)) {
        client = QLIST_FIRST(&map_client_list);
3629
        client->callback(client->opaque);
3630
        cpu_unregister_map_client(client);
3631 3632 3633
    }
}

3634 3635 3636 3637
/* Map a physical memory region into a host virtual address.
 * May map a subset of the requested range, given by and returned in *plen.
 * May return NULL if resources needed to perform the mapping are exhausted.
 * Use only for reads OR writes - not for read-modify-write operations.
3638 3639
 * Use cpu_register_map_client() to know when retrying the map operation is
 * likely to succeed.
3640
 */
A
Anthony Liguori 已提交
3641 3642
void *cpu_physical_memory_map(target_phys_addr_t addr,
                              target_phys_addr_t *plen,
3643 3644
                              int is_write)
{
A
Anthony Liguori 已提交
3645 3646
    target_phys_addr_t len = *plen;
    target_phys_addr_t done = 0;
3647 3648 3649
    int l;
    uint8_t *ret = NULL;
    uint8_t *ptr;
A
Anthony Liguori 已提交
3650
    target_phys_addr_t page;
3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679
    unsigned long pd;
    PhysPageDesc *p;
    unsigned long addr1;

    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
        p = phys_page_find(page >> TARGET_PAGE_BITS);
        if (!p) {
            pd = IO_MEM_UNASSIGNED;
        } else {
            pd = p->phys_offset;
        }

        if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
            if (done || bounce.buffer) {
                break;
            }
            bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
            bounce.addr = addr;
            bounce.len = l;
            if (!is_write) {
                cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
            }
            ptr = bounce.buffer;
        } else {
            addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
P
pbrook 已提交
3680
            ptr = qemu_get_ram_ptr(addr1);
3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699
        }
        if (!done) {
            ret = ptr;
        } else if (ret + done != ptr) {
            break;
        }

        len -= l;
        addr += l;
        done += l;
    }
    *plen = done;
    return ret;
}

/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
 * Will also mark the memory as dirty if is_write == 1.  access_len gives
 * the amount of memory that was actually read or written by the caller.
 */
A
Anthony Liguori 已提交
3700 3701
void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
                               int is_write, target_phys_addr_t access_len)
3702 3703 3704
{
    if (buffer != bounce.buffer) {
        if (is_write) {
A
Anthony Liguori 已提交
3705
            ram_addr_t addr1 = qemu_ram_addr_from_host(buffer);
3706 3707 3708 3709 3710 3711 3712 3713 3714
            while (access_len) {
                unsigned l;
                l = TARGET_PAGE_SIZE;
                if (l > access_len)
                    l = access_len;
                if (!cpu_physical_memory_is_dirty(addr1)) {
                    /* invalidate code */
                    tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
                    /* set dirty bit */
3715 3716
                    cpu_physical_memory_set_dirty_flags(
                        addr1, (0xff & ~CODE_DIRTY_FLAG));
3717 3718 3719 3720 3721 3722 3723 3724 3725 3726
                }
                addr1 += l;
                access_len -= l;
            }
        }
        return;
    }
    if (is_write) {
        cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
    }
3727
    qemu_vfree(bounce.buffer);
3728
    bounce.buffer = NULL;
3729
    cpu_notify_map_clients();
3730
}
B
bellard 已提交
3731

B
bellard 已提交
3732
/* warning: addr must be aligned */
A
Anthony Liguori 已提交
3733
uint32_t ldl_phys(target_phys_addr_t addr)
B
bellard 已提交
3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746
{
    int io_index;
    uint8_t *ptr;
    uint32_t val;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3747

3748
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3749
        !(pd & IO_MEM_ROMD)) {
B
bellard 已提交
3750 3751
        /* I/O case */
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3752 3753
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
3754 3755 3756
        val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
    } else {
        /* RAM case */
P
pbrook 已提交
3757
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
B
bellard 已提交
3758 3759 3760 3761 3762 3763
            (addr & ~TARGET_PAGE_MASK);
        val = ldl_p(ptr);
    }
    return val;
}

B
bellard 已提交
3764
/* warning: addr must be aligned */
A
Anthony Liguori 已提交
3765
uint64_t ldq_phys(target_phys_addr_t addr)
B
bellard 已提交
3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778
{
    int io_index;
    uint8_t *ptr;
    uint64_t val;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3779

3780 3781
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
        !(pd & IO_MEM_ROMD)) {
B
bellard 已提交
3782 3783
        /* I/O case */
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3784 3785
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
3786 3787 3788 3789 3790 3791 3792 3793 3794
#ifdef TARGET_WORDS_BIGENDIAN
        val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
        val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
#else
        val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
        val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
#endif
    } else {
        /* RAM case */
P
pbrook 已提交
3795
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
B
bellard 已提交
3796 3797 3798 3799 3800 3801
            (addr & ~TARGET_PAGE_MASK);
        val = ldq_p(ptr);
    }
    return val;
}

B
bellard 已提交
3802
/* XXX: optimize */
A
Anthony Liguori 已提交
3803
uint32_t ldub_phys(target_phys_addr_t addr)
B
bellard 已提交
3804 3805 3806 3807 3808 3809
{
    uint8_t val;
    cpu_physical_memory_read(addr, &val, 1);
    return val;
}

3810
/* warning: addr must be aligned */
A
Anthony Liguori 已提交
3811
uint32_t lduw_phys(target_phys_addr_t addr)
B
bellard 已提交
3812
{
3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839
    int io_index;
    uint8_t *ptr;
    uint64_t val;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }

    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
        !(pd & IO_MEM_ROMD)) {
        /* I/O case */
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
        val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
    } else {
        /* RAM case */
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
            (addr & ~TARGET_PAGE_MASK);
        val = lduw_p(ptr);
    }
    return val;
B
bellard 已提交
3840 3841
}

B
bellard 已提交
3842 3843 3844
/* warning: addr must be aligned. The ram page is not masked as dirty
   and the code inside is not invalidated. It is useful if the dirty
   bits are used to track modified PTEs */
A
Anthony Liguori 已提交
3845
void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
B
bellard 已提交
3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857
{
    int io_index;
    uint8_t *ptr;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3858

3859
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
B
bellard 已提交
3860
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3861 3862
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
3863 3864
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
    } else {
A
aliguori 已提交
3865
        unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
P
pbrook 已提交
3866
        ptr = qemu_get_ram_ptr(addr1);
B
bellard 已提交
3867
        stl_p(ptr, val);
A
aliguori 已提交
3868 3869 3870 3871 3872 3873

        if (unlikely(in_migration)) {
            if (!cpu_physical_memory_is_dirty(addr1)) {
                /* invalidate code */
                tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
                /* set dirty bit */
3874 3875
                cpu_physical_memory_set_dirty_flags(
                    addr1, (0xff & ~CODE_DIRTY_FLAG));
A
aliguori 已提交
3876 3877
            }
        }
B
bellard 已提交
3878 3879 3880
    }
}

A
Anthony Liguori 已提交
3881
void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
J
j_mayer 已提交
3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893
{
    int io_index;
    uint8_t *ptr;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3894

J
j_mayer 已提交
3895 3896
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3897 3898
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
J
j_mayer 已提交
3899 3900 3901 3902 3903 3904 3905 3906
#ifdef TARGET_WORDS_BIGENDIAN
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
#else
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
#endif
    } else {
P
pbrook 已提交
3907
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
J
j_mayer 已提交
3908 3909 3910 3911 3912
            (addr & ~TARGET_PAGE_MASK);
        stq_p(ptr, val);
    }
}

B
bellard 已提交
3913
/* warning: addr must be aligned */
A
Anthony Liguori 已提交
3914
void stl_phys(target_phys_addr_t addr, uint32_t val)
B
bellard 已提交
3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926
{
    int io_index;
    uint8_t *ptr;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3927

3928
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
B
bellard 已提交
3929
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3930 3931
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
3932 3933 3934 3935 3936
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
    } else {
        unsigned long addr1;
        addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
        /* RAM case */
P
pbrook 已提交
3937
        ptr = qemu_get_ram_ptr(addr1);
B
bellard 已提交
3938
        stl_p(ptr, val);
3939 3940 3941 3942
        if (!cpu_physical_memory_is_dirty(addr1)) {
            /* invalidate code */
            tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
            /* set dirty bit */
3943 3944
            cpu_physical_memory_set_dirty_flags(addr1,
                (0xff & ~CODE_DIRTY_FLAG));
3945
        }
B
bellard 已提交
3946 3947 3948
    }
}

B
bellard 已提交
3949
/* XXX: optimize */
A
Anthony Liguori 已提交
3950
void stb_phys(target_phys_addr_t addr, uint32_t val)
B
bellard 已提交
3951 3952 3953 3954 3955
{
    uint8_t v = val;
    cpu_physical_memory_write(addr, &v, 1);
}

3956
/* warning: addr must be aligned */
A
Anthony Liguori 已提交
3957
void stw_phys(target_phys_addr_t addr, uint32_t val)
B
bellard 已提交
3958
{
3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989
    int io_index;
    uint8_t *ptr;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }

    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
        io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
    } else {
        unsigned long addr1;
        addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
        /* RAM case */
        ptr = qemu_get_ram_ptr(addr1);
        stw_p(ptr, val);
        if (!cpu_physical_memory_is_dirty(addr1)) {
            /* invalidate code */
            tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
            /* set dirty bit */
            cpu_physical_memory_set_dirty_flags(addr1,
                (0xff & ~CODE_DIRTY_FLAG));
        }
    }
B
bellard 已提交
3990 3991 3992
}

/* XXX: optimize */
A
Anthony Liguori 已提交
3993
void stq_phys(target_phys_addr_t addr, uint64_t val)
B
bellard 已提交
3994 3995 3996 3997 3998
{
    val = tswap64(val);
    cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
}

3999
/* virtual memory access for debug (includes writing to ROM) */
4000
int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
4001
                        uint8_t *buf, int len, int is_write)
B
bellard 已提交
4002 4003
{
    int l;
A
Anthony Liguori 已提交
4004
    target_phys_addr_t phys_addr;
4005
    target_ulong page;
B
bellard 已提交
4006 4007 4008 4009 4010 4011 4012 4013 4014 4015

    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        phys_addr = cpu_get_phys_page_debug(env, page);
        /* if no physical page mapped, return an error */
        if (phys_addr == -1)
            return -1;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
4016 4017 4018 4019 4020
        phys_addr += (addr & ~TARGET_PAGE_MASK);
        if (is_write)
            cpu_physical_memory_write_rom(phys_addr, buf, l);
        else
            cpu_physical_memory_rw(phys_addr, buf, l, is_write);
B
bellard 已提交
4021 4022 4023 4024 4025 4026
        len -= l;
        buf += l;
        addr += l;
    }
    return 0;
}
P
Paul Brook 已提交
4027
#endif
B
bellard 已提交
4028

P
pbrook 已提交
4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045
/* in deterministic execution mode, instructions doing device I/Os
   must be at the end of the TB */
void cpu_io_recompile(CPUState *env, void *retaddr)
{
    TranslationBlock *tb;
    uint32_t n, cflags;
    target_ulong pc, cs_base;
    uint64_t flags;

    tb = tb_find_pc((unsigned long)retaddr);
    if (!tb) {
        cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p", 
                  retaddr);
    }
    n = env->icount_decr.u16.low + tb->icount;
    cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
    /* Calculate how many instructions had been executed before the fault
T
ths 已提交
4046
       occurred.  */
P
pbrook 已提交
4047 4048 4049 4050 4051
    n = n - env->icount_decr.u16.low;
    /* Generate a new TB ending on the I/O insn.  */
    n++;
    /* On MIPS and SH, delay slot instructions can only be restarted if
       they were already the first instruction in the TB.  If this is not
T
ths 已提交
4052
       the first instruction in a TB then re-execute the preceding
P
pbrook 已提交
4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079
       branch.  */
#if defined(TARGET_MIPS)
    if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
        env->active_tc.PC -= 4;
        env->icount_decr.u16.low++;
        env->hflags &= ~MIPS_HFLAG_BMASK;
    }
#elif defined(TARGET_SH4)
    if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
            && n > 1) {
        env->pc -= 2;
        env->icount_decr.u16.low++;
        env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
    }
#endif
    /* This should never happen.  */
    if (n > CF_COUNT_MASK)
        cpu_abort(env, "TB too big during recompile");

    cflags = n | CF_LAST_IO;
    pc = tb->pc;
    cs_base = tb->cs_base;
    flags = tb->flags;
    tb_phys_invalidate(tb, -1);
    /* FIXME: In theory this could raise an exception.  In practice
       we have already translated the block once so it's probably ok.  */
    tb_gen_code(env, pc, cs_base, flags, cflags);
T
ths 已提交
4080
    /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
P
pbrook 已提交
4081 4082 4083 4084 4085 4086 4087
       the first in the TB) then we end up generating a whole new TB and
       repeating the fault, which is horribly inefficient.
       Better would be to execute just this insn uncached, or generate a
       second new TB.  */
    cpu_resume_from_signal(env, NULL);
}

4088 4089
#if !defined(CONFIG_USER_ONLY)

B
bellard 已提交
4090 4091 4092 4093 4094 4095
void dump_exec_info(FILE *f,
                    int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
{
    int i, target_code_size, max_target_code_size;
    int direct_jmp_count, direct_jmp2_count, cross_page;
    TranslationBlock *tb;
4096

B
bellard 已提交
4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116
    target_code_size = 0;
    max_target_code_size = 0;
    cross_page = 0;
    direct_jmp_count = 0;
    direct_jmp2_count = 0;
    for(i = 0; i < nb_tbs; i++) {
        tb = &tbs[i];
        target_code_size += tb->size;
        if (tb->size > max_target_code_size)
            max_target_code_size = tb->size;
        if (tb->page_addr[1] != -1)
            cross_page++;
        if (tb->tb_next_offset[0] != 0xffff) {
            direct_jmp_count++;
            if (tb->tb_next_offset[1] != 0xffff) {
                direct_jmp2_count++;
            }
        }
    }
    /* XXX: avoid using doubles ? */
B
bellard 已提交
4117
    cpu_fprintf(f, "Translation buffer state:\n");
4118 4119 4120 4121
    cpu_fprintf(f, "gen code size       %ld/%ld\n",
                code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
    cpu_fprintf(f, "TB count            %d/%d\n", 
                nb_tbs, code_gen_max_blocks);
4122
    cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
B
bellard 已提交
4123 4124
                nb_tbs ? target_code_size / nb_tbs : 0,
                max_target_code_size);
4125
    cpu_fprintf(f, "TB avg host size    %d bytes (expansion ratio: %0.1f)\n",
B
bellard 已提交
4126 4127
                nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
                target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
4128 4129
    cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
            cross_page,
B
bellard 已提交
4130 4131
            nb_tbs ? (cross_page * 100) / nb_tbs : 0);
    cpu_fprintf(f, "direct jump count   %d (%d%%) (2 jumps=%d %d%%)\n",
4132
                direct_jmp_count,
B
bellard 已提交
4133 4134 4135
                nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
                direct_jmp2_count,
                nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
B
bellard 已提交
4136
    cpu_fprintf(f, "\nStatistics:\n");
B
bellard 已提交
4137 4138 4139
    cpu_fprintf(f, "TB flush count      %d\n", tb_flush_count);
    cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
    cpu_fprintf(f, "TLB flush count     %d\n", tlb_flush_count);
B
bellard 已提交
4140
    tcg_dump_info(f, cpu_fprintf);
B
bellard 已提交
4141 4142
}

B
bellard 已提交
4143 4144 4145
#define MMUSUFFIX _cmmu
#define GETPC() NULL
#define env cpu_single_env
B
bellard 已提交
4146
#define SOFTMMU_CODE_ACCESS
B
bellard 已提交
4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162

#define SHIFT 0
#include "softmmu_template.h"

#define SHIFT 1
#include "softmmu_template.h"

#define SHIFT 2
#include "softmmu_template.h"

#define SHIFT 3
#include "softmmu_template.h"

#undef env

#endif