exec.c 102.2 KB
Newer Older
B
bellard 已提交
1
/*
B
bellard 已提交
2
 *  virtual page mapping and translated block handling
3
 *
B
bellard 已提交
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
 *  Copyright (c) 2003 Fabrice Bellard
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
B
bellard 已提交
20
#include "config.h"
B
bellard 已提交
21
#ifdef _WIN32
T
ths 已提交
22
#define WIN32_LEAN_AND_MEAN
B
bellard 已提交
23 24
#include <windows.h>
#else
B
bellard 已提交
25
#include <sys/types.h>
B
bellard 已提交
26 27
#include <sys/mman.h>
#endif
B
bellard 已提交
28 29 30 31 32 33 34 35
#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <inttypes.h>

B
bellard 已提交
36 37
#include "cpu.h"
#include "exec-all.h"
38
#include "qemu-common.h"
B
bellard 已提交
39
#include "tcg.h"
40
#include "hw/hw.h"
A
aliguori 已提交
41
#include "osdep.h"
A
aliguori 已提交
42
#include "kvm.h"
43 44 45
#if defined(CONFIG_USER_ONLY)
#include <qemu.h>
#endif
B
bellard 已提交
46

B
bellard 已提交
47
//#define DEBUG_TB_INVALIDATE
B
bellard 已提交
48
//#define DEBUG_FLUSH
49
//#define DEBUG_TLB
P
pbrook 已提交
50
//#define DEBUG_UNASSIGNED
B
bellard 已提交
51 52

/* make various TB consistency checks */
53 54
//#define DEBUG_TB_CHECK
//#define DEBUG_TLB_CHECK
B
bellard 已提交
55

T
ths 已提交
56
//#define DEBUG_IOPORT
57
//#define DEBUG_SUBPAGE
T
ths 已提交
58

59 60 61 62 63
#if !defined(CONFIG_USER_ONLY)
/* TB consistency checks only implemented for usermode emulation.  */
#undef DEBUG_TB_CHECK
#endif

64 65 66 67
#define SMC_BITMAP_USE_THRESHOLD 10

#define MMAP_AREA_START        0x00000000
#define MMAP_AREA_END          0xa8000000
B
bellard 已提交
68

69 70
#if defined(TARGET_SPARC64)
#define TARGET_PHYS_ADDR_SPACE_BITS 41
71 72
#elif defined(TARGET_SPARC)
#define TARGET_PHYS_ADDR_SPACE_BITS 36
73 74 75
#elif defined(TARGET_ALPHA)
#define TARGET_PHYS_ADDR_SPACE_BITS 42
#define TARGET_VIRT_ADDR_SPACE_BITS 42
76 77
#elif defined(TARGET_PPC64)
#define TARGET_PHYS_ADDR_SPACE_BITS 42
78 79 80 81
#elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
#define TARGET_PHYS_ADDR_SPACE_BITS 42
#elif defined(TARGET_I386) && !defined(USE_KQEMU)
#define TARGET_PHYS_ADDR_SPACE_BITS 36
82 83 84 85 86
#else
/* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
#define TARGET_PHYS_ADDR_SPACE_BITS 32
#endif

B
blueswir1 已提交
87
static TranslationBlock *tbs;
88
int code_gen_max_blocks;
89
TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
B
blueswir1 已提交
90
static int nb_tbs;
B
bellard 已提交
91 92
/* any access to the tbs or the page table must use this lock */
spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
B
bellard 已提交
93

B
blueswir1 已提交
94 95 96
#if defined(__arm__) || defined(__sparc_v9__)
/* The prologue must be reachable with a direct jump. ARM and Sparc64
 have limited branch ranges (possibly also PPC) so place it in a
97 98 99 100 101 102 103 104 105 106
 section close to code segment. */
#define code_gen_section                                \
    __attribute__((__section__(".gen_code")))           \
    __attribute__((aligned (32)))
#else
#define code_gen_section                                \
    __attribute__((aligned (32)))
#endif

uint8_t code_gen_prologue[1024] code_gen_section;
B
blueswir1 已提交
107 108
static uint8_t *code_gen_buffer;
static unsigned long code_gen_buffer_size;
109
/* threshold to flush the translated code buffer */
B
blueswir1 已提交
110
static unsigned long code_gen_buffer_max_size;
B
bellard 已提交
111 112
uint8_t *code_gen_ptr;

113
#if !defined(CONFIG_USER_ONLY)
114
ram_addr_t phys_ram_size;
115 116
int phys_ram_fd;
uint8_t *phys_ram_base;
117
uint8_t *phys_ram_dirty;
A
aliguori 已提交
118
static int in_migration;
B
bellard 已提交
119
static ram_addr_t phys_ram_alloc_offset = 0;
120
#endif
121

B
bellard 已提交
122 123 124
CPUState *first_cpu;
/* current CPU in the current thread. It is only valid inside
   cpu_exec() */
125
CPUState *cpu_single_env;
P
pbrook 已提交
126
/* 0 = Do not count executed instructions.
T
ths 已提交
127
   1 = Precise instruction counting.
P
pbrook 已提交
128 129 130 131 132
   2 = Adaptive rate instruction counting.  */
int use_icount = 0;
/* Current instruction counter.  While executing translated code this may
   include some instructions that have not yet been executed.  */
int64_t qemu_icount;
B
bellard 已提交
133

B
bellard 已提交
134
typedef struct PageDesc {
B
bellard 已提交
135
    /* list of TBs intersecting this ram page */
B
bellard 已提交
136
    TranslationBlock *first_tb;
137 138 139 140 141 142 143
    /* in order to optimize self modifying code, we count the number
       of lookups we do to a given page to use a bitmap */
    unsigned int code_write_count;
    uint8_t *code_bitmap;
#if defined(CONFIG_USER_ONLY)
    unsigned long flags;
#endif
B
bellard 已提交
144 145
} PageDesc;

B
bellard 已提交
146
typedef struct PhysPageDesc {
P
pbrook 已提交
147
    /* offset in host memory of the page + io_index in the low bits */
148
    ram_addr_t phys_offset;
149
    ram_addr_t region_offset;
B
bellard 已提交
150 151
} PhysPageDesc;

B
bellard 已提交
152
#define L2_BITS 10
153 154 155 156 157 158 159
#if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
/* XXX: this is a temporary hack for alpha target.
 *      In the future, this is to be replaced by a multi-level table
 *      to actually be able to handle the complete 64 bits address space.
 */
#define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
#else
160
#define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
161
#endif
B
bellard 已提交
162 163 164 165

#define L1_SIZE (1 << L1_BITS)
#define L2_SIZE (1 << L2_BITS)

166 167 168 169
unsigned long qemu_real_host_page_size;
unsigned long qemu_host_page_bits;
unsigned long qemu_host_page_size;
unsigned long qemu_host_page_mask;
B
bellard 已提交
170

B
bellard 已提交
171
/* XXX: for system emulation, it could just be an array */
B
bellard 已提交
172
static PageDesc *l1_map[L1_SIZE];
B
blueswir1 已提交
173
static PhysPageDesc **l1_phys_map;
B
bellard 已提交
174

175 176 177
#if !defined(CONFIG_USER_ONLY)
static void io_mem_init(void);

178 179 180
/* io memory support */
CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
B
bellard 已提交
181
void *io_mem_opaque[IO_MEM_NB_ENTRIES];
182
static int io_mem_nb;
183 184
static int io_mem_watch;
#endif
185

186
/* log support */
187
static const char *logfilename = "/tmp/qemu.log";
188 189
FILE *logfile;
int loglevel;
P
pbrook 已提交
190
static int log_append = 0;
191

B
bellard 已提交
192 193 194 195 196
/* statistics */
static int tlb_flush_count;
static int tb_flush_count;
static int tb_phys_invalidate_count;

197 198 199
#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
typedef struct subpage_t {
    target_phys_addr_t base;
200 201 202
    CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
    CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
    void *opaque[TARGET_PAGE_SIZE][2][4];
203
    ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
204 205
} subpage_t;

206 207 208 209 210 211 212 213 214 215 216
#ifdef _WIN32
static void map_exec(void *addr, long size)
{
    DWORD old_protect;
    VirtualProtect(addr, size,
                   PAGE_EXECUTE_READWRITE, &old_protect);
    
}
#else
static void map_exec(void *addr, long size)
{
217
    unsigned long start, end, page_size;
218
    
219
    page_size = getpagesize();
220
    start = (unsigned long)addr;
221
    start &= ~(page_size - 1);
222 223
    
    end = (unsigned long)addr + size;
224 225
    end += page_size - 1;
    end &= ~(page_size - 1);
226 227 228 229 230 231
    
    mprotect((void *)start, end - start,
             PROT_READ | PROT_WRITE | PROT_EXEC);
}
#endif

B
bellard 已提交
232
static void page_init(void)
B
bellard 已提交
233
{
234
    /* NOTE: we can always suppose that qemu_host_page_size >=
B
bellard 已提交
235
       TARGET_PAGE_SIZE */
236 237 238 239 240 241 242 243 244 245
#ifdef _WIN32
    {
        SYSTEM_INFO system_info;

        GetSystemInfo(&system_info);
        qemu_real_host_page_size = system_info.dwPageSize;
    }
#else
    qemu_real_host_page_size = getpagesize();
#endif
246 247 248 249 250 251 252 253
    if (qemu_host_page_size == 0)
        qemu_host_page_size = qemu_real_host_page_size;
    if (qemu_host_page_size < TARGET_PAGE_SIZE)
        qemu_host_page_size = TARGET_PAGE_SIZE;
    qemu_host_page_bits = 0;
    while ((1 << qemu_host_page_bits) < qemu_host_page_size)
        qemu_host_page_bits++;
    qemu_host_page_mask = ~(qemu_host_page_size - 1);
254 255
    l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
    memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
256 257 258 259 260 261 262

#if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
    {
        long long startaddr, endaddr;
        FILE *f;
        int n;

P
pbrook 已提交
263
        mmap_lock();
P
pbrook 已提交
264
        last_brk = (unsigned long)sbrk(0);
265 266 267 268 269
        f = fopen("/proc/self/maps", "r");
        if (f) {
            do {
                n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
                if (n == 2) {
270 271 272 273
                    startaddr = MIN(startaddr,
                                    (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
                    endaddr = MIN(endaddr,
                                    (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
P
pbrook 已提交
274
                    page_set_flags(startaddr & TARGET_PAGE_MASK,
275 276 277 278 279 280
                                   TARGET_PAGE_ALIGN(endaddr),
                                   PAGE_RESERVED); 
                }
            } while (!feof(f));
            fclose(f);
        }
P
pbrook 已提交
281
        mmap_unlock();
282 283
    }
#endif
B
bellard 已提交
284 285
}

286
static inline PageDesc **page_l1_map(target_ulong index)
B
bellard 已提交
287
{
288 289 290
#if TARGET_LONG_BITS > 32
    /* Host memory outside guest VM.  For 32-bit targets we have already
       excluded high addresses.  */
T
ths 已提交
291
    if (index > ((target_ulong)L2_SIZE * L1_SIZE))
292 293
        return NULL;
#endif
294 295 296 297 298 299 300 301 302 303
    return &l1_map[index >> L2_BITS];
}

static inline PageDesc *page_find_alloc(target_ulong index)
{
    PageDesc **lp, *p;
    lp = page_l1_map(index);
    if (!lp)
        return NULL;

B
bellard 已提交
304 305 306
    p = *lp;
    if (!p) {
        /* allocate if not found */
307 308 309 310 311
#if defined(CONFIG_USER_ONLY)
        size_t len = sizeof(PageDesc) * L2_SIZE;
        /* Don't use qemu_malloc because it may recurse.  */
        p = mmap(0, len, PROT_READ | PROT_WRITE,
                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
B
bellard 已提交
312
        *lp = p;
313 314
        if (h2g_valid(p)) {
            unsigned long addr = h2g(p);
315 316 317 318 319 320 321 322
            page_set_flags(addr & TARGET_PAGE_MASK,
                           TARGET_PAGE_ALIGN(addr + len),
                           PAGE_RESERVED); 
        }
#else
        p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
        *lp = p;
#endif
B
bellard 已提交
323 324 325 326
    }
    return p + (index & (L2_SIZE - 1));
}

327
static inline PageDesc *page_find(target_ulong index)
B
bellard 已提交
328
{
329 330 331 332
    PageDesc **lp, *p;
    lp = page_l1_map(index);
    if (!lp)
        return NULL;
B
bellard 已提交
333

334
    p = *lp;
B
bellard 已提交
335 336
    if (!p)
        return 0;
B
bellard 已提交
337 338 339
    return p + (index & (L2_SIZE - 1));
}

340
static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
B
bellard 已提交
341
{
342
    void **lp, **p;
343
    PhysPageDesc *pd;
B
bellard 已提交
344

345 346 347 348 349 350 351
    p = (void **)l1_phys_map;
#if TARGET_PHYS_ADDR_SPACE_BITS > 32

#if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
#error unsupported TARGET_PHYS_ADDR_SPACE_BITS
#endif
    lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
B
bellard 已提交
352 353 354
    p = *lp;
    if (!p) {
        /* allocate if not found */
355 356 357 358 359 360 361 362
        if (!alloc)
            return NULL;
        p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
        memset(p, 0, sizeof(void *) * L1_SIZE);
        *lp = p;
    }
#endif
    lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
363 364 365
    pd = *lp;
    if (!pd) {
        int i;
366 367 368
        /* allocate if not found */
        if (!alloc)
            return NULL;
369 370 371 372
        pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
        *lp = pd;
        for (i = 0; i < L2_SIZE; i++)
          pd[i].phys_offset = IO_MEM_UNASSIGNED;
B
bellard 已提交
373
    }
374
    return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
B
bellard 已提交
375 376
}

377
static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
B
bellard 已提交
378
{
379
    return phys_page_find_alloc(index, 0);
B
bellard 已提交
380 381
}

382
#if !defined(CONFIG_USER_ONLY)
B
bellard 已提交
383
static void tlb_protect_code(ram_addr_t ram_addr);
384
static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
385
                                    target_ulong vaddr);
P
pbrook 已提交
386 387
#define mmap_lock() do { } while(0)
#define mmap_unlock() do { } while(0)
388
#endif
B
bellard 已提交
389

390 391 392 393 394 395 396 397 398 399 400 401
#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)

#if defined(CONFIG_USER_ONLY)
/* Currently it is not recommanded to allocate big chunks of data in
   user mode. It will change when a dedicated libc will be used */
#define USE_STATIC_CODE_GEN_BUFFER
#endif

#ifdef USE_STATIC_CODE_GEN_BUFFER
static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
#endif

402
static void code_gen_alloc(unsigned long tb_size)
403
{
404 405 406 407 408
#ifdef USE_STATIC_CODE_GEN_BUFFER
    code_gen_buffer = static_code_gen_buffer;
    code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
    map_exec(code_gen_buffer, code_gen_buffer_size);
#else
409 410
    code_gen_buffer_size = tb_size;
    if (code_gen_buffer_size == 0) {
411 412 413 414
#if defined(CONFIG_USER_ONLY)
        /* in user mode, phys_ram_size is not meaningful */
        code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
#else
415
        /* XXX: needs ajustments */
416
        code_gen_buffer_size = (unsigned long)(phys_ram_size / 4);
417
#endif
418 419 420 421 422 423 424 425
    }
    if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
        code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
    /* The code gen buffer location may have constraints depending on
       the host cpu and OS */
#if defined(__linux__) 
    {
        int flags;
B
blueswir1 已提交
426 427
        void *start = NULL;

428 429 430 431 432 433
        flags = MAP_PRIVATE | MAP_ANONYMOUS;
#if defined(__x86_64__)
        flags |= MAP_32BIT;
        /* Cannot map more than that */
        if (code_gen_buffer_size > (800 * 1024 * 1024))
            code_gen_buffer_size = (800 * 1024 * 1024);
B
blueswir1 已提交
434 435 436 437 438 439
#elif defined(__sparc_v9__)
        // Map the buffer below 2G, so we can use direct calls and branches
        flags |= MAP_FIXED;
        start = (void *) 0x60000000UL;
        if (code_gen_buffer_size > (512 * 1024 * 1024))
            code_gen_buffer_size = (512 * 1024 * 1024);
440
#elif defined(__arm__)
B
balrog 已提交
441
        /* Map the buffer below 32M, so we can use direct calls and branches */
442 443 444 445
        flags |= MAP_FIXED;
        start = (void *) 0x01000000UL;
        if (code_gen_buffer_size > 16 * 1024 * 1024)
            code_gen_buffer_size = 16 * 1024 * 1024;
446
#endif
B
blueswir1 已提交
447 448
        code_gen_buffer = mmap(start, code_gen_buffer_size,
                               PROT_WRITE | PROT_READ | PROT_EXEC,
449 450 451 452 453 454
                               flags, -1, 0);
        if (code_gen_buffer == MAP_FAILED) {
            fprintf(stderr, "Could not allocate dynamic translator buffer\n");
            exit(1);
        }
    }
455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476
#elif defined(__FreeBSD__)
    {
        int flags;
        void *addr = NULL;
        flags = MAP_PRIVATE | MAP_ANONYMOUS;
#if defined(__x86_64__)
        /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
         * 0x40000000 is free */
        flags |= MAP_FIXED;
        addr = (void *)0x40000000;
        /* Cannot map more than that */
        if (code_gen_buffer_size > (800 * 1024 * 1024))
            code_gen_buffer_size = (800 * 1024 * 1024);
#endif
        code_gen_buffer = mmap(addr, code_gen_buffer_size,
                               PROT_WRITE | PROT_READ | PROT_EXEC, 
                               flags, -1, 0);
        if (code_gen_buffer == MAP_FAILED) {
            fprintf(stderr, "Could not allocate dynamic translator buffer\n");
            exit(1);
        }
    }
477 478 479 480 481 482 483 484
#else
    code_gen_buffer = qemu_malloc(code_gen_buffer_size);
    if (!code_gen_buffer) {
        fprintf(stderr, "Could not allocate dynamic translator buffer\n");
        exit(1);
    }
    map_exec(code_gen_buffer, code_gen_buffer_size);
#endif
485
#endif /* !USE_STATIC_CODE_GEN_BUFFER */
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500
    map_exec(code_gen_prologue, sizeof(code_gen_prologue));
    code_gen_buffer_max_size = code_gen_buffer_size - 
        code_gen_max_block_size();
    code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
    tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
}

/* Must be called before using the QEMU cpus. 'tb_size' is the size
   (in bytes) allocated to the translation buffer. Zero means default
   size. */
void cpu_exec_init_all(unsigned long tb_size)
{
    cpu_gen_init();
    code_gen_alloc(tb_size);
    code_gen_ptr = code_gen_buffer;
501
    page_init();
502
#if !defined(CONFIG_USER_ONLY)
503
    io_mem_init();
504
#endif
505 506
}

507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526
#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)

#define CPU_COMMON_SAVE_VERSION 1

static void cpu_common_save(QEMUFile *f, void *opaque)
{
    CPUState *env = opaque;

    qemu_put_be32s(f, &env->halted);
    qemu_put_be32s(f, &env->interrupt_request);
}

static int cpu_common_load(QEMUFile *f, void *opaque, int version_id)
{
    CPUState *env = opaque;

    if (version_id != CPU_COMMON_SAVE_VERSION)
        return -EINVAL;

    qemu_get_be32s(f, &env->halted);
P
pbrook 已提交
527
    qemu_get_be32s(f, &env->interrupt_request);
528 529 530 531 532 533
    tlb_flush(env, 1);

    return 0;
}
#endif

B
bellard 已提交
534
void cpu_exec_init(CPUState *env)
B
bellard 已提交
535
{
B
bellard 已提交
536 537 538 539 540 541 542 543 544 545 546
    CPUState **penv;
    int cpu_index;

    env->next_cpu = NULL;
    penv = &first_cpu;
    cpu_index = 0;
    while (*penv != NULL) {
        penv = (CPUState **)&(*penv)->next_cpu;
        cpu_index++;
    }
    env->cpu_index = cpu_index;
547 548
    TAILQ_INIT(&env->breakpoints);
    TAILQ_INIT(&env->watchpoints);
B
bellard 已提交
549
    *penv = env;
550
#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
551 552
    register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
                    cpu_common_save, cpu_common_load, env);
553 554 555
    register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
                    cpu_save, cpu_load, env);
#endif
B
bellard 已提交
556 557
}

558 559 560
static inline void invalidate_page_bitmap(PageDesc *p)
{
    if (p->code_bitmap) {
561
        qemu_free(p->code_bitmap);
562 563 564 565 566
        p->code_bitmap = NULL;
    }
    p->code_write_count = 0;
}

B
bellard 已提交
567 568 569 570 571 572 573 574 575
/* set to NULL all the 'first_tb' fields in all PageDescs */
static void page_flush_tb(void)
{
    int i, j;
    PageDesc *p;

    for(i = 0; i < L1_SIZE; i++) {
        p = l1_map[i];
        if (p) {
576 577 578 579 580
            for(j = 0; j < L2_SIZE; j++) {
                p->first_tb = NULL;
                invalidate_page_bitmap(p);
                p++;
            }
B
bellard 已提交
581 582 583 584 585
        }
    }
}

/* flush all the translation blocks */
B
bellard 已提交
586
/* XXX: tb_flush is currently not thread safe */
B
bellard 已提交
587
void tb_flush(CPUState *env1)
B
bellard 已提交
588
{
B
bellard 已提交
589
    CPUState *env;
590
#if defined(DEBUG_FLUSH)
B
blueswir1 已提交
591 592 593 594
    printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
           (unsigned long)(code_gen_ptr - code_gen_buffer),
           nb_tbs, nb_tbs > 0 ?
           ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
B
bellard 已提交
595
#endif
596
    if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
P
pbrook 已提交
597 598
        cpu_abort(env1, "Internal error: code buffer overflow\n");

B
bellard 已提交
599
    nb_tbs = 0;
600

B
bellard 已提交
601 602 603
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
        memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
    }
604

B
bellard 已提交
605
    memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
B
bellard 已提交
606
    page_flush_tb();
607

B
bellard 已提交
608
    code_gen_ptr = code_gen_buffer;
B
bellard 已提交
609 610
    /* XXX: flush processor icache at this point if cache flush is
       expensive */
B
bellard 已提交
611
    tb_flush_count++;
B
bellard 已提交
612 613 614 615
}

#ifdef DEBUG_TB_CHECK

J
j_mayer 已提交
616
static void tb_invalidate_check(target_ulong address)
B
bellard 已提交
617 618 619 620
{
    TranslationBlock *tb;
    int i;
    address &= TARGET_PAGE_MASK;
621 622
    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
B
bellard 已提交
623 624 625
            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
                  address >= tb->pc + tb->size)) {
                printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
626
                       address, (long)tb->pc, tb->size);
B
bellard 已提交
627 628 629 630 631 632 633 634 635 636
            }
        }
    }
}

/* verify that all the pages have correct rights for code */
static void tb_page_check(void)
{
    TranslationBlock *tb;
    int i, flags1, flags2;
637

638 639
    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
B
bellard 已提交
640 641 642 643
            flags1 = page_get_flags(tb->pc);
            flags2 = page_get_flags(tb->pc + tb->size - 1);
            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
644
                       (long)tb->pc, tb->size, flags1, flags2);
B
bellard 已提交
645 646 647 648 649
            }
        }
    }
}

B
blueswir1 已提交
650
static void tb_jmp_check(TranslationBlock *tb)
B
bellard 已提交
651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669
{
    TranslationBlock *tb1;
    unsigned int n1;

    /* suppress any remaining jumps to this TB */
    tb1 = tb->jmp_first;
    for(;;) {
        n1 = (long)tb1 & 3;
        tb1 = (TranslationBlock *)((long)tb1 & ~3);
        if (n1 == 2)
            break;
        tb1 = tb1->jmp_next[n1];
    }
    /* check end of list */
    if (tb1 != tb) {
        printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);
    }
}

B
bellard 已提交
670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686
#endif

/* invalidate one TB */
static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
                             int next_offset)
{
    TranslationBlock *tb1;
    for(;;) {
        tb1 = *ptb;
        if (tb1 == tb) {
            *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
            break;
        }
        ptb = (TranslationBlock **)((char *)tb1 + next_offset);
    }
}

687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703
static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
{
    TranslationBlock *tb1;
    unsigned int n1;

    for(;;) {
        tb1 = *ptb;
        n1 = (long)tb1 & 3;
        tb1 = (TranslationBlock *)((long)tb1 & ~3);
        if (tb1 == tb) {
            *ptb = tb1->page_next[n1];
            break;
        }
        ptb = &tb1->page_next[n1];
    }
}

B
bellard 已提交
704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738
static inline void tb_jmp_remove(TranslationBlock *tb, int n)
{
    TranslationBlock *tb1, **ptb;
    unsigned int n1;

    ptb = &tb->jmp_next[n];
    tb1 = *ptb;
    if (tb1) {
        /* find tb(n) in circular list */
        for(;;) {
            tb1 = *ptb;
            n1 = (long)tb1 & 3;
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
            if (n1 == n && tb1 == tb)
                break;
            if (n1 == 2) {
                ptb = &tb1->jmp_first;
            } else {
                ptb = &tb1->jmp_next[n1];
            }
        }
        /* now we can suppress tb(n) from the list */
        *ptb = tb->jmp_next[n];

        tb->jmp_next[n] = NULL;
    }
}

/* reset the jump entry 'n' of a TB so that it is not chained to
   another TB */
static inline void tb_reset_jump(TranslationBlock *tb, int n)
{
    tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
}

P
pbrook 已提交
739
void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
B
bellard 已提交
740
{
B
bellard 已提交
741
    CPUState *env;
742
    PageDesc *p;
B
bellard 已提交
743
    unsigned int h, n1;
744
    target_phys_addr_t phys_pc;
745
    TranslationBlock *tb1, *tb2;
746

747 748 749
    /* remove the TB from the hash list */
    phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
    h = tb_phys_hash_func(phys_pc);
750
    tb_remove(&tb_phys_hash[h], tb,
751 752 753 754 755 756 757 758 759 760 761 762 763 764
              offsetof(TranslationBlock, phys_hash_next));

    /* remove the TB from the page list */
    if (tb->page_addr[0] != page_addr) {
        p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
        tb_page_remove(&p->first_tb, tb);
        invalidate_page_bitmap(p);
    }
    if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
        p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
        tb_page_remove(&p->first_tb, tb);
        invalidate_page_bitmap(p);
    }

765
    tb_invalidated_flag = 1;
766

B
bellard 已提交
767
    /* remove the TB from the hash list */
768
    h = tb_jmp_cache_hash_func(tb->pc);
B
bellard 已提交
769 770 771 772
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
        if (env->tb_jmp_cache[h] == tb)
            env->tb_jmp_cache[h] = NULL;
    }
B
bellard 已提交
773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790

    /* suppress this TB from the two jump lists */
    tb_jmp_remove(tb, 0);
    tb_jmp_remove(tb, 1);

    /* suppress any remaining jumps to this TB */
    tb1 = tb->jmp_first;
    for(;;) {
        n1 = (long)tb1 & 3;
        if (n1 == 2)
            break;
        tb1 = (TranslationBlock *)((long)tb1 & ~3);
        tb2 = tb1->jmp_next[n1];
        tb_reset_jump(tb1, n1);
        tb1->jmp_next[n1] = NULL;
        tb1 = tb2;
    }
    tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
791

B
bellard 已提交
792
    tb_phys_invalidate_count++;
793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825
}

static inline void set_bits(uint8_t *tab, int start, int len)
{
    int end, mask, end1;

    end = start + len;
    tab += start >> 3;
    mask = 0xff << (start & 7);
    if ((start & ~7) == (end & ~7)) {
        if (start < end) {
            mask &= ~(0xff << (end & 7));
            *tab |= mask;
        }
    } else {
        *tab++ |= mask;
        start = (start + 8) & ~7;
        end1 = end & ~7;
        while (start < end1) {
            *tab++ = 0xff;
            start += 8;
        }
        if (start < end) {
            mask = ~(0xff << (end & 7));
            *tab |= mask;
        }
    }
}

static void build_page_bitmap(PageDesc *p)
{
    int n, tb_start, tb_end;
    TranslationBlock *tb;
826

P
pbrook 已提交
827
    p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851
    if (!p->code_bitmap)
        return;

    tb = p->first_tb;
    while (tb != NULL) {
        n = (long)tb & 3;
        tb = (TranslationBlock *)((long)tb & ~3);
        /* NOTE: this is subtle as a TB may span two physical pages */
        if (n == 0) {
            /* NOTE: tb_end may be after the end of the page, but
               it is not a problem */
            tb_start = tb->pc & ~TARGET_PAGE_MASK;
            tb_end = tb_start + tb->size;
            if (tb_end > TARGET_PAGE_SIZE)
                tb_end = TARGET_PAGE_SIZE;
        } else {
            tb_start = 0;
            tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
        }
        set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
        tb = tb->page_next[n];
    }
}

P
pbrook 已提交
852 853 854
TranslationBlock *tb_gen_code(CPUState *env,
                              target_ulong pc, target_ulong cs_base,
                              int flags, int cflags)
B
bellard 已提交
855 856 857 858 859 860
{
    TranslationBlock *tb;
    uint8_t *tc_ptr;
    target_ulong phys_pc, phys_page2, virt_page2;
    int code_gen_size;

B
bellard 已提交
861 862
    phys_pc = get_phys_addr_code(env, pc);
    tb = tb_alloc(pc);
B
bellard 已提交
863 864 865 866
    if (!tb) {
        /* flush must be done */
        tb_flush(env);
        /* cannot fail at this point */
B
bellard 已提交
867
        tb = tb_alloc(pc);
P
pbrook 已提交
868 869
        /* Don't forget to invalidate previous TB info.  */
        tb_invalidated_flag = 1;
B
bellard 已提交
870 871 872 873 874 875
    }
    tc_ptr = code_gen_ptr;
    tb->tc_ptr = tc_ptr;
    tb->cs_base = cs_base;
    tb->flags = flags;
    tb->cflags = cflags;
876
    cpu_gen_code(env, tb, &code_gen_size);
B
bellard 已提交
877
    code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
878

B
bellard 已提交
879
    /* check next page if needed */
B
bellard 已提交
880
    virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
B
bellard 已提交
881
    phys_page2 = -1;
B
bellard 已提交
882
    if ((pc & TARGET_PAGE_MASK) != virt_page2) {
B
bellard 已提交
883 884 885
        phys_page2 = get_phys_addr_code(env, virt_page2);
    }
    tb_link_phys(tb, phys_pc, phys_page2);
P
pbrook 已提交
886
    return tb;
B
bellard 已提交
887
}
888

889 890
/* invalidate all TBs which intersect with the target physical page
   starting in range [start;end[. NOTE: start and end must refer to
B
bellard 已提交
891 892 893
   the same physical page. 'is_cpu_write_access' should be true if called
   from a real cpu write access: the virtual CPU will exit the current
   TB if code is modified inside this TB. */
894
void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
B
bellard 已提交
895 896
                                   int is_cpu_write_access)
{
897
    TranslationBlock *tb, *tb_next, *saved_tb;
B
bellard 已提交
898
    CPUState *env = cpu_single_env;
899
    target_ulong tb_start, tb_end;
900 901 902 903 904 905 906 907 908 909
    PageDesc *p;
    int n;
#ifdef TARGET_HAS_PRECISE_SMC
    int current_tb_not_found = is_cpu_write_access;
    TranslationBlock *current_tb = NULL;
    int current_tb_modified = 0;
    target_ulong current_pc = 0;
    target_ulong current_cs_base = 0;
    int current_flags = 0;
#endif /* TARGET_HAS_PRECISE_SMC */
910 911

    p = page_find(start >> TARGET_PAGE_BITS);
912
    if (!p)
913
        return;
914
    if (!p->code_bitmap &&
B
bellard 已提交
915 916
        ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
        is_cpu_write_access) {
917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938
        /* build code bitmap */
        build_page_bitmap(p);
    }

    /* we remove all the TBs in the range [start, end[ */
    /* XXX: see if in some cases it could be faster to invalidate all the code */
    tb = p->first_tb;
    while (tb != NULL) {
        n = (long)tb & 3;
        tb = (TranslationBlock *)((long)tb & ~3);
        tb_next = tb->page_next[n];
        /* NOTE: this is subtle as a TB may span two physical pages */
        if (n == 0) {
            /* NOTE: tb_end may be after the end of the page, but
               it is not a problem */
            tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
            tb_end = tb_start + tb->size;
        } else {
            tb_start = tb->page_addr[1];
            tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
        }
        if (!(tb_end <= start || tb_start >= end)) {
B
bellard 已提交
939 940 941 942
#ifdef TARGET_HAS_PRECISE_SMC
            if (current_tb_not_found) {
                current_tb_not_found = 0;
                current_tb = NULL;
P
pbrook 已提交
943
                if (env->mem_io_pc) {
B
bellard 已提交
944
                    /* now we have a real cpu fault */
P
pbrook 已提交
945
                    current_tb = tb_find_pc(env->mem_io_pc);
B
bellard 已提交
946 947 948
                }
            }
            if (current_tb == tb &&
P
pbrook 已提交
949
                (current_tb->cflags & CF_COUNT_MASK) != 1) {
B
bellard 已提交
950 951 952 953 954
                /* If we are modifying the current TB, we must stop
                its execution. We could be more precise by checking
                that the modification is after the current PC, but it
                would require a specialized function to partially
                restore the CPU state */
955

B
bellard 已提交
956
                current_tb_modified = 1;
957
                cpu_restore_state(current_tb, env,
P
pbrook 已提交
958
                                  env->mem_io_pc, NULL);
959 960
                cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
                                     &current_flags);
B
bellard 已提交
961 962
            }
#endif /* TARGET_HAS_PRECISE_SMC */
963 964 965 966 967 968 969
            /* we need to do that to handle the case where a signal
               occurs while doing tb_phys_invalidate() */
            saved_tb = NULL;
            if (env) {
                saved_tb = env->current_tb;
                env->current_tb = NULL;
            }
970
            tb_phys_invalidate(tb, -1);
971 972 973 974 975
            if (env) {
                env->current_tb = saved_tb;
                if (env->interrupt_request && env->current_tb)
                    cpu_interrupt(env, env->interrupt_request);
            }
976 977 978 979 980 981 982
        }
        tb = tb_next;
    }
#if !defined(CONFIG_USER_ONLY)
    /* if no code remaining, no need to continue to use slow writes */
    if (!p->first_tb) {
        invalidate_page_bitmap(p);
B
bellard 已提交
983
        if (is_cpu_write_access) {
P
pbrook 已提交
984
            tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
B
bellard 已提交
985 986 987 988 989 990 991 992
        }
    }
#endif
#ifdef TARGET_HAS_PRECISE_SMC
    if (current_tb_modified) {
        /* we generate a block containing just the instruction
           modifying the memory. It will ensure that it cannot modify
           itself */
993
        env->current_tb = NULL;
P
pbrook 已提交
994
        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
B
bellard 已提交
995
        cpu_resume_from_signal(env, NULL);
996
    }
B
bellard 已提交
997
#endif
998
}
B
bellard 已提交
999

1000
/* len must be <= 8 and start must be a multiple of len */
1001
static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1002 1003 1004
{
    PageDesc *p;
    int offset, b;
1005
#if 0
B
bellard 已提交
1006 1007
    if (1) {
        if (loglevel) {
1008
            fprintf(logfile, "modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
P
pbrook 已提交
1009
                   cpu_single_env->mem_io_vaddr, len,
1010
                   cpu_single_env->eip,
B
bellard 已提交
1011 1012
                   cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
        }
1013 1014
    }
#endif
1015
    p = page_find(start >> TARGET_PAGE_BITS);
1016
    if (!p)
1017 1018 1019 1020 1021 1022 1023 1024
        return;
    if (p->code_bitmap) {
        offset = start & ~TARGET_PAGE_MASK;
        b = p->code_bitmap[offset >> 3] >> (offset & 7);
        if (b & ((1 << len) - 1))
            goto do_invalidate;
    } else {
    do_invalidate:
B
bellard 已提交
1025
        tb_invalidate_phys_page_range(start, start + len, 1);
1026 1027 1028 1029
    }
}

#if !defined(CONFIG_SOFTMMU)
1030
static void tb_invalidate_phys_page(target_phys_addr_t addr,
B
bellard 已提交
1031
                                    unsigned long pc, void *puc)
1032
{
1033
    TranslationBlock *tb;
1034
    PageDesc *p;
1035
    int n;
B
bellard 已提交
1036
#ifdef TARGET_HAS_PRECISE_SMC
1037
    TranslationBlock *current_tb = NULL;
B
bellard 已提交
1038
    CPUState *env = cpu_single_env;
1039 1040 1041 1042
    int current_tb_modified = 0;
    target_ulong current_pc = 0;
    target_ulong current_cs_base = 0;
    int current_flags = 0;
B
bellard 已提交
1043
#endif
1044 1045 1046

    addr &= TARGET_PAGE_MASK;
    p = page_find(addr >> TARGET_PAGE_BITS);
1047
    if (!p)
1048 1049
        return;
    tb = p->first_tb;
B
bellard 已提交
1050 1051 1052 1053 1054
#ifdef TARGET_HAS_PRECISE_SMC
    if (tb && pc != 0) {
        current_tb = tb_find_pc(pc);
    }
#endif
1055 1056 1057
    while (tb != NULL) {
        n = (long)tb & 3;
        tb = (TranslationBlock *)((long)tb & ~3);
B
bellard 已提交
1058 1059
#ifdef TARGET_HAS_PRECISE_SMC
        if (current_tb == tb &&
P
pbrook 已提交
1060
            (current_tb->cflags & CF_COUNT_MASK) != 1) {
B
bellard 已提交
1061 1062 1063 1064 1065
                /* If we are modifying the current TB, we must stop
                   its execution. We could be more precise by checking
                   that the modification is after the current PC, but it
                   would require a specialized function to partially
                   restore the CPU state */
1066

B
bellard 已提交
1067 1068
            current_tb_modified = 1;
            cpu_restore_state(current_tb, env, pc, puc);
1069 1070
            cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
                                 &current_flags);
B
bellard 已提交
1071 1072
        }
#endif /* TARGET_HAS_PRECISE_SMC */
1073 1074 1075
        tb_phys_invalidate(tb, addr);
        tb = tb->page_next[n];
    }
B
bellard 已提交
1076
    p->first_tb = NULL;
B
bellard 已提交
1077 1078 1079 1080 1081
#ifdef TARGET_HAS_PRECISE_SMC
    if (current_tb_modified) {
        /* we generate a block containing just the instruction
           modifying the memory. It will ensure that it cannot modify
           itself */
1082
        env->current_tb = NULL;
P
pbrook 已提交
1083
        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
B
bellard 已提交
1084 1085 1086
        cpu_resume_from_signal(env, puc);
    }
#endif
B
bellard 已提交
1087
}
1088
#endif
B
bellard 已提交
1089 1090

/* add the tb in the target page and protect it if necessary */
1091
static inline void tb_alloc_page(TranslationBlock *tb,
1092
                                 unsigned int n, target_ulong page_addr)
B
bellard 已提交
1093 1094
{
    PageDesc *p;
1095 1096 1097
    TranslationBlock *last_first_tb;

    tb->page_addr[n] = page_addr;
1098
    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1099 1100 1101 1102
    tb->page_next[n] = p->first_tb;
    last_first_tb = p->first_tb;
    p->first_tb = (TranslationBlock *)((long)tb | n);
    invalidate_page_bitmap(p);
B
bellard 已提交
1103

1104
#if defined(TARGET_HAS_SMC) || 1
B
bellard 已提交
1105

1106
#if defined(CONFIG_USER_ONLY)
B
bellard 已提交
1107
    if (p->flags & PAGE_WRITE) {
1108 1109
        target_ulong addr;
        PageDesc *p2;
1110 1111
        int prot;

B
bellard 已提交
1112 1113
        /* force the host page as non writable (writes will have a
           page fault + mprotect overhead) */
1114
        page_addr &= qemu_host_page_mask;
B
bellard 已提交
1115
        prot = 0;
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
        for(addr = page_addr; addr < page_addr + qemu_host_page_size;
            addr += TARGET_PAGE_SIZE) {

            p2 = page_find (addr >> TARGET_PAGE_BITS);
            if (!p2)
                continue;
            prot |= p2->flags;
            p2->flags &= ~PAGE_WRITE;
            page_get_flags(addr);
          }
1126
        mprotect(g2h(page_addr), qemu_host_page_size,
B
bellard 已提交
1127 1128
                 (prot & PAGE_BITS) & ~PAGE_WRITE);
#ifdef DEBUG_TB_INVALIDATE
B
blueswir1 已提交
1129
        printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1130
               page_addr);
B
bellard 已提交
1131 1132
#endif
    }
1133 1134 1135 1136 1137
#else
    /* if some code is already present, then the pages are already
       protected. So we handle the case where only the first TB is
       allocated in a physical page */
    if (!last_first_tb) {
B
bellard 已提交
1138
        tlb_protect_code(page_addr);
1139 1140
    }
#endif
B
bellard 已提交
1141 1142

#endif /* TARGET_HAS_SMC */
B
bellard 已提交
1143 1144 1145 1146
}

/* Allocate a new translation block. Flush the translation buffer if
   too many translation blocks or too much generated code. */
B
bellard 已提交
1147
TranslationBlock *tb_alloc(target_ulong pc)
B
bellard 已提交
1148 1149 1150
{
    TranslationBlock *tb;

1151 1152
    if (nb_tbs >= code_gen_max_blocks ||
        (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
B
bellard 已提交
1153
        return NULL;
B
bellard 已提交
1154 1155
    tb = &tbs[nb_tbs++];
    tb->pc = pc;
1156
    tb->cflags = 0;
B
bellard 已提交
1157 1158 1159
    return tb;
}

P
pbrook 已提交
1160 1161
void tb_free(TranslationBlock *tb)
{
T
ths 已提交
1162
    /* In practice this is mostly used for single use temporary TB
P
pbrook 已提交
1163 1164 1165 1166 1167 1168 1169 1170
       Ignore the hard cases and just back up if this TB happens to
       be the last one generated.  */
    if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
        code_gen_ptr = tb->tc_ptr;
        nb_tbs--;
    }
}

1171 1172
/* add a new TB and link it to the physical page tables. phys_page2 is
   (-1) to indicate that only one page contains the TB. */
1173
void tb_link_phys(TranslationBlock *tb,
1174
                  target_ulong phys_pc, target_ulong phys_page2)
B
bellard 已提交
1175
{
1176 1177 1178
    unsigned int h;
    TranslationBlock **ptb;

P
pbrook 已提交
1179 1180 1181
    /* Grab the mmap lock to stop another thread invalidating this TB
       before we are done.  */
    mmap_lock();
1182 1183 1184 1185 1186
    /* add in the physical hash table */
    h = tb_phys_hash_func(phys_pc);
    ptb = &tb_phys_hash[h];
    tb->phys_hash_next = *ptb;
    *ptb = tb;
B
bellard 已提交
1187 1188

    /* add in the page list */
1189 1190 1191 1192 1193 1194
    tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
    if (phys_page2 != -1)
        tb_alloc_page(tb, 1, phys_page2);
    else
        tb->page_addr[1] = -1;

B
bellard 已提交
1195 1196 1197 1198 1199 1200 1201 1202 1203
    tb->jmp_first = (TranslationBlock *)((long)tb | 2);
    tb->jmp_next[0] = NULL;
    tb->jmp_next[1] = NULL;

    /* init original jump addresses */
    if (tb->tb_next_offset[0] != 0xffff)
        tb_reset_jump(tb, 0);
    if (tb->tb_next_offset[1] != 0xffff)
        tb_reset_jump(tb, 1);
1204 1205 1206 1207

#ifdef DEBUG_TB_CHECK
    tb_page_check();
#endif
P
pbrook 已提交
1208
    mmap_unlock();
B
bellard 已提交
1209 1210
}

1211 1212 1213
/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
   tb[1].tc_ptr. Return NULL if not found */
TranslationBlock *tb_find_pc(unsigned long tc_ptr)
B
bellard 已提交
1214
{
1215 1216 1217
    int m_min, m_max, m;
    unsigned long v;
    TranslationBlock *tb;
B
bellard 已提交
1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237

    if (nb_tbs <= 0)
        return NULL;
    if (tc_ptr < (unsigned long)code_gen_buffer ||
        tc_ptr >= (unsigned long)code_gen_ptr)
        return NULL;
    /* binary search (cf Knuth) */
    m_min = 0;
    m_max = nb_tbs - 1;
    while (m_min <= m_max) {
        m = (m_min + m_max) >> 1;
        tb = &tbs[m];
        v = (unsigned long)tb->tc_ptr;
        if (v == tc_ptr)
            return tb;
        else if (tc_ptr < v) {
            m_max = m - 1;
        } else {
            m_min = m + 1;
        }
1238
    }
B
bellard 已提交
1239 1240
    return &tbs[m_max];
}
B
bellard 已提交
1241

B
bellard 已提交
1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273
static void tb_reset_jump_recursive(TranslationBlock *tb);

static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
{
    TranslationBlock *tb1, *tb_next, **ptb;
    unsigned int n1;

    tb1 = tb->jmp_next[n];
    if (tb1 != NULL) {
        /* find head of list */
        for(;;) {
            n1 = (long)tb1 & 3;
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
            if (n1 == 2)
                break;
            tb1 = tb1->jmp_next[n1];
        }
        /* we are now sure now that tb jumps to tb1 */
        tb_next = tb1;

        /* remove tb from the jmp_first list */
        ptb = &tb_next->jmp_first;
        for(;;) {
            tb1 = *ptb;
            n1 = (long)tb1 & 3;
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
            if (n1 == n && tb1 == tb)
                break;
            ptb = &tb1->jmp_next[n1];
        }
        *ptb = tb->jmp_next[n];
        tb->jmp_next[n] = NULL;
1274

B
bellard 已提交
1275 1276 1277
        /* suppress the jump to next tb in generated code */
        tb_reset_jump(tb, n);

1278
        /* suppress jumps in the tb on which we could have jumped */
B
bellard 已提交
1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
        tb_reset_jump_recursive(tb_next);
    }
}

static void tb_reset_jump_recursive(TranslationBlock *tb)
{
    tb_reset_jump_recursive2(tb, 0);
    tb_reset_jump_recursive2(tb, 1);
}

B
bellard 已提交
1289
#if defined(TARGET_HAS_ICE)
B
bellard 已提交
1290 1291
static void breakpoint_invalidate(CPUState *env, target_ulong pc)
{
1292 1293
    target_phys_addr_t addr;
    target_ulong pd;
P
pbrook 已提交
1294 1295
    ram_addr_t ram_addr;
    PhysPageDesc *p;
B
bellard 已提交
1296

P
pbrook 已提交
1297 1298 1299 1300 1301 1302 1303 1304
    addr = cpu_get_phys_page_debug(env, pc);
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
    ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
P
pbrook 已提交
1305
    tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
B
bellard 已提交
1306
}
B
bellard 已提交
1307
#endif
B
bellard 已提交
1308

1309
/* Add a watchpoint.  */
1310 1311
int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
                          int flags, CPUWatchpoint **watchpoint)
1312
{
1313
    target_ulong len_mask = ~(len - 1);
1314
    CPUWatchpoint *wp;
1315

1316 1317 1318 1319 1320 1321
    /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
    if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
        fprintf(stderr, "qemu: tried to set invalid watchpoint at "
                TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
        return -EINVAL;
    }
1322 1323
    wp = qemu_malloc(sizeof(*wp));
    if (!wp)
A
aliguori 已提交
1324
        return -ENOMEM;
1325 1326

    wp->vaddr = addr;
1327
    wp->len_mask = len_mask;
1328 1329
    wp->flags = flags;

1330
    /* keep all GDB-injected watchpoints in front */
1331 1332 1333 1334
    if (flags & BP_GDB)
        TAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
    else
        TAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1335 1336

    tlb_flush_page(env, addr);
1337 1338 1339 1340

    if (watchpoint)
        *watchpoint = wp;
    return 0;
1341 1342
}

1343 1344 1345
/* Remove a specific watchpoint.  */
int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
                          int flags)
1346
{
1347
    target_ulong len_mask = ~(len - 1);
1348
    CPUWatchpoint *wp;
1349

1350
    TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1351
        if (addr == wp->vaddr && len_mask == wp->len_mask
1352
                && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1353
            cpu_watchpoint_remove_by_ref(env, wp);
1354 1355 1356
            return 0;
        }
    }
1357
    return -ENOENT;
1358 1359
}

1360 1361 1362
/* Remove a specific watchpoint by reference.  */
void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
{
1363
    TAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1364

1365 1366 1367 1368 1369 1370 1371 1372
    tlb_flush_page(env, watchpoint->vaddr);

    qemu_free(watchpoint);
}

/* Remove all matching watchpoints.  */
void cpu_watchpoint_remove_all(CPUState *env, int mask)
{
1373
    CPUWatchpoint *wp, *next;
1374

1375
    TAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1376 1377
        if (wp->flags & mask)
            cpu_watchpoint_remove_by_ref(env, wp);
1378
    }
1379 1380
}

1381 1382 1383
/* Add a breakpoint.  */
int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
                          CPUBreakpoint **breakpoint)
B
bellard 已提交
1384
{
B
bellard 已提交
1385
#if defined(TARGET_HAS_ICE)
1386
    CPUBreakpoint *bp;
1387

1388 1389
    bp = qemu_malloc(sizeof(*bp));
    if (!bp)
A
aliguori 已提交
1390
        return -ENOMEM;
B
bellard 已提交
1391

1392 1393 1394
    bp->pc = pc;
    bp->flags = flags;

1395
    /* keep all GDB-injected breakpoints in front */
1396 1397 1398 1399
    if (flags & BP_GDB)
        TAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
    else
        TAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1400

B
bellard 已提交
1401
    breakpoint_invalidate(env, pc);
1402 1403 1404

    if (breakpoint)
        *breakpoint = bp;
B
bellard 已提交
1405 1406
    return 0;
#else
1407
    return -ENOSYS;
B
bellard 已提交
1408 1409 1410
#endif
}

1411 1412 1413
/* Remove a specific breakpoint.  */
int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
{
1414
#if defined(TARGET_HAS_ICE)
1415 1416
    CPUBreakpoint *bp;

1417
    TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1418 1419 1420 1421
        if (bp->pc == pc && bp->flags == flags) {
            cpu_breakpoint_remove_by_ref(env, bp);
            return 0;
        }
1422
    }
1423 1424 1425
    return -ENOENT;
#else
    return -ENOSYS;
1426 1427 1428
#endif
}

1429 1430
/* Remove a specific breakpoint by reference.  */
void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
B
bellard 已提交
1431
{
B
bellard 已提交
1432
#if defined(TARGET_HAS_ICE)
1433
    TAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
B
bellard 已提交
1434

1435 1436 1437 1438 1439 1440 1441 1442 1443 1444
    breakpoint_invalidate(env, breakpoint->pc);

    qemu_free(breakpoint);
#endif
}

/* Remove all matching breakpoints. */
void cpu_breakpoint_remove_all(CPUState *env, int mask)
{
#if defined(TARGET_HAS_ICE)
1445
    CPUBreakpoint *bp, *next;
1446

1447
    TAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1448 1449
        if (bp->flags & mask)
            cpu_breakpoint_remove_by_ref(env, bp);
1450
    }
B
bellard 已提交
1451 1452 1453
#endif
}

B
bellard 已提交
1454 1455 1456 1457
/* enable or disable single step mode. EXCP_DEBUG is returned by the
   CPU loop after each instruction */
void cpu_single_step(CPUState *env, int enabled)
{
B
bellard 已提交
1458
#if defined(TARGET_HAS_ICE)
B
bellard 已提交
1459 1460 1461
    if (env->singlestep_enabled != enabled) {
        env->singlestep_enabled = enabled;
        /* must flush all the translated code to avoid inconsistancies */
1462
        /* XXX: only flush what is necessary */
1463
        tb_flush(env);
B
bellard 已提交
1464 1465 1466 1467
    }
#endif
}

1468 1469 1470 1471 1472
/* enable or disable low levels log */
void cpu_set_log(int log_flags)
{
    loglevel = log_flags;
    if (loglevel && !logfile) {
P
pbrook 已提交
1473
        logfile = fopen(logfilename, log_append ? "a" : "w");
1474 1475 1476 1477
        if (!logfile) {
            perror(logfilename);
            _exit(1);
        }
1478 1479 1480
#if !defined(CONFIG_SOFTMMU)
        /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
        {
1481
            static char logfile_buf[4096];
1482 1483 1484
            setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
        }
#else
1485
        setvbuf(logfile, NULL, _IOLBF, 0);
1486
#endif
P
pbrook 已提交
1487 1488 1489 1490 1491
        log_append = 1;
    }
    if (!loglevel && logfile) {
        fclose(logfile);
        logfile = NULL;
1492 1493 1494 1495 1496 1497
    }
}

void cpu_set_log_filename(const char *filename)
{
    logfilename = strdup(filename);
P
pbrook 已提交
1498 1499 1500 1501 1502
    if (logfile) {
        fclose(logfile);
        logfile = NULL;
    }
    cpu_set_log(loglevel);
1503
}
B
bellard 已提交
1504

1505
/* mask must never be zero, except for A20 change call */
B
bellard 已提交
1506
void cpu_interrupt(CPUState *env, int mask)
B
bellard 已提交
1507
{
P
pbrook 已提交
1508
#if !defined(USE_NPTL)
B
bellard 已提交
1509
    TranslationBlock *tb;
1510
    static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
P
pbrook 已提交
1511
#endif
P
pbrook 已提交
1512
    int old_mask;
1513

P
pbrook 已提交
1514
    old_mask = env->interrupt_request;
P
pbrook 已提交
1515
    /* FIXME: This is probably not threadsafe.  A different thread could
T
ths 已提交
1516
       be in the middle of a read-modify-write operation.  */
B
bellard 已提交
1517
    env->interrupt_request |= mask;
P
pbrook 已提交
1518 1519 1520 1521 1522 1523
#if defined(USE_NPTL)
    /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
       problem and hope the cpu will stop of its own accord.  For userspace
       emulation this often isn't actually as bad as it sounds.  Often
       signals are used primarily to interrupt blocking syscalls.  */
#else
P
pbrook 已提交
1524
    if (use_icount) {
P
pbrook 已提交
1525
        env->icount_decr.u16.high = 0xffff;
P
pbrook 已提交
1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542
#ifndef CONFIG_USER_ONLY
        /* CPU_INTERRUPT_EXIT isn't a real interrupt.  It just means
           an async event happened and we need to process it.  */
        if (!can_do_io(env)
            && (mask & ~(old_mask | CPU_INTERRUPT_EXIT)) != 0) {
            cpu_abort(env, "Raised interrupt while not in I/O function");
        }
#endif
    } else {
        tb = env->current_tb;
        /* if the cpu is currently executing code, we must unlink it and
           all the potentially executing TB */
        if (tb && !testandset(&interrupt_lock)) {
            env->current_tb = NULL;
            tb_reset_jump_recursive(tb);
            resetlock(&interrupt_lock);
        }
B
bellard 已提交
1543
    }
P
pbrook 已提交
1544
#endif
B
bellard 已提交
1545 1546
}

1547 1548 1549 1550 1551
void cpu_reset_interrupt(CPUState *env, int mask)
{
    env->interrupt_request &= ~mask;
}

B
blueswir1 已提交
1552
const CPULogItem cpu_log_items[] = {
1553
    { CPU_LOG_TB_OUT_ASM, "out_asm",
1554 1555 1556
      "show generated host assembly code for each compiled TB" },
    { CPU_LOG_TB_IN_ASM, "in_asm",
      "show target assembly code for each compiled TB" },
1557
    { CPU_LOG_TB_OP, "op",
B
bellard 已提交
1558
      "show micro ops for each compiled TB" },
1559
    { CPU_LOG_TB_OP_OPT, "op_opt",
B
blueswir1 已提交
1560 1561 1562
      "show micro ops "
#ifdef TARGET_I386
      "before eflags optimization and "
1563
#endif
B
blueswir1 已提交
1564
      "after liveness analysis" },
1565 1566 1567 1568
    { CPU_LOG_INT, "int",
      "show interrupts/exceptions in short format" },
    { CPU_LOG_EXEC, "exec",
      "show trace before each executed TB (lots of logs)" },
1569
    { CPU_LOG_TB_CPU, "cpu",
T
ths 已提交
1570
      "show CPU state before block translation" },
1571 1572 1573 1574
#ifdef TARGET_I386
    { CPU_LOG_PCALL, "pcall",
      "show protected mode far calls/returns/exceptions" },
#endif
B
bellard 已提交
1575
#ifdef DEBUG_IOPORT
1576 1577
    { CPU_LOG_IOPORT, "ioport",
      "show all i/o ports accesses" },
B
bellard 已提交
1578
#endif
1579 1580 1581 1582 1583 1584 1585 1586 1587
    { 0, NULL, NULL },
};

static int cmp1(const char *s1, int n, const char *s2)
{
    if (strlen(s2) != n)
        return 0;
    return memcmp(s1, s2, n) == 0;
}
1588

1589 1590 1591
/* takes a comma separated list of log masks. Return 0 if error. */
int cpu_str_to_log_mask(const char *str)
{
B
blueswir1 已提交
1592
    const CPULogItem *item;
1593 1594 1595 1596 1597 1598 1599 1600 1601
    int mask;
    const char *p, *p1;

    p = str;
    mask = 0;
    for(;;) {
        p1 = strchr(p, ',');
        if (!p1)
            p1 = p + strlen(p);
B
bellard 已提交
1602 1603 1604 1605 1606
	if(cmp1(p,p1-p,"all")) {
		for(item = cpu_log_items; item->mask != 0; item++) {
			mask |= item->mask;
		}
	} else {
1607 1608 1609 1610 1611
        for(item = cpu_log_items; item->mask != 0; item++) {
            if (cmp1(p, p1 - p, item->name))
                goto found;
        }
        return 0;
B
bellard 已提交
1612
	}
1613 1614 1615 1616 1617 1618 1619 1620
    found:
        mask |= item->mask;
        if (*p1 != ',')
            break;
        p = p1 + 1;
    }
    return mask;
}
B
bellard 已提交
1621

B
bellard 已提交
1622 1623 1624
void cpu_abort(CPUState *env, const char *fmt, ...)
{
    va_list ap;
P
pbrook 已提交
1625
    va_list ap2;
B
bellard 已提交
1626 1627

    va_start(ap, fmt);
P
pbrook 已提交
1628
    va_copy(ap2, ap);
B
bellard 已提交
1629 1630 1631 1632
    fprintf(stderr, "qemu: fatal: ");
    vfprintf(stderr, fmt, ap);
    fprintf(stderr, "\n");
#ifdef TARGET_I386
B
bellard 已提交
1633 1634 1635
    cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
#else
    cpu_dump_state(env, stderr, fprintf, 0);
B
bellard 已提交
1636
#endif
1637
    if (logfile) {
1638
        fprintf(logfile, "qemu: fatal: ");
P
pbrook 已提交
1639
        vfprintf(logfile, fmt, ap2);
1640 1641 1642 1643 1644 1645
        fprintf(logfile, "\n");
#ifdef TARGET_I386
        cpu_dump_state(env, logfile, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
#else
        cpu_dump_state(env, logfile, fprintf, 0);
#endif
1646 1647 1648
        fflush(logfile);
        fclose(logfile);
    }
P
pbrook 已提交
1649
    va_end(ap2);
1650
    va_end(ap);
B
bellard 已提交
1651 1652 1653
    abort();
}

1654 1655
CPUState *cpu_copy(CPUState *env)
{
1656
    CPUState *new_env = cpu_init(env->cpu_model_str);
1657 1658 1659 1660 1661 1662 1663 1664 1665
    /* preserve chaining and index */
    CPUState *next_cpu = new_env->next_cpu;
    int cpu_index = new_env->cpu_index;
    memcpy(new_env, env, sizeof(CPUState));
    new_env->next_cpu = next_cpu;
    new_env->cpu_index = cpu_index;
    return new_env;
}

1666 1667
#if !defined(CONFIG_USER_ONLY)

1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682
static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
{
    unsigned int i;

    /* Discard jump cache entries for any tb which might potentially
       overlap the flushed page.  */
    i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
    memset (&env->tb_jmp_cache[i], 0, 
	    TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));

    i = tb_jmp_cache_hash_page(addr);
    memset (&env->tb_jmp_cache[i], 0, 
	    TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
}

1683 1684 1685
/* NOTE: if flush_global is true, also flush global entries (not
   implemented yet) */
void tlb_flush(CPUState *env, int flush_global)
1686 1687
{
    int i;
1688

1689 1690 1691
#if defined(DEBUG_TLB)
    printf("tlb_flush:\n");
#endif
1692 1693 1694 1695
    /* must reset current TB so that interrupts cannot modify the
       links while we are modifying them */
    env->current_tb = NULL;

1696
    for(i = 0; i < CPU_TLB_SIZE; i++) {
B
bellard 已提交
1697 1698 1699 1700 1701 1702
        env->tlb_table[0][i].addr_read = -1;
        env->tlb_table[0][i].addr_write = -1;
        env->tlb_table[0][i].addr_code = -1;
        env->tlb_table[1][i].addr_read = -1;
        env->tlb_table[1][i].addr_write = -1;
        env->tlb_table[1][i].addr_code = -1;
1703 1704 1705 1706 1707 1708 1709 1710 1711 1712
#if (NB_MMU_MODES >= 3)
        env->tlb_table[2][i].addr_read = -1;
        env->tlb_table[2][i].addr_write = -1;
        env->tlb_table[2][i].addr_code = -1;
#if (NB_MMU_MODES == 4)
        env->tlb_table[3][i].addr_read = -1;
        env->tlb_table[3][i].addr_write = -1;
        env->tlb_table[3][i].addr_code = -1;
#endif
#endif
1713
    }
1714

1715
    memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1716

B
bellard 已提交
1717 1718 1719 1720
#ifdef USE_KQEMU
    if (env->kqemu_enabled) {
        kqemu_flush(env, flush_global);
    }
1721
#endif
B
bellard 已提交
1722
    tlb_flush_count++;
1723 1724
}

B
bellard 已提交
1725
static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
B
bellard 已提交
1726
{
1727
    if (addr == (tlb_entry->addr_read &
B
bellard 已提交
1728
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1729
        addr == (tlb_entry->addr_write &
B
bellard 已提交
1730
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1731
        addr == (tlb_entry->addr_code &
B
bellard 已提交
1732 1733 1734 1735 1736
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
        tlb_entry->addr_read = -1;
        tlb_entry->addr_write = -1;
        tlb_entry->addr_code = -1;
    }
B
bellard 已提交
1737 1738
}

1739
void tlb_flush_page(CPUState *env, target_ulong addr)
1740
{
1741
    int i;
1742

1743
#if defined(DEBUG_TLB)
1744
    printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1745
#endif
1746 1747 1748
    /* must reset current TB so that interrupts cannot modify the
       links while we are modifying them */
    env->current_tb = NULL;
B
bellard 已提交
1749 1750 1751

    addr &= TARGET_PAGE_MASK;
    i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
B
bellard 已提交
1752 1753
    tlb_flush_entry(&env->tlb_table[0][i], addr);
    tlb_flush_entry(&env->tlb_table[1][i], addr);
1754 1755 1756 1757 1758 1759
#if (NB_MMU_MODES >= 3)
    tlb_flush_entry(&env->tlb_table[2][i], addr);
#if (NB_MMU_MODES == 4)
    tlb_flush_entry(&env->tlb_table[3][i], addr);
#endif
#endif
1760

1761
    tlb_flush_jmp_cache(env, addr);
1762

B
bellard 已提交
1763 1764 1765 1766 1767
#ifdef USE_KQEMU
    if (env->kqemu_enabled) {
        kqemu_flush_page(env, addr);
    }
#endif
1768 1769 1770 1771
}

/* update the TLBs so that writes to code in the virtual page 'addr'
   can be detected */
B
bellard 已提交
1772
static void tlb_protect_code(ram_addr_t ram_addr)
1773
{
1774
    cpu_physical_memory_reset_dirty(ram_addr,
B
bellard 已提交
1775 1776
                                    ram_addr + TARGET_PAGE_SIZE,
                                    CODE_DIRTY_FLAG);
1777 1778 1779
}

/* update the TLB so that writes in physical page 'phys_addr' are no longer
1780
   tested for self modifying code */
1781
static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1782
                                    target_ulong vaddr)
1783
{
1784
    phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1785 1786
}

1787
static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1788 1789 1790
                                         unsigned long start, unsigned long length)
{
    unsigned long addr;
B
bellard 已提交
1791 1792
    if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
        addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1793
        if ((addr - start) < length) {
P
pbrook 已提交
1794
            tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1795 1796 1797 1798
        }
    }
}

1799
void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
B
bellard 已提交
1800
                                     int dirty_flags)
1801 1802
{
    CPUState *env;
B
bellard 已提交
1803
    unsigned long length, start1;
B
bellard 已提交
1804 1805
    int i, mask, len;
    uint8_t *p;
1806 1807 1808 1809 1810 1811 1812

    start &= TARGET_PAGE_MASK;
    end = TARGET_PAGE_ALIGN(end);

    length = end - start;
    if (length == 0)
        return;
B
bellard 已提交
1813
    len = length >> TARGET_PAGE_BITS;
1814
#ifdef USE_KQEMU
B
bellard 已提交
1815 1816
    /* XXX: should not depend on cpu context */
    env = first_cpu;
1817
    if (env->kqemu_enabled) {
B
bellard 已提交
1818 1819 1820 1821 1822 1823
        ram_addr_t addr;
        addr = start;
        for(i = 0; i < len; i++) {
            kqemu_set_notdirty(env, addr);
            addr += TARGET_PAGE_SIZE;
        }
1824 1825
    }
#endif
B
bellard 已提交
1826 1827 1828 1829 1830
    mask = ~dirty_flags;
    p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
    for(i = 0; i < len; i++)
        p[i] &= mask;

1831 1832
    /* we modify the TLB cache so that the dirty bit will be set again
       when accessing the range */
1833
    start1 = start + (unsigned long)phys_ram_base;
B
bellard 已提交
1834 1835
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
        for(i = 0; i < CPU_TLB_SIZE; i++)
B
bellard 已提交
1836
            tlb_reset_dirty_range(&env->tlb_table[0][i], start1, length);
B
bellard 已提交
1837
        for(i = 0; i < CPU_TLB_SIZE; i++)
B
bellard 已提交
1838
            tlb_reset_dirty_range(&env->tlb_table[1][i], start1, length);
1839 1840 1841 1842 1843 1844 1845 1846
#if (NB_MMU_MODES >= 3)
        for(i = 0; i < CPU_TLB_SIZE; i++)
            tlb_reset_dirty_range(&env->tlb_table[2][i], start1, length);
#if (NB_MMU_MODES == 4)
        for(i = 0; i < CPU_TLB_SIZE; i++)
            tlb_reset_dirty_range(&env->tlb_table[3][i], start1, length);
#endif
#endif
B
bellard 已提交
1847
    }
1848 1849
}

A
aliguori 已提交
1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860
int cpu_physical_memory_set_dirty_tracking(int enable)
{
    in_migration = enable;
    return 0;
}

int cpu_physical_memory_get_dirty_tracking(void)
{
    return in_migration;
}

A
aliguori 已提交
1861 1862 1863 1864 1865 1866
void cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr, target_phys_addr_t end_addr)
{
    if (kvm_enabled())
        kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
}

1867 1868 1869 1870
static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
{
    ram_addr_t ram_addr;

B
bellard 已提交
1871
    if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1872
        ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) +
1873 1874
            tlb_entry->addend - (unsigned long)phys_ram_base;
        if (!cpu_physical_memory_is_dirty(ram_addr)) {
P
pbrook 已提交
1875
            tlb_entry->addr_write |= TLB_NOTDIRTY;
1876 1877 1878 1879 1880 1881 1882 1883 1884
        }
    }
}

/* update the TLB according to the current state of the dirty bits */
void cpu_tlb_update_dirty(CPUState *env)
{
    int i;
    for(i = 0; i < CPU_TLB_SIZE; i++)
B
bellard 已提交
1885
        tlb_update_dirty(&env->tlb_table[0][i]);
1886
    for(i = 0; i < CPU_TLB_SIZE; i++)
B
bellard 已提交
1887
        tlb_update_dirty(&env->tlb_table[1][i]);
1888 1889 1890 1891 1892 1893 1894 1895
#if (NB_MMU_MODES >= 3)
    for(i = 0; i < CPU_TLB_SIZE; i++)
        tlb_update_dirty(&env->tlb_table[2][i]);
#if (NB_MMU_MODES == 4)
    for(i = 0; i < CPU_TLB_SIZE; i++)
        tlb_update_dirty(&env->tlb_table[3][i]);
#endif
#endif
1896 1897
}

P
pbrook 已提交
1898
static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1899
{
P
pbrook 已提交
1900 1901
    if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
        tlb_entry->addr_write = vaddr;
1902 1903
}

P
pbrook 已提交
1904 1905 1906
/* update the TLB corresponding to virtual page vaddr
   so that it is no longer dirty */
static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1907 1908 1909
{
    int i;

P
pbrook 已提交
1910
    vaddr &= TARGET_PAGE_MASK;
1911
    i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
P
pbrook 已提交
1912 1913
    tlb_set_dirty1(&env->tlb_table[0][i], vaddr);
    tlb_set_dirty1(&env->tlb_table[1][i], vaddr);
1914
#if (NB_MMU_MODES >= 3)
P
pbrook 已提交
1915
    tlb_set_dirty1(&env->tlb_table[2][i], vaddr);
1916
#if (NB_MMU_MODES == 4)
P
pbrook 已提交
1917
    tlb_set_dirty1(&env->tlb_table[3][i], vaddr);
1918 1919
#endif
#endif
1920 1921
}

1922 1923 1924 1925
/* add a new TLB entry. At most one entry for a given virtual address
   is permitted. Return 0 if OK or 2 if the page could not be mapped
   (can only happen in non SOFTMMU mode for I/O pages or pages
   conflicting with the host address space). */
1926 1927
int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
                      target_phys_addr_t paddr, int prot,
1928
                      int mmu_idx, int is_softmmu)
1929
{
B
bellard 已提交
1930
    PhysPageDesc *p;
B
bellard 已提交
1931
    unsigned long pd;
1932
    unsigned int index;
B
bellard 已提交
1933
    target_ulong address;
P
pbrook 已提交
1934
    target_ulong code_address;
1935
    target_phys_addr_t addend;
1936
    int ret;
B
bellard 已提交
1937
    CPUTLBEntry *te;
1938
    CPUWatchpoint *wp;
P
pbrook 已提交
1939
    target_phys_addr_t iotlb;
1940

B
bellard 已提交
1941
    p = phys_page_find(paddr >> TARGET_PAGE_BITS);
1942 1943 1944 1945 1946 1947
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
#if defined(DEBUG_TLB)
1948 1949
    printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
           vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
1950 1951 1952
#endif

    ret = 0;
P
pbrook 已提交
1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972
    address = vaddr;
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
        /* IO memory case (romd handled later) */
        address |= TLB_MMIO;
    }
    addend = (unsigned long)phys_ram_base + (pd & TARGET_PAGE_MASK);
    if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
        /* Normal RAM.  */
        iotlb = pd & TARGET_PAGE_MASK;
        if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
            iotlb |= IO_MEM_NOTDIRTY;
        else
            iotlb |= IO_MEM_ROM;
    } else {
        /* IO handlers are currently passed a phsical address.
           It would be nice to pass an offset from the base address
           of that region.  This would avoid having to special case RAM,
           and avoid full address decoding in every device.
           We can't use the high bits of pd for this because
           IO_MEM_ROMD uses these as a ram address.  */
1973 1974 1975 1976 1977 1978
        iotlb = (pd & ~TARGET_PAGE_MASK);
        if (p) {
            iotlb += p->region_offset;
        } else {
            iotlb += paddr;
        }
P
pbrook 已提交
1979 1980 1981 1982 1983
    }

    code_address = address;
    /* Make accesses to pages with watchpoints go via the
       watchpoint trap routines.  */
1984
    TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1985
        if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
P
pbrook 已提交
1986 1987 1988 1989
            iotlb = io_mem_watch + paddr;
            /* TODO: The memory case can be optimized by not trapping
               reads of pages with a write breakpoint.  */
            address |= TLB_MMIO;
1990
        }
P
pbrook 已提交
1991
    }
1992

P
pbrook 已提交
1993 1994 1995 1996 1997 1998 1999 2000 2001
    index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    env->iotlb[mmu_idx][index] = iotlb - vaddr;
    te = &env->tlb_table[mmu_idx][index];
    te->addend = addend - vaddr;
    if (prot & PAGE_READ) {
        te->addr_read = address;
    } else {
        te->addr_read = -1;
    }
2002

P
pbrook 已提交
2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015
    if (prot & PAGE_EXEC) {
        te->addr_code = code_address;
    } else {
        te->addr_code = -1;
    }
    if (prot & PAGE_WRITE) {
        if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
            (pd & IO_MEM_ROMD)) {
            /* Write access calls the I/O callback.  */
            te->addr_write = address | TLB_MMIO;
        } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
                   !cpu_physical_memory_is_dirty(pd)) {
            te->addr_write = address | TLB_NOTDIRTY;
2016
        } else {
P
pbrook 已提交
2017
            te->addr_write = address;
2018
        }
P
pbrook 已提交
2019 2020
    } else {
        te->addr_write = -1;
2021 2022 2023 2024
    }
    return ret;
}

2025 2026
#else

2027
void tlb_flush(CPUState *env, int flush_global)
2028 2029 2030
{
}

2031
void tlb_flush_page(CPUState *env, target_ulong addr)
2032 2033 2034
{
}

2035 2036
int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
                      target_phys_addr_t paddr, int prot,
2037
                      int mmu_idx, int is_softmmu)
2038 2039 2040
{
    return 0;
}
2041

2042 2043
/* dump memory mappings */
void page_dump(FILE *f)
2044
{
2045 2046 2047
    unsigned long start, end;
    int i, j, prot, prot1;
    PageDesc *p;
2048

2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067
    fprintf(f, "%-8s %-8s %-8s %s\n",
            "start", "end", "size", "prot");
    start = -1;
    end = -1;
    prot = 0;
    for(i = 0; i <= L1_SIZE; i++) {
        if (i < L1_SIZE)
            p = l1_map[i];
        else
            p = NULL;
        for(j = 0;j < L2_SIZE; j++) {
            if (!p)
                prot1 = 0;
            else
                prot1 = p[j].flags;
            if (prot1 != prot) {
                end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
                if (start != -1) {
                    fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2068
                            start, end, end - start,
2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081
                            prot & PAGE_READ ? 'r' : '-',
                            prot & PAGE_WRITE ? 'w' : '-',
                            prot & PAGE_EXEC ? 'x' : '-');
                }
                if (prot1 != 0)
                    start = end;
                else
                    start = -1;
                prot = prot1;
            }
            if (!p)
                break;
        }
2082 2083 2084
    }
}

2085
int page_get_flags(target_ulong address)
2086
{
2087 2088 2089
    PageDesc *p;

    p = page_find(address >> TARGET_PAGE_BITS);
2090
    if (!p)
2091 2092 2093 2094 2095 2096 2097
        return 0;
    return p->flags;
}

/* modify the flags of a page and invalidate the code if
   necessary. The flag PAGE_WRITE_ORG is positionned automatically
   depending on PAGE_WRITE */
2098
void page_set_flags(target_ulong start, target_ulong end, int flags)
2099 2100
{
    PageDesc *p;
2101
    target_ulong addr;
2102

P
pbrook 已提交
2103
    /* mmap_lock should already be held.  */
2104 2105 2106 2107 2108 2109
    start = start & TARGET_PAGE_MASK;
    end = TARGET_PAGE_ALIGN(end);
    if (flags & PAGE_WRITE)
        flags |= PAGE_WRITE_ORG;
    for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
        p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2110 2111 2112 2113
        /* We may be called for host regions that are outside guest
           address space.  */
        if (!p)
            return;
2114 2115
        /* if the write protection is set, then we invalidate the code
           inside */
2116
        if (!(p->flags & PAGE_WRITE) &&
2117 2118
            (flags & PAGE_WRITE) &&
            p->first_tb) {
B
bellard 已提交
2119
            tb_invalidate_phys_page(addr, 0, NULL);
2120 2121 2122
        }
        p->flags = flags;
    }
2123 2124
}

2125 2126 2127 2128 2129 2130
int page_check_range(target_ulong start, target_ulong len, int flags)
{
    PageDesc *p;
    target_ulong end;
    target_ulong addr;

2131 2132 2133 2134
    if (start + len < start)
        /* we've wrapped around */
        return -1;

2135 2136 2137 2138 2139 2140 2141 2142 2143 2144
    end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
    start = start & TARGET_PAGE_MASK;

    for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
        p = page_find(addr >> TARGET_PAGE_BITS);
        if( !p )
            return -1;
        if( !(p->flags & PAGE_VALID) )
            return -1;

2145
        if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2146
            return -1;
2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157
        if (flags & PAGE_WRITE) {
            if (!(p->flags & PAGE_WRITE_ORG))
                return -1;
            /* unprotect the page if it was put read-only because it
               contains translated code */
            if (!(p->flags & PAGE_WRITE)) {
                if (!page_unprotect(addr, 0, NULL))
                    return -1;
            }
            return 0;
        }
2158 2159 2160 2161
    }
    return 0;
}

2162 2163
/* called from signal handler: invalidate the code and unprotect the
   page. Return TRUE if the fault was succesfully handled. */
2164
int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2165 2166 2167
{
    unsigned int page_index, prot, pindex;
    PageDesc *p, *p1;
2168
    target_ulong host_start, host_end, addr;
2169

P
pbrook 已提交
2170 2171 2172 2173 2174
    /* Technically this isn't safe inside a signal handler.  However we
       know this only ever happens in a synchronous SEGV handler, so in
       practice it seems to be ok.  */
    mmap_lock();

2175
    host_start = address & qemu_host_page_mask;
2176 2177
    page_index = host_start >> TARGET_PAGE_BITS;
    p1 = page_find(page_index);
P
pbrook 已提交
2178 2179
    if (!p1) {
        mmap_unlock();
2180
        return 0;
P
pbrook 已提交
2181
    }
2182
    host_end = host_start + qemu_host_page_size;
2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193
    p = p1;
    prot = 0;
    for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
        prot |= p->flags;
        p++;
    }
    /* if the page was really writable, then we change its
       protection back to writable */
    if (prot & PAGE_WRITE_ORG) {
        pindex = (address - host_start) >> TARGET_PAGE_BITS;
        if (!(p1[pindex].flags & PAGE_WRITE)) {
2194
            mprotect((void *)g2h(host_start), qemu_host_page_size,
2195 2196 2197 2198
                     (prot & PAGE_BITS) | PAGE_WRITE);
            p1[pindex].flags |= PAGE_WRITE;
            /* and since the content will be modified, we must invalidate
               the corresponding translated code. */
B
bellard 已提交
2199
            tb_invalidate_phys_page(address, pc, puc);
2200 2201 2202
#ifdef DEBUG_TB_CHECK
            tb_invalidate_check(address);
#endif
P
pbrook 已提交
2203
            mmap_unlock();
2204 2205 2206
            return 1;
        }
    }
P
pbrook 已提交
2207
    mmap_unlock();
2208 2209 2210
    return 0;
}

B
bellard 已提交
2211 2212
static inline void tlb_set_dirty(CPUState *env,
                                 unsigned long addr, target_ulong vaddr)
2213 2214
{
}
2215 2216
#endif /* defined(CONFIG_USER_ONLY) */

2217
#if !defined(CONFIG_USER_ONLY)
2218

2219
static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2220
                             ram_addr_t memory, ram_addr_t region_offset);
2221
static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2222
                           ram_addr_t orig_memory, ram_addr_t region_offset);
2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233
#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
                      need_subpage)                                     \
    do {                                                                \
        if (addr > start_addr)                                          \
            start_addr2 = 0;                                            \
        else {                                                          \
            start_addr2 = start_addr & ~TARGET_PAGE_MASK;               \
            if (start_addr2 > 0)                                        \
                need_subpage = 1;                                       \
        }                                                               \
                                                                        \
2234
        if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE)        \
2235 2236 2237 2238 2239 2240 2241 2242
            end_addr2 = TARGET_PAGE_SIZE - 1;                           \
        else {                                                          \
            end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
            if (end_addr2 < TARGET_PAGE_SIZE - 1)                       \
                need_subpage = 1;                                       \
        }                                                               \
    } while (0)

2243 2244
/* register physical memory. 'size' must be a multiple of the target
   page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2245 2246 2247 2248 2249 2250 2251 2252 2253
   io memory page.  The address used when calling the IO function is
   the offset from the start of the region, plus region_offset.  Both
   start_region and regon_offset are rounded down to a page boundary
   before calculating this offset.  This should not be a problem unless
   the low bits of start_addr and region_offset differ.  */
void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
                                         ram_addr_t size,
                                         ram_addr_t phys_offset,
                                         ram_addr_t region_offset)
2254
{
2255
    target_phys_addr_t addr, end_addr;
B
bellard 已提交
2256
    PhysPageDesc *p;
2257
    CPUState *env;
2258
    ram_addr_t orig_size = size;
2259
    void *subpage;
2260

2261 2262 2263 2264 2265 2266 2267
#ifdef USE_KQEMU
    /* XXX: should not depend on cpu context */
    env = first_cpu;
    if (env->kqemu_enabled) {
        kqemu_set_phys_mem(start_addr, size, phys_offset);
    }
#endif
A
aliguori 已提交
2268 2269 2270
    if (kvm_enabled())
        kvm_set_phys_mem(start_addr, size, phys_offset);

2271
    region_offset &= TARGET_PAGE_MASK;
B
bellard 已提交
2272
    size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2273 2274
    end_addr = start_addr + (target_phys_addr_t)size;
    for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2275 2276
        p = phys_page_find(addr >> TARGET_PAGE_BITS);
        if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2277
            ram_addr_t orig_memory = p->phys_offset;
2278 2279 2280 2281 2282
            target_phys_addr_t start_addr2, end_addr2;
            int need_subpage = 0;

            CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
                          need_subpage);
2283
            if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2284 2285
                if (!(orig_memory & IO_MEM_SUBPAGE)) {
                    subpage = subpage_init((addr & TARGET_PAGE_MASK),
2286 2287
                                           &p->phys_offset, orig_memory,
                                           p->region_offset);
2288 2289 2290 2291
                } else {
                    subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
                                            >> IO_MEM_SHIFT];
                }
2292 2293 2294
                subpage_register(subpage, start_addr2, end_addr2, phys_offset,
                                 region_offset);
                p->region_offset = 0;
2295 2296 2297 2298 2299 2300 2301 2302 2303
            } else {
                p->phys_offset = phys_offset;
                if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
                    (phys_offset & IO_MEM_ROMD))
                    phys_offset += TARGET_PAGE_SIZE;
            }
        } else {
            p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
            p->phys_offset = phys_offset;
2304
            p->region_offset = region_offset;
2305
            if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2306
                (phys_offset & IO_MEM_ROMD)) {
2307
                phys_offset += TARGET_PAGE_SIZE;
P
pbrook 已提交
2308
            } else {
2309 2310 2311 2312 2313 2314
                target_phys_addr_t start_addr2, end_addr2;
                int need_subpage = 0;

                CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
                              end_addr2, need_subpage);

2315
                if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2316
                    subpage = subpage_init((addr & TARGET_PAGE_MASK),
2317 2318
                                           &p->phys_offset, IO_MEM_UNASSIGNED,
                                           0);
2319
                    subpage_register(subpage, start_addr2, end_addr2,
2320 2321
                                     phys_offset, region_offset);
                    p->region_offset = 0;
2322 2323 2324
                }
            }
        }
2325
        region_offset += TARGET_PAGE_SIZE;
2326
    }
2327

2328 2329 2330 2331 2332 2333
    /* since each CPU stores ram addresses in its TLB cache, we must
       reset the modified entries */
    /* XXX: slow ! */
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
        tlb_flush(env, 1);
    }
2334 2335
}

B
bellard 已提交
2336
/* XXX: temporary until new memory mapping API */
2337
ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
B
bellard 已提交
2338 2339 2340 2341 2342 2343 2344 2345 2346
{
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p)
        return IO_MEM_UNASSIGNED;
    return p->phys_offset;
}

B
bellard 已提交
2347
/* XXX: better than nothing */
2348
ram_addr_t qemu_ram_alloc(ram_addr_t size)
B
bellard 已提交
2349 2350
{
    ram_addr_t addr;
2351
    if ((phys_ram_alloc_offset + size) > phys_ram_size) {
T
ths 已提交
2352
        fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
B
bellard 已提交
2353
                (uint64_t)size, (uint64_t)phys_ram_size);
B
bellard 已提交
2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364
        abort();
    }
    addr = phys_ram_alloc_offset;
    phys_ram_alloc_offset = TARGET_PAGE_ALIGN(phys_ram_alloc_offset + size);
    return addr;
}

void qemu_ram_free(ram_addr_t addr)
{
}

B
bellard 已提交
2365
static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2366
{
P
pbrook 已提交
2367
#ifdef DEBUG_UNASSIGNED
B
blueswir1 已提交
2368
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2369
#endif
2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393
#if defined(TARGET_SPARC) || defined(TARGET_CRIS)
    do_unassigned_access(addr, 0, 0, 0, 1);
#endif
    return 0;
}

static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
{
#ifdef DEBUG_UNASSIGNED
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
#endif
#if defined(TARGET_SPARC) || defined(TARGET_CRIS)
    do_unassigned_access(addr, 0, 0, 0, 2);
#endif
    return 0;
}

static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
{
#ifdef DEBUG_UNASSIGNED
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
#endif
#if defined(TARGET_SPARC) || defined(TARGET_CRIS)
    do_unassigned_access(addr, 0, 0, 0, 4);
P
pbrook 已提交
2394
#endif
2395 2396 2397
    return 0;
}

B
bellard 已提交
2398
static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2399
{
P
pbrook 已提交
2400
#ifdef DEBUG_UNASSIGNED
B
blueswir1 已提交
2401
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
P
pbrook 已提交
2402
#endif
2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424
#if defined(TARGET_SPARC) || defined(TARGET_CRIS)
    do_unassigned_access(addr, 1, 0, 0, 1);
#endif
}

static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
{
#ifdef DEBUG_UNASSIGNED
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
#endif
#if defined(TARGET_SPARC) || defined(TARGET_CRIS)
    do_unassigned_access(addr, 1, 0, 0, 2);
#endif
}

static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
{
#ifdef DEBUG_UNASSIGNED
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
#endif
#if defined(TARGET_SPARC) || defined(TARGET_CRIS)
    do_unassigned_access(addr, 1, 0, 0, 4);
2425
#endif
2426 2427 2428 2429
}

static CPUReadMemoryFunc *unassigned_mem_read[3] = {
    unassigned_mem_readb,
2430 2431
    unassigned_mem_readw,
    unassigned_mem_readl,
2432 2433 2434 2435
};

static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
    unassigned_mem_writeb,
2436 2437
    unassigned_mem_writew,
    unassigned_mem_writel,
2438 2439
};

P
pbrook 已提交
2440 2441
static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
                                uint32_t val)
2442
{
2443 2444 2445
    int dirty_flags;
    dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2446
#if !defined(CONFIG_USER_ONLY)
2447 2448
        tb_invalidate_phys_page_fast(ram_addr, 1);
        dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2449
#endif
2450
    }
P
pbrook 已提交
2451
    stb_p(phys_ram_base + ram_addr, val);
2452 2453 2454 2455 2456
#ifdef USE_KQEMU
    if (cpu_single_env->kqemu_enabled &&
        (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
        kqemu_modify_page(cpu_single_env, ram_addr);
#endif
B
bellard 已提交
2457 2458 2459 2460 2461
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
    phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
    /* we remove the notdirty callback only if the code has been
       flushed */
    if (dirty_flags == 0xff)
P
pbrook 已提交
2462
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2463 2464
}

P
pbrook 已提交
2465 2466
static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
                                uint32_t val)
2467
{
2468 2469 2470
    int dirty_flags;
    dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2471
#if !defined(CONFIG_USER_ONLY)
2472 2473
        tb_invalidate_phys_page_fast(ram_addr, 2);
        dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2474
#endif
2475
    }
P
pbrook 已提交
2476
    stw_p(phys_ram_base + ram_addr, val);
2477 2478 2479 2480 2481
#ifdef USE_KQEMU
    if (cpu_single_env->kqemu_enabled &&
        (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
        kqemu_modify_page(cpu_single_env, ram_addr);
#endif
B
bellard 已提交
2482 2483 2484 2485 2486
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
    phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
    /* we remove the notdirty callback only if the code has been
       flushed */
    if (dirty_flags == 0xff)
P
pbrook 已提交
2487
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2488 2489
}

P
pbrook 已提交
2490 2491
static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
                                uint32_t val)
2492
{
2493 2494 2495
    int dirty_flags;
    dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2496
#if !defined(CONFIG_USER_ONLY)
2497 2498
        tb_invalidate_phys_page_fast(ram_addr, 4);
        dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2499
#endif
2500
    }
P
pbrook 已提交
2501
    stl_p(phys_ram_base + ram_addr, val);
2502 2503 2504 2505 2506
#ifdef USE_KQEMU
    if (cpu_single_env->kqemu_enabled &&
        (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
        kqemu_modify_page(cpu_single_env, ram_addr);
#endif
B
bellard 已提交
2507 2508 2509 2510 2511
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
    phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
    /* we remove the notdirty callback only if the code has been
       flushed */
    if (dirty_flags == 0xff)
P
pbrook 已提交
2512
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2513 2514
}

2515
static CPUReadMemoryFunc *error_mem_read[3] = {
2516 2517 2518 2519 2520
    NULL, /* never used */
    NULL, /* never used */
    NULL, /* never used */
};

2521 2522 2523 2524 2525 2526
static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
    notdirty_mem_writeb,
    notdirty_mem_writew,
    notdirty_mem_writel,
};

P
pbrook 已提交
2527
/* Generate a debug exception if a watchpoint has been hit.  */
2528
static void check_watchpoint(int offset, int len_mask, int flags)
P
pbrook 已提交
2529 2530
{
    CPUState *env = cpu_single_env;
2531 2532
    target_ulong pc, cs_base;
    TranslationBlock *tb;
P
pbrook 已提交
2533
    target_ulong vaddr;
2534
    CPUWatchpoint *wp;
2535
    int cpu_flags;
P
pbrook 已提交
2536

2537 2538 2539 2540 2541 2542 2543
    if (env->watchpoint_hit) {
        /* We re-entered the check after replacing the TB. Now raise
         * the debug interrupt so that is will trigger after the
         * current instruction. */
        cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
        return;
    }
P
pbrook 已提交
2544
    vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2545
    TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2546 2547
        if ((vaddr == (wp->vaddr & len_mask) ||
             (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564
            wp->flags |= BP_WATCHPOINT_HIT;
            if (!env->watchpoint_hit) {
                env->watchpoint_hit = wp;
                tb = tb_find_pc(env->mem_io_pc);
                if (!tb) {
                    cpu_abort(env, "check_watchpoint: could not find TB for "
                              "pc=%p", (void *)env->mem_io_pc);
                }
                cpu_restore_state(tb, env, env->mem_io_pc, NULL);
                tb_phys_invalidate(tb, -1);
                if (wp->flags & BP_STOP_BEFORE_ACCESS) {
                    env->exception_index = EXCP_DEBUG;
                } else {
                    cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
                    tb_gen_code(env, pc, cs_base, cpu_flags, 1);
                }
                cpu_resume_from_signal(env, NULL);
2565
            }
2566 2567
        } else {
            wp->flags &= ~BP_WATCHPOINT_HIT;
P
pbrook 已提交
2568 2569 2570 2571
        }
    }
}

2572 2573 2574 2575 2576
/* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
   so these check for a hit then pass through to the normal out-of-line
   phys routines.  */
static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
{
2577
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
2578 2579 2580 2581 2582
    return ldub_phys(addr);
}

static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
{
2583
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
2584 2585 2586 2587 2588
    return lduw_phys(addr);
}

static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
{
2589
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
2590 2591 2592 2593 2594 2595
    return ldl_phys(addr);
}

static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
                             uint32_t val)
{
2596
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
2597 2598 2599 2600 2601 2602
    stb_phys(addr, val);
}

static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
                             uint32_t val)
{
2603
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
2604 2605 2606 2607 2608 2609
    stw_phys(addr, val);
}

static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
                             uint32_t val)
{
2610
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625
    stl_phys(addr, val);
}

static CPUReadMemoryFunc *watch_mem_read[3] = {
    watch_mem_readb,
    watch_mem_readw,
    watch_mem_readl,
};

static CPUWriteMemoryFunc *watch_mem_write[3] = {
    watch_mem_writeb,
    watch_mem_writew,
    watch_mem_writel,
};

2626 2627 2628 2629 2630 2631
static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
                                 unsigned int len)
{
    uint32_t ret;
    unsigned int idx;

2632
    idx = SUBPAGE_IDX(addr);
2633 2634 2635 2636
#if defined(DEBUG_SUBPAGE)
    printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
           mmio, len, addr, idx);
#endif
2637 2638
    ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
                                       addr + mmio->region_offset[idx][0][len]);
2639 2640 2641 2642 2643 2644 2645 2646 2647

    return ret;
}

static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
                              uint32_t value, unsigned int len)
{
    unsigned int idx;

2648
    idx = SUBPAGE_IDX(addr);
2649 2650 2651 2652
#if defined(DEBUG_SUBPAGE)
    printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
           mmio, len, addr, idx, value);
#endif
2653 2654 2655
    (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
                                  addr + mmio->region_offset[idx][1][len],
                                  value);
2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724
}

static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
{
#if defined(DEBUG_SUBPAGE)
    printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
#endif

    return subpage_readlen(opaque, addr, 0);
}

static void subpage_writeb (void *opaque, target_phys_addr_t addr,
                            uint32_t value)
{
#if defined(DEBUG_SUBPAGE)
    printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
#endif
    subpage_writelen(opaque, addr, value, 0);
}

static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
{
#if defined(DEBUG_SUBPAGE)
    printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
#endif

    return subpage_readlen(opaque, addr, 1);
}

static void subpage_writew (void *opaque, target_phys_addr_t addr,
                            uint32_t value)
{
#if defined(DEBUG_SUBPAGE)
    printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
#endif
    subpage_writelen(opaque, addr, value, 1);
}

static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
{
#if defined(DEBUG_SUBPAGE)
    printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
#endif

    return subpage_readlen(opaque, addr, 2);
}

static void subpage_writel (void *opaque,
                         target_phys_addr_t addr, uint32_t value)
{
#if defined(DEBUG_SUBPAGE)
    printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
#endif
    subpage_writelen(opaque, addr, value, 2);
}

static CPUReadMemoryFunc *subpage_read[] = {
    &subpage_readb,
    &subpage_readw,
    &subpage_readl,
};

static CPUWriteMemoryFunc *subpage_write[] = {
    &subpage_writeb,
    &subpage_writew,
    &subpage_writel,
};

static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2725
                             ram_addr_t memory, ram_addr_t region_offset)
2726 2727
{
    int idx, eidx;
2728
    unsigned int i;
2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739

    if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
        return -1;
    idx = SUBPAGE_IDX(start);
    eidx = SUBPAGE_IDX(end);
#if defined(DEBUG_SUBPAGE)
    printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__,
           mmio, start, end, idx, eidx, memory);
#endif
    memory >>= IO_MEM_SHIFT;
    for (; idx <= eidx; idx++) {
2740
        for (i = 0; i < 4; i++) {
2741 2742 2743
            if (io_mem_read[memory][i]) {
                mmio->mem_read[idx][i] = &io_mem_read[memory][i];
                mmio->opaque[idx][0][i] = io_mem_opaque[memory];
2744
                mmio->region_offset[idx][0][i] = region_offset;
2745 2746 2747 2748
            }
            if (io_mem_write[memory][i]) {
                mmio->mem_write[idx][i] = &io_mem_write[memory][i];
                mmio->opaque[idx][1][i] = io_mem_opaque[memory];
2749
                mmio->region_offset[idx][1][i] = region_offset;
2750
            }
2751
        }
2752 2753 2754 2755 2756
    }

    return 0;
}

2757
static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2758
                           ram_addr_t orig_memory, ram_addr_t region_offset)
2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771
{
    subpage_t *mmio;
    int subpage_memory;

    mmio = qemu_mallocz(sizeof(subpage_t));
    if (mmio != NULL) {
        mmio->base = base;
        subpage_memory = cpu_register_io_memory(0, subpage_read, subpage_write, mmio);
#if defined(DEBUG_SUBPAGE)
        printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
               mmio, base, TARGET_PAGE_SIZE, subpage_memory);
#endif
        *phys = subpage_memory | IO_MEM_SUBPAGE;
2772 2773
        subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
                         region_offset);
2774 2775 2776 2777 2778
    }

    return mmio;
}

2779 2780
static void io_mem_init(void)
{
2781
    cpu_register_io_memory(IO_MEM_ROM >> IO_MEM_SHIFT, error_mem_read, unassigned_mem_write, NULL);
B
bellard 已提交
2782
    cpu_register_io_memory(IO_MEM_UNASSIGNED >> IO_MEM_SHIFT, unassigned_mem_read, unassigned_mem_write, NULL);
2783
    cpu_register_io_memory(IO_MEM_NOTDIRTY >> IO_MEM_SHIFT, error_mem_read, notdirty_mem_write, NULL);
2784 2785
    io_mem_nb = 5;

P
pbrook 已提交
2786
    io_mem_watch = cpu_register_io_memory(0, watch_mem_read,
2787
                                          watch_mem_write, NULL);
2788
    /* alloc dirty bits array */
B
bellard 已提交
2789
    phys_ram_dirty = qemu_vmalloc(phys_ram_size >> TARGET_PAGE_BITS);
2790
    memset(phys_ram_dirty, 0xff, phys_ram_size >> TARGET_PAGE_BITS);
2791 2792 2793 2794
}

/* mem_read and mem_write are arrays of functions containing the
   function to access byte (index 0), word (index 1) and dword (index
2795 2796 2797
   2). Functions can be omitted with a NULL function pointer. The
   registered functions may be modified dynamically later.
   If io_index is non zero, the corresponding io zone is
2798 2799 2800
   modified. If it is zero, a new io zone is allocated. The return
   value can be used with cpu_register_physical_memory(). (-1) is
   returned if error. */
2801 2802
int cpu_register_io_memory(int io_index,
                           CPUReadMemoryFunc **mem_read,
B
bellard 已提交
2803 2804
                           CPUWriteMemoryFunc **mem_write,
                           void *opaque)
2805
{
2806
    int i, subwidth = 0;
2807 2808

    if (io_index <= 0) {
B
bellard 已提交
2809
        if (io_mem_nb >= IO_MEM_NB_ENTRIES)
2810 2811 2812 2813 2814 2815
            return -1;
        io_index = io_mem_nb++;
    } else {
        if (io_index >= IO_MEM_NB_ENTRIES)
            return -1;
    }
B
bellard 已提交
2816

2817
    for(i = 0;i < 3; i++) {
2818 2819
        if (!mem_read[i] || !mem_write[i])
            subwidth = IO_MEM_SUBWIDTH;
2820 2821 2822
        io_mem_read[io_index][i] = mem_read[i];
        io_mem_write[io_index][i] = mem_write[i];
    }
B
bellard 已提交
2823
    io_mem_opaque[io_index] = opaque;
2824
    return (io_index << IO_MEM_SHIFT) | subwidth;
2825
}
B
bellard 已提交
2826

B
bellard 已提交
2827 2828 2829 2830 2831 2832 2833 2834 2835 2836
CPUWriteMemoryFunc **cpu_get_io_memory_write(int io_index)
{
    return io_mem_write[io_index >> IO_MEM_SHIFT];
}

CPUReadMemoryFunc **cpu_get_io_memory_read(int io_index)
{
    return io_mem_read[io_index >> IO_MEM_SHIFT];
}

2837 2838
#endif /* !defined(CONFIG_USER_ONLY) */

B
bellard 已提交
2839 2840
/* physical memory access (slow version, mainly for debug) */
#if defined(CONFIG_USER_ONLY)
2841
void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
B
bellard 已提交
2842 2843 2844 2845
                            int len, int is_write)
{
    int l, flags;
    target_ulong page;
2846
    void * p;
B
bellard 已提交
2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858

    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
        flags = page_get_flags(page);
        if (!(flags & PAGE_VALID))
            return;
        if (is_write) {
            if (!(flags & PAGE_WRITE))
                return;
2859
            /* XXX: this code should not depend on lock_user */
A
aurel32 已提交
2860
            if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2861 2862
                /* FIXME - should this return an error rather than just fail? */
                return;
A
aurel32 已提交
2863 2864
            memcpy(p, buf, l);
            unlock_user(p, addr, l);
B
bellard 已提交
2865 2866 2867
        } else {
            if (!(flags & PAGE_READ))
                return;
2868
            /* XXX: this code should not depend on lock_user */
A
aurel32 已提交
2869
            if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2870 2871
                /* FIXME - should this return an error rather than just fail? */
                return;
A
aurel32 已提交
2872
            memcpy(buf, p, l);
A
aurel32 已提交
2873
            unlock_user(p, addr, 0);
B
bellard 已提交
2874 2875 2876 2877 2878 2879
        }
        len -= l;
        buf += l;
        addr += l;
    }
}
B
bellard 已提交
2880

B
bellard 已提交
2881
#else
2882
void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
B
bellard 已提交
2883 2884 2885 2886 2887
                            int len, int is_write)
{
    int l, io_index;
    uint8_t *ptr;
    uint32_t val;
2888 2889
    target_phys_addr_t page;
    unsigned long pd;
B
bellard 已提交
2890
    PhysPageDesc *p;
2891

B
bellard 已提交
2892 2893 2894 2895 2896
    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
B
bellard 已提交
2897
        p = phys_page_find(page >> TARGET_PAGE_BITS);
B
bellard 已提交
2898 2899 2900 2901 2902
        if (!p) {
            pd = IO_MEM_UNASSIGNED;
        } else {
            pd = p->phys_offset;
        }
2903

B
bellard 已提交
2904
        if (is_write) {
2905
            if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
B
bellard 已提交
2906
                io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
2907 2908
                if (p)
                    addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
2909 2910
                /* XXX: could force cpu_single_env to NULL to avoid
                   potential bugs */
B
bellard 已提交
2911
                if (l >= 4 && ((addr & 3) == 0)) {
B
bellard 已提交
2912
                    /* 32 bit write access */
B
bellard 已提交
2913
                    val = ldl_p(buf);
B
bellard 已提交
2914
                    io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
B
bellard 已提交
2915 2916
                    l = 4;
                } else if (l >= 2 && ((addr & 1) == 0)) {
B
bellard 已提交
2917
                    /* 16 bit write access */
B
bellard 已提交
2918
                    val = lduw_p(buf);
B
bellard 已提交
2919
                    io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
B
bellard 已提交
2920 2921
                    l = 2;
                } else {
B
bellard 已提交
2922
                    /* 8 bit write access */
B
bellard 已提交
2923
                    val = ldub_p(buf);
B
bellard 已提交
2924
                    io_mem_write[io_index][0](io_mem_opaque[io_index], addr, val);
B
bellard 已提交
2925 2926 2927
                    l = 1;
                }
            } else {
2928 2929
                unsigned long addr1;
                addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
B
bellard 已提交
2930
                /* RAM case */
2931
                ptr = phys_ram_base + addr1;
B
bellard 已提交
2932
                memcpy(ptr, buf, l);
2933 2934 2935 2936
                if (!cpu_physical_memory_is_dirty(addr1)) {
                    /* invalidate code */
                    tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
                    /* set dirty bit */
2937
                    phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
B
bellard 已提交
2938
                        (0xff & ~CODE_DIRTY_FLAG);
2939
                }
B
bellard 已提交
2940 2941
            }
        } else {
2942
            if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
2943
                !(pd & IO_MEM_ROMD)) {
B
bellard 已提交
2944 2945
                /* I/O case */
                io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
2946 2947
                if (p)
                    addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
2948 2949
                if (l >= 4 && ((addr & 3) == 0)) {
                    /* 32 bit read access */
B
bellard 已提交
2950
                    val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
B
bellard 已提交
2951
                    stl_p(buf, val);
B
bellard 已提交
2952 2953 2954
                    l = 4;
                } else if (l >= 2 && ((addr & 1) == 0)) {
                    /* 16 bit read access */
B
bellard 已提交
2955
                    val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
B
bellard 已提交
2956
                    stw_p(buf, val);
B
bellard 已提交
2957 2958
                    l = 2;
                } else {
B
bellard 已提交
2959
                    /* 8 bit read access */
B
bellard 已提交
2960
                    val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr);
B
bellard 已提交
2961
                    stb_p(buf, val);
B
bellard 已提交
2962 2963 2964 2965
                    l = 1;
                }
            } else {
                /* RAM case */
2966
                ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
B
bellard 已提交
2967 2968 2969 2970 2971 2972 2973 2974 2975
                    (addr & ~TARGET_PAGE_MASK);
                memcpy(buf, ptr, l);
            }
        }
        len -= l;
        buf += l;
        addr += l;
    }
}
B
bellard 已提交
2976

B
bellard 已提交
2977
/* used for ROM loading : can write in RAM and ROM */
2978
void cpu_physical_memory_write_rom(target_phys_addr_t addr,
B
bellard 已提交
2979 2980 2981 2982 2983 2984 2985
                                   const uint8_t *buf, int len)
{
    int l;
    uint8_t *ptr;
    target_phys_addr_t page;
    unsigned long pd;
    PhysPageDesc *p;
2986

B
bellard 已提交
2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997
    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
        p = phys_page_find(page >> TARGET_PAGE_BITS);
        if (!p) {
            pd = IO_MEM_UNASSIGNED;
        } else {
            pd = p->phys_offset;
        }
2998

B
bellard 已提交
2999
        if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3000 3001
            (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
            !(pd & IO_MEM_ROMD)) {
B
bellard 已提交
3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016
            /* do nothing */
        } else {
            unsigned long addr1;
            addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
            /* ROM/RAM case */
            ptr = phys_ram_base + addr1;
            memcpy(ptr, buf, l);
        }
        len -= l;
        buf += l;
        addr += l;
    }
}


B
bellard 已提交
3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031
/* warning: addr must be aligned */
uint32_t ldl_phys(target_phys_addr_t addr)
{
    int io_index;
    uint8_t *ptr;
    uint32_t val;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3032

3033
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3034
        !(pd & IO_MEM_ROMD)) {
B
bellard 已提交
3035 3036
        /* I/O case */
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3037 3038
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
3039 3040 3041
        val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
    } else {
        /* RAM case */
3042
        ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
B
bellard 已提交
3043 3044 3045 3046 3047 3048
            (addr & ~TARGET_PAGE_MASK);
        val = ldl_p(ptr);
    }
    return val;
}

B
bellard 已提交
3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063
/* warning: addr must be aligned */
uint64_t ldq_phys(target_phys_addr_t addr)
{
    int io_index;
    uint8_t *ptr;
    uint64_t val;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3064

3065 3066
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
        !(pd & IO_MEM_ROMD)) {
B
bellard 已提交
3067 3068
        /* I/O case */
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3069 3070
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
3071 3072 3073 3074 3075 3076 3077 3078 3079
#ifdef TARGET_WORDS_BIGENDIAN
        val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
        val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
#else
        val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
        val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
#endif
    } else {
        /* RAM case */
3080
        ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
B
bellard 已提交
3081 3082 3083 3084 3085 3086
            (addr & ~TARGET_PAGE_MASK);
        val = ldq_p(ptr);
    }
    return val;
}

B
bellard 已提交
3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102
/* XXX: optimize */
uint32_t ldub_phys(target_phys_addr_t addr)
{
    uint8_t val;
    cpu_physical_memory_read(addr, &val, 1);
    return val;
}

/* XXX: optimize */
uint32_t lduw_phys(target_phys_addr_t addr)
{
    uint16_t val;
    cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
    return tswap16(val);
}

B
bellard 已提交
3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118
/* warning: addr must be aligned. The ram page is not masked as dirty
   and the code inside is not invalidated. It is useful if the dirty
   bits are used to track modified PTEs */
void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
{
    int io_index;
    uint8_t *ptr;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3119

3120
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
B
bellard 已提交
3121
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3122 3123
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
3124 3125
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
    } else {
A
aliguori 已提交
3126 3127
        unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
        ptr = phys_ram_base + addr1;
B
bellard 已提交
3128
        stl_p(ptr, val);
A
aliguori 已提交
3129 3130 3131 3132 3133 3134 3135 3136 3137 3138

        if (unlikely(in_migration)) {
            if (!cpu_physical_memory_is_dirty(addr1)) {
                /* invalidate code */
                tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
                /* set dirty bit */
                phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
                    (0xff & ~CODE_DIRTY_FLAG);
            }
        }
B
bellard 已提交
3139 3140 3141
    }
}

J
j_mayer 已提交
3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154
void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
{
    int io_index;
    uint8_t *ptr;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3155

J
j_mayer 已提交
3156 3157
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3158 3159
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
J
j_mayer 已提交
3160 3161 3162 3163 3164 3165 3166 3167
#ifdef TARGET_WORDS_BIGENDIAN
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
#else
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
#endif
    } else {
3168
        ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
J
j_mayer 已提交
3169 3170 3171 3172 3173
            (addr & ~TARGET_PAGE_MASK);
        stq_p(ptr, val);
    }
}

B
bellard 已提交
3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187
/* warning: addr must be aligned */
void stl_phys(target_phys_addr_t addr, uint32_t val)
{
    int io_index;
    uint8_t *ptr;
    unsigned long pd;
    PhysPageDesc *p;

    p = phys_page_find(addr >> TARGET_PAGE_BITS);
    if (!p) {
        pd = IO_MEM_UNASSIGNED;
    } else {
        pd = p->phys_offset;
    }
3188

3189
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
B
bellard 已提交
3190
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3191 3192
        if (p)
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
B
bellard 已提交
3193 3194 3195 3196 3197 3198 3199
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
    } else {
        unsigned long addr1;
        addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
        /* RAM case */
        ptr = phys_ram_base + addr1;
        stl_p(ptr, val);
3200 3201 3202 3203
        if (!cpu_physical_memory_is_dirty(addr1)) {
            /* invalidate code */
            tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
            /* set dirty bit */
B
bellard 已提交
3204 3205
            phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
                (0xff & ~CODE_DIRTY_FLAG);
3206
        }
B
bellard 已提交
3207 3208 3209
    }
}

B
bellard 已提交
3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230
/* XXX: optimize */
void stb_phys(target_phys_addr_t addr, uint32_t val)
{
    uint8_t v = val;
    cpu_physical_memory_write(addr, &v, 1);
}

/* XXX: optimize */
void stw_phys(target_phys_addr_t addr, uint32_t val)
{
    uint16_t v = tswap16(val);
    cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
}

/* XXX: optimize */
void stq_phys(target_phys_addr_t addr, uint64_t val)
{
    val = tswap64(val);
    cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
}

B
bellard 已提交
3231 3232 3233
#endif

/* virtual memory access for debug */
3234
int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3235
                        uint8_t *buf, int len, int is_write)
B
bellard 已提交
3236 3237
{
    int l;
3238 3239
    target_phys_addr_t phys_addr;
    target_ulong page;
B
bellard 已提交
3240 3241 3242 3243 3244 3245 3246 3247 3248 3249

    while (len > 0) {
        page = addr & TARGET_PAGE_MASK;
        phys_addr = cpu_get_phys_page_debug(env, page);
        /* if no physical page mapped, return an error */
        if (phys_addr == -1)
            return -1;
        l = (page + TARGET_PAGE_SIZE) - addr;
        if (l > len)
            l = len;
3250
        cpu_physical_memory_rw(phys_addr + (addr & ~TARGET_PAGE_MASK),
3251
                               buf, l, is_write);
B
bellard 已提交
3252 3253 3254 3255 3256 3257 3258
        len -= l;
        buf += l;
        addr += l;
    }
    return 0;
}

P
pbrook 已提交
3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275
/* in deterministic execution mode, instructions doing device I/Os
   must be at the end of the TB */
void cpu_io_recompile(CPUState *env, void *retaddr)
{
    TranslationBlock *tb;
    uint32_t n, cflags;
    target_ulong pc, cs_base;
    uint64_t flags;

    tb = tb_find_pc((unsigned long)retaddr);
    if (!tb) {
        cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p", 
                  retaddr);
    }
    n = env->icount_decr.u16.low + tb->icount;
    cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
    /* Calculate how many instructions had been executed before the fault
T
ths 已提交
3276
       occurred.  */
P
pbrook 已提交
3277 3278 3279 3280 3281
    n = n - env->icount_decr.u16.low;
    /* Generate a new TB ending on the I/O insn.  */
    n++;
    /* On MIPS and SH, delay slot instructions can only be restarted if
       they were already the first instruction in the TB.  If this is not
T
ths 已提交
3282
       the first instruction in a TB then re-execute the preceding
P
pbrook 已提交
3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309
       branch.  */
#if defined(TARGET_MIPS)
    if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
        env->active_tc.PC -= 4;
        env->icount_decr.u16.low++;
        env->hflags &= ~MIPS_HFLAG_BMASK;
    }
#elif defined(TARGET_SH4)
    if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
            && n > 1) {
        env->pc -= 2;
        env->icount_decr.u16.low++;
        env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
    }
#endif
    /* This should never happen.  */
    if (n > CF_COUNT_MASK)
        cpu_abort(env, "TB too big during recompile");

    cflags = n | CF_LAST_IO;
    pc = tb->pc;
    cs_base = tb->cs_base;
    flags = tb->flags;
    tb_phys_invalidate(tb, -1);
    /* FIXME: In theory this could raise an exception.  In practice
       we have already translated the block once so it's probably ok.  */
    tb_gen_code(env, pc, cs_base, flags, cflags);
T
ths 已提交
3310
    /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
P
pbrook 已提交
3311 3312 3313 3314 3315 3316 3317
       the first in the TB) then we end up generating a whole new TB and
       repeating the fault, which is horribly inefficient.
       Better would be to execute just this insn uncached, or generate a
       second new TB.  */
    cpu_resume_from_signal(env, NULL);
}

B
bellard 已提交
3318 3319 3320 3321 3322 3323
void dump_exec_info(FILE *f,
                    int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
{
    int i, target_code_size, max_target_code_size;
    int direct_jmp_count, direct_jmp2_count, cross_page;
    TranslationBlock *tb;
3324

B
bellard 已提交
3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344
    target_code_size = 0;
    max_target_code_size = 0;
    cross_page = 0;
    direct_jmp_count = 0;
    direct_jmp2_count = 0;
    for(i = 0; i < nb_tbs; i++) {
        tb = &tbs[i];
        target_code_size += tb->size;
        if (tb->size > max_target_code_size)
            max_target_code_size = tb->size;
        if (tb->page_addr[1] != -1)
            cross_page++;
        if (tb->tb_next_offset[0] != 0xffff) {
            direct_jmp_count++;
            if (tb->tb_next_offset[1] != 0xffff) {
                direct_jmp2_count++;
            }
        }
    }
    /* XXX: avoid using doubles ? */
B
bellard 已提交
3345
    cpu_fprintf(f, "Translation buffer state:\n");
3346 3347 3348 3349
    cpu_fprintf(f, "gen code size       %ld/%ld\n",
                code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
    cpu_fprintf(f, "TB count            %d/%d\n", 
                nb_tbs, code_gen_max_blocks);
3350
    cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
B
bellard 已提交
3351 3352
                nb_tbs ? target_code_size / nb_tbs : 0,
                max_target_code_size);
3353
    cpu_fprintf(f, "TB avg host size    %d bytes (expansion ratio: %0.1f)\n",
B
bellard 已提交
3354 3355
                nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
                target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3356 3357
    cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
            cross_page,
B
bellard 已提交
3358 3359
            nb_tbs ? (cross_page * 100) / nb_tbs : 0);
    cpu_fprintf(f, "direct jump count   %d (%d%%) (2 jumps=%d %d%%)\n",
3360
                direct_jmp_count,
B
bellard 已提交
3361 3362 3363
                nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
                direct_jmp2_count,
                nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
B
bellard 已提交
3364
    cpu_fprintf(f, "\nStatistics:\n");
B
bellard 已提交
3365 3366 3367
    cpu_fprintf(f, "TB flush count      %d\n", tb_flush_count);
    cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
    cpu_fprintf(f, "TLB flush count     %d\n", tlb_flush_count);
B
bellard 已提交
3368
    tcg_dump_info(f, cpu_fprintf);
B
bellard 已提交
3369 3370
}

3371
#if !defined(CONFIG_USER_ONLY)
B
bellard 已提交
3372 3373 3374 3375

#define MMUSUFFIX _cmmu
#define GETPC() NULL
#define env cpu_single_env
B
bellard 已提交
3376
#define SOFTMMU_CODE_ACCESS
B
bellard 已提交
3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392

#define SHIFT 0
#include "softmmu_template.h"

#define SHIFT 1
#include "softmmu_template.h"

#define SHIFT 2
#include "softmmu_template.h"

#define SHIFT 3
#include "softmmu_template.h"

#undef env

#endif