Statistics
| Branch: | Revision:

root / exec.c @ 91dbed4b

History | View | Annotate | Download (118.5 kB)

1
/*
2
 *  virtual page mapping and translated block handling
3
 *
4
 *  Copyright (c) 2003 Fabrice Bellard
5
 *
6
 * This library is free software; you can redistribute it and/or
7
 * modify it under the terms of the GNU Lesser General Public
8
 * License as published by the Free Software Foundation; either
9
 * version 2 of the License, or (at your option) any later version.
10
 *
11
 * This library is distributed in the hope that it will be useful,
12
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
 * Lesser General Public License for more details.
15
 *
16
 * You should have received a copy of the GNU Lesser General Public
17
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18
 */
19
#include "config.h"
20
#ifdef _WIN32
21
#include <windows.h>
22
#else
23
#include <sys/types.h>
24
#include <sys/mman.h>
25
#endif
26
#include <stdlib.h>
27
#include <stdio.h>
28
#include <stdarg.h>
29
#include <string.h>
30
#include <errno.h>
31
#include <unistd.h>
32
#include <inttypes.h>
33

    
34
#include "cpu.h"
35
#include "exec-all.h"
36
#include "qemu-common.h"
37
#include "tcg.h"
38
#include "hw/hw.h"
39
#include "osdep.h"
40
#include "kvm.h"
41
#if defined(CONFIG_USER_ONLY)
42
#include <qemu.h>
43
#include <signal.h>
44
#endif
45

    
46
//#define DEBUG_TB_INVALIDATE
47
//#define DEBUG_FLUSH
48
//#define DEBUG_TLB
49
//#define DEBUG_UNASSIGNED
50

    
51
/* make various TB consistency checks */
52
//#define DEBUG_TB_CHECK
53
//#define DEBUG_TLB_CHECK
54

    
55
//#define DEBUG_IOPORT
56
//#define DEBUG_SUBPAGE
57

    
58
#if !defined(CONFIG_USER_ONLY)
59
/* TB consistency checks only implemented for usermode emulation.  */
60
#undef DEBUG_TB_CHECK
61
#endif
62

    
63
#define SMC_BITMAP_USE_THRESHOLD 10
64

    
65
static TranslationBlock *tbs;
66
int code_gen_max_blocks;
67
TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
68
static int nb_tbs;
69
/* any access to the tbs or the page table must use this lock */
70
spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
71

    
72
#if defined(__arm__) || defined(__sparc_v9__)
73
/* The prologue must be reachable with a direct jump. ARM and Sparc64
74
 have limited branch ranges (possibly also PPC) so place it in a
75
 section close to code segment. */
76
#define code_gen_section                                \
77
    __attribute__((__section__(".gen_code")))           \
78
    __attribute__((aligned (32)))
79
#elif defined(_WIN32)
80
/* Maximum alignment for Win32 is 16. */
81
#define code_gen_section                                \
82
    __attribute__((aligned (16)))
83
#else
84
#define code_gen_section                                \
85
    __attribute__((aligned (32)))
86
#endif
87

    
88
uint8_t code_gen_prologue[1024] code_gen_section;
89
static uint8_t *code_gen_buffer;
90
static unsigned long code_gen_buffer_size;
91
/* threshold to flush the translated code buffer */
92
static unsigned long code_gen_buffer_max_size;
93
uint8_t *code_gen_ptr;
94

    
95
#if !defined(CONFIG_USER_ONLY)
96
int phys_ram_fd;
97
uint8_t *phys_ram_dirty;
98
static int in_migration;
99

    
100
typedef struct RAMBlock {
101
    uint8_t *host;
102
    ram_addr_t offset;
103
    ram_addr_t length;
104
    struct RAMBlock *next;
105
} RAMBlock;
106

    
107
static RAMBlock *ram_blocks;
108
/* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
109
   then we can no longer assume contiguous ram offsets, and external uses
110
   of this variable will break.  */
111
ram_addr_t last_ram_offset;
112
#endif
113

    
114
CPUState *first_cpu;
115
/* current CPU in the current thread. It is only valid inside
116
   cpu_exec() */
117
CPUState *cpu_single_env;
118
/* 0 = Do not count executed instructions.
119
   1 = Precise instruction counting.
120
   2 = Adaptive rate instruction counting.  */
121
int use_icount = 0;
122
/* Current instruction counter.  While executing translated code this may
123
   include some instructions that have not yet been executed.  */
124
int64_t qemu_icount;
125

    
126
typedef struct PageDesc {
127
    /* list of TBs intersecting this ram page */
128
    TranslationBlock *first_tb;
129
    /* in order to optimize self modifying code, we count the number
130
       of lookups we do to a given page to use a bitmap */
131
    unsigned int code_write_count;
132
    uint8_t *code_bitmap;
133
#if defined(CONFIG_USER_ONLY)
134
    unsigned long flags;
135
#endif
136
} PageDesc;
137

    
138
/* In system mode we want L1_MAP to be based on ram offsets,
139
   while in user mode we want it to be based on virtual addresses.  */
140
#if !defined(CONFIG_USER_ONLY)
141
#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
142
# define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
143
#else
144
# define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
145
#endif
146
#else
147
# define L1_MAP_ADDR_SPACE_BITS  TARGET_VIRT_ADDR_SPACE_BITS
148
#endif
149

    
150
/* Size of the L2 (and L3, etc) page tables.  */
151
#define L2_BITS 10
152
#define L2_SIZE (1 << L2_BITS)
153

    
154
/* The bits remaining after N lower levels of page tables.  */
155
#define P_L1_BITS_REM \
156
    ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
157
#define V_L1_BITS_REM \
158
    ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
159

    
160
/* Size of the L1 page table.  Avoid silly small sizes.  */
161
#if P_L1_BITS_REM < 4
162
#define P_L1_BITS  (P_L1_BITS_REM + L2_BITS)
163
#else
164
#define P_L1_BITS  P_L1_BITS_REM
165
#endif
166

    
167
#if V_L1_BITS_REM < 4
168
#define V_L1_BITS  (V_L1_BITS_REM + L2_BITS)
169
#else
170
#define V_L1_BITS  V_L1_BITS_REM
171
#endif
172

    
173
#define P_L1_SIZE  ((target_phys_addr_t)1 << P_L1_BITS)
174
#define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)
175

    
176
#define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
177
#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
178

    
179
unsigned long qemu_real_host_page_size;
180
unsigned long qemu_host_page_bits;
181
unsigned long qemu_host_page_size;
182
unsigned long qemu_host_page_mask;
183

    
184
/* This is a multi-level map on the virtual address space.
185
   The bottom level has pointers to PageDesc.  */
186
static void *l1_map[V_L1_SIZE];
187

    
188
#if !defined(CONFIG_USER_ONLY)
189
typedef struct PhysPageDesc {
190
    /* offset in host memory of the page + io_index in the low bits */
191
    ram_addr_t phys_offset;
192
    ram_addr_t region_offset;
193
} PhysPageDesc;
194

    
195
/* This is a multi-level map on the physical address space.
196
   The bottom level has pointers to PhysPageDesc.  */
197
static void *l1_phys_map[P_L1_SIZE];
198

    
199
static void io_mem_init(void);
200

    
201
/* io memory support */
202
CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
203
CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
204
void *io_mem_opaque[IO_MEM_NB_ENTRIES];
205
static char io_mem_used[IO_MEM_NB_ENTRIES];
206
static int io_mem_watch;
207
#endif
208

    
209
/* log support */
210
#ifdef WIN32
211
static const char *logfilename = "qemu.log";
212
#else
213
static const char *logfilename = "/tmp/qemu.log";
214
#endif
215
FILE *logfile;
216
int loglevel;
217
static int log_append = 0;
218

    
219
/* statistics */
220
#if !defined(CONFIG_USER_ONLY)
221
static int tlb_flush_count;
222
#endif
223
static int tb_flush_count;
224
static int tb_phys_invalidate_count;
225

    
226
#ifdef _WIN32
227
static void map_exec(void *addr, long size)
228
{
229
    DWORD old_protect;
230
    VirtualProtect(addr, size,
231
                   PAGE_EXECUTE_READWRITE, &old_protect);
232
    
233
}
234
#else
235
static void map_exec(void *addr, long size)
236
{
237
    unsigned long start, end, page_size;
238
    
239
    page_size = getpagesize();
240
    start = (unsigned long)addr;
241
    start &= ~(page_size - 1);
242
    
243
    end = (unsigned long)addr + size;
244
    end += page_size - 1;
245
    end &= ~(page_size - 1);
246
    
247
    mprotect((void *)start, end - start,
248
             PROT_READ | PROT_WRITE | PROT_EXEC);
249
}
250
#endif
251

    
252
static void page_init(void)
253
{
254
    /* NOTE: we can always suppose that qemu_host_page_size >=
255
       TARGET_PAGE_SIZE */
256
#ifdef _WIN32
257
    {
258
        SYSTEM_INFO system_info;
259

    
260
        GetSystemInfo(&system_info);
261
        qemu_real_host_page_size = system_info.dwPageSize;
262
    }
263
#else
264
    qemu_real_host_page_size = getpagesize();
265
#endif
266
    if (qemu_host_page_size == 0)
267
        qemu_host_page_size = qemu_real_host_page_size;
268
    if (qemu_host_page_size < TARGET_PAGE_SIZE)
269
        qemu_host_page_size = TARGET_PAGE_SIZE;
270
    qemu_host_page_bits = 0;
271
    while ((1 << qemu_host_page_bits) < qemu_host_page_size)
272
        qemu_host_page_bits++;
273
    qemu_host_page_mask = ~(qemu_host_page_size - 1);
274

    
275
#if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
276
    {
277
        FILE *f;
278

    
279
        last_brk = (unsigned long)sbrk(0);
280

    
281
        f = fopen("/proc/self/maps", "r");
282
        if (f) {
283
            mmap_lock();
284

    
285
            do {
286
                unsigned long startaddr, endaddr;
287
                int n;
288

    
289
                n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
290

    
291
                if (n == 2 && h2g_valid(startaddr)) {
292
                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
293

    
294
                    if (h2g_valid(endaddr)) {
295
                        endaddr = h2g(endaddr);
296
                    } else {
297
                        endaddr = ~0ul;
298
                    }
299
                    page_set_flags(startaddr, endaddr, PAGE_RESERVED);
300
                }
301
            } while (!feof(f));
302

    
303
            fclose(f);
304
            mmap_unlock();
305
        }
306
    }
307
#endif
308
}
309

    
310
static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
311
{
312
    PageDesc *pd;
313
    void **lp;
314
    int i;
315

    
316
#if defined(CONFIG_USER_ONLY)
317
    /* We can't use qemu_malloc because it may recurse into a locked mutex.
318
       Neither can we record the new pages we reserve while allocating a
319
       given page because that may recurse into an unallocated page table
320
       entry.  Stuff the allocations we do make into a queue and process
321
       them after having completed one entire page table allocation.  */
322

    
323
    unsigned long reserve[2 * (V_L1_SHIFT / L2_BITS)];
324
    int reserve_idx = 0;
325

    
326
# define ALLOC(P, SIZE)                                 \
327
    do {                                                \
328
        P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,    \
329
                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);   \
330
        if (h2g_valid(P)) {                             \
331
            reserve[reserve_idx] = h2g(P);              \
332
            reserve[reserve_idx + 1] = SIZE;            \
333
            reserve_idx += 2;                           \
334
        }                                               \
335
    } while (0)
336
#else
337
# define ALLOC(P, SIZE) \
338
    do { P = qemu_mallocz(SIZE); } while (0)
339
#endif
340

    
341
    /* Level 1.  Always allocated.  */
342
    lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
343

    
344
    /* Level 2..N-1.  */
345
    for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
346
        void **p = *lp;
347

    
348
        if (p == NULL) {
349
            if (!alloc) {
350
                return NULL;
351
            }
352
            ALLOC(p, sizeof(void *) * L2_SIZE);
353
            *lp = p;
354
        }
355

    
356
        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
357
    }
358

    
359
    pd = *lp;
360
    if (pd == NULL) {
361
        if (!alloc) {
362
            return NULL;
363
        }
364
        ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
365
        *lp = pd;
366
    }
367

    
368
#undef ALLOC
369
#if defined(CONFIG_USER_ONLY)
370
    for (i = 0; i < reserve_idx; i += 2) {
371
        unsigned long addr = reserve[i];
372
        unsigned long len = reserve[i + 1];
373

    
374
        page_set_flags(addr & TARGET_PAGE_MASK,
375
                       TARGET_PAGE_ALIGN(addr + len),
376
                       PAGE_RESERVED);
377
    }
378
#endif
379

    
380
    return pd + (index & (L2_SIZE - 1));
381
}
382

    
383
static inline PageDesc *page_find(tb_page_addr_t index)
384
{
385
    return page_find_alloc(index, 0);
386
}
387

    
388
#if !defined(CONFIG_USER_ONLY)
389
static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
390
{
391
    PhysPageDesc *pd;
392
    void **lp;
393
    int i;
394

    
395
    /* Level 1.  Always allocated.  */
396
    lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
397

    
398
    /* Level 2..N-1.  */
399
    for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
400
        void **p = *lp;
401
        if (p == NULL) {
402
            if (!alloc) {
403
                return NULL;
404
            }
405
            *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
406
        }
407
        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
408
    }
409

    
410
    pd = *lp;
411
    if (pd == NULL) {
412
        int i;
413

    
414
        if (!alloc) {
415
            return NULL;
416
        }
417

    
418
        *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);
419

    
420
        for (i = 0; i < L2_SIZE; i++) {
421
            pd[i].phys_offset = IO_MEM_UNASSIGNED;
422
            pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
423
        }
424
    }
425

    
426
    return pd + (index & (L2_SIZE - 1));
427
}
428

    
429
static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
430
{
431
    return phys_page_find_alloc(index, 0);
432
}
433

    
434
static void tlb_protect_code(ram_addr_t ram_addr);
435
static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
436
                                    target_ulong vaddr);
437
#define mmap_lock() do { } while(0)
438
#define mmap_unlock() do { } while(0)
439
#endif
440

    
441
#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
442

    
443
#if defined(CONFIG_USER_ONLY)
444
/* Currently it is not recommended to allocate big chunks of data in
445
   user mode. It will change when a dedicated libc will be used */
446
#define USE_STATIC_CODE_GEN_BUFFER
447
#endif
448

    
449
#ifdef USE_STATIC_CODE_GEN_BUFFER
450
static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
451
#endif
452

    
453
static void code_gen_alloc(unsigned long tb_size)
454
{
455
#ifdef USE_STATIC_CODE_GEN_BUFFER
456
    code_gen_buffer = static_code_gen_buffer;
457
    code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
458
    map_exec(code_gen_buffer, code_gen_buffer_size);
459
#else
460
    code_gen_buffer_size = tb_size;
461
    if (code_gen_buffer_size == 0) {
462
#if defined(CONFIG_USER_ONLY)
463
        /* in user mode, phys_ram_size is not meaningful */
464
        code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
465
#else
466
        /* XXX: needs adjustments */
467
        code_gen_buffer_size = (unsigned long)(ram_size / 4);
468
#endif
469
    }
470
    if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
471
        code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
472
    /* The code gen buffer location may have constraints depending on
473
       the host cpu and OS */
474
#if defined(__linux__) 
475
    {
476
        int flags;
477
        void *start = NULL;
478

    
479
        flags = MAP_PRIVATE | MAP_ANONYMOUS;
480
#if defined(__x86_64__)
481
        flags |= MAP_32BIT;
482
        /* Cannot map more than that */
483
        if (code_gen_buffer_size > (800 * 1024 * 1024))
484
            code_gen_buffer_size = (800 * 1024 * 1024);
485
#elif defined(__sparc_v9__)
486
        // Map the buffer below 2G, so we can use direct calls and branches
487
        flags |= MAP_FIXED;
488
        start = (void *) 0x60000000UL;
489
        if (code_gen_buffer_size > (512 * 1024 * 1024))
490
            code_gen_buffer_size = (512 * 1024 * 1024);
491
#elif defined(__arm__)
492
        /* Map the buffer below 32M, so we can use direct calls and branches */
493
        flags |= MAP_FIXED;
494
        start = (void *) 0x01000000UL;
495
        if (code_gen_buffer_size > 16 * 1024 * 1024)
496
            code_gen_buffer_size = 16 * 1024 * 1024;
497
#endif
498
        code_gen_buffer = mmap(start, code_gen_buffer_size,
499
                               PROT_WRITE | PROT_READ | PROT_EXEC,
500
                               flags, -1, 0);
501
        if (code_gen_buffer == MAP_FAILED) {
502
            fprintf(stderr, "Could not allocate dynamic translator buffer\n");
503
            exit(1);
504
        }
505
    }
506
#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
507
    {
508
        int flags;
509
        void *addr = NULL;
510
        flags = MAP_PRIVATE | MAP_ANONYMOUS;
511
#if defined(__x86_64__)
512
        /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
513
         * 0x40000000 is free */
514
        flags |= MAP_FIXED;
515
        addr = (void *)0x40000000;
516
        /* Cannot map more than that */
517
        if (code_gen_buffer_size > (800 * 1024 * 1024))
518
            code_gen_buffer_size = (800 * 1024 * 1024);
519
#endif
520
        code_gen_buffer = mmap(addr, code_gen_buffer_size,
521
                               PROT_WRITE | PROT_READ | PROT_EXEC, 
522
                               flags, -1, 0);
523
        if (code_gen_buffer == MAP_FAILED) {
524
            fprintf(stderr, "Could not allocate dynamic translator buffer\n");
525
            exit(1);
526
        }
527
    }
528
#else
529
    code_gen_buffer = qemu_malloc(code_gen_buffer_size);
530
    map_exec(code_gen_buffer, code_gen_buffer_size);
531
#endif
532
#endif /* !USE_STATIC_CODE_GEN_BUFFER */
533
    map_exec(code_gen_prologue, sizeof(code_gen_prologue));
534
    code_gen_buffer_max_size = code_gen_buffer_size - 
535
        code_gen_max_block_size();
536
    code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
537
    tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
538
}
539

    
540
/* Must be called before using the QEMU cpus. 'tb_size' is the size
541
   (in bytes) allocated to the translation buffer. Zero means default
542
   size. */
543
void cpu_exec_init_all(unsigned long tb_size)
544
{
545
    cpu_gen_init();
546
    code_gen_alloc(tb_size);
547
    code_gen_ptr = code_gen_buffer;
548
    page_init();
549
#if !defined(CONFIG_USER_ONLY)
550
    io_mem_init();
551
#endif
552
}
553

    
554
#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
555

    
556
static int cpu_common_post_load(void *opaque, int version_id)
557
{
558
    CPUState *env = opaque;
559

    
560
    /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
561
       version_id is increased. */
562
    env->interrupt_request &= ~0x01;
563
    tlb_flush(env, 1);
564

    
565
    return 0;
566
}
567

    
568
static const VMStateDescription vmstate_cpu_common = {
569
    .name = "cpu_common",
570
    .version_id = 1,
571
    .minimum_version_id = 1,
572
    .minimum_version_id_old = 1,
573
    .post_load = cpu_common_post_load,
574
    .fields      = (VMStateField []) {
575
        VMSTATE_UINT32(halted, CPUState),
576
        VMSTATE_UINT32(interrupt_request, CPUState),
577
        VMSTATE_END_OF_LIST()
578
    }
579
};
580
#endif
581

    
582
CPUState *qemu_get_cpu(int cpu)
583
{
584
    CPUState *env = first_cpu;
585

    
586
    while (env) {
587
        if (env->cpu_index == cpu)
588
            break;
589
        env = env->next_cpu;
590
    }
591

    
592
    return env;
593
}
594

    
595
void cpu_exec_init(CPUState *env)
596
{
597
    CPUState **penv;
598
    int cpu_index;
599

    
600
#if defined(CONFIG_USER_ONLY)
601
    cpu_list_lock();
602
#endif
603
    env->next_cpu = NULL;
604
    penv = &first_cpu;
605
    cpu_index = 0;
606
    while (*penv != NULL) {
607
        penv = &(*penv)->next_cpu;
608
        cpu_index++;
609
    }
610
    env->cpu_index = cpu_index;
611
    env->numa_node = 0;
612
    QTAILQ_INIT(&env->breakpoints);
613
    QTAILQ_INIT(&env->watchpoints);
614
    *penv = env;
615
#if defined(CONFIG_USER_ONLY)
616
    cpu_list_unlock();
617
#endif
618
#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
619
    vmstate_register(cpu_index, &vmstate_cpu_common, env);
620
    register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
621
                    cpu_save, cpu_load, env);
622
#endif
623
}
624

    
625
static inline void invalidate_page_bitmap(PageDesc *p)
626
{
627
    if (p->code_bitmap) {
628
        qemu_free(p->code_bitmap);
629
        p->code_bitmap = NULL;
630
    }
631
    p->code_write_count = 0;
632
}
633

    
634
/* Set to NULL all the 'first_tb' fields in all PageDescs. */
635

    
636
static void page_flush_tb_1 (int level, void **lp)
637
{
638
    int i;
639

    
640
    if (*lp == NULL) {
641
        return;
642
    }
643
    if (level == 0) {
644
        PageDesc *pd = *lp;
645
        for (i = 0; i < L2_SIZE; ++i) {
646
            pd[i].first_tb = NULL;
647
            invalidate_page_bitmap(pd + i);
648
        }
649
    } else {
650
        void **pp = *lp;
651
        for (i = 0; i < L2_SIZE; ++i) {
652
            page_flush_tb_1 (level - 1, pp + i);
653
        }
654
    }
655
}
656

    
657
static void page_flush_tb(void)
658
{
659
    int i;
660
    for (i = 0; i < V_L1_SIZE; i++) {
661
        page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
662
    }
663
}
664

    
665
/* flush all the translation blocks */
666
/* XXX: tb_flush is currently not thread safe */
667
void tb_flush(CPUState *env1)
668
{
669
    CPUState *env;
670
#if defined(DEBUG_FLUSH)
671
    printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
672
           (unsigned long)(code_gen_ptr - code_gen_buffer),
673
           nb_tbs, nb_tbs > 0 ?
674
           ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
675
#endif
676
    if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
677
        cpu_abort(env1, "Internal error: code buffer overflow\n");
678

    
679
    nb_tbs = 0;
680

    
681
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
682
        memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
683
    }
684

    
685
    memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
686
    page_flush_tb();
687

    
688
    code_gen_ptr = code_gen_buffer;
689
    /* XXX: flush processor icache at this point if cache flush is
690
       expensive */
691
    tb_flush_count++;
692
}
693

    
694
#ifdef DEBUG_TB_CHECK
695

    
696
static void tb_invalidate_check(target_ulong address)
697
{
698
    TranslationBlock *tb;
699
    int i;
700
    address &= TARGET_PAGE_MASK;
701
    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
702
        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
703
            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
704
                  address >= tb->pc + tb->size)) {
705
                printf("ERROR invalidate: address=" TARGET_FMT_lx
706
                       " PC=%08lx size=%04x\n",
707
                       address, (long)tb->pc, tb->size);
708
            }
709
        }
710
    }
711
}
712

    
713
/* verify that all the pages have correct rights for code */
714
static void tb_page_check(void)
715
{
716
    TranslationBlock *tb;
717
    int i, flags1, flags2;
718

    
719
    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
720
        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
721
            flags1 = page_get_flags(tb->pc);
722
            flags2 = page_get_flags(tb->pc + tb->size - 1);
723
            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
724
                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
725
                       (long)tb->pc, tb->size, flags1, flags2);
726
            }
727
        }
728
    }
729
}
730

    
731
#endif
732

    
733
/* invalidate one TB */
734
static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
735
                             int next_offset)
736
{
737
    TranslationBlock *tb1;
738
    for(;;) {
739
        tb1 = *ptb;
740
        if (tb1 == tb) {
741
            *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
742
            break;
743
        }
744
        ptb = (TranslationBlock **)((char *)tb1 + next_offset);
745
    }
746
}
747

    
748
static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
749
{
750
    TranslationBlock *tb1;
751
    unsigned int n1;
752

    
753
    for(;;) {
754
        tb1 = *ptb;
755
        n1 = (long)tb1 & 3;
756
        tb1 = (TranslationBlock *)((long)tb1 & ~3);
757
        if (tb1 == tb) {
758
            *ptb = tb1->page_next[n1];
759
            break;
760
        }
761
        ptb = &tb1->page_next[n1];
762
    }
763
}
764

    
765
static inline void tb_jmp_remove(TranslationBlock *tb, int n)
766
{
767
    TranslationBlock *tb1, **ptb;
768
    unsigned int n1;
769

    
770
    ptb = &tb->jmp_next[n];
771
    tb1 = *ptb;
772
    if (tb1) {
773
        /* find tb(n) in circular list */
774
        for(;;) {
775
            tb1 = *ptb;
776
            n1 = (long)tb1 & 3;
777
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
778
            if (n1 == n && tb1 == tb)
779
                break;
780
            if (n1 == 2) {
781
                ptb = &tb1->jmp_first;
782
            } else {
783
                ptb = &tb1->jmp_next[n1];
784
            }
785
        }
786
        /* now we can suppress tb(n) from the list */
787
        *ptb = tb->jmp_next[n];
788

    
789
        tb->jmp_next[n] = NULL;
790
    }
791
}
792

    
793
/* reset the jump entry 'n' of a TB so that it is not chained to
794
   another TB */
795
static inline void tb_reset_jump(TranslationBlock *tb, int n)
796
{
797
    tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
798
}
799

    
800
void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
801
{
802
    CPUState *env;
803
    PageDesc *p;
804
    unsigned int h, n1;
805
    tb_page_addr_t phys_pc;
806
    TranslationBlock *tb1, *tb2;
807

    
808
    /* remove the TB from the hash list */
809
    phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
810
    h = tb_phys_hash_func(phys_pc);
811
    tb_remove(&tb_phys_hash[h], tb,
812
              offsetof(TranslationBlock, phys_hash_next));
813

    
814
    /* remove the TB from the page list */
815
    if (tb->page_addr[0] != page_addr) {
816
        p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
817
        tb_page_remove(&p->first_tb, tb);
818
        invalidate_page_bitmap(p);
819
    }
820
    if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
821
        p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
822
        tb_page_remove(&p->first_tb, tb);
823
        invalidate_page_bitmap(p);
824
    }
825

    
826
    tb_invalidated_flag = 1;
827

    
828
    /* remove the TB from the hash list */
829
    h = tb_jmp_cache_hash_func(tb->pc);
830
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
831
        if (env->tb_jmp_cache[h] == tb)
832
            env->tb_jmp_cache[h] = NULL;
833
    }
834

    
835
    /* suppress this TB from the two jump lists */
836
    tb_jmp_remove(tb, 0);
837
    tb_jmp_remove(tb, 1);
838

    
839
    /* suppress any remaining jumps to this TB */
840
    tb1 = tb->jmp_first;
841
    for(;;) {
842
        n1 = (long)tb1 & 3;
843
        if (n1 == 2)
844
            break;
845
        tb1 = (TranslationBlock *)((long)tb1 & ~3);
846
        tb2 = tb1->jmp_next[n1];
847
        tb_reset_jump(tb1, n1);
848
        tb1->jmp_next[n1] = NULL;
849
        tb1 = tb2;
850
    }
851
    tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
852

    
853
    tb_phys_invalidate_count++;
854
}
855

    
856
static inline void set_bits(uint8_t *tab, int start, int len)
857
{
858
    int end, mask, end1;
859

    
860
    end = start + len;
861
    tab += start >> 3;
862
    mask = 0xff << (start & 7);
863
    if ((start & ~7) == (end & ~7)) {
864
        if (start < end) {
865
            mask &= ~(0xff << (end & 7));
866
            *tab |= mask;
867
        }
868
    } else {
869
        *tab++ |= mask;
870
        start = (start + 8) & ~7;
871
        end1 = end & ~7;
872
        while (start < end1) {
873
            *tab++ = 0xff;
874
            start += 8;
875
        }
876
        if (start < end) {
877
            mask = ~(0xff << (end & 7));
878
            *tab |= mask;
879
        }
880
    }
881
}
882

    
883
static void build_page_bitmap(PageDesc *p)
884
{
885
    int n, tb_start, tb_end;
886
    TranslationBlock *tb;
887

    
888
    p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
889

    
890
    tb = p->first_tb;
891
    while (tb != NULL) {
892
        n = (long)tb & 3;
893
        tb = (TranslationBlock *)((long)tb & ~3);
894
        /* NOTE: this is subtle as a TB may span two physical pages */
895
        if (n == 0) {
896
            /* NOTE: tb_end may be after the end of the page, but
897
               it is not a problem */
898
            tb_start = tb->pc & ~TARGET_PAGE_MASK;
899
            tb_end = tb_start + tb->size;
900
            if (tb_end > TARGET_PAGE_SIZE)
901
                tb_end = TARGET_PAGE_SIZE;
902
        } else {
903
            tb_start = 0;
904
            tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
905
        }
906
        set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
907
        tb = tb->page_next[n];
908
    }
909
}
910

    
911
TranslationBlock *tb_gen_code(CPUState *env,
912
                              target_ulong pc, target_ulong cs_base,
913
                              int flags, int cflags)
914
{
915
    TranslationBlock *tb;
916
    uint8_t *tc_ptr;
917
    tb_page_addr_t phys_pc, phys_page2;
918
    target_ulong virt_page2;
919
    int code_gen_size;
920

    
921
    phys_pc = get_page_addr_code(env, pc);
922
    tb = tb_alloc(pc);
923
    if (!tb) {
924
        /* flush must be done */
925
        tb_flush(env);
926
        /* cannot fail at this point */
927
        tb = tb_alloc(pc);
928
        /* Don't forget to invalidate previous TB info.  */
929
        tb_invalidated_flag = 1;
930
    }
931
    tc_ptr = code_gen_ptr;
932
    tb->tc_ptr = tc_ptr;
933
    tb->cs_base = cs_base;
934
    tb->flags = flags;
935
    tb->cflags = cflags;
936
    cpu_gen_code(env, tb, &code_gen_size);
937
    code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
938

    
939
    /* check next page if needed */
940
    virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
941
    phys_page2 = -1;
942
    if ((pc & TARGET_PAGE_MASK) != virt_page2) {
943
        phys_page2 = get_page_addr_code(env, virt_page2);
944
    }
945
    tb_link_page(tb, phys_pc, phys_page2);
946
    return tb;
947
}
948

    
949
/* invalidate all TBs which intersect with the target physical page
950
   starting in range [start;end[. NOTE: start and end must refer to
951
   the same physical page. 'is_cpu_write_access' should be true if called
952
   from a real cpu write access: the virtual CPU will exit the current
953
   TB if code is modified inside this TB. */
954
void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
955
                                   int is_cpu_write_access)
956
{
957
    TranslationBlock *tb, *tb_next, *saved_tb;
958
    CPUState *env = cpu_single_env;
959
    tb_page_addr_t tb_start, tb_end;
960
    PageDesc *p;
961
    int n;
962
#ifdef TARGET_HAS_PRECISE_SMC
963
    int current_tb_not_found = is_cpu_write_access;
964
    TranslationBlock *current_tb = NULL;
965
    int current_tb_modified = 0;
966
    target_ulong current_pc = 0;
967
    target_ulong current_cs_base = 0;
968
    int current_flags = 0;
969
#endif /* TARGET_HAS_PRECISE_SMC */
970

    
971
    p = page_find(start >> TARGET_PAGE_BITS);
972
    if (!p)
973
        return;
974
    if (!p->code_bitmap &&
975
        ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
976
        is_cpu_write_access) {
977
        /* build code bitmap */
978
        build_page_bitmap(p);
979
    }
980

    
981
    /* we remove all the TBs in the range [start, end[ */
982
    /* XXX: see if in some cases it could be faster to invalidate all the code */
983
    tb = p->first_tb;
984
    while (tb != NULL) {
985
        n = (long)tb & 3;
986
        tb = (TranslationBlock *)((long)tb & ~3);
987
        tb_next = tb->page_next[n];
988
        /* NOTE: this is subtle as a TB may span two physical pages */
989
        if (n == 0) {
990
            /* NOTE: tb_end may be after the end of the page, but
991
               it is not a problem */
992
            tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
993
            tb_end = tb_start + tb->size;
994
        } else {
995
            tb_start = tb->page_addr[1];
996
            tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
997
        }
998
        if (!(tb_end <= start || tb_start >= end)) {
999
#ifdef TARGET_HAS_PRECISE_SMC
1000
            if (current_tb_not_found) {
1001
                current_tb_not_found = 0;
1002
                current_tb = NULL;
1003
                if (env->mem_io_pc) {
1004
                    /* now we have a real cpu fault */
1005
                    current_tb = tb_find_pc(env->mem_io_pc);
1006
                }
1007
            }
1008
            if (current_tb == tb &&
1009
                (current_tb->cflags & CF_COUNT_MASK) != 1) {
1010
                /* If we are modifying the current TB, we must stop
1011
                its execution. We could be more precise by checking
1012
                that the modification is after the current PC, but it
1013
                would require a specialized function to partially
1014
                restore the CPU state */
1015

    
1016
                current_tb_modified = 1;
1017
                cpu_restore_state(current_tb, env,
1018
                                  env->mem_io_pc, NULL);
1019
                cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1020
                                     &current_flags);
1021
            }
1022
#endif /* TARGET_HAS_PRECISE_SMC */
1023
            /* we need to do that to handle the case where a signal
1024
               occurs while doing tb_phys_invalidate() */
1025
            saved_tb = NULL;
1026
            if (env) {
1027
                saved_tb = env->current_tb;
1028
                env->current_tb = NULL;
1029
            }
1030
            tb_phys_invalidate(tb, -1);
1031
            if (env) {
1032
                env->current_tb = saved_tb;
1033
                if (env->interrupt_request && env->current_tb)
1034
                    cpu_interrupt(env, env->interrupt_request);
1035
            }
1036
        }
1037
        tb = tb_next;
1038
    }
1039
#if !defined(CONFIG_USER_ONLY)
1040
    /* if no code remaining, no need to continue to use slow writes */
1041
    if (!p->first_tb) {
1042
        invalidate_page_bitmap(p);
1043
        if (is_cpu_write_access) {
1044
            tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1045
        }
1046
    }
1047
#endif
1048
#ifdef TARGET_HAS_PRECISE_SMC
1049
    if (current_tb_modified) {
1050
        /* we generate a block containing just the instruction
1051
           modifying the memory. It will ensure that it cannot modify
1052
           itself */
1053
        env->current_tb = NULL;
1054
        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1055
        cpu_resume_from_signal(env, NULL);
1056
    }
1057
#endif
1058
}
1059

    
1060
/* len must be <= 8 and start must be a multiple of len */
1061
static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1062
{
1063
    PageDesc *p;
1064
    int offset, b;
1065
#if 0
1066
    if (1) {
1067
        qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1068
                  cpu_single_env->mem_io_vaddr, len,
1069
                  cpu_single_env->eip,
1070
                  cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1071
    }
1072
#endif
1073
    p = page_find(start >> TARGET_PAGE_BITS);
1074
    if (!p)
1075
        return;
1076
    if (p->code_bitmap) {
1077
        offset = start & ~TARGET_PAGE_MASK;
1078
        b = p->code_bitmap[offset >> 3] >> (offset & 7);
1079
        if (b & ((1 << len) - 1))
1080
            goto do_invalidate;
1081
    } else {
1082
    do_invalidate:
1083
        tb_invalidate_phys_page_range(start, start + len, 1);
1084
    }
1085
}
1086

    
1087
#if !defined(CONFIG_SOFTMMU)
1088
static void tb_invalidate_phys_page(tb_page_addr_t addr,
1089
                                    unsigned long pc, void *puc)
1090
{
1091
    TranslationBlock *tb;
1092
    PageDesc *p;
1093
    int n;
1094
#ifdef TARGET_HAS_PRECISE_SMC
1095
    TranslationBlock *current_tb = NULL;
1096
    CPUState *env = cpu_single_env;
1097
    int current_tb_modified = 0;
1098
    target_ulong current_pc = 0;
1099
    target_ulong current_cs_base = 0;
1100
    int current_flags = 0;
1101
#endif
1102

    
1103
    addr &= TARGET_PAGE_MASK;
1104
    p = page_find(addr >> TARGET_PAGE_BITS);
1105
    if (!p)
1106
        return;
1107
    tb = p->first_tb;
1108
#ifdef TARGET_HAS_PRECISE_SMC
1109
    if (tb && pc != 0) {
1110
        current_tb = tb_find_pc(pc);
1111
    }
1112
#endif
1113
    while (tb != NULL) {
1114
        n = (long)tb & 3;
1115
        tb = (TranslationBlock *)((long)tb & ~3);
1116
#ifdef TARGET_HAS_PRECISE_SMC
1117
        if (current_tb == tb &&
1118
            (current_tb->cflags & CF_COUNT_MASK) != 1) {
1119
                /* If we are modifying the current TB, we must stop
1120
                   its execution. We could be more precise by checking
1121
                   that the modification is after the current PC, but it
1122
                   would require a specialized function to partially
1123
                   restore the CPU state */
1124

    
1125
            current_tb_modified = 1;
1126
            cpu_restore_state(current_tb, env, pc, puc);
1127
            cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1128
                                 &current_flags);
1129
        }
1130
#endif /* TARGET_HAS_PRECISE_SMC */
1131
        tb_phys_invalidate(tb, addr);
1132
        tb = tb->page_next[n];
1133
    }
1134
    p->first_tb = NULL;
1135
#ifdef TARGET_HAS_PRECISE_SMC
1136
    if (current_tb_modified) {
1137
        /* we generate a block containing just the instruction
1138
           modifying the memory. It will ensure that it cannot modify
1139
           itself */
1140
        env->current_tb = NULL;
1141
        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1142
        cpu_resume_from_signal(env, puc);
1143
    }
1144
#endif
1145
}
1146
#endif
1147

    
1148
/* add the tb in the target page and protect it if necessary */
1149
static inline void tb_alloc_page(TranslationBlock *tb,
1150
                                 unsigned int n, tb_page_addr_t page_addr)
1151
{
1152
    PageDesc *p;
1153
    TranslationBlock *last_first_tb;
1154

    
1155
    tb->page_addr[n] = page_addr;
1156
    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1157
    tb->page_next[n] = p->first_tb;
1158
    last_first_tb = p->first_tb;
1159
    p->first_tb = (TranslationBlock *)((long)tb | n);
1160
    invalidate_page_bitmap(p);
1161

    
1162
#if defined(TARGET_HAS_SMC) || 1
1163

    
1164
#if defined(CONFIG_USER_ONLY)
1165
    if (p->flags & PAGE_WRITE) {
1166
        target_ulong addr;
1167
        PageDesc *p2;
1168
        int prot;
1169

    
1170
        /* force the host page as non writable (writes will have a
1171
           page fault + mprotect overhead) */
1172
        page_addr &= qemu_host_page_mask;
1173
        prot = 0;
1174
        for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1175
            addr += TARGET_PAGE_SIZE) {
1176

    
1177
            p2 = page_find (addr >> TARGET_PAGE_BITS);
1178
            if (!p2)
1179
                continue;
1180
            prot |= p2->flags;
1181
            p2->flags &= ~PAGE_WRITE;
1182
          }
1183
        mprotect(g2h(page_addr), qemu_host_page_size,
1184
                 (prot & PAGE_BITS) & ~PAGE_WRITE);
1185
#ifdef DEBUG_TB_INVALIDATE
1186
        printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1187
               page_addr);
1188
#endif
1189
    }
1190
#else
1191
    /* if some code is already present, then the pages are already
1192
       protected. So we handle the case where only the first TB is
1193
       allocated in a physical page */
1194
    if (!last_first_tb) {
1195
        tlb_protect_code(page_addr);
1196
    }
1197
#endif
1198

    
1199
#endif /* TARGET_HAS_SMC */
1200
}
1201

    
1202
/* Allocate a new translation block. Flush the translation buffer if
1203
   too many translation blocks or too much generated code. */
1204
TranslationBlock *tb_alloc(target_ulong pc)
1205
{
1206
    TranslationBlock *tb;
1207

    
1208
    if (nb_tbs >= code_gen_max_blocks ||
1209
        (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1210
        return NULL;
1211
    tb = &tbs[nb_tbs++];
1212
    tb->pc = pc;
1213
    tb->cflags = 0;
1214
    return tb;
1215
}
1216

    
1217
void tb_free(TranslationBlock *tb)
1218
{
1219
    /* In practice this is mostly used for single use temporary TB
1220
       Ignore the hard cases and just back up if this TB happens to
1221
       be the last one generated.  */
1222
    if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1223
        code_gen_ptr = tb->tc_ptr;
1224
        nb_tbs--;
1225
    }
1226
}
1227

    
1228
/* add a new TB and link it to the physical page tables. phys_page2 is
1229
   (-1) to indicate that only one page contains the TB. */
1230
void tb_link_page(TranslationBlock *tb,
1231
                  tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
1232
{
1233
    unsigned int h;
1234
    TranslationBlock **ptb;
1235

    
1236
    /* Grab the mmap lock to stop another thread invalidating this TB
1237
       before we are done.  */
1238
    mmap_lock();
1239
    /* add in the physical hash table */
1240
    h = tb_phys_hash_func(phys_pc);
1241
    ptb = &tb_phys_hash[h];
1242
    tb->phys_hash_next = *ptb;
1243
    *ptb = tb;
1244

    
1245
    /* add in the page list */
1246
    tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1247
    if (phys_page2 != -1)
1248
        tb_alloc_page(tb, 1, phys_page2);
1249
    else
1250
        tb->page_addr[1] = -1;
1251

    
1252
    tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1253
    tb->jmp_next[0] = NULL;
1254
    tb->jmp_next[1] = NULL;
1255

    
1256
    /* init original jump addresses */
1257
    if (tb->tb_next_offset[0] != 0xffff)
1258
        tb_reset_jump(tb, 0);
1259
    if (tb->tb_next_offset[1] != 0xffff)
1260
        tb_reset_jump(tb, 1);
1261

    
1262
#ifdef DEBUG_TB_CHECK
1263
    tb_page_check();
1264
#endif
1265
    mmap_unlock();
1266
}
1267

    
1268
/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1269
   tb[1].tc_ptr. Return NULL if not found */
1270
TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1271
{
1272
    int m_min, m_max, m;
1273
    unsigned long v;
1274
    TranslationBlock *tb;
1275

    
1276
    if (nb_tbs <= 0)
1277
        return NULL;
1278
    if (tc_ptr < (unsigned long)code_gen_buffer ||
1279
        tc_ptr >= (unsigned long)code_gen_ptr)
1280
        return NULL;
1281
    /* binary search (cf Knuth) */
1282
    m_min = 0;
1283
    m_max = nb_tbs - 1;
1284
    while (m_min <= m_max) {
1285
        m = (m_min + m_max) >> 1;
1286
        tb = &tbs[m];
1287
        v = (unsigned long)tb->tc_ptr;
1288
        if (v == tc_ptr)
1289
            return tb;
1290
        else if (tc_ptr < v) {
1291
            m_max = m - 1;
1292
        } else {
1293
            m_min = m + 1;
1294
        }
1295
    }
1296
    return &tbs[m_max];
1297
}
1298

    
1299
static void tb_reset_jump_recursive(TranslationBlock *tb);
1300

    
1301
static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1302
{
1303
    TranslationBlock *tb1, *tb_next, **ptb;
1304
    unsigned int n1;
1305

    
1306
    tb1 = tb->jmp_next[n];
1307
    if (tb1 != NULL) {
1308
        /* find head of list */
1309
        for(;;) {
1310
            n1 = (long)tb1 & 3;
1311
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
1312
            if (n1 == 2)
1313
                break;
1314
            tb1 = tb1->jmp_next[n1];
1315
        }
1316
        /* we are now sure now that tb jumps to tb1 */
1317
        tb_next = tb1;
1318

    
1319
        /* remove tb from the jmp_first list */
1320
        ptb = &tb_next->jmp_first;
1321
        for(;;) {
1322
            tb1 = *ptb;
1323
            n1 = (long)tb1 & 3;
1324
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
1325
            if (n1 == n && tb1 == tb)
1326
                break;
1327
            ptb = &tb1->jmp_next[n1];
1328
        }
1329
        *ptb = tb->jmp_next[n];
1330
        tb->jmp_next[n] = NULL;
1331

    
1332
        /* suppress the jump to next tb in generated code */
1333
        tb_reset_jump(tb, n);
1334

    
1335
        /* suppress jumps in the tb on which we could have jumped */
1336
        tb_reset_jump_recursive(tb_next);
1337
    }
1338
}
1339

    
1340
static void tb_reset_jump_recursive(TranslationBlock *tb)
1341
{
1342
    tb_reset_jump_recursive2(tb, 0);
1343
    tb_reset_jump_recursive2(tb, 1);
1344
}
1345

    
1346
#if defined(TARGET_HAS_ICE)
1347
#if defined(CONFIG_USER_ONLY)
1348
static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1349
{
1350
    tb_invalidate_phys_page_range(pc, pc + 1, 0);
1351
}
1352
#else
1353
static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1354
{
1355
    target_phys_addr_t addr;
1356
    target_ulong pd;
1357
    ram_addr_t ram_addr;
1358
    PhysPageDesc *p;
1359

    
1360
    addr = cpu_get_phys_page_debug(env, pc);
1361
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
1362
    if (!p) {
1363
        pd = IO_MEM_UNASSIGNED;
1364
    } else {
1365
        pd = p->phys_offset;
1366
    }
1367
    ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1368
    tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1369
}
1370
#endif
1371
#endif /* TARGET_HAS_ICE */
1372

    
1373
#if defined(CONFIG_USER_ONLY)
1374
void cpu_watchpoint_remove_all(CPUState *env, int mask)
1375

    
1376
{
1377
}
1378

    
1379
int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1380
                          int flags, CPUWatchpoint **watchpoint)
1381
{
1382
    return -ENOSYS;
1383
}
1384
#else
1385
/* Add a watchpoint.  */
1386
int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1387
                          int flags, CPUWatchpoint **watchpoint)
1388
{
1389
    target_ulong len_mask = ~(len - 1);
1390
    CPUWatchpoint *wp;
1391

    
1392
    /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1393
    if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1394
        fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1395
                TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1396
        return -EINVAL;
1397
    }
1398
    wp = qemu_malloc(sizeof(*wp));
1399

    
1400
    wp->vaddr = addr;
1401
    wp->len_mask = len_mask;
1402
    wp->flags = flags;
1403

    
1404
    /* keep all GDB-injected watchpoints in front */
1405
    if (flags & BP_GDB)
1406
        QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1407
    else
1408
        QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1409

    
1410
    tlb_flush_page(env, addr);
1411

    
1412
    if (watchpoint)
1413
        *watchpoint = wp;
1414
    return 0;
1415
}
1416

    
1417
/* Remove a specific watchpoint.  */
1418
int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1419
                          int flags)
1420
{
1421
    target_ulong len_mask = ~(len - 1);
1422
    CPUWatchpoint *wp;
1423

    
1424
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1425
        if (addr == wp->vaddr && len_mask == wp->len_mask
1426
                && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1427
            cpu_watchpoint_remove_by_ref(env, wp);
1428
            return 0;
1429
        }
1430
    }
1431
    return -ENOENT;
1432
}
1433

    
1434
/* Remove a specific watchpoint by reference.  */
1435
void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1436
{
1437
    QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1438

    
1439
    tlb_flush_page(env, watchpoint->vaddr);
1440

    
1441
    qemu_free(watchpoint);
1442
}
1443

    
1444
/* Remove all matching watchpoints.  */
1445
void cpu_watchpoint_remove_all(CPUState *env, int mask)
1446
{
1447
    CPUWatchpoint *wp, *next;
1448

    
1449
    QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1450
        if (wp->flags & mask)
1451
            cpu_watchpoint_remove_by_ref(env, wp);
1452
    }
1453
}
1454
#endif
1455

    
1456
/* Add a breakpoint.  */
1457
int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1458
                          CPUBreakpoint **breakpoint)
1459
{
1460
#if defined(TARGET_HAS_ICE)
1461
    CPUBreakpoint *bp;
1462

    
1463
    bp = qemu_malloc(sizeof(*bp));
1464

    
1465
    bp->pc = pc;
1466
    bp->flags = flags;
1467

    
1468
    /* keep all GDB-injected breakpoints in front */
1469
    if (flags & BP_GDB)
1470
        QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1471
    else
1472
        QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1473

    
1474
    breakpoint_invalidate(env, pc);
1475

    
1476
    if (breakpoint)
1477
        *breakpoint = bp;
1478
    return 0;
1479
#else
1480
    return -ENOSYS;
1481
#endif
1482
}
1483

    
1484
/* Remove a specific breakpoint.  */
1485
int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1486
{
1487
#if defined(TARGET_HAS_ICE)
1488
    CPUBreakpoint *bp;
1489

    
1490
    QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1491
        if (bp->pc == pc && bp->flags == flags) {
1492
            cpu_breakpoint_remove_by_ref(env, bp);
1493
            return 0;
1494
        }
1495
    }
1496
    return -ENOENT;
1497
#else
1498
    return -ENOSYS;
1499
#endif
1500
}
1501

    
1502
/* Remove a specific breakpoint by reference.  */
1503
void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1504
{
1505
#if defined(TARGET_HAS_ICE)
1506
    QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1507

    
1508
    breakpoint_invalidate(env, breakpoint->pc);
1509

    
1510
    qemu_free(breakpoint);
1511
#endif
1512
}
1513

    
1514
/* Remove all matching breakpoints. */
1515
void cpu_breakpoint_remove_all(CPUState *env, int mask)
1516
{
1517
#if defined(TARGET_HAS_ICE)
1518
    CPUBreakpoint *bp, *next;
1519

    
1520
    QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1521
        if (bp->flags & mask)
1522
            cpu_breakpoint_remove_by_ref(env, bp);
1523
    }
1524
#endif
1525
}
1526

    
1527
/* enable or disable single step mode. EXCP_DEBUG is returned by the
1528
   CPU loop after each instruction */
1529
void cpu_single_step(CPUState *env, int enabled)
1530
{
1531
#if defined(TARGET_HAS_ICE)
1532
    if (env->singlestep_enabled != enabled) {
1533
        env->singlestep_enabled = enabled;
1534
        if (kvm_enabled())
1535
            kvm_update_guest_debug(env, 0);
1536
        else {
1537
            /* must flush all the translated code to avoid inconsistencies */
1538
            /* XXX: only flush what is necessary */
1539
            tb_flush(env);
1540
        }
1541
    }
1542
#endif
1543
}
1544

    
1545
/* enable or disable low levels log */
1546
void cpu_set_log(int log_flags)
1547
{
1548
    loglevel = log_flags;
1549
    if (loglevel && !logfile) {
1550
        logfile = fopen(logfilename, log_append ? "a" : "w");
1551
        if (!logfile) {
1552
            perror(logfilename);
1553
            _exit(1);
1554
        }
1555
#if !defined(CONFIG_SOFTMMU)
1556
        /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1557
        {
1558
            static char logfile_buf[4096];
1559
            setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1560
        }
1561
#elif !defined(_WIN32)
1562
        /* Win32 doesn't support line-buffering and requires size >= 2 */
1563
        setvbuf(logfile, NULL, _IOLBF, 0);
1564
#endif
1565
        log_append = 1;
1566
    }
1567
    if (!loglevel && logfile) {
1568
        fclose(logfile);
1569
        logfile = NULL;
1570
    }
1571
}
1572

    
1573
void cpu_set_log_filename(const char *filename)
1574
{
1575
    logfilename = strdup(filename);
1576
    if (logfile) {
1577
        fclose(logfile);
1578
        logfile = NULL;
1579
    }
1580
    cpu_set_log(loglevel);
1581
}
1582

    
1583
static void cpu_unlink_tb(CPUState *env)
1584
{
1585
    /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
1586
       problem and hope the cpu will stop of its own accord.  For userspace
1587
       emulation this often isn't actually as bad as it sounds.  Often
1588
       signals are used primarily to interrupt blocking syscalls.  */
1589
    TranslationBlock *tb;
1590
    static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1591

    
1592
    spin_lock(&interrupt_lock);
1593
    tb = env->current_tb;
1594
    /* if the cpu is currently executing code, we must unlink it and
1595
       all the potentially executing TB */
1596
    if (tb) {
1597
        env->current_tb = NULL;
1598
        tb_reset_jump_recursive(tb);
1599
    }
1600
    spin_unlock(&interrupt_lock);
1601
}
1602

    
1603
/* mask must never be zero, except for A20 change call */
1604
void cpu_interrupt(CPUState *env, int mask)
1605
{
1606
    int old_mask;
1607

    
1608
    old_mask = env->interrupt_request;
1609
    env->interrupt_request |= mask;
1610

    
1611
#ifndef CONFIG_USER_ONLY
1612
    /*
1613
     * If called from iothread context, wake the target cpu in
1614
     * case its halted.
1615
     */
1616
    if (!qemu_cpu_self(env)) {
1617
        qemu_cpu_kick(env);
1618
        return;
1619
    }
1620
#endif
1621

    
1622
    if (use_icount) {
1623
        env->icount_decr.u16.high = 0xffff;
1624
#ifndef CONFIG_USER_ONLY
1625
        if (!can_do_io(env)
1626
            && (mask & ~old_mask) != 0) {
1627
            cpu_abort(env, "Raised interrupt while not in I/O function");
1628
        }
1629
#endif
1630
    } else {
1631
        cpu_unlink_tb(env);
1632
    }
1633
}
1634

    
1635
void cpu_reset_interrupt(CPUState *env, int mask)
1636
{
1637
    env->interrupt_request &= ~mask;
1638
}
1639

    
1640
void cpu_exit(CPUState *env)
1641
{
1642
    env->exit_request = 1;
1643
    cpu_unlink_tb(env);
1644
}
1645

    
1646
const CPULogItem cpu_log_items[] = {
1647
    { CPU_LOG_TB_OUT_ASM, "out_asm",
1648
      "show generated host assembly code for each compiled TB" },
1649
    { CPU_LOG_TB_IN_ASM, "in_asm",
1650
      "show target assembly code for each compiled TB" },
1651
    { CPU_LOG_TB_OP, "op",
1652
      "show micro ops for each compiled TB" },
1653
    { CPU_LOG_TB_OP_OPT, "op_opt",
1654
      "show micro ops "
1655
#ifdef TARGET_I386
1656
      "before eflags optimization and "
1657
#endif
1658
      "after liveness analysis" },
1659
    { CPU_LOG_INT, "int",
1660
      "show interrupts/exceptions in short format" },
1661
    { CPU_LOG_EXEC, "exec",
1662
      "show trace before each executed TB (lots of logs)" },
1663
    { CPU_LOG_TB_CPU, "cpu",
1664
      "show CPU state before block translation" },
1665
#ifdef TARGET_I386
1666
    { CPU_LOG_PCALL, "pcall",
1667
      "show protected mode far calls/returns/exceptions" },
1668
    { CPU_LOG_RESET, "cpu_reset",
1669
      "show CPU state before CPU resets" },
1670
#endif
1671
#ifdef DEBUG_IOPORT
1672
    { CPU_LOG_IOPORT, "ioport",
1673
      "show all i/o ports accesses" },
1674
#endif
1675
    { 0, NULL, NULL },
1676
};
1677

    
1678
#ifndef CONFIG_USER_ONLY
1679
static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
1680
    = QLIST_HEAD_INITIALIZER(memory_client_list);
1681

    
1682
static void cpu_notify_set_memory(target_phys_addr_t start_addr,
1683
                                  ram_addr_t size,
1684
                                  ram_addr_t phys_offset)
1685
{
1686
    CPUPhysMemoryClient *client;
1687
    QLIST_FOREACH(client, &memory_client_list, list) {
1688
        client->set_memory(client, start_addr, size, phys_offset);
1689
    }
1690
}
1691

    
1692
static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
1693
                                        target_phys_addr_t end)
1694
{
1695
    CPUPhysMemoryClient *client;
1696
    QLIST_FOREACH(client, &memory_client_list, list) {
1697
        int r = client->sync_dirty_bitmap(client, start, end);
1698
        if (r < 0)
1699
            return r;
1700
    }
1701
    return 0;
1702
}
1703

    
1704
static int cpu_notify_migration_log(int enable)
1705
{
1706
    CPUPhysMemoryClient *client;
1707
    QLIST_FOREACH(client, &memory_client_list, list) {
1708
        int r = client->migration_log(client, enable);
1709
        if (r < 0)
1710
            return r;
1711
    }
1712
    return 0;
1713
}
1714

    
1715
static void phys_page_for_each_1(CPUPhysMemoryClient *client,
1716
                                 int level, void **lp)
1717
{
1718
    int i;
1719

    
1720
    if (*lp == NULL) {
1721
        return;
1722
    }
1723
    if (level == 0) {
1724
        PhysPageDesc *pd = *lp;
1725
        for (i = 0; i < L2_SIZE; ++i) {
1726
            if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
1727
                client->set_memory(client, pd[i].region_offset,
1728
                                   TARGET_PAGE_SIZE, pd[i].phys_offset);
1729
            }
1730
        }
1731
    } else {
1732
        void **pp = *lp;
1733
        for (i = 0; i < L2_SIZE; ++i) {
1734
            phys_page_for_each_1(client, level - 1, pp + i);
1735
        }
1736
    }
1737
}
1738

    
1739
static void phys_page_for_each(CPUPhysMemoryClient *client)
1740
{
1741
    int i;
1742
    for (i = 0; i < P_L1_SIZE; ++i) {
1743
        phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
1744
                             l1_phys_map + 1);
1745
    }
1746
}
1747

    
1748
void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
1749
{
1750
    QLIST_INSERT_HEAD(&memory_client_list, client, list);
1751
    phys_page_for_each(client);
1752
}
1753

    
1754
void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
1755
{
1756
    QLIST_REMOVE(client, list);
1757
}
1758
#endif
1759

    
1760
static int cmp1(const char *s1, int n, const char *s2)
1761
{
1762
    if (strlen(s2) != n)
1763
        return 0;
1764
    return memcmp(s1, s2, n) == 0;
1765
}
1766

    
1767
/* takes a comma separated list of log masks. Return 0 if error. */
1768
int cpu_str_to_log_mask(const char *str)
1769
{
1770
    const CPULogItem *item;
1771
    int mask;
1772
    const char *p, *p1;
1773

    
1774
    p = str;
1775
    mask = 0;
1776
    for(;;) {
1777
        p1 = strchr(p, ',');
1778
        if (!p1)
1779
            p1 = p + strlen(p);
1780
        if(cmp1(p,p1-p,"all")) {
1781
                for(item = cpu_log_items; item->mask != 0; item++) {
1782
                        mask |= item->mask;
1783
                }
1784
        } else {
1785
        for(item = cpu_log_items; item->mask != 0; item++) {
1786
            if (cmp1(p, p1 - p, item->name))
1787
                goto found;
1788
        }
1789
        return 0;
1790
        }
1791
    found:
1792
        mask |= item->mask;
1793
        if (*p1 != ',')
1794
            break;
1795
        p = p1 + 1;
1796
    }
1797
    return mask;
1798
}
1799

    
1800
void cpu_abort(CPUState *env, const char *fmt, ...)
1801
{
1802
    va_list ap;
1803
    va_list ap2;
1804

    
1805
    va_start(ap, fmt);
1806
    va_copy(ap2, ap);
1807
    fprintf(stderr, "qemu: fatal: ");
1808
    vfprintf(stderr, fmt, ap);
1809
    fprintf(stderr, "\n");
1810
#ifdef TARGET_I386
1811
    cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1812
#else
1813
    cpu_dump_state(env, stderr, fprintf, 0);
1814
#endif
1815
    if (qemu_log_enabled()) {
1816
        qemu_log("qemu: fatal: ");
1817
        qemu_log_vprintf(fmt, ap2);
1818
        qemu_log("\n");
1819
#ifdef TARGET_I386
1820
        log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1821
#else
1822
        log_cpu_state(env, 0);
1823
#endif
1824
        qemu_log_flush();
1825
        qemu_log_close();
1826
    }
1827
    va_end(ap2);
1828
    va_end(ap);
1829
#if defined(CONFIG_USER_ONLY)
1830
    {
1831
        struct sigaction act;
1832
        sigfillset(&act.sa_mask);
1833
        act.sa_handler = SIG_DFL;
1834
        sigaction(SIGABRT, &act, NULL);
1835
    }
1836
#endif
1837
    abort();
1838
}
1839

    
1840
CPUState *cpu_copy(CPUState *env)
1841
{
1842
    CPUState *new_env = cpu_init(env->cpu_model_str);
1843
    CPUState *next_cpu = new_env->next_cpu;
1844
    int cpu_index = new_env->cpu_index;
1845
#if defined(TARGET_HAS_ICE)
1846
    CPUBreakpoint *bp;
1847
    CPUWatchpoint *wp;
1848
#endif
1849

    
1850
    memcpy(new_env, env, sizeof(CPUState));
1851

    
1852
    /* Preserve chaining and index. */
1853
    new_env->next_cpu = next_cpu;
1854
    new_env->cpu_index = cpu_index;
1855

    
1856
    /* Clone all break/watchpoints.
1857
       Note: Once we support ptrace with hw-debug register access, make sure
1858
       BP_CPU break/watchpoints are handled correctly on clone. */
1859
    QTAILQ_INIT(&env->breakpoints);
1860
    QTAILQ_INIT(&env->watchpoints);
1861
#if defined(TARGET_HAS_ICE)
1862
    QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1863
        cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1864
    }
1865
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1866
        cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1867
                              wp->flags, NULL);
1868
    }
1869
#endif
1870

    
1871
    return new_env;
1872
}
1873

    
1874
#if !defined(CONFIG_USER_ONLY)
1875

    
1876
static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1877
{
1878
    unsigned int i;
1879

    
1880
    /* Discard jump cache entries for any tb which might potentially
1881
       overlap the flushed page.  */
1882
    i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1883
    memset (&env->tb_jmp_cache[i], 0, 
1884
            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1885

    
1886
    i = tb_jmp_cache_hash_page(addr);
1887
    memset (&env->tb_jmp_cache[i], 0, 
1888
            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1889
}
1890

    
1891
static CPUTLBEntry s_cputlb_empty_entry = {
1892
    .addr_read  = -1,
1893
    .addr_write = -1,
1894
    .addr_code  = -1,
1895
    .addend     = -1,
1896
};
1897

    
1898
/* NOTE: if flush_global is true, also flush global entries (not
1899
   implemented yet) */
1900
void tlb_flush(CPUState *env, int flush_global)
1901
{
1902
    int i;
1903

    
1904
#if defined(DEBUG_TLB)
1905
    printf("tlb_flush:\n");
1906
#endif
1907
    /* must reset current TB so that interrupts cannot modify the
1908
       links while we are modifying them */
1909
    env->current_tb = NULL;
1910

    
1911
    for(i = 0; i < CPU_TLB_SIZE; i++) {
1912
        int mmu_idx;
1913
        for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1914
            env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1915
        }
1916
    }
1917

    
1918
    memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1919

    
1920
    env->tlb_flush_addr = -1;
1921
    env->tlb_flush_mask = 0;
1922
    tlb_flush_count++;
1923
}
1924

    
1925
static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1926
{
1927
    if (addr == (tlb_entry->addr_read &
1928
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1929
        addr == (tlb_entry->addr_write &
1930
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1931
        addr == (tlb_entry->addr_code &
1932
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1933
        *tlb_entry = s_cputlb_empty_entry;
1934
    }
1935
}
1936

    
1937
void tlb_flush_page(CPUState *env, target_ulong addr)
1938
{
1939
    int i;
1940
    int mmu_idx;
1941

    
1942
#if defined(DEBUG_TLB)
1943
    printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1944
#endif
1945
    /* Check if we need to flush due to large pages.  */
1946
    if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
1947
#if defined(DEBUG_TLB)
1948
        printf("tlb_flush_page: forced full flush ("
1949
               TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
1950
               env->tlb_flush_addr, env->tlb_flush_mask);
1951
#endif
1952
        tlb_flush(env, 1);
1953
        return;
1954
    }
1955
    /* must reset current TB so that interrupts cannot modify the
1956
       links while we are modifying them */
1957
    env->current_tb = NULL;
1958

    
1959
    addr &= TARGET_PAGE_MASK;
1960
    i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1961
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
1962
        tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
1963

    
1964
    tlb_flush_jmp_cache(env, addr);
1965
}
1966

    
1967
/* update the TLBs so that writes to code in the virtual page 'addr'
1968
   can be detected */
1969
static void tlb_protect_code(ram_addr_t ram_addr)
1970
{
1971
    cpu_physical_memory_reset_dirty(ram_addr,
1972
                                    ram_addr + TARGET_PAGE_SIZE,
1973
                                    CODE_DIRTY_FLAG);
1974
}
1975

    
1976
/* update the TLB so that writes in physical page 'phys_addr' are no longer
1977
   tested for self modifying code */
1978
static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1979
                                    target_ulong vaddr)
1980
{
1981
    phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1982
}
1983

    
1984
static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1985
                                         unsigned long start, unsigned long length)
1986
{
1987
    unsigned long addr;
1988
    if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1989
        addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1990
        if ((addr - start) < length) {
1991
            tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1992
        }
1993
    }
1994
}
1995

    
1996
/* Note: start and end must be within the same ram block.  */
1997
void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1998
                                     int dirty_flags)
1999
{
2000
    CPUState *env;
2001
    unsigned long length, start1;
2002
    int i, mask, len;
2003
    uint8_t *p;
2004

    
2005
    start &= TARGET_PAGE_MASK;
2006
    end = TARGET_PAGE_ALIGN(end);
2007

    
2008
    length = end - start;
2009
    if (length == 0)
2010
        return;
2011
    len = length >> TARGET_PAGE_BITS;
2012
    mask = ~dirty_flags;
2013
    p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
2014
    for(i = 0; i < len; i++)
2015
        p[i] &= mask;
2016

    
2017
    /* we modify the TLB cache so that the dirty bit will be set again
2018
       when accessing the range */
2019
    start1 = (unsigned long)qemu_get_ram_ptr(start);
2020
    /* Chek that we don't span multiple blocks - this breaks the
2021
       address comparisons below.  */
2022
    if ((unsigned long)qemu_get_ram_ptr(end - 1) - start1
2023
            != (end - 1) - start) {
2024
        abort();
2025
    }
2026

    
2027
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
2028
        int mmu_idx;
2029
        for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2030
            for(i = 0; i < CPU_TLB_SIZE; i++)
2031
                tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
2032
                                      start1, length);
2033
        }
2034
    }
2035
}
2036

    
2037
int cpu_physical_memory_set_dirty_tracking(int enable)
2038
{
2039
    int ret = 0;
2040
    in_migration = enable;
2041
    ret = cpu_notify_migration_log(!!enable);
2042
    return ret;
2043
}
2044

    
2045
int cpu_physical_memory_get_dirty_tracking(void)
2046
{
2047
    return in_migration;
2048
}
2049

    
2050
int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2051
                                   target_phys_addr_t end_addr)
2052
{
2053
    int ret;
2054

    
2055
    ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
2056
    return ret;
2057
}
2058

    
2059
static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2060
{
2061
    ram_addr_t ram_addr;
2062
    void *p;
2063

    
2064
    if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2065
        p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2066
            + tlb_entry->addend);
2067
        ram_addr = qemu_ram_addr_from_host(p);
2068
        if (!cpu_physical_memory_is_dirty(ram_addr)) {
2069
            tlb_entry->addr_write |= TLB_NOTDIRTY;
2070
        }
2071
    }
2072
}
2073

    
2074
/* update the TLB according to the current state of the dirty bits */
2075
void cpu_tlb_update_dirty(CPUState *env)
2076
{
2077
    int i;
2078
    int mmu_idx;
2079
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2080
        for(i = 0; i < CPU_TLB_SIZE; i++)
2081
            tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2082
    }
2083
}
2084

    
2085
static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2086
{
2087
    if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
2088
        tlb_entry->addr_write = vaddr;
2089
}
2090

    
2091
/* update the TLB corresponding to virtual page vaddr
2092
   so that it is no longer dirty */
2093
static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2094
{
2095
    int i;
2096
    int mmu_idx;
2097

    
2098
    vaddr &= TARGET_PAGE_MASK;
2099
    i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2100
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2101
        tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2102
}
2103

    
2104
/* Our TLB does not support large pages, so remember the area covered by
2105
   large pages and trigger a full TLB flush if these are invalidated.  */
2106
static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
2107
                               target_ulong size)
2108
{
2109
    target_ulong mask = ~(size - 1);
2110

    
2111
    if (env->tlb_flush_addr == (target_ulong)-1) {
2112
        env->tlb_flush_addr = vaddr & mask;
2113
        env->tlb_flush_mask = mask;
2114
        return;
2115
    }
2116
    /* Extend the existing region to include the new page.
2117
       This is a compromise between unnecessary flushes and the cost
2118
       of maintaining a full variable size TLB.  */
2119
    mask &= env->tlb_flush_mask;
2120
    while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
2121
        mask <<= 1;
2122
    }
2123
    env->tlb_flush_addr &= mask;
2124
    env->tlb_flush_mask = mask;
2125
}
2126

    
2127
/* Add a new TLB entry. At most one entry for a given virtual address
2128
   is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2129
   supplied size is only used by tlb_flush_page.  */
2130
void tlb_set_page(CPUState *env, target_ulong vaddr,
2131
                  target_phys_addr_t paddr, int prot,
2132
                  int mmu_idx, target_ulong size)
2133
{
2134
    PhysPageDesc *p;
2135
    unsigned long pd;
2136
    unsigned int index;
2137
    target_ulong address;
2138
    target_ulong code_address;
2139
    target_phys_addr_t addend;
2140
    CPUTLBEntry *te;
2141
    CPUWatchpoint *wp;
2142
    target_phys_addr_t iotlb;
2143

    
2144
    assert(size >= TARGET_PAGE_SIZE);
2145
    if (size != TARGET_PAGE_SIZE) {
2146
        tlb_add_large_page(env, vaddr, size);
2147
    }
2148
    p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2149
    if (!p) {
2150
        pd = IO_MEM_UNASSIGNED;
2151
    } else {
2152
        pd = p->phys_offset;
2153
    }
2154
#if defined(DEBUG_TLB)
2155
    printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
2156
           vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
2157
#endif
2158

    
2159
    address = vaddr;
2160
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2161
        /* IO memory case (romd handled later) */
2162
        address |= TLB_MMIO;
2163
    }
2164
    addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2165
    if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2166
        /* Normal RAM.  */
2167
        iotlb = pd & TARGET_PAGE_MASK;
2168
        if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2169
            iotlb |= IO_MEM_NOTDIRTY;
2170
        else
2171
            iotlb |= IO_MEM_ROM;
2172
    } else {
2173
        /* IO handlers are currently passed a physical address.
2174
           It would be nice to pass an offset from the base address
2175
           of that region.  This would avoid having to special case RAM,
2176
           and avoid full address decoding in every device.
2177
           We can't use the high bits of pd for this because
2178
           IO_MEM_ROMD uses these as a ram address.  */
2179
        iotlb = (pd & ~TARGET_PAGE_MASK);
2180
        if (p) {
2181
            iotlb += p->region_offset;
2182
        } else {
2183
            iotlb += paddr;
2184
        }
2185
    }
2186

    
2187
    code_address = address;
2188
    /* Make accesses to pages with watchpoints go via the
2189
       watchpoint trap routines.  */
2190
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2191
        if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2192
            iotlb = io_mem_watch + paddr;
2193
            /* TODO: The memory case can be optimized by not trapping
2194
               reads of pages with a write breakpoint.  */
2195
            address |= TLB_MMIO;
2196
        }
2197
    }
2198

    
2199
    index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2200
    env->iotlb[mmu_idx][index] = iotlb - vaddr;
2201
    te = &env->tlb_table[mmu_idx][index];
2202
    te->addend = addend - vaddr;
2203
    if (prot & PAGE_READ) {
2204
        te->addr_read = address;
2205
    } else {
2206
        te->addr_read = -1;
2207
    }
2208

    
2209
    if (prot & PAGE_EXEC) {
2210
        te->addr_code = code_address;
2211
    } else {
2212
        te->addr_code = -1;
2213
    }
2214
    if (prot & PAGE_WRITE) {
2215
        if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2216
            (pd & IO_MEM_ROMD)) {
2217
            /* Write access calls the I/O callback.  */
2218
            te->addr_write = address | TLB_MMIO;
2219
        } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2220
                   !cpu_physical_memory_is_dirty(pd)) {
2221
            te->addr_write = address | TLB_NOTDIRTY;
2222
        } else {
2223
            te->addr_write = address;
2224
        }
2225
    } else {
2226
        te->addr_write = -1;
2227
    }
2228
}
2229

    
2230
#else
2231

    
2232
void tlb_flush(CPUState *env, int flush_global)
2233
{
2234
}
2235

    
2236
void tlb_flush_page(CPUState *env, target_ulong addr)
2237
{
2238
}
2239

    
2240
/*
2241
 * Walks guest process memory "regions" one by one
2242
 * and calls callback function 'fn' for each region.
2243
 */
2244

    
2245
struct walk_memory_regions_data
2246
{
2247
    walk_memory_regions_fn fn;
2248
    void *priv;
2249
    unsigned long start;
2250
    int prot;
2251
};
2252

    
2253
static int walk_memory_regions_end(struct walk_memory_regions_data *data,
2254
                                   abi_ulong end, int new_prot)
2255
{
2256
    if (data->start != -1ul) {
2257
        int rc = data->fn(data->priv, data->start, end, data->prot);
2258
        if (rc != 0) {
2259
            return rc;
2260
        }
2261
    }
2262

    
2263
    data->start = (new_prot ? end : -1ul);
2264
    data->prot = new_prot;
2265

    
2266
    return 0;
2267
}
2268

    
2269
static int walk_memory_regions_1(struct walk_memory_regions_data *data,
2270
                                 abi_ulong base, int level, void **lp)
2271
{
2272
    abi_ulong pa;
2273
    int i, rc;
2274

    
2275
    if (*lp == NULL) {
2276
        return walk_memory_regions_end(data, base, 0);
2277
    }
2278

    
2279
    if (level == 0) {
2280
        PageDesc *pd = *lp;
2281
        for (i = 0; i < L2_SIZE; ++i) {
2282
            int prot = pd[i].flags;
2283

    
2284
            pa = base | (i << TARGET_PAGE_BITS);
2285
            if (prot != data->prot) {
2286
                rc = walk_memory_regions_end(data, pa, prot);
2287
                if (rc != 0) {
2288
                    return rc;
2289
                }
2290
            }
2291
        }
2292
    } else {
2293
        void **pp = *lp;
2294
        for (i = 0; i < L2_SIZE; ++i) {
2295
            pa = base | ((abi_ulong)i <<
2296
                (TARGET_PAGE_BITS + L2_BITS * level));
2297
            rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2298
            if (rc != 0) {
2299
                return rc;
2300
            }
2301
        }
2302
    }
2303

    
2304
    return 0;
2305
}
2306

    
2307
int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2308
{
2309
    struct walk_memory_regions_data data;
2310
    unsigned long i;
2311

    
2312
    data.fn = fn;
2313
    data.priv = priv;
2314
    data.start = -1ul;
2315
    data.prot = 0;
2316

    
2317
    for (i = 0; i < V_L1_SIZE; i++) {
2318
        int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
2319
                                       V_L1_SHIFT / L2_BITS - 1, l1_map + i);
2320
        if (rc != 0) {
2321
            return rc;
2322
        }
2323
    }
2324

    
2325
    return walk_memory_regions_end(&data, 0, 0);
2326
}
2327

    
2328
static int dump_region(void *priv, abi_ulong start,
2329
    abi_ulong end, unsigned long prot)
2330
{
2331
    FILE *f = (FILE *)priv;
2332

    
2333
    (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
2334
        " "TARGET_ABI_FMT_lx" %c%c%c\n",
2335
        start, end, end - start,
2336
        ((prot & PAGE_READ) ? 'r' : '-'),
2337
        ((prot & PAGE_WRITE) ? 'w' : '-'),
2338
        ((prot & PAGE_EXEC) ? 'x' : '-'));
2339

    
2340
    return (0);
2341
}
2342

    
2343
/* dump memory mappings */
2344
void page_dump(FILE *f)
2345
{
2346
    (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2347
            "start", "end", "size", "prot");
2348
    walk_memory_regions(f, dump_region);
2349
}
2350

    
2351
int page_get_flags(target_ulong address)
2352
{
2353
    PageDesc *p;
2354

    
2355
    p = page_find(address >> TARGET_PAGE_BITS);
2356
    if (!p)
2357
        return 0;
2358
    return p->flags;
2359
}
2360

    
2361
/* Modify the flags of a page and invalidate the code if necessary.
2362
   The flag PAGE_WRITE_ORG is positioned automatically depending
2363
   on PAGE_WRITE.  The mmap_lock should already be held.  */
2364
void page_set_flags(target_ulong start, target_ulong end, int flags)
2365
{
2366
    target_ulong addr, len;
2367

    
2368
    /* This function should never be called with addresses outside the
2369
       guest address space.  If this assert fires, it probably indicates
2370
       a missing call to h2g_valid.  */
2371
#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2372
    assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2373
#endif
2374
    assert(start < end);
2375

    
2376
    start = start & TARGET_PAGE_MASK;
2377
    end = TARGET_PAGE_ALIGN(end);
2378

    
2379
    if (flags & PAGE_WRITE) {
2380
        flags |= PAGE_WRITE_ORG;
2381
    }
2382

    
2383
    for (addr = start, len = end - start;
2384
         len != 0;
2385
         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2386
        PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2387

    
2388
        /* If the write protection bit is set, then we invalidate
2389
           the code inside.  */
2390
        if (!(p->flags & PAGE_WRITE) &&
2391
            (flags & PAGE_WRITE) &&
2392
            p->first_tb) {
2393
            tb_invalidate_phys_page(addr, 0, NULL);
2394
        }
2395
        p->flags = flags;
2396
    }
2397
}
2398

    
2399
int page_check_range(target_ulong start, target_ulong len, int flags)
2400
{
2401
    PageDesc *p;
2402
    target_ulong end;
2403
    target_ulong addr;
2404

    
2405
    /* This function should never be called with addresses outside the
2406
       guest address space.  If this assert fires, it probably indicates
2407
       a missing call to h2g_valid.  */
2408
#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2409
    assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2410
#endif
2411

    
2412
    if (start + len - 1 < start) {
2413
        /* We've wrapped around.  */
2414
        return -1;
2415
    }
2416

    
2417
    end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2418
    start = start & TARGET_PAGE_MASK;
2419

    
2420
    for (addr = start, len = end - start;
2421
         len != 0;
2422
         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2423
        p = page_find(addr >> TARGET_PAGE_BITS);
2424
        if( !p )
2425
            return -1;
2426
        if( !(p->flags & PAGE_VALID) )
2427
            return -1;
2428

    
2429
        if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2430
            return -1;
2431
        if (flags & PAGE_WRITE) {
2432
            if (!(p->flags & PAGE_WRITE_ORG))
2433
                return -1;
2434
            /* unprotect the page if it was put read-only because it
2435
               contains translated code */
2436
            if (!(p->flags & PAGE_WRITE)) {
2437
                if (!page_unprotect(addr, 0, NULL))
2438
                    return -1;
2439
            }
2440
            return 0;
2441
        }
2442
    }
2443
    return 0;
2444
}
2445

    
2446
/* called from signal handler: invalidate the code and unprotect the
2447
   page. Return TRUE if the fault was successfully handled. */
2448
int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2449
{
2450
    unsigned int page_index, prot, pindex;
2451
    PageDesc *p, *p1;
2452
    target_ulong host_start, host_end, addr;
2453

    
2454
    /* Technically this isn't safe inside a signal handler.  However we
2455
       know this only ever happens in a synchronous SEGV handler, so in
2456
       practice it seems to be ok.  */
2457
    mmap_lock();
2458

    
2459
    host_start = address & qemu_host_page_mask;
2460
    page_index = host_start >> TARGET_PAGE_BITS;
2461
    p1 = page_find(page_index);
2462
    if (!p1) {
2463
        mmap_unlock();
2464
        return 0;
2465
    }
2466
    host_end = host_start + qemu_host_page_size;
2467
    p = p1;
2468
    prot = 0;
2469
    for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2470
        prot |= p->flags;
2471
        p++;
2472
    }
2473
    /* if the page was really writable, then we change its
2474
       protection back to writable */
2475
    if (prot & PAGE_WRITE_ORG) {
2476
        pindex = (address - host_start) >> TARGET_PAGE_BITS;
2477
        if (!(p1[pindex].flags & PAGE_WRITE)) {
2478
            mprotect((void *)g2h(host_start), qemu_host_page_size,
2479
                     (prot & PAGE_BITS) | PAGE_WRITE);
2480
            p1[pindex].flags |= PAGE_WRITE;
2481
            /* and since the content will be modified, we must invalidate
2482
               the corresponding translated code. */
2483
            tb_invalidate_phys_page(address, pc, puc);
2484
#ifdef DEBUG_TB_CHECK
2485
            tb_invalidate_check(address);
2486
#endif
2487
            mmap_unlock();
2488
            return 1;
2489
        }
2490
    }
2491
    mmap_unlock();
2492
    return 0;
2493
}
2494

    
2495
static inline void tlb_set_dirty(CPUState *env,
2496
                                 unsigned long addr, target_ulong vaddr)
2497
{
2498
}
2499
#endif /* defined(CONFIG_USER_ONLY) */
2500

    
2501
#if !defined(CONFIG_USER_ONLY)
2502

    
2503
#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2504
typedef struct subpage_t {
2505
    target_phys_addr_t base;
2506
    CPUReadMemoryFunc * const *mem_read[TARGET_PAGE_SIZE][4];
2507
    CPUWriteMemoryFunc * const *mem_write[TARGET_PAGE_SIZE][4];
2508
    void *opaque[TARGET_PAGE_SIZE][2][4];
2509
    ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
2510
} subpage_t;
2511

    
2512
static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2513
                             ram_addr_t memory, ram_addr_t region_offset);
2514
static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2515
                           ram_addr_t orig_memory, ram_addr_t region_offset);
2516
#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2517
                      need_subpage)                                     \
2518
    do {                                                                \
2519
        if (addr > start_addr)                                          \
2520
            start_addr2 = 0;                                            \
2521
        else {                                                          \
2522
            start_addr2 = start_addr & ~TARGET_PAGE_MASK;               \
2523
            if (start_addr2 > 0)                                        \
2524
                need_subpage = 1;                                       \
2525
        }                                                               \
2526
                                                                        \
2527
        if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE)        \
2528
            end_addr2 = TARGET_PAGE_SIZE - 1;                           \
2529
        else {                                                          \
2530
            end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2531
            if (end_addr2 < TARGET_PAGE_SIZE - 1)                       \
2532
                need_subpage = 1;                                       \
2533
        }                                                               \
2534
    } while (0)
2535

    
2536
/* register physical memory.
2537
   For RAM, 'size' must be a multiple of the target page size.
2538
   If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2539
   io memory page.  The address used when calling the IO function is
2540
   the offset from the start of the region, plus region_offset.  Both
2541
   start_addr and region_offset are rounded down to a page boundary
2542
   before calculating this offset.  This should not be a problem unless
2543
   the low bits of start_addr and region_offset differ.  */
2544
void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2545
                                         ram_addr_t size,
2546
                                         ram_addr_t phys_offset,
2547
                                         ram_addr_t region_offset)
2548
{
2549
    target_phys_addr_t addr, end_addr;
2550
    PhysPageDesc *p;
2551
    CPUState *env;
2552
    ram_addr_t orig_size = size;
2553
    void *subpage;
2554

    
2555
    cpu_notify_set_memory(start_addr, size, phys_offset);
2556

    
2557
    if (phys_offset == IO_MEM_UNASSIGNED) {
2558
        region_offset = start_addr;
2559
    }
2560
    region_offset &= TARGET_PAGE_MASK;
2561
    size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2562
    end_addr = start_addr + (target_phys_addr_t)size;
2563
    for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2564
        p = phys_page_find(addr >> TARGET_PAGE_BITS);
2565
        if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2566
            ram_addr_t orig_memory = p->phys_offset;
2567
            target_phys_addr_t start_addr2, end_addr2;
2568
            int need_subpage = 0;
2569

    
2570
            CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2571
                          need_subpage);
2572
            if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2573
                if (!(orig_memory & IO_MEM_SUBPAGE)) {
2574
                    subpage = subpage_init((addr & TARGET_PAGE_MASK),
2575
                                           &p->phys_offset, orig_memory,
2576
                                           p->region_offset);
2577
                } else {
2578
                    subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2579
                                            >> IO_MEM_SHIFT];
2580
                }
2581
                subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2582
                                 region_offset);
2583
                p->region_offset = 0;
2584
            } else {
2585
                p->phys_offset = phys_offset;
2586
                if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2587
                    (phys_offset & IO_MEM_ROMD))
2588
                    phys_offset += TARGET_PAGE_SIZE;
2589
            }
2590
        } else {
2591
            p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2592
            p->phys_offset = phys_offset;
2593
            p->region_offset = region_offset;
2594
            if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2595
                (phys_offset & IO_MEM_ROMD)) {
2596
                phys_offset += TARGET_PAGE_SIZE;
2597
            } else {
2598
                target_phys_addr_t start_addr2, end_addr2;
2599
                int need_subpage = 0;
2600

    
2601
                CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2602
                              end_addr2, need_subpage);
2603

    
2604
                if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2605
                    subpage = subpage_init((addr & TARGET_PAGE_MASK),
2606
                                           &p->phys_offset, IO_MEM_UNASSIGNED,
2607
                                           addr & TARGET_PAGE_MASK);
2608
                    subpage_register(subpage, start_addr2, end_addr2,
2609
                                     phys_offset, region_offset);
2610
                    p->region_offset = 0;
2611
                }
2612
            }
2613
        }
2614
        region_offset += TARGET_PAGE_SIZE;
2615
    }
2616

    
2617
    /* since each CPU stores ram addresses in its TLB cache, we must
2618
       reset the modified entries */
2619
    /* XXX: slow ! */
2620
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
2621
        tlb_flush(env, 1);
2622
    }
2623
}
2624

    
2625
/* XXX: temporary until new memory mapping API */
2626
ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2627
{
2628
    PhysPageDesc *p;
2629

    
2630
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
2631
    if (!p)
2632
        return IO_MEM_UNASSIGNED;
2633
    return p->phys_offset;
2634
}
2635

    
2636
void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2637
{
2638
    if (kvm_enabled())
2639
        kvm_coalesce_mmio_region(addr, size);
2640
}
2641

    
2642
void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2643
{
2644
    if (kvm_enabled())
2645
        kvm_uncoalesce_mmio_region(addr, size);
2646
}
2647

    
2648
void qemu_flush_coalesced_mmio_buffer(void)
2649
{
2650
    if (kvm_enabled())
2651
        kvm_flush_coalesced_mmio_buffer();
2652
}
2653

    
2654
#if defined(__linux__) && !defined(TARGET_S390X)
2655

    
2656
#include <sys/vfs.h>
2657

    
2658
#define HUGETLBFS_MAGIC       0x958458f6
2659

    
2660
static long gethugepagesize(const char *path)
2661
{
2662
    struct statfs fs;
2663
    int ret;
2664

    
2665
    do {
2666
            ret = statfs(path, &fs);
2667
    } while (ret != 0 && errno == EINTR);
2668

    
2669
    if (ret != 0) {
2670
            perror(path);
2671
            return 0;
2672
    }
2673

    
2674
    if (fs.f_type != HUGETLBFS_MAGIC)
2675
            fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
2676

    
2677
    return fs.f_bsize;
2678
}
2679

    
2680
static void *file_ram_alloc(ram_addr_t memory, const char *path)
2681
{
2682
    char *filename;
2683
    void *area;
2684
    int fd;
2685
#ifdef MAP_POPULATE
2686
    int flags;
2687
#endif
2688
    unsigned long hpagesize;
2689

    
2690
    hpagesize = gethugepagesize(path);
2691
    if (!hpagesize) {
2692
        return NULL;
2693
    }
2694

    
2695
    if (memory < hpagesize) {
2696
        return NULL;
2697
    }
2698

    
2699
    if (kvm_enabled() && !kvm_has_sync_mmu()) {
2700
        fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2701
        return NULL;
2702
    }
2703

    
2704
    if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
2705
        return NULL;
2706
    }
2707

    
2708
    fd = mkstemp(filename);
2709
    if (fd < 0) {
2710
        perror("unable to create backing store for hugepages");
2711
        free(filename);
2712
        return NULL;
2713
    }
2714
    unlink(filename);
2715
    free(filename);
2716

    
2717
    memory = (memory+hpagesize-1) & ~(hpagesize-1);
2718

    
2719
    /*
2720
     * ftruncate is not supported by hugetlbfs in older
2721
     * hosts, so don't bother bailing out on errors.
2722
     * If anything goes wrong with it under other filesystems,
2723
     * mmap will fail.
2724
     */
2725
    if (ftruncate(fd, memory))
2726
        perror("ftruncate");
2727

    
2728
#ifdef MAP_POPULATE
2729
    /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2730
     * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
2731
     * to sidestep this quirk.
2732
     */
2733
    flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
2734
    area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
2735
#else
2736
    area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
2737
#endif
2738
    if (area == MAP_FAILED) {
2739
        perror("file_ram_alloc: can't mmap RAM pages");
2740
        close(fd);
2741
        return (NULL);
2742
    }
2743
    return area;
2744
}
2745
#endif
2746

    
2747
ram_addr_t qemu_ram_alloc(ram_addr_t size)
2748
{
2749
    RAMBlock *new_block;
2750

    
2751
    size = TARGET_PAGE_ALIGN(size);
2752
    new_block = qemu_malloc(sizeof(*new_block));
2753

    
2754
    if (mem_path) {
2755
#if defined (__linux__) && !defined(TARGET_S390X)
2756
        new_block->host = file_ram_alloc(size, mem_path);
2757
        if (!new_block->host)
2758
            exit(1);
2759
#else
2760
        fprintf(stderr, "-mem-path option unsupported\n");
2761
        exit(1);
2762
#endif
2763
    } else {
2764
#if defined(TARGET_S390X) && defined(CONFIG_KVM)
2765
        /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2766
        new_block->host = mmap((void*)0x1000000, size,
2767
                                PROT_EXEC|PROT_READ|PROT_WRITE,
2768
                                MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2769
#else
2770
        new_block->host = qemu_vmalloc(size);
2771
#endif
2772
#ifdef MADV_MERGEABLE
2773
        madvise(new_block->host, size, MADV_MERGEABLE);
2774
#endif
2775
    }
2776
    new_block->offset = last_ram_offset;
2777
    new_block->length = size;
2778

    
2779
    new_block->next = ram_blocks;
2780
    ram_blocks = new_block;
2781

    
2782
    phys_ram_dirty = qemu_realloc(phys_ram_dirty,
2783
        (last_ram_offset + size) >> TARGET_PAGE_BITS);
2784
    memset(phys_ram_dirty + (last_ram_offset >> TARGET_PAGE_BITS),
2785
           0xff, size >> TARGET_PAGE_BITS);
2786

    
2787
    last_ram_offset += size;
2788

    
2789
    if (kvm_enabled())
2790
        kvm_setup_guest_memory(new_block->host, size);
2791

    
2792
    return new_block->offset;
2793
}
2794

    
2795
void qemu_ram_free(ram_addr_t addr)
2796
{
2797
    /* TODO: implement this.  */
2798
}
2799

    
2800
/* Return a host pointer to ram allocated with qemu_ram_alloc.
2801
   With the exception of the softmmu code in this file, this should
2802
   only be used for local memory (e.g. video ram) that the device owns,
2803
   and knows it isn't going to access beyond the end of the block.
2804

2805
   It should not be used for general purpose DMA.
2806
   Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2807
 */
2808
void *qemu_get_ram_ptr(ram_addr_t addr)
2809
{
2810
    RAMBlock *prev;
2811
    RAMBlock **prevp;
2812
    RAMBlock *block;
2813

    
2814
    prev = NULL;
2815
    prevp = &ram_blocks;
2816
    block = ram_blocks;
2817
    while (block && (block->offset > addr
2818
                     || block->offset + block->length <= addr)) {
2819
        if (prev)
2820
          prevp = &prev->next;
2821
        prev = block;
2822
        block = block->next;
2823
    }
2824
    if (!block) {
2825
        fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
2826
        abort();
2827
    }
2828
    /* Move this entry to to start of the list.  */
2829
    if (prev) {
2830
        prev->next = block->next;
2831
        block->next = *prevp;
2832
        *prevp = block;
2833
    }
2834
    return block->host + (addr - block->offset);
2835
}
2836

    
2837
/* Some of the softmmu routines need to translate from a host pointer
2838
   (typically a TLB entry) back to a ram offset.  */
2839
ram_addr_t qemu_ram_addr_from_host(void *ptr)
2840
{
2841
    RAMBlock *prev;
2842
    RAMBlock *block;
2843
    uint8_t *host = ptr;
2844

    
2845
    prev = NULL;
2846
    block = ram_blocks;
2847
    while (block && (block->host > host
2848
                     || block->host + block->length <= host)) {
2849
        prev = block;
2850
        block = block->next;
2851
    }
2852
    if (!block) {
2853
        fprintf(stderr, "Bad ram pointer %p\n", ptr);
2854
        abort();
2855
    }
2856
    return block->offset + (host - block->host);
2857
}
2858

    
2859
static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2860
{
2861
#ifdef DEBUG_UNASSIGNED
2862
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2863
#endif
2864
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2865
    do_unassigned_access(addr, 0, 0, 0, 1);
2866
#endif
2867
    return 0;
2868
}
2869

    
2870
static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2871
{
2872
#ifdef DEBUG_UNASSIGNED
2873
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2874
#endif
2875
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2876
    do_unassigned_access(addr, 0, 0, 0, 2);
2877
#endif
2878
    return 0;
2879
}
2880

    
2881
static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2882
{
2883
#ifdef DEBUG_UNASSIGNED
2884
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2885
#endif
2886
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2887
    do_unassigned_access(addr, 0, 0, 0, 4);
2888
#endif
2889
    return 0;
2890
}
2891

    
2892
static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2893
{
2894
#ifdef DEBUG_UNASSIGNED
2895
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2896
#endif
2897
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2898
    do_unassigned_access(addr, 1, 0, 0, 1);
2899
#endif
2900
}
2901

    
2902
static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2903
{
2904
#ifdef DEBUG_UNASSIGNED
2905
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2906
#endif
2907
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2908
    do_unassigned_access(addr, 1, 0, 0, 2);
2909
#endif
2910
}
2911

    
2912
static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2913
{
2914
#ifdef DEBUG_UNASSIGNED
2915
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2916
#endif
2917
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2918
    do_unassigned_access(addr, 1, 0, 0, 4);
2919
#endif
2920
}
2921

    
2922
static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
2923
    unassigned_mem_readb,
2924
    unassigned_mem_readw,
2925
    unassigned_mem_readl,
2926
};
2927

    
2928
static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
2929
    unassigned_mem_writeb,
2930
    unassigned_mem_writew,
2931
    unassigned_mem_writel,
2932
};
2933

    
2934
static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2935
                                uint32_t val)
2936
{
2937
    int dirty_flags;
2938
    dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2939
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2940
#if !defined(CONFIG_USER_ONLY)
2941
        tb_invalidate_phys_page_fast(ram_addr, 1);
2942
        dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2943
#endif
2944
    }
2945
    stb_p(qemu_get_ram_ptr(ram_addr), val);
2946
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2947
    phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2948
    /* we remove the notdirty callback only if the code has been
2949
       flushed */
2950
    if (dirty_flags == 0xff)
2951
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2952
}
2953

    
2954
static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2955
                                uint32_t val)
2956
{
2957
    int dirty_flags;
2958
    dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2959
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2960
#if !defined(CONFIG_USER_ONLY)
2961
        tb_invalidate_phys_page_fast(ram_addr, 2);
2962
        dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2963
#endif
2964
    }
2965
    stw_p(qemu_get_ram_ptr(ram_addr), val);
2966
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2967
    phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2968
    /* we remove the notdirty callback only if the code has been
2969
       flushed */
2970
    if (dirty_flags == 0xff)
2971
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2972
}
2973

    
2974
static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2975
                                uint32_t val)
2976
{
2977
    int dirty_flags;
2978
    dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2979
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2980
#if !defined(CONFIG_USER_ONLY)
2981
        tb_invalidate_phys_page_fast(ram_addr, 4);
2982
        dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2983
#endif
2984
    }
2985
    stl_p(qemu_get_ram_ptr(ram_addr), val);
2986
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2987
    phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2988
    /* we remove the notdirty callback only if the code has been
2989
       flushed */
2990
    if (dirty_flags == 0xff)
2991
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2992
}
2993

    
2994
static CPUReadMemoryFunc * const error_mem_read[3] = {
2995
    NULL, /* never used */
2996
    NULL, /* never used */
2997
    NULL, /* never used */
2998
};
2999

    
3000
static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
3001
    notdirty_mem_writeb,
3002
    notdirty_mem_writew,
3003
    notdirty_mem_writel,
3004
};
3005

    
3006
/* Generate a debug exception if a watchpoint has been hit.  */
3007
static void check_watchpoint(int offset, int len_mask, int flags)
3008
{
3009
    CPUState *env = cpu_single_env;
3010
    target_ulong pc, cs_base;
3011
    TranslationBlock *tb;
3012
    target_ulong vaddr;
3013
    CPUWatchpoint *wp;
3014
    int cpu_flags;
3015

    
3016
    if (env->watchpoint_hit) {
3017
        /* We re-entered the check after replacing the TB. Now raise
3018
         * the debug interrupt so that is will trigger after the
3019
         * current instruction. */
3020
        cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
3021
        return;
3022
    }
3023
    vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
3024
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
3025
        if ((vaddr == (wp->vaddr & len_mask) ||
3026
             (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
3027
            wp->flags |= BP_WATCHPOINT_HIT;
3028
            if (!env->watchpoint_hit) {
3029
                env->watchpoint_hit = wp;
3030
                tb = tb_find_pc(env->mem_io_pc);
3031
                if (!tb) {
3032
                    cpu_abort(env, "check_watchpoint: could not find TB for "
3033
                              "pc=%p", (void *)env->mem_io_pc);
3034
                }
3035
                cpu_restore_state(tb, env, env->mem_io_pc, NULL);
3036
                tb_phys_invalidate(tb, -1);
3037
                if (wp->flags & BP_STOP_BEFORE_ACCESS) {
3038
                    env->exception_index = EXCP_DEBUG;
3039
                } else {
3040
                    cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
3041
                    tb_gen_code(env, pc, cs_base, cpu_flags, 1);
3042
                }
3043
                cpu_resume_from_signal(env, NULL);
3044
            }
3045
        } else {
3046
            wp->flags &= ~BP_WATCHPOINT_HIT;
3047
        }
3048
    }
3049
}
3050

    
3051
/* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
3052
   so these check for a hit then pass through to the normal out-of-line
3053
   phys routines.  */
3054
static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
3055
{
3056
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
3057
    return ldub_phys(addr);
3058
}
3059

    
3060
static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
3061
{
3062
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
3063
    return lduw_phys(addr);
3064
}
3065

    
3066
static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
3067
{
3068
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
3069
    return ldl_phys(addr);
3070
}
3071

    
3072
static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
3073
                             uint32_t val)
3074
{
3075
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
3076
    stb_phys(addr, val);
3077
}
3078

    
3079
static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
3080
                             uint32_t val)
3081
{
3082
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
3083
    stw_phys(addr, val);
3084
}
3085

    
3086
static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
3087
                             uint32_t val)
3088
{
3089
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
3090
    stl_phys(addr, val);
3091
}
3092

    
3093
static CPUReadMemoryFunc * const watch_mem_read[3] = {
3094
    watch_mem_readb,
3095
    watch_mem_readw,
3096
    watch_mem_readl,
3097
};
3098

    
3099
static CPUWriteMemoryFunc * const watch_mem_write[3] = {
3100
    watch_mem_writeb,
3101
    watch_mem_writew,
3102
    watch_mem_writel,
3103
};
3104

    
3105
static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
3106
                                 unsigned int len)
3107
{
3108
    uint32_t ret;
3109
    unsigned int idx;
3110

    
3111
    idx = SUBPAGE_IDX(addr);
3112
#if defined(DEBUG_SUBPAGE)
3113
    printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
3114
           mmio, len, addr, idx);
3115
#endif
3116
    ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
3117
                                       addr + mmio->region_offset[idx][0][len]);
3118

    
3119
    return ret;
3120
}
3121

    
3122
static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
3123
                              uint32_t value, unsigned int len)
3124
{
3125
    unsigned int idx;
3126

    
3127
    idx = SUBPAGE_IDX(addr);
3128
#if defined(DEBUG_SUBPAGE)
3129
    printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
3130
           mmio, len, addr, idx, value);
3131
#endif
3132
    (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
3133
                                  addr + mmio->region_offset[idx][1][len],
3134
                                  value);
3135
}
3136

    
3137
static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
3138
{
3139
#if defined(DEBUG_SUBPAGE)
3140
    printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
3141
#endif
3142

    
3143
    return subpage_readlen(opaque, addr, 0);
3144
}
3145

    
3146
static void subpage_writeb (void *opaque, target_phys_addr_t addr,
3147
                            uint32_t value)
3148
{
3149
#if defined(DEBUG_SUBPAGE)
3150
    printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
3151
#endif
3152
    subpage_writelen(opaque, addr, value, 0);
3153
}
3154

    
3155
static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
3156
{
3157
#if defined(DEBUG_SUBPAGE)
3158
    printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
3159
#endif
3160

    
3161
    return subpage_readlen(opaque, addr, 1);
3162
}
3163

    
3164
static void subpage_writew (void *opaque, target_phys_addr_t addr,
3165
                            uint32_t value)
3166
{
3167
#if defined(DEBUG_SUBPAGE)
3168
    printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
3169
#endif
3170
    subpage_writelen(opaque, addr, value, 1);
3171
}
3172

    
3173
static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
3174
{
3175
#if defined(DEBUG_SUBPAGE)
3176
    printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
3177
#endif
3178

    
3179
    return subpage_readlen(opaque, addr, 2);
3180
}
3181

    
3182
static void subpage_writel (void *opaque,
3183
                         target_phys_addr_t addr, uint32_t value)
3184
{
3185
#if defined(DEBUG_SUBPAGE)
3186
    printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
3187
#endif
3188
    subpage_writelen(opaque, addr, value, 2);
3189
}
3190

    
3191
static CPUReadMemoryFunc * const subpage_read[] = {
3192
    &subpage_readb,
3193
    &subpage_readw,
3194
    &subpage_readl,
3195
};
3196

    
3197
static CPUWriteMemoryFunc * const subpage_write[] = {
3198
    &subpage_writeb,
3199
    &subpage_writew,
3200
    &subpage_writel,
3201
};
3202

    
3203
static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
3204
                             ram_addr_t memory, ram_addr_t region_offset)
3205
{
3206
    int idx, eidx;
3207
    unsigned int i;
3208

    
3209
    if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
3210
        return -1;
3211
    idx = SUBPAGE_IDX(start);
3212
    eidx = SUBPAGE_IDX(end);
3213
#if defined(DEBUG_SUBPAGE)
3214
    printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
3215
           mmio, start, end, idx, eidx, memory);
3216
#endif
3217
    memory >>= IO_MEM_SHIFT;
3218
    for (; idx <= eidx; idx++) {
3219
        for (i = 0; i < 4; i++) {
3220
            if (io_mem_read[memory][i]) {
3221
                mmio->mem_read[idx][i] = &io_mem_read[memory][i];
3222
                mmio->opaque[idx][0][i] = io_mem_opaque[memory];
3223
                mmio->region_offset[idx][0][i] = region_offset;
3224
            }
3225
            if (io_mem_write[memory][i]) {
3226
                mmio->mem_write[idx][i] = &io_mem_write[memory][i];
3227
                mmio->opaque[idx][1][i] = io_mem_opaque[memory];
3228
                mmio->region_offset[idx][1][i] = region_offset;
3229
            }
3230
        }
3231
    }
3232

    
3233
    return 0;
3234
}
3235

    
3236
static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3237
                           ram_addr_t orig_memory, ram_addr_t region_offset)
3238
{
3239
    subpage_t *mmio;
3240
    int subpage_memory;
3241

    
3242
    mmio = qemu_mallocz(sizeof(subpage_t));
3243

    
3244
    mmio->base = base;
3245
    subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio);
3246
#if defined(DEBUG_SUBPAGE)
3247
    printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3248
           mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3249
#endif
3250
    *phys = subpage_memory | IO_MEM_SUBPAGE;
3251
    subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
3252
                         region_offset);
3253

    
3254
    return mmio;
3255
}
3256

    
3257
static int get_free_io_mem_idx(void)
3258
{
3259
    int i;
3260

    
3261
    for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3262
        if (!io_mem_used[i]) {
3263
            io_mem_used[i] = 1;
3264
            return i;
3265
        }
3266
    fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
3267
    return -1;
3268
}
3269

    
3270
/* mem_read and mem_write are arrays of functions containing the
3271
   function to access byte (index 0), word (index 1) and dword (index
3272
   2). Functions can be omitted with a NULL function pointer.
3273
   If io_index is non zero, the corresponding io zone is
3274
   modified. If it is zero, a new io zone is allocated. The return
3275
   value can be used with cpu_register_physical_memory(). (-1) is
3276
   returned if error. */
3277
static int cpu_register_io_memory_fixed(int io_index,
3278
                                        CPUReadMemoryFunc * const *mem_read,
3279
                                        CPUWriteMemoryFunc * const *mem_write,
3280
                                        void *opaque)
3281
{
3282
    int i, subwidth = 0;
3283

    
3284
    if (io_index <= 0) {
3285
        io_index = get_free_io_mem_idx();
3286
        if (io_index == -1)
3287
            return io_index;
3288
    } else {
3289
        io_index >>= IO_MEM_SHIFT;
3290
        if (io_index >= IO_MEM_NB_ENTRIES)
3291
            return -1;
3292
    }
3293

    
3294
    for(i = 0;i < 3; i++) {
3295
        if (!mem_read[i] || !mem_write[i])
3296
            subwidth = IO_MEM_SUBWIDTH;
3297
        io_mem_read[io_index][i] = mem_read[i];
3298
        io_mem_write[io_index][i] = mem_write[i];
3299
    }
3300
    io_mem_opaque[io_index] = opaque;
3301
    return (io_index << IO_MEM_SHIFT) | subwidth;
3302
}
3303

    
3304
int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
3305
                           CPUWriteMemoryFunc * const *mem_write,
3306
                           void *opaque)
3307
{
3308
    return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque);
3309
}
3310

    
3311
void cpu_unregister_io_memory(int io_table_address)
3312
{
3313
    int i;
3314
    int io_index = io_table_address >> IO_MEM_SHIFT;
3315

    
3316
    for (i=0;i < 3; i++) {
3317
        io_mem_read[io_index][i] = unassigned_mem_read[i];
3318
        io_mem_write[io_index][i] = unassigned_mem_write[i];
3319
    }
3320
    io_mem_opaque[io_index] = NULL;
3321
    io_mem_used[io_index] = 0;
3322
}
3323

    
3324
static void io_mem_init(void)
3325
{
3326
    int i;
3327

    
3328
    cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read, unassigned_mem_write, NULL);
3329
    cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read, unassigned_mem_write, NULL);
3330
    cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read, notdirty_mem_write, NULL);
3331
    for (i=0; i<5; i++)
3332
        io_mem_used[i] = 1;
3333

    
3334
    io_mem_watch = cpu_register_io_memory(watch_mem_read,
3335
                                          watch_mem_write, NULL);
3336
}
3337

    
3338
#endif /* !defined(CONFIG_USER_ONLY) */
3339

    
3340
/* physical memory access (slow version, mainly for debug) */
3341
#if defined(CONFIG_USER_ONLY)
3342
int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3343
                        uint8_t *buf, int len, int is_write)
3344
{
3345
    int l, flags;
3346
    target_ulong page;
3347
    void * p;
3348

    
3349
    while (len > 0) {
3350
        page = addr & TARGET_PAGE_MASK;
3351
        l = (page + TARGET_PAGE_SIZE) - addr;
3352
        if (l > len)
3353
            l = len;
3354
        flags = page_get_flags(page);
3355
        if (!(flags & PAGE_VALID))
3356
            return -1;
3357
        if (is_write) {
3358
            if (!(flags & PAGE_WRITE))
3359
                return -1;
3360
            /* XXX: this code should not depend on lock_user */
3361
            if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3362
                return -1;
3363
            memcpy(p, buf, l);
3364
            unlock_user(p, addr, l);
3365
        } else {
3366
            if (!(flags & PAGE_READ))
3367
                return -1;
3368
            /* XXX: this code should not depend on lock_user */
3369
            if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3370
                return -1;
3371
            memcpy(buf, p, l);
3372
            unlock_user(p, addr, 0);
3373
        }
3374
        len -= l;
3375
        buf += l;
3376
        addr += l;
3377
    }
3378
    return 0;
3379
}
3380

    
3381
#else
3382
void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3383
                            int len, int is_write)
3384
{
3385
    int l, io_index;
3386
    uint8_t *ptr;
3387
    uint32_t val;
3388
    target_phys_addr_t page;
3389
    unsigned long pd;
3390
    PhysPageDesc *p;
3391

    
3392
    while (len > 0) {
3393
        page = addr & TARGET_PAGE_MASK;
3394
        l = (page + TARGET_PAGE_SIZE) - addr;
3395
        if (l > len)
3396
            l = len;
3397
        p = phys_page_find(page >> TARGET_PAGE_BITS);
3398
        if (!p) {
3399
            pd = IO_MEM_UNASSIGNED;
3400
        } else {
3401
            pd = p->phys_offset;
3402
        }
3403

    
3404
        if (is_write) {
3405
            if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3406
                target_phys_addr_t addr1 = addr;
3407
                io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3408
                if (p)
3409
                    addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3410
                /* XXX: could force cpu_single_env to NULL to avoid
3411
                   potential bugs */
3412
                if (l >= 4 && ((addr1 & 3) == 0)) {
3413
                    /* 32 bit write access */
3414
                    val = ldl_p(buf);
3415
                    io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3416
                    l = 4;
3417
                } else if (l >= 2 && ((addr1 & 1) == 0)) {
3418
                    /* 16 bit write access */
3419
                    val = lduw_p(buf);
3420
                    io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3421
                    l = 2;
3422
                } else {
3423
                    /* 8 bit write access */
3424
                    val = ldub_p(buf);
3425
                    io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3426
                    l = 1;
3427
                }
3428
            } else {
3429
                unsigned long addr1;
3430
                addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3431
                /* RAM case */
3432
                ptr = qemu_get_ram_ptr(addr1);
3433
                memcpy(ptr, buf, l);
3434
                if (!cpu_physical_memory_is_dirty(addr1)) {
3435
                    /* invalidate code */
3436
                    tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3437
                    /* set dirty bit */
3438
                    phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3439
                        (0xff & ~CODE_DIRTY_FLAG);
3440
                }
3441
            }
3442
        } else {
3443
            if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3444
                !(pd & IO_MEM_ROMD)) {
3445
                target_phys_addr_t addr1 = addr;
3446
                /* I/O case */
3447
                io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3448
                if (p)
3449
                    addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3450
                if (l >= 4 && ((addr1 & 3) == 0)) {
3451
                    /* 32 bit read access */
3452
                    val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3453
                    stl_p(buf, val);
3454
                    l = 4;
3455
                } else if (l >= 2 && ((addr1 & 1) == 0)) {
3456
                    /* 16 bit read access */
3457
                    val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3458
                    stw_p(buf, val);
3459
                    l = 2;
3460
                } else {
3461
                    /* 8 bit read access */
3462
                    val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3463
                    stb_p(buf, val);
3464
                    l = 1;
3465
                }
3466
            } else {
3467
                /* RAM case */
3468
                ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3469
                    (addr & ~TARGET_PAGE_MASK);
3470
                memcpy(buf, ptr, l);
3471
            }
3472
        }
3473
        len -= l;
3474
        buf += l;
3475
        addr += l;
3476
    }
3477
}
3478

    
3479
/* used for ROM loading : can write in RAM and ROM */
3480
void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3481
                                   const uint8_t *buf, int len)
3482
{
3483
    int l;
3484
    uint8_t *ptr;
3485
    target_phys_addr_t page;
3486
    unsigned long pd;
3487
    PhysPageDesc *p;
3488

    
3489
    while (len > 0) {
3490
        page = addr & TARGET_PAGE_MASK;
3491
        l = (page + TARGET_PAGE_SIZE) - addr;
3492
        if (l > len)
3493
            l = len;
3494
        p = phys_page_find(page >> TARGET_PAGE_BITS);
3495
        if (!p) {
3496
            pd = IO_MEM_UNASSIGNED;
3497
        } else {
3498
            pd = p->phys_offset;
3499
        }
3500

    
3501
        if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3502
            (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3503
            !(pd & IO_MEM_ROMD)) {
3504
            /* do nothing */
3505
        } else {
3506
            unsigned long addr1;
3507
            addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3508
            /* ROM/RAM case */
3509
            ptr = qemu_get_ram_ptr(addr1);
3510
            memcpy(ptr, buf, l);
3511
        }
3512
        len -= l;
3513
        buf += l;
3514
        addr += l;
3515
    }
3516
}
3517

    
3518
typedef struct {
3519
    void *buffer;
3520
    target_phys_addr_t addr;
3521
    target_phys_addr_t len;
3522
} BounceBuffer;
3523

    
3524
static BounceBuffer bounce;
3525

    
3526
typedef struct MapClient {
3527
    void *opaque;
3528
    void (*callback)(void *opaque);
3529
    QLIST_ENTRY(MapClient) link;
3530
} MapClient;
3531

    
3532
static QLIST_HEAD(map_client_list, MapClient) map_client_list
3533
    = QLIST_HEAD_INITIALIZER(map_client_list);
3534

    
3535
void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3536
{
3537
    MapClient *client = qemu_malloc(sizeof(*client));
3538

    
3539
    client->opaque = opaque;
3540
    client->callback = callback;
3541
    QLIST_INSERT_HEAD(&map_client_list, client, link);
3542
    return client;
3543
}
3544

    
3545
void cpu_unregister_map_client(void *_client)
3546
{
3547
    MapClient *client = (MapClient *)_client;
3548

    
3549
    QLIST_REMOVE(client, link);
3550
    qemu_free(client);
3551
}
3552

    
3553
static void cpu_notify_map_clients(void)
3554
{
3555
    MapClient *client;
3556

    
3557
    while (!QLIST_EMPTY(&map_client_list)) {
3558
        client = QLIST_FIRST(&map_client_list);
3559
        client->callback(client->opaque);
3560
        cpu_unregister_map_client(client);
3561
    }
3562
}
3563

    
3564
/* Map a physical memory region into a host virtual address.
3565
 * May map a subset of the requested range, given by and returned in *plen.
3566
 * May return NULL if resources needed to perform the mapping are exhausted.
3567
 * Use only for reads OR writes - not for read-modify-write operations.
3568
 * Use cpu_register_map_client() to know when retrying the map operation is
3569
 * likely to succeed.
3570
 */
3571
void *cpu_physical_memory_map(target_phys_addr_t addr,
3572
                              target_phys_addr_t *plen,
3573
                              int is_write)
3574
{
3575
    target_phys_addr_t len = *plen;
3576
    target_phys_addr_t done = 0;
3577
    int l;
3578
    uint8_t *ret = NULL;
3579
    uint8_t *ptr;
3580
    target_phys_addr_t page;
3581
    unsigned long pd;
3582
    PhysPageDesc *p;
3583
    unsigned long addr1;
3584

    
3585
    while (len > 0) {
3586
        page = addr & TARGET_PAGE_MASK;
3587
        l = (page + TARGET_PAGE_SIZE) - addr;
3588
        if (l > len)
3589
            l = len;
3590
        p = phys_page_find(page >> TARGET_PAGE_BITS);
3591
        if (!p) {
3592
            pd = IO_MEM_UNASSIGNED;
3593
        } else {
3594
            pd = p->phys_offset;
3595
        }
3596

    
3597
        if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3598
            if (done || bounce.buffer) {
3599
                break;
3600
            }
3601
            bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3602
            bounce.addr = addr;
3603
            bounce.len = l;
3604
            if (!is_write) {
3605
                cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3606
            }
3607
            ptr = bounce.buffer;
3608
        } else {
3609
            addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3610
            ptr = qemu_get_ram_ptr(addr1);
3611
        }
3612
        if (!done) {
3613
            ret = ptr;
3614
        } else if (ret + done != ptr) {
3615
            break;
3616
        }
3617

    
3618
        len -= l;
3619
        addr += l;
3620
        done += l;
3621
    }
3622
    *plen = done;
3623
    return ret;
3624
}
3625

    
3626
/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3627
 * Will also mark the memory as dirty if is_write == 1.  access_len gives
3628
 * the amount of memory that was actually read or written by the caller.
3629
 */
3630
void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3631
                               int is_write, target_phys_addr_t access_len)
3632
{
3633
    if (buffer != bounce.buffer) {
3634
        if (is_write) {
3635
            ram_addr_t addr1 = qemu_ram_addr_from_host(buffer);
3636
            while (access_len) {
3637
                unsigned l;
3638
                l = TARGET_PAGE_SIZE;
3639
                if (l > access_len)
3640
                    l = access_len;
3641
                if (!cpu_physical_memory_is_dirty(addr1)) {
3642
                    /* invalidate code */
3643
                    tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3644
                    /* set dirty bit */
3645
                    phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3646
                        (0xff & ~CODE_DIRTY_FLAG);
3647
                }
3648
                addr1 += l;
3649
                access_len -= l;
3650
            }
3651
        }
3652
        return;
3653
    }
3654
    if (is_write) {
3655
        cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3656
    }
3657
    qemu_vfree(bounce.buffer);
3658
    bounce.buffer = NULL;
3659
    cpu_notify_map_clients();
3660
}
3661

    
3662
/* warning: addr must be aligned */
3663
uint32_t ldl_phys(target_phys_addr_t addr)
3664
{
3665
    int io_index;
3666
    uint8_t *ptr;
3667
    uint32_t val;
3668
    unsigned long pd;
3669
    PhysPageDesc *p;
3670

    
3671
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
3672
    if (!p) {
3673
        pd = IO_MEM_UNASSIGNED;
3674
    } else {
3675
        pd = p->phys_offset;
3676
    }
3677

    
3678
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3679
        !(pd & IO_MEM_ROMD)) {
3680
        /* I/O case */
3681
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3682
        if (p)
3683
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3684
        val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3685
    } else {
3686
        /* RAM case */
3687
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3688
            (addr & ~TARGET_PAGE_MASK);
3689
        val = ldl_p(ptr);
3690
    }
3691
    return val;
3692
}
3693

    
3694
/* warning: addr must be aligned */
3695
uint64_t ldq_phys(target_phys_addr_t addr)
3696
{
3697
    int io_index;
3698
    uint8_t *ptr;
3699
    uint64_t val;
3700
    unsigned long pd;
3701
    PhysPageDesc *p;
3702

    
3703
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
3704
    if (!p) {
3705
        pd = IO_MEM_UNASSIGNED;
3706
    } else {
3707
        pd = p->phys_offset;
3708
    }
3709

    
3710
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3711
        !(pd & IO_MEM_ROMD)) {
3712
        /* I/O case */
3713
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3714
        if (p)
3715
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3716
#ifdef TARGET_WORDS_BIGENDIAN
3717
        val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3718
        val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3719
#else
3720
        val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3721
        val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3722
#endif
3723
    } else {
3724
        /* RAM case */
3725
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3726
            (addr & ~TARGET_PAGE_MASK);
3727
        val = ldq_p(ptr);
3728
    }
3729
    return val;
3730
}
3731

    
3732
/* XXX: optimize */
3733
uint32_t ldub_phys(target_phys_addr_t addr)
3734
{
3735
    uint8_t val;
3736
    cpu_physical_memory_read(addr, &val, 1);
3737
    return val;
3738
}
3739

    
3740
/* XXX: optimize */
3741
uint32_t lduw_phys(target_phys_addr_t addr)
3742
{
3743
    uint16_t val;
3744
    cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3745
    return tswap16(val);
3746
}
3747

    
3748
/* warning: addr must be aligned. The ram page is not masked as dirty
3749
   and the code inside is not invalidated. It is useful if the dirty
3750
   bits are used to track modified PTEs */
3751
void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3752
{
3753
    int io_index;
3754
    uint8_t *ptr;
3755
    unsigned long pd;
3756
    PhysPageDesc *p;
3757

    
3758
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
3759
    if (!p) {
3760
        pd = IO_MEM_UNASSIGNED;
3761
    } else {
3762
        pd = p->phys_offset;
3763
    }
3764

    
3765
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3766
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3767
        if (p)
3768
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3769
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3770
    } else {
3771
        unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3772
        ptr = qemu_get_ram_ptr(addr1);
3773
        stl_p(ptr, val);
3774

    
3775
        if (unlikely(in_migration)) {
3776
            if (!cpu_physical_memory_is_dirty(addr1)) {
3777
                /* invalidate code */
3778
                tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3779
                /* set dirty bit */
3780
                phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3781
                    (0xff & ~CODE_DIRTY_FLAG);
3782
            }
3783
        }
3784
    }
3785
}
3786

    
3787
void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3788
{
3789
    int io_index;
3790
    uint8_t *ptr;
3791
    unsigned long pd;
3792
    PhysPageDesc *p;
3793

    
3794
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
3795
    if (!p) {
3796
        pd = IO_MEM_UNASSIGNED;
3797
    } else {
3798
        pd = p->phys_offset;
3799
    }
3800

    
3801
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3802
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3803
        if (p)
3804
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3805
#ifdef TARGET_WORDS_BIGENDIAN
3806
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3807
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3808
#else
3809
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3810
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3811
#endif
3812
    } else {
3813
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3814
            (addr & ~TARGET_PAGE_MASK);
3815
        stq_p(ptr, val);
3816
    }
3817
}
3818

    
3819
/* warning: addr must be aligned */
3820
void stl_phys(target_phys_addr_t addr, uint32_t val)
3821
{
3822
    int io_index;
3823
    uint8_t *ptr;
3824
    unsigned long pd;
3825
    PhysPageDesc *p;
3826

    
3827
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
3828
    if (!p) {
3829
        pd = IO_MEM_UNASSIGNED;
3830
    } else {
3831
        pd = p->phys_offset;
3832
    }
3833

    
3834
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3835
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3836
        if (p)
3837
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3838
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3839
    } else {
3840
        unsigned long addr1;
3841
        addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3842
        /* RAM case */
3843
        ptr = qemu_get_ram_ptr(addr1);
3844
        stl_p(ptr, val);
3845
        if (!cpu_physical_memory_is_dirty(addr1)) {
3846
            /* invalidate code */
3847
            tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3848
            /* set dirty bit */
3849
            phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3850
                (0xff & ~CODE_DIRTY_FLAG);
3851
        }
3852
    }
3853
}
3854

    
3855
/* XXX: optimize */
3856
void stb_phys(target_phys_addr_t addr, uint32_t val)
3857
{
3858
    uint8_t v = val;
3859
    cpu_physical_memory_write(addr, &v, 1);
3860
}
3861

    
3862
/* XXX: optimize */
3863
void stw_phys(target_phys_addr_t addr, uint32_t val)
3864
{
3865
    uint16_t v = tswap16(val);
3866
    cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3867
}
3868

    
3869
/* XXX: optimize */
3870
void stq_phys(target_phys_addr_t addr, uint64_t val)
3871
{
3872
    val = tswap64(val);
3873
    cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3874
}
3875

    
3876
/* virtual memory access for debug (includes writing to ROM) */
3877
int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3878
                        uint8_t *buf, int len, int is_write)
3879
{
3880
    int l;
3881
    target_phys_addr_t phys_addr;
3882
    target_ulong page;
3883

    
3884
    while (len > 0) {
3885
        page = addr & TARGET_PAGE_MASK;
3886
        phys_addr = cpu_get_phys_page_debug(env, page);
3887
        /* if no physical page mapped, return an error */
3888
        if (phys_addr == -1)
3889
            return -1;
3890
        l = (page + TARGET_PAGE_SIZE) - addr;
3891
        if (l > len)
3892
            l = len;
3893
        phys_addr += (addr & ~TARGET_PAGE_MASK);
3894
        if (is_write)
3895
            cpu_physical_memory_write_rom(phys_addr, buf, l);
3896
        else
3897
            cpu_physical_memory_rw(phys_addr, buf, l, is_write);
3898
        len -= l;
3899
        buf += l;
3900
        addr += l;
3901
    }
3902
    return 0;
3903
}
3904
#endif
3905

    
3906
/* in deterministic execution mode, instructions doing device I/Os
3907
   must be at the end of the TB */
3908
void cpu_io_recompile(CPUState *env, void *retaddr)
3909
{
3910
    TranslationBlock *tb;
3911
    uint32_t n, cflags;
3912
    target_ulong pc, cs_base;
3913
    uint64_t flags;
3914

    
3915
    tb = tb_find_pc((unsigned long)retaddr);
3916
    if (!tb) {
3917
        cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p", 
3918
                  retaddr);
3919
    }
3920
    n = env->icount_decr.u16.low + tb->icount;
3921
    cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3922
    /* Calculate how many instructions had been executed before the fault
3923
       occurred.  */
3924
    n = n - env->icount_decr.u16.low;
3925
    /* Generate a new TB ending on the I/O insn.  */
3926
    n++;
3927
    /* On MIPS and SH, delay slot instructions can only be restarted if
3928
       they were already the first instruction in the TB.  If this is not
3929
       the first instruction in a TB then re-execute the preceding
3930
       branch.  */
3931
#if defined(TARGET_MIPS)
3932
    if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3933
        env->active_tc.PC -= 4;
3934
        env->icount_decr.u16.low++;
3935
        env->hflags &= ~MIPS_HFLAG_BMASK;
3936
    }
3937
#elif defined(TARGET_SH4)
3938
    if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3939
            && n > 1) {
3940
        env->pc -= 2;
3941
        env->icount_decr.u16.low++;
3942
        env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3943
    }
3944
#endif
3945
    /* This should never happen.  */
3946
    if (n > CF_COUNT_MASK)
3947
        cpu_abort(env, "TB too big during recompile");
3948

    
3949
    cflags = n | CF_LAST_IO;
3950
    pc = tb->pc;
3951
    cs_base = tb->cs_base;
3952
    flags = tb->flags;
3953
    tb_phys_invalidate(tb, -1);
3954
    /* FIXME: In theory this could raise an exception.  In practice
3955
       we have already translated the block once so it's probably ok.  */
3956
    tb_gen_code(env, pc, cs_base, flags, cflags);
3957
    /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3958
       the first in the TB) then we end up generating a whole new TB and
3959
       repeating the fault, which is horribly inefficient.
3960
       Better would be to execute just this insn uncached, or generate a
3961
       second new TB.  */
3962
    cpu_resume_from_signal(env, NULL);
3963
}
3964

    
3965
#if !defined(CONFIG_USER_ONLY)
3966

    
3967
void dump_exec_info(FILE *f,
3968
                    int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3969
{
3970
    int i, target_code_size, max_target_code_size;
3971
    int direct_jmp_count, direct_jmp2_count, cross_page;
3972
    TranslationBlock *tb;
3973

    
3974
    target_code_size = 0;
3975
    max_target_code_size = 0;
3976
    cross_page = 0;
3977
    direct_jmp_count = 0;
3978
    direct_jmp2_count = 0;
3979
    for(i = 0; i < nb_tbs; i++) {
3980
        tb = &tbs[i];
3981
        target_code_size += tb->size;
3982
        if (tb->size > max_target_code_size)
3983
            max_target_code_size = tb->size;
3984
        if (tb->page_addr[1] != -1)
3985
            cross_page++;
3986
        if (tb->tb_next_offset[0] != 0xffff) {
3987
            direct_jmp_count++;
3988
            if (tb->tb_next_offset[1] != 0xffff) {
3989
                direct_jmp2_count++;
3990
            }
3991
        }
3992
    }
3993
    /* XXX: avoid using doubles ? */
3994
    cpu_fprintf(f, "Translation buffer state:\n");
3995
    cpu_fprintf(f, "gen code size       %ld/%ld\n",
3996
                code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3997
    cpu_fprintf(f, "TB count            %d/%d\n", 
3998
                nb_tbs, code_gen_max_blocks);
3999
    cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
4000
                nb_tbs ? target_code_size / nb_tbs : 0,
4001
                max_target_code_size);
4002
    cpu_fprintf(f, "TB avg host size    %d bytes (expansion ratio: %0.1f)\n",
4003
                nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
4004
                target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
4005
    cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
4006
            cross_page,
4007
            nb_tbs ? (cross_page * 100) / nb_tbs : 0);
4008
    cpu_fprintf(f, "direct jump count   %d (%d%%) (2 jumps=%d %d%%)\n",
4009
                direct_jmp_count,
4010
                nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
4011
                direct_jmp2_count,
4012
                nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
4013
    cpu_fprintf(f, "\nStatistics:\n");
4014
    cpu_fprintf(f, "TB flush count      %d\n", tb_flush_count);
4015
    cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
4016
    cpu_fprintf(f, "TLB flush count     %d\n", tlb_flush_count);
4017
    tcg_dump_info(f, cpu_fprintf);
4018
}
4019

    
4020
#define MMUSUFFIX _cmmu
4021
#define GETPC() NULL
4022
#define env cpu_single_env
4023
#define SOFTMMU_CODE_ACCESS
4024

    
4025
#define SHIFT 0
4026
#include "softmmu_template.h"
4027

    
4028
#define SHIFT 1
4029
#include "softmmu_template.h"
4030

    
4031
#define SHIFT 2
4032
#include "softmmu_template.h"
4033

    
4034
#define SHIFT 3
4035
#include "softmmu_template.h"
4036

    
4037
#undef env
4038

    
4039
#endif