Statistics
| Branch: | Revision:

root / exec.c @ 7fd3f494

History | View | Annotate | Download (122.1 kB)

1
/*
2
 *  virtual page mapping and translated block handling
3
 *
4
 *  Copyright (c) 2003 Fabrice Bellard
5
 *
6
 * This library is free software; you can redistribute it and/or
7
 * modify it under the terms of the GNU Lesser General Public
8
 * License as published by the Free Software Foundation; either
9
 * version 2 of the License, or (at your option) any later version.
10
 *
11
 * This library is distributed in the hope that it will be useful,
12
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
 * Lesser General Public License for more details.
15
 *
16
 * You should have received a copy of the GNU Lesser General Public
17
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18
 */
19
#include "config.h"
20
#ifdef _WIN32
21
#include <windows.h>
22
#else
23
#include <sys/types.h>
24
#include <sys/mman.h>
25
#endif
26
#include <stdlib.h>
27
#include <stdio.h>
28
#include <stdarg.h>
29
#include <string.h>
30
#include <errno.h>
31
#include <unistd.h>
32
#include <inttypes.h>
33

    
34
#include "cpu.h"
35
#include "exec-all.h"
36
#include "qemu-common.h"
37
#include "tcg.h"
38
#include "hw/hw.h"
39
#include "hw/qdev.h"
40
#include "osdep.h"
41
#include "kvm.h"
42
#include "qemu-timer.h"
43
#if defined(CONFIG_USER_ONLY)
44
#include <qemu.h>
45
#include <signal.h>
46
#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
47
#include <sys/param.h>
48
#if __FreeBSD_version >= 700104
49
#define HAVE_KINFO_GETVMMAP
50
#define sigqueue sigqueue_freebsd  /* avoid redefinition */
51
#include <sys/time.h>
52
#include <sys/proc.h>
53
#include <machine/profile.h>
54
#define _KERNEL
55
#include <sys/user.h>
56
#undef _KERNEL
57
#undef sigqueue
58
#include <libutil.h>
59
#endif
60
#endif
61
#endif
62

    
63
//#define DEBUG_TB_INVALIDATE
64
//#define DEBUG_FLUSH
65
//#define DEBUG_TLB
66
//#define DEBUG_UNASSIGNED
67

    
68
/* make various TB consistency checks */
69
//#define DEBUG_TB_CHECK
70
//#define DEBUG_TLB_CHECK
71

    
72
//#define DEBUG_IOPORT
73
//#define DEBUG_SUBPAGE
74

    
75
#if !defined(CONFIG_USER_ONLY)
76
/* TB consistency checks only implemented for usermode emulation.  */
77
#undef DEBUG_TB_CHECK
78
#endif
79

    
80
#define SMC_BITMAP_USE_THRESHOLD 10
81

    
82
static TranslationBlock *tbs;
83
static int code_gen_max_blocks;
84
TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
85
static int nb_tbs;
86
/* any access to the tbs or the page table must use this lock */
87
spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
88

    
89
#if defined(__arm__) || defined(__sparc_v9__)
90
/* The prologue must be reachable with a direct jump. ARM and Sparc64
91
 have limited branch ranges (possibly also PPC) so place it in a
92
 section close to code segment. */
93
#define code_gen_section                                \
94
    __attribute__((__section__(".gen_code")))           \
95
    __attribute__((aligned (32)))
96
#elif defined(_WIN32)
97
/* Maximum alignment for Win32 is 16. */
98
#define code_gen_section                                \
99
    __attribute__((aligned (16)))
100
#else
101
#define code_gen_section                                \
102
    __attribute__((aligned (32)))
103
#endif
104

    
105
uint8_t code_gen_prologue[1024] code_gen_section;
106
static uint8_t *code_gen_buffer;
107
static unsigned long code_gen_buffer_size;
108
/* threshold to flush the translated code buffer */
109
static unsigned long code_gen_buffer_max_size;
110
static uint8_t *code_gen_ptr;
111

    
112
#if !defined(CONFIG_USER_ONLY)
113
int phys_ram_fd;
114
static int in_migration;
115

    
116
RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list) };
117
#endif
118

    
119
CPUState *first_cpu;
120
/* current CPU in the current thread. It is only valid inside
121
   cpu_exec() */
122
CPUState *cpu_single_env;
123
/* 0 = Do not count executed instructions.
124
   1 = Precise instruction counting.
125
   2 = Adaptive rate instruction counting.  */
126
int use_icount = 0;
127
/* Current instruction counter.  While executing translated code this may
128
   include some instructions that have not yet been executed.  */
129
int64_t qemu_icount;
130

    
131
typedef struct PageDesc {
132
    /* list of TBs intersecting this ram page */
133
    TranslationBlock *first_tb;
134
    /* in order to optimize self modifying code, we count the number
135
       of lookups we do to a given page to use a bitmap */
136
    unsigned int code_write_count;
137
    uint8_t *code_bitmap;
138
#if defined(CONFIG_USER_ONLY)
139
    unsigned long flags;
140
#endif
141
} PageDesc;
142

    
143
/* In system mode we want L1_MAP to be based on ram offsets,
144
   while in user mode we want it to be based on virtual addresses.  */
145
#if !defined(CONFIG_USER_ONLY)
146
#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
147
# define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
148
#else
149
# define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
150
#endif
151
#else
152
# define L1_MAP_ADDR_SPACE_BITS  TARGET_VIRT_ADDR_SPACE_BITS
153
#endif
154

    
155
/* Size of the L2 (and L3, etc) page tables.  */
156
#define L2_BITS 10
157
#define L2_SIZE (1 << L2_BITS)
158

    
159
/* The bits remaining after N lower levels of page tables.  */
160
#define P_L1_BITS_REM \
161
    ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
162
#define V_L1_BITS_REM \
163
    ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
164

    
165
/* Size of the L1 page table.  Avoid silly small sizes.  */
166
#if P_L1_BITS_REM < 4
167
#define P_L1_BITS  (P_L1_BITS_REM + L2_BITS)
168
#else
169
#define P_L1_BITS  P_L1_BITS_REM
170
#endif
171

    
172
#if V_L1_BITS_REM < 4
173
#define V_L1_BITS  (V_L1_BITS_REM + L2_BITS)
174
#else
175
#define V_L1_BITS  V_L1_BITS_REM
176
#endif
177

    
178
#define P_L1_SIZE  ((target_phys_addr_t)1 << P_L1_BITS)
179
#define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)
180

    
181
#define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
182
#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
183

    
184
unsigned long qemu_real_host_page_size;
185
unsigned long qemu_host_page_bits;
186
unsigned long qemu_host_page_size;
187
unsigned long qemu_host_page_mask;
188

    
189
/* This is a multi-level map on the virtual address space.
190
   The bottom level has pointers to PageDesc.  */
191
static void *l1_map[V_L1_SIZE];
192

    
193
#if !defined(CONFIG_USER_ONLY)
194
typedef struct PhysPageDesc {
195
    /* offset in host memory of the page + io_index in the low bits */
196
    ram_addr_t phys_offset;
197
    ram_addr_t region_offset;
198
} PhysPageDesc;
199

    
200
/* This is a multi-level map on the physical address space.
201
   The bottom level has pointers to PhysPageDesc.  */
202
static void *l1_phys_map[P_L1_SIZE];
203

    
204
static void io_mem_init(void);
205

    
206
/* io memory support */
207
CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
208
CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
209
void *io_mem_opaque[IO_MEM_NB_ENTRIES];
210
static char io_mem_used[IO_MEM_NB_ENTRIES];
211
static int io_mem_watch;
212
#endif
213

    
214
/* log support */
215
#ifdef WIN32
216
static const char *logfilename = "qemu.log";
217
#else
218
static const char *logfilename = "/tmp/qemu.log";
219
#endif
220
FILE *logfile;
221
int loglevel;
222
static int log_append = 0;
223

    
224
/* statistics */
225
#if !defined(CONFIG_USER_ONLY)
226
static int tlb_flush_count;
227
#endif
228
static int tb_flush_count;
229
static int tb_phys_invalidate_count;
230

    
231
#ifdef _WIN32
232
static void map_exec(void *addr, long size)
233
{
234
    DWORD old_protect;
235
    VirtualProtect(addr, size,
236
                   PAGE_EXECUTE_READWRITE, &old_protect);
237
    
238
}
239
#else
240
static void map_exec(void *addr, long size)
241
{
242
    unsigned long start, end, page_size;
243
    
244
    page_size = getpagesize();
245
    start = (unsigned long)addr;
246
    start &= ~(page_size - 1);
247
    
248
    end = (unsigned long)addr + size;
249
    end += page_size - 1;
250
    end &= ~(page_size - 1);
251
    
252
    mprotect((void *)start, end - start,
253
             PROT_READ | PROT_WRITE | PROT_EXEC);
254
}
255
#endif
256

    
257
static void page_init(void)
258
{
259
    /* NOTE: we can always suppose that qemu_host_page_size >=
260
       TARGET_PAGE_SIZE */
261
#ifdef _WIN32
262
    {
263
        SYSTEM_INFO system_info;
264

    
265
        GetSystemInfo(&system_info);
266
        qemu_real_host_page_size = system_info.dwPageSize;
267
    }
268
#else
269
    qemu_real_host_page_size = getpagesize();
270
#endif
271
    if (qemu_host_page_size == 0)
272
        qemu_host_page_size = qemu_real_host_page_size;
273
    if (qemu_host_page_size < TARGET_PAGE_SIZE)
274
        qemu_host_page_size = TARGET_PAGE_SIZE;
275
    qemu_host_page_bits = 0;
276
    while ((1 << qemu_host_page_bits) < qemu_host_page_size)
277
        qemu_host_page_bits++;
278
    qemu_host_page_mask = ~(qemu_host_page_size - 1);
279

    
280
#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
281
    {
282
#ifdef HAVE_KINFO_GETVMMAP
283
        struct kinfo_vmentry *freep;
284
        int i, cnt;
285

    
286
        freep = kinfo_getvmmap(getpid(), &cnt);
287
        if (freep) {
288
            mmap_lock();
289
            for (i = 0; i < cnt; i++) {
290
                unsigned long startaddr, endaddr;
291

    
292
                startaddr = freep[i].kve_start;
293
                endaddr = freep[i].kve_end;
294
                if (h2g_valid(startaddr)) {
295
                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
296

    
297
                    if (h2g_valid(endaddr)) {
298
                        endaddr = h2g(endaddr);
299
                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
300
                    } else {
301
#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
302
                        endaddr = ~0ul;
303
                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);
304
#endif
305
                    }
306
                }
307
            }
308
            free(freep);
309
            mmap_unlock();
310
        }
311
#else
312
        FILE *f;
313

    
314
        last_brk = (unsigned long)sbrk(0);
315

    
316
        f = fopen("/compat/linux/proc/self/maps", "r");
317
        if (f) {
318
            mmap_lock();
319

    
320
            do {
321
                unsigned long startaddr, endaddr;
322
                int n;
323

    
324
                n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
325

    
326
                if (n == 2 && h2g_valid(startaddr)) {
327
                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
328

    
329
                    if (h2g_valid(endaddr)) {
330
                        endaddr = h2g(endaddr);
331
                    } else {
332
                        endaddr = ~0ul;
333
                    }
334
                    page_set_flags(startaddr, endaddr, PAGE_RESERVED);
335
                }
336
            } while (!feof(f));
337

    
338
            fclose(f);
339
            mmap_unlock();
340
        }
341
#endif
342
    }
343
#endif
344
}
345

    
346
static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
347
{
348
    PageDesc *pd;
349
    void **lp;
350
    int i;
351

    
352
#if defined(CONFIG_USER_ONLY)
353
    /* We can't use qemu_malloc because it may recurse into a locked mutex. */
354
# define ALLOC(P, SIZE)                                 \
355
    do {                                                \
356
        P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,    \
357
                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);   \
358
    } while (0)
359
#else
360
# define ALLOC(P, SIZE) \
361
    do { P = qemu_mallocz(SIZE); } while (0)
362
#endif
363

    
364
    /* Level 1.  Always allocated.  */
365
    lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
366

    
367
    /* Level 2..N-1.  */
368
    for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
369
        void **p = *lp;
370

    
371
        if (p == NULL) {
372
            if (!alloc) {
373
                return NULL;
374
            }
375
            ALLOC(p, sizeof(void *) * L2_SIZE);
376
            *lp = p;
377
        }
378

    
379
        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
380
    }
381

    
382
    pd = *lp;
383
    if (pd == NULL) {
384
        if (!alloc) {
385
            return NULL;
386
        }
387
        ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
388
        *lp = pd;
389
    }
390

    
391
#undef ALLOC
392

    
393
    return pd + (index & (L2_SIZE - 1));
394
}
395

    
396
static inline PageDesc *page_find(tb_page_addr_t index)
397
{
398
    return page_find_alloc(index, 0);
399
}
400

    
401
#if !defined(CONFIG_USER_ONLY)
402
static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
403
{
404
    PhysPageDesc *pd;
405
    void **lp;
406
    int i;
407

    
408
    /* Level 1.  Always allocated.  */
409
    lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
410

    
411
    /* Level 2..N-1.  */
412
    for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
413
        void **p = *lp;
414
        if (p == NULL) {
415
            if (!alloc) {
416
                return NULL;
417
            }
418
            *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
419
        }
420
        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
421
    }
422

    
423
    pd = *lp;
424
    if (pd == NULL) {
425
        int i;
426

    
427
        if (!alloc) {
428
            return NULL;
429
        }
430

    
431
        *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);
432

    
433
        for (i = 0; i < L2_SIZE; i++) {
434
            pd[i].phys_offset = IO_MEM_UNASSIGNED;
435
            pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
436
        }
437
    }
438

    
439
    return pd + (index & (L2_SIZE - 1));
440
}
441

    
442
static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
443
{
444
    return phys_page_find_alloc(index, 0);
445
}
446

    
447
static void tlb_protect_code(ram_addr_t ram_addr);
448
static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
449
                                    target_ulong vaddr);
450
#define mmap_lock() do { } while(0)
451
#define mmap_unlock() do { } while(0)
452
#endif
453

    
454
#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
455

    
456
#if defined(CONFIG_USER_ONLY)
457
/* Currently it is not recommended to allocate big chunks of data in
458
   user mode. It will change when a dedicated libc will be used */
459
#define USE_STATIC_CODE_GEN_BUFFER
460
#endif
461

    
462
#ifdef USE_STATIC_CODE_GEN_BUFFER
463
static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
464
               __attribute__((aligned (CODE_GEN_ALIGN)));
465
#endif
466

    
467
static void code_gen_alloc(unsigned long tb_size)
468
{
469
#ifdef USE_STATIC_CODE_GEN_BUFFER
470
    code_gen_buffer = static_code_gen_buffer;
471
    code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
472
    map_exec(code_gen_buffer, code_gen_buffer_size);
473
#else
474
    code_gen_buffer_size = tb_size;
475
    if (code_gen_buffer_size == 0) {
476
#if defined(CONFIG_USER_ONLY)
477
        /* in user mode, phys_ram_size is not meaningful */
478
        code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
479
#else
480
        /* XXX: needs adjustments */
481
        code_gen_buffer_size = (unsigned long)(ram_size / 4);
482
#endif
483
    }
484
    if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
485
        code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
486
    /* The code gen buffer location may have constraints depending on
487
       the host cpu and OS */
488
#if defined(__linux__) 
489
    {
490
        int flags;
491
        void *start = NULL;
492

    
493
        flags = MAP_PRIVATE | MAP_ANONYMOUS;
494
#if defined(__x86_64__)
495
        flags |= MAP_32BIT;
496
        /* Cannot map more than that */
497
        if (code_gen_buffer_size > (800 * 1024 * 1024))
498
            code_gen_buffer_size = (800 * 1024 * 1024);
499
#elif defined(__sparc_v9__)
500
        // Map the buffer below 2G, so we can use direct calls and branches
501
        flags |= MAP_FIXED;
502
        start = (void *) 0x60000000UL;
503
        if (code_gen_buffer_size > (512 * 1024 * 1024))
504
            code_gen_buffer_size = (512 * 1024 * 1024);
505
#elif defined(__arm__)
506
        /* Map the buffer below 32M, so we can use direct calls and branches */
507
        flags |= MAP_FIXED;
508
        start = (void *) 0x01000000UL;
509
        if (code_gen_buffer_size > 16 * 1024 * 1024)
510
            code_gen_buffer_size = 16 * 1024 * 1024;
511
#elif defined(__s390x__)
512
        /* Map the buffer so that we can use direct calls and branches.  */
513
        /* We have a +- 4GB range on the branches; leave some slop.  */
514
        if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
515
            code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
516
        }
517
        start = (void *)0x90000000UL;
518
#endif
519
        code_gen_buffer = mmap(start, code_gen_buffer_size,
520
                               PROT_WRITE | PROT_READ | PROT_EXEC,
521
                               flags, -1, 0);
522
        if (code_gen_buffer == MAP_FAILED) {
523
            fprintf(stderr, "Could not allocate dynamic translator buffer\n");
524
            exit(1);
525
        }
526
    }
527
#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
528
    {
529
        int flags;
530
        void *addr = NULL;
531
        flags = MAP_PRIVATE | MAP_ANONYMOUS;
532
#if defined(__x86_64__)
533
        /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
534
         * 0x40000000 is free */
535
        flags |= MAP_FIXED;
536
        addr = (void *)0x40000000;
537
        /* Cannot map more than that */
538
        if (code_gen_buffer_size > (800 * 1024 * 1024))
539
            code_gen_buffer_size = (800 * 1024 * 1024);
540
#endif
541
        code_gen_buffer = mmap(addr, code_gen_buffer_size,
542
                               PROT_WRITE | PROT_READ | PROT_EXEC, 
543
                               flags, -1, 0);
544
        if (code_gen_buffer == MAP_FAILED) {
545
            fprintf(stderr, "Could not allocate dynamic translator buffer\n");
546
            exit(1);
547
        }
548
    }
549
#else
550
    code_gen_buffer = qemu_malloc(code_gen_buffer_size);
551
    map_exec(code_gen_buffer, code_gen_buffer_size);
552
#endif
553
#endif /* !USE_STATIC_CODE_GEN_BUFFER */
554
    map_exec(code_gen_prologue, sizeof(code_gen_prologue));
555
    code_gen_buffer_max_size = code_gen_buffer_size - 
556
        (TCG_MAX_OP_SIZE * OPC_MAX_SIZE);
557
    code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
558
    tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
559
}
560

    
561
/* Must be called before using the QEMU cpus. 'tb_size' is the size
562
   (in bytes) allocated to the translation buffer. Zero means default
563
   size. */
564
void cpu_exec_init_all(unsigned long tb_size)
565
{
566
    cpu_gen_init();
567
    code_gen_alloc(tb_size);
568
    code_gen_ptr = code_gen_buffer;
569
    page_init();
570
#if !defined(CONFIG_USER_ONLY)
571
    io_mem_init();
572
#endif
573
#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
574
    /* There's no guest base to take into account, so go ahead and
575
       initialize the prologue now.  */
576
    tcg_prologue_init(&tcg_ctx);
577
#endif
578
}
579

    
580
#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
581

    
582
static int cpu_common_post_load(void *opaque, int version_id)
583
{
584
    CPUState *env = opaque;
585

    
586
    /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
587
       version_id is increased. */
588
    env->interrupt_request &= ~0x01;
589
    tlb_flush(env, 1);
590

    
591
    return 0;
592
}
593

    
594
static const VMStateDescription vmstate_cpu_common = {
595
    .name = "cpu_common",
596
    .version_id = 1,
597
    .minimum_version_id = 1,
598
    .minimum_version_id_old = 1,
599
    .post_load = cpu_common_post_load,
600
    .fields      = (VMStateField []) {
601
        VMSTATE_UINT32(halted, CPUState),
602
        VMSTATE_UINT32(interrupt_request, CPUState),
603
        VMSTATE_END_OF_LIST()
604
    }
605
};
606
#endif
607

    
608
CPUState *qemu_get_cpu(int cpu)
609
{
610
    CPUState *env = first_cpu;
611

    
612
    while (env) {
613
        if (env->cpu_index == cpu)
614
            break;
615
        env = env->next_cpu;
616
    }
617

    
618
    return env;
619
}
620

    
621
void cpu_exec_init(CPUState *env)
622
{
623
    CPUState **penv;
624
    int cpu_index;
625

    
626
#if defined(CONFIG_USER_ONLY)
627
    cpu_list_lock();
628
#endif
629
    env->next_cpu = NULL;
630
    penv = &first_cpu;
631
    cpu_index = 0;
632
    while (*penv != NULL) {
633
        penv = &(*penv)->next_cpu;
634
        cpu_index++;
635
    }
636
    env->cpu_index = cpu_index;
637
    env->numa_node = 0;
638
    QTAILQ_INIT(&env->breakpoints);
639
    QTAILQ_INIT(&env->watchpoints);
640
    *penv = env;
641
#if defined(CONFIG_USER_ONLY)
642
    cpu_list_unlock();
643
#endif
644
#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
645
    vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
646
    register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
647
                    cpu_save, cpu_load, env);
648
#endif
649
}
650

    
651
static inline void invalidate_page_bitmap(PageDesc *p)
652
{
653
    if (p->code_bitmap) {
654
        qemu_free(p->code_bitmap);
655
        p->code_bitmap = NULL;
656
    }
657
    p->code_write_count = 0;
658
}
659

    
660
/* Set to NULL all the 'first_tb' fields in all PageDescs. */
661

    
662
static void page_flush_tb_1 (int level, void **lp)
663
{
664
    int i;
665

    
666
    if (*lp == NULL) {
667
        return;
668
    }
669
    if (level == 0) {
670
        PageDesc *pd = *lp;
671
        for (i = 0; i < L2_SIZE; ++i) {
672
            pd[i].first_tb = NULL;
673
            invalidate_page_bitmap(pd + i);
674
        }
675
    } else {
676
        void **pp = *lp;
677
        for (i = 0; i < L2_SIZE; ++i) {
678
            page_flush_tb_1 (level - 1, pp + i);
679
        }
680
    }
681
}
682

    
683
static void page_flush_tb(void)
684
{
685
    int i;
686
    for (i = 0; i < V_L1_SIZE; i++) {
687
        page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
688
    }
689
}
690

    
691
/* flush all the translation blocks */
692
/* XXX: tb_flush is currently not thread safe */
693
void tb_flush(CPUState *env1)
694
{
695
    CPUState *env;
696
#if defined(DEBUG_FLUSH)
697
    printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
698
           (unsigned long)(code_gen_ptr - code_gen_buffer),
699
           nb_tbs, nb_tbs > 0 ?
700
           ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
701
#endif
702
    if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
703
        cpu_abort(env1, "Internal error: code buffer overflow\n");
704

    
705
    nb_tbs = 0;
706

    
707
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
708
        memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
709
    }
710

    
711
    memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
712
    page_flush_tb();
713

    
714
    code_gen_ptr = code_gen_buffer;
715
    /* XXX: flush processor icache at this point if cache flush is
716
       expensive */
717
    tb_flush_count++;
718
}
719

    
720
#ifdef DEBUG_TB_CHECK
721

    
722
static void tb_invalidate_check(target_ulong address)
723
{
724
    TranslationBlock *tb;
725
    int i;
726
    address &= TARGET_PAGE_MASK;
727
    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
728
        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
729
            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
730
                  address >= tb->pc + tb->size)) {
731
                printf("ERROR invalidate: address=" TARGET_FMT_lx
732
                       " PC=%08lx size=%04x\n",
733
                       address, (long)tb->pc, tb->size);
734
            }
735
        }
736
    }
737
}
738

    
739
/* verify that all the pages have correct rights for code */
740
static void tb_page_check(void)
741
{
742
    TranslationBlock *tb;
743
    int i, flags1, flags2;
744

    
745
    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
746
        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
747
            flags1 = page_get_flags(tb->pc);
748
            flags2 = page_get_flags(tb->pc + tb->size - 1);
749
            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
750
                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
751
                       (long)tb->pc, tb->size, flags1, flags2);
752
            }
753
        }
754
    }
755
}
756

    
757
#endif
758

    
759
/* invalidate one TB */
760
static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
761
                             int next_offset)
762
{
763
    TranslationBlock *tb1;
764
    for(;;) {
765
        tb1 = *ptb;
766
        if (tb1 == tb) {
767
            *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
768
            break;
769
        }
770
        ptb = (TranslationBlock **)((char *)tb1 + next_offset);
771
    }
772
}
773

    
774
static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
775
{
776
    TranslationBlock *tb1;
777
    unsigned int n1;
778

    
779
    for(;;) {
780
        tb1 = *ptb;
781
        n1 = (long)tb1 & 3;
782
        tb1 = (TranslationBlock *)((long)tb1 & ~3);
783
        if (tb1 == tb) {
784
            *ptb = tb1->page_next[n1];
785
            break;
786
        }
787
        ptb = &tb1->page_next[n1];
788
    }
789
}
790

    
791
static inline void tb_jmp_remove(TranslationBlock *tb, int n)
792
{
793
    TranslationBlock *tb1, **ptb;
794
    unsigned int n1;
795

    
796
    ptb = &tb->jmp_next[n];
797
    tb1 = *ptb;
798
    if (tb1) {
799
        /* find tb(n) in circular list */
800
        for(;;) {
801
            tb1 = *ptb;
802
            n1 = (long)tb1 & 3;
803
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
804
            if (n1 == n && tb1 == tb)
805
                break;
806
            if (n1 == 2) {
807
                ptb = &tb1->jmp_first;
808
            } else {
809
                ptb = &tb1->jmp_next[n1];
810
            }
811
        }
812
        /* now we can suppress tb(n) from the list */
813
        *ptb = tb->jmp_next[n];
814

    
815
        tb->jmp_next[n] = NULL;
816
    }
817
}
818

    
819
/* reset the jump entry 'n' of a TB so that it is not chained to
820
   another TB */
821
static inline void tb_reset_jump(TranslationBlock *tb, int n)
822
{
823
    tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
824
}
825

    
826
void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
827
{
828
    CPUState *env;
829
    PageDesc *p;
830
    unsigned int h, n1;
831
    tb_page_addr_t phys_pc;
832
    TranslationBlock *tb1, *tb2;
833

    
834
    /* remove the TB from the hash list */
835
    phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
836
    h = tb_phys_hash_func(phys_pc);
837
    tb_remove(&tb_phys_hash[h], tb,
838
              offsetof(TranslationBlock, phys_hash_next));
839

    
840
    /* remove the TB from the page list */
841
    if (tb->page_addr[0] != page_addr) {
842
        p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
843
        tb_page_remove(&p->first_tb, tb);
844
        invalidate_page_bitmap(p);
845
    }
846
    if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
847
        p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
848
        tb_page_remove(&p->first_tb, tb);
849
        invalidate_page_bitmap(p);
850
    }
851

    
852
    tb_invalidated_flag = 1;
853

    
854
    /* remove the TB from the hash list */
855
    h = tb_jmp_cache_hash_func(tb->pc);
856
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
857
        if (env->tb_jmp_cache[h] == tb)
858
            env->tb_jmp_cache[h] = NULL;
859
    }
860

    
861
    /* suppress this TB from the two jump lists */
862
    tb_jmp_remove(tb, 0);
863
    tb_jmp_remove(tb, 1);
864

    
865
    /* suppress any remaining jumps to this TB */
866
    tb1 = tb->jmp_first;
867
    for(;;) {
868
        n1 = (long)tb1 & 3;
869
        if (n1 == 2)
870
            break;
871
        tb1 = (TranslationBlock *)((long)tb1 & ~3);
872
        tb2 = tb1->jmp_next[n1];
873
        tb_reset_jump(tb1, n1);
874
        tb1->jmp_next[n1] = NULL;
875
        tb1 = tb2;
876
    }
877
    tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
878

    
879
    tb_phys_invalidate_count++;
880
}
881

    
882
static inline void set_bits(uint8_t *tab, int start, int len)
883
{
884
    int end, mask, end1;
885

    
886
    end = start + len;
887
    tab += start >> 3;
888
    mask = 0xff << (start & 7);
889
    if ((start & ~7) == (end & ~7)) {
890
        if (start < end) {
891
            mask &= ~(0xff << (end & 7));
892
            *tab |= mask;
893
        }
894
    } else {
895
        *tab++ |= mask;
896
        start = (start + 8) & ~7;
897
        end1 = end & ~7;
898
        while (start < end1) {
899
            *tab++ = 0xff;
900
            start += 8;
901
        }
902
        if (start < end) {
903
            mask = ~(0xff << (end & 7));
904
            *tab |= mask;
905
        }
906
    }
907
}
908

    
909
static void build_page_bitmap(PageDesc *p)
910
{
911
    int n, tb_start, tb_end;
912
    TranslationBlock *tb;
913

    
914
    p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
915

    
916
    tb = p->first_tb;
917
    while (tb != NULL) {
918
        n = (long)tb & 3;
919
        tb = (TranslationBlock *)((long)tb & ~3);
920
        /* NOTE: this is subtle as a TB may span two physical pages */
921
        if (n == 0) {
922
            /* NOTE: tb_end may be after the end of the page, but
923
               it is not a problem */
924
            tb_start = tb->pc & ~TARGET_PAGE_MASK;
925
            tb_end = tb_start + tb->size;
926
            if (tb_end > TARGET_PAGE_SIZE)
927
                tb_end = TARGET_PAGE_SIZE;
928
        } else {
929
            tb_start = 0;
930
            tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
931
        }
932
        set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
933
        tb = tb->page_next[n];
934
    }
935
}
936

    
937
TranslationBlock *tb_gen_code(CPUState *env,
938
                              target_ulong pc, target_ulong cs_base,
939
                              int flags, int cflags)
940
{
941
    TranslationBlock *tb;
942
    uint8_t *tc_ptr;
943
    tb_page_addr_t phys_pc, phys_page2;
944
    target_ulong virt_page2;
945
    int code_gen_size;
946

    
947
    phys_pc = get_page_addr_code(env, pc);
948
    tb = tb_alloc(pc);
949
    if (!tb) {
950
        /* flush must be done */
951
        tb_flush(env);
952
        /* cannot fail at this point */
953
        tb = tb_alloc(pc);
954
        /* Don't forget to invalidate previous TB info.  */
955
        tb_invalidated_flag = 1;
956
    }
957
    tc_ptr = code_gen_ptr;
958
    tb->tc_ptr = tc_ptr;
959
    tb->cs_base = cs_base;
960
    tb->flags = flags;
961
    tb->cflags = cflags;
962
    cpu_gen_code(env, tb, &code_gen_size);
963
    code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
964

    
965
    /* check next page if needed */
966
    virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
967
    phys_page2 = -1;
968
    if ((pc & TARGET_PAGE_MASK) != virt_page2) {
969
        phys_page2 = get_page_addr_code(env, virt_page2);
970
    }
971
    tb_link_page(tb, phys_pc, phys_page2);
972
    return tb;
973
}
974

    
975
/* invalidate all TBs which intersect with the target physical page
976
   starting in range [start;end[. NOTE: start and end must refer to
977
   the same physical page. 'is_cpu_write_access' should be true if called
978
   from a real cpu write access: the virtual CPU will exit the current
979
   TB if code is modified inside this TB. */
980
void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
981
                                   int is_cpu_write_access)
982
{
983
    TranslationBlock *tb, *tb_next, *saved_tb;
984
    CPUState *env = cpu_single_env;
985
    tb_page_addr_t tb_start, tb_end;
986
    PageDesc *p;
987
    int n;
988
#ifdef TARGET_HAS_PRECISE_SMC
989
    int current_tb_not_found = is_cpu_write_access;
990
    TranslationBlock *current_tb = NULL;
991
    int current_tb_modified = 0;
992
    target_ulong current_pc = 0;
993
    target_ulong current_cs_base = 0;
994
    int current_flags = 0;
995
#endif /* TARGET_HAS_PRECISE_SMC */
996

    
997
    p = page_find(start >> TARGET_PAGE_BITS);
998
    if (!p)
999
        return;
1000
    if (!p->code_bitmap &&
1001
        ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1002
        is_cpu_write_access) {
1003
        /* build code bitmap */
1004
        build_page_bitmap(p);
1005
    }
1006

    
1007
    /* we remove all the TBs in the range [start, end[ */
1008
    /* XXX: see if in some cases it could be faster to invalidate all the code */
1009
    tb = p->first_tb;
1010
    while (tb != NULL) {
1011
        n = (long)tb & 3;
1012
        tb = (TranslationBlock *)((long)tb & ~3);
1013
        tb_next = tb->page_next[n];
1014
        /* NOTE: this is subtle as a TB may span two physical pages */
1015
        if (n == 0) {
1016
            /* NOTE: tb_end may be after the end of the page, but
1017
               it is not a problem */
1018
            tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1019
            tb_end = tb_start + tb->size;
1020
        } else {
1021
            tb_start = tb->page_addr[1];
1022
            tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1023
        }
1024
        if (!(tb_end <= start || tb_start >= end)) {
1025
#ifdef TARGET_HAS_PRECISE_SMC
1026
            if (current_tb_not_found) {
1027
                current_tb_not_found = 0;
1028
                current_tb = NULL;
1029
                if (env->mem_io_pc) {
1030
                    /* now we have a real cpu fault */
1031
                    current_tb = tb_find_pc(env->mem_io_pc);
1032
                }
1033
            }
1034
            if (current_tb == tb &&
1035
                (current_tb->cflags & CF_COUNT_MASK) != 1) {
1036
                /* If we are modifying the current TB, we must stop
1037
                its execution. We could be more precise by checking
1038
                that the modification is after the current PC, but it
1039
                would require a specialized function to partially
1040
                restore the CPU state */
1041

    
1042
                current_tb_modified = 1;
1043
                cpu_restore_state(current_tb, env,
1044
                                  env->mem_io_pc, NULL);
1045
                cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1046
                                     &current_flags);
1047
            }
1048
#endif /* TARGET_HAS_PRECISE_SMC */
1049
            /* we need to do that to handle the case where a signal
1050
               occurs while doing tb_phys_invalidate() */
1051
            saved_tb = NULL;
1052
            if (env) {
1053
                saved_tb = env->current_tb;
1054
                env->current_tb = NULL;
1055
            }
1056
            tb_phys_invalidate(tb, -1);
1057
            if (env) {
1058
                env->current_tb = saved_tb;
1059
                if (env->interrupt_request && env->current_tb)
1060
                    cpu_interrupt(env, env->interrupt_request);
1061
            }
1062
        }
1063
        tb = tb_next;
1064
    }
1065
#if !defined(CONFIG_USER_ONLY)
1066
    /* if no code remaining, no need to continue to use slow writes */
1067
    if (!p->first_tb) {
1068
        invalidate_page_bitmap(p);
1069
        if (is_cpu_write_access) {
1070
            tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1071
        }
1072
    }
1073
#endif
1074
#ifdef TARGET_HAS_PRECISE_SMC
1075
    if (current_tb_modified) {
1076
        /* we generate a block containing just the instruction
1077
           modifying the memory. It will ensure that it cannot modify
1078
           itself */
1079
        env->current_tb = NULL;
1080
        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1081
        cpu_resume_from_signal(env, NULL);
1082
    }
1083
#endif
1084
}
1085

    
1086
/* len must be <= 8 and start must be a multiple of len */
1087
static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1088
{
1089
    PageDesc *p;
1090
    int offset, b;
1091
#if 0
1092
    if (1) {
1093
        qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1094
                  cpu_single_env->mem_io_vaddr, len,
1095
                  cpu_single_env->eip,
1096
                  cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1097
    }
1098
#endif
1099
    p = page_find(start >> TARGET_PAGE_BITS);
1100
    if (!p)
1101
        return;
1102
    if (p->code_bitmap) {
1103
        offset = start & ~TARGET_PAGE_MASK;
1104
        b = p->code_bitmap[offset >> 3] >> (offset & 7);
1105
        if (b & ((1 << len) - 1))
1106
            goto do_invalidate;
1107
    } else {
1108
    do_invalidate:
1109
        tb_invalidate_phys_page_range(start, start + len, 1);
1110
    }
1111
}
1112

    
1113
#if !defined(CONFIG_SOFTMMU)
1114
static void tb_invalidate_phys_page(tb_page_addr_t addr,
1115
                                    unsigned long pc, void *puc)
1116
{
1117
    TranslationBlock *tb;
1118
    PageDesc *p;
1119
    int n;
1120
#ifdef TARGET_HAS_PRECISE_SMC
1121
    TranslationBlock *current_tb = NULL;
1122
    CPUState *env = cpu_single_env;
1123
    int current_tb_modified = 0;
1124
    target_ulong current_pc = 0;
1125
    target_ulong current_cs_base = 0;
1126
    int current_flags = 0;
1127
#endif
1128

    
1129
    addr &= TARGET_PAGE_MASK;
1130
    p = page_find(addr >> TARGET_PAGE_BITS);
1131
    if (!p)
1132
        return;
1133
    tb = p->first_tb;
1134
#ifdef TARGET_HAS_PRECISE_SMC
1135
    if (tb && pc != 0) {
1136
        current_tb = tb_find_pc(pc);
1137
    }
1138
#endif
1139
    while (tb != NULL) {
1140
        n = (long)tb & 3;
1141
        tb = (TranslationBlock *)((long)tb & ~3);
1142
#ifdef TARGET_HAS_PRECISE_SMC
1143
        if (current_tb == tb &&
1144
            (current_tb->cflags & CF_COUNT_MASK) != 1) {
1145
                /* If we are modifying the current TB, we must stop
1146
                   its execution. We could be more precise by checking
1147
                   that the modification is after the current PC, but it
1148
                   would require a specialized function to partially
1149
                   restore the CPU state */
1150

    
1151
            current_tb_modified = 1;
1152
            cpu_restore_state(current_tb, env, pc, puc);
1153
            cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1154
                                 &current_flags);
1155
        }
1156
#endif /* TARGET_HAS_PRECISE_SMC */
1157
        tb_phys_invalidate(tb, addr);
1158
        tb = tb->page_next[n];
1159
    }
1160
    p->first_tb = NULL;
1161
#ifdef TARGET_HAS_PRECISE_SMC
1162
    if (current_tb_modified) {
1163
        /* we generate a block containing just the instruction
1164
           modifying the memory. It will ensure that it cannot modify
1165
           itself */
1166
        env->current_tb = NULL;
1167
        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1168
        cpu_resume_from_signal(env, puc);
1169
    }
1170
#endif
1171
}
1172
#endif
1173

    
1174
/* add the tb in the target page and protect it if necessary */
1175
static inline void tb_alloc_page(TranslationBlock *tb,
1176
                                 unsigned int n, tb_page_addr_t page_addr)
1177
{
1178
    PageDesc *p;
1179
    TranslationBlock *last_first_tb;
1180

    
1181
    tb->page_addr[n] = page_addr;
1182
    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1183
    tb->page_next[n] = p->first_tb;
1184
    last_first_tb = p->first_tb;
1185
    p->first_tb = (TranslationBlock *)((long)tb | n);
1186
    invalidate_page_bitmap(p);
1187

    
1188
#if defined(TARGET_HAS_SMC) || 1
1189

    
1190
#if defined(CONFIG_USER_ONLY)
1191
    if (p->flags & PAGE_WRITE) {
1192
        target_ulong addr;
1193
        PageDesc *p2;
1194
        int prot;
1195

    
1196
        /* force the host page as non writable (writes will have a
1197
           page fault + mprotect overhead) */
1198
        page_addr &= qemu_host_page_mask;
1199
        prot = 0;
1200
        for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1201
            addr += TARGET_PAGE_SIZE) {
1202

    
1203
            p2 = page_find (addr >> TARGET_PAGE_BITS);
1204
            if (!p2)
1205
                continue;
1206
            prot |= p2->flags;
1207
            p2->flags &= ~PAGE_WRITE;
1208
          }
1209
        mprotect(g2h(page_addr), qemu_host_page_size,
1210
                 (prot & PAGE_BITS) & ~PAGE_WRITE);
1211
#ifdef DEBUG_TB_INVALIDATE
1212
        printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1213
               page_addr);
1214
#endif
1215
    }
1216
#else
1217
    /* if some code is already present, then the pages are already
1218
       protected. So we handle the case where only the first TB is
1219
       allocated in a physical page */
1220
    if (!last_first_tb) {
1221
        tlb_protect_code(page_addr);
1222
    }
1223
#endif
1224

    
1225
#endif /* TARGET_HAS_SMC */
1226
}
1227

    
1228
/* Allocate a new translation block. Flush the translation buffer if
1229
   too many translation blocks or too much generated code. */
1230
TranslationBlock *tb_alloc(target_ulong pc)
1231
{
1232
    TranslationBlock *tb;
1233

    
1234
    if (nb_tbs >= code_gen_max_blocks ||
1235
        (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1236
        return NULL;
1237
    tb = &tbs[nb_tbs++];
1238
    tb->pc = pc;
1239
    tb->cflags = 0;
1240
    return tb;
1241
}
1242

    
1243
void tb_free(TranslationBlock *tb)
1244
{
1245
    /* In practice this is mostly used for single use temporary TB
1246
       Ignore the hard cases and just back up if this TB happens to
1247
       be the last one generated.  */
1248
    if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1249
        code_gen_ptr = tb->tc_ptr;
1250
        nb_tbs--;
1251
    }
1252
}
1253

    
1254
/* add a new TB and link it to the physical page tables. phys_page2 is
1255
   (-1) to indicate that only one page contains the TB. */
1256
void tb_link_page(TranslationBlock *tb,
1257
                  tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
1258
{
1259
    unsigned int h;
1260
    TranslationBlock **ptb;
1261

    
1262
    /* Grab the mmap lock to stop another thread invalidating this TB
1263
       before we are done.  */
1264
    mmap_lock();
1265
    /* add in the physical hash table */
1266
    h = tb_phys_hash_func(phys_pc);
1267
    ptb = &tb_phys_hash[h];
1268
    tb->phys_hash_next = *ptb;
1269
    *ptb = tb;
1270

    
1271
    /* add in the page list */
1272
    tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1273
    if (phys_page2 != -1)
1274
        tb_alloc_page(tb, 1, phys_page2);
1275
    else
1276
        tb->page_addr[1] = -1;
1277

    
1278
    tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1279
    tb->jmp_next[0] = NULL;
1280
    tb->jmp_next[1] = NULL;
1281

    
1282
    /* init original jump addresses */
1283
    if (tb->tb_next_offset[0] != 0xffff)
1284
        tb_reset_jump(tb, 0);
1285
    if (tb->tb_next_offset[1] != 0xffff)
1286
        tb_reset_jump(tb, 1);
1287

    
1288
#ifdef DEBUG_TB_CHECK
1289
    tb_page_check();
1290
#endif
1291
    mmap_unlock();
1292
}
1293

    
1294
/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1295
   tb[1].tc_ptr. Return NULL if not found */
1296
TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1297
{
1298
    int m_min, m_max, m;
1299
    unsigned long v;
1300
    TranslationBlock *tb;
1301

    
1302
    if (nb_tbs <= 0)
1303
        return NULL;
1304
    if (tc_ptr < (unsigned long)code_gen_buffer ||
1305
        tc_ptr >= (unsigned long)code_gen_ptr)
1306
        return NULL;
1307
    /* binary search (cf Knuth) */
1308
    m_min = 0;
1309
    m_max = nb_tbs - 1;
1310
    while (m_min <= m_max) {
1311
        m = (m_min + m_max) >> 1;
1312
        tb = &tbs[m];
1313
        v = (unsigned long)tb->tc_ptr;
1314
        if (v == tc_ptr)
1315
            return tb;
1316
        else if (tc_ptr < v) {
1317
            m_max = m - 1;
1318
        } else {
1319
            m_min = m + 1;
1320
        }
1321
    }
1322
    return &tbs[m_max];
1323
}
1324

    
1325
static void tb_reset_jump_recursive(TranslationBlock *tb);
1326

    
1327
static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1328
{
1329
    TranslationBlock *tb1, *tb_next, **ptb;
1330
    unsigned int n1;
1331

    
1332
    tb1 = tb->jmp_next[n];
1333
    if (tb1 != NULL) {
1334
        /* find head of list */
1335
        for(;;) {
1336
            n1 = (long)tb1 & 3;
1337
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
1338
            if (n1 == 2)
1339
                break;
1340
            tb1 = tb1->jmp_next[n1];
1341
        }
1342
        /* we are now sure now that tb jumps to tb1 */
1343
        tb_next = tb1;
1344

    
1345
        /* remove tb from the jmp_first list */
1346
        ptb = &tb_next->jmp_first;
1347
        for(;;) {
1348
            tb1 = *ptb;
1349
            n1 = (long)tb1 & 3;
1350
            tb1 = (TranslationBlock *)((long)tb1 & ~3);
1351
            if (n1 == n && tb1 == tb)
1352
                break;
1353
            ptb = &tb1->jmp_next[n1];
1354
        }
1355
        *ptb = tb->jmp_next[n];
1356
        tb->jmp_next[n] = NULL;
1357

    
1358
        /* suppress the jump to next tb in generated code */
1359
        tb_reset_jump(tb, n);
1360

    
1361
        /* suppress jumps in the tb on which we could have jumped */
1362
        tb_reset_jump_recursive(tb_next);
1363
    }
1364
}
1365

    
1366
static void tb_reset_jump_recursive(TranslationBlock *tb)
1367
{
1368
    tb_reset_jump_recursive2(tb, 0);
1369
    tb_reset_jump_recursive2(tb, 1);
1370
}
1371

    
1372
#if defined(TARGET_HAS_ICE)
1373
#if defined(CONFIG_USER_ONLY)
1374
static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1375
{
1376
    tb_invalidate_phys_page_range(pc, pc + 1, 0);
1377
}
1378
#else
1379
static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1380
{
1381
    target_phys_addr_t addr;
1382
    target_ulong pd;
1383
    ram_addr_t ram_addr;
1384
    PhysPageDesc *p;
1385

    
1386
    addr = cpu_get_phys_page_debug(env, pc);
1387
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
1388
    if (!p) {
1389
        pd = IO_MEM_UNASSIGNED;
1390
    } else {
1391
        pd = p->phys_offset;
1392
    }
1393
    ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1394
    tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1395
}
1396
#endif
1397
#endif /* TARGET_HAS_ICE */
1398

    
1399
#if defined(CONFIG_USER_ONLY)
1400
void cpu_watchpoint_remove_all(CPUState *env, int mask)
1401

    
1402
{
1403
}
1404

    
1405
int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1406
                          int flags, CPUWatchpoint **watchpoint)
1407
{
1408
    return -ENOSYS;
1409
}
1410
#else
1411
/* Add a watchpoint.  */
1412
int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1413
                          int flags, CPUWatchpoint **watchpoint)
1414
{
1415
    target_ulong len_mask = ~(len - 1);
1416
    CPUWatchpoint *wp;
1417

    
1418
    /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1419
    if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1420
        fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1421
                TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1422
        return -EINVAL;
1423
    }
1424
    wp = qemu_malloc(sizeof(*wp));
1425

    
1426
    wp->vaddr = addr;
1427
    wp->len_mask = len_mask;
1428
    wp->flags = flags;
1429

    
1430
    /* keep all GDB-injected watchpoints in front */
1431
    if (flags & BP_GDB)
1432
        QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1433
    else
1434
        QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1435

    
1436
    tlb_flush_page(env, addr);
1437

    
1438
    if (watchpoint)
1439
        *watchpoint = wp;
1440
    return 0;
1441
}
1442

    
1443
/* Remove a specific watchpoint.  */
1444
int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1445
                          int flags)
1446
{
1447
    target_ulong len_mask = ~(len - 1);
1448
    CPUWatchpoint *wp;
1449

    
1450
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1451
        if (addr == wp->vaddr && len_mask == wp->len_mask
1452
                && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1453
            cpu_watchpoint_remove_by_ref(env, wp);
1454
            return 0;
1455
        }
1456
    }
1457
    return -ENOENT;
1458
}
1459

    
1460
/* Remove a specific watchpoint by reference.  */
1461
void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1462
{
1463
    QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1464

    
1465
    tlb_flush_page(env, watchpoint->vaddr);
1466

    
1467
    qemu_free(watchpoint);
1468
}
1469

    
1470
/* Remove all matching watchpoints.  */
1471
void cpu_watchpoint_remove_all(CPUState *env, int mask)
1472
{
1473
    CPUWatchpoint *wp, *next;
1474

    
1475
    QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1476
        if (wp->flags & mask)
1477
            cpu_watchpoint_remove_by_ref(env, wp);
1478
    }
1479
}
1480
#endif
1481

    
1482
/* Add a breakpoint.  */
1483
int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1484
                          CPUBreakpoint **breakpoint)
1485
{
1486
#if defined(TARGET_HAS_ICE)
1487
    CPUBreakpoint *bp;
1488

    
1489
    bp = qemu_malloc(sizeof(*bp));
1490

    
1491
    bp->pc = pc;
1492
    bp->flags = flags;
1493

    
1494
    /* keep all GDB-injected breakpoints in front */
1495
    if (flags & BP_GDB)
1496
        QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1497
    else
1498
        QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1499

    
1500
    breakpoint_invalidate(env, pc);
1501

    
1502
    if (breakpoint)
1503
        *breakpoint = bp;
1504
    return 0;
1505
#else
1506
    return -ENOSYS;
1507
#endif
1508
}
1509

    
1510
/* Remove a specific breakpoint.  */
1511
int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1512
{
1513
#if defined(TARGET_HAS_ICE)
1514
    CPUBreakpoint *bp;
1515

    
1516
    QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1517
        if (bp->pc == pc && bp->flags == flags) {
1518
            cpu_breakpoint_remove_by_ref(env, bp);
1519
            return 0;
1520
        }
1521
    }
1522
    return -ENOENT;
1523
#else
1524
    return -ENOSYS;
1525
#endif
1526
}
1527

    
1528
/* Remove a specific breakpoint by reference.  */
1529
void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1530
{
1531
#if defined(TARGET_HAS_ICE)
1532
    QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1533

    
1534
    breakpoint_invalidate(env, breakpoint->pc);
1535

    
1536
    qemu_free(breakpoint);
1537
#endif
1538
}
1539

    
1540
/* Remove all matching breakpoints. */
1541
void cpu_breakpoint_remove_all(CPUState *env, int mask)
1542
{
1543
#if defined(TARGET_HAS_ICE)
1544
    CPUBreakpoint *bp, *next;
1545

    
1546
    QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1547
        if (bp->flags & mask)
1548
            cpu_breakpoint_remove_by_ref(env, bp);
1549
    }
1550
#endif
1551
}
1552

    
1553
/* enable or disable single step mode. EXCP_DEBUG is returned by the
1554
   CPU loop after each instruction */
1555
void cpu_single_step(CPUState *env, int enabled)
1556
{
1557
#if defined(TARGET_HAS_ICE)
1558
    if (env->singlestep_enabled != enabled) {
1559
        env->singlestep_enabled = enabled;
1560
        if (kvm_enabled())
1561
            kvm_update_guest_debug(env, 0);
1562
        else {
1563
            /* must flush all the translated code to avoid inconsistencies */
1564
            /* XXX: only flush what is necessary */
1565
            tb_flush(env);
1566
        }
1567
    }
1568
#endif
1569
}
1570

    
1571
/* enable or disable low levels log */
1572
void cpu_set_log(int log_flags)
1573
{
1574
    loglevel = log_flags;
1575
    if (loglevel && !logfile) {
1576
        logfile = fopen(logfilename, log_append ? "a" : "w");
1577
        if (!logfile) {
1578
            perror(logfilename);
1579
            _exit(1);
1580
        }
1581
#if !defined(CONFIG_SOFTMMU)
1582
        /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1583
        {
1584
            static char logfile_buf[4096];
1585
            setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1586
        }
1587
#elif !defined(_WIN32)
1588
        /* Win32 doesn't support line-buffering and requires size >= 2 */
1589
        setvbuf(logfile, NULL, _IOLBF, 0);
1590
#endif
1591
        log_append = 1;
1592
    }
1593
    if (!loglevel && logfile) {
1594
        fclose(logfile);
1595
        logfile = NULL;
1596
    }
1597
}
1598

    
1599
void cpu_set_log_filename(const char *filename)
1600
{
1601
    logfilename = strdup(filename);
1602
    if (logfile) {
1603
        fclose(logfile);
1604
        logfile = NULL;
1605
    }
1606
    cpu_set_log(loglevel);
1607
}
1608

    
1609
static void cpu_unlink_tb(CPUState *env)
1610
{
1611
    /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
1612
       problem and hope the cpu will stop of its own accord.  For userspace
1613
       emulation this often isn't actually as bad as it sounds.  Often
1614
       signals are used primarily to interrupt blocking syscalls.  */
1615
    TranslationBlock *tb;
1616
    static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1617

    
1618
    spin_lock(&interrupt_lock);
1619
    tb = env->current_tb;
1620
    /* if the cpu is currently executing code, we must unlink it and
1621
       all the potentially executing TB */
1622
    if (tb) {
1623
        env->current_tb = NULL;
1624
        tb_reset_jump_recursive(tb);
1625
    }
1626
    spin_unlock(&interrupt_lock);
1627
}
1628

    
1629
/* mask must never be zero, except for A20 change call */
1630
void cpu_interrupt(CPUState *env, int mask)
1631
{
1632
    int old_mask;
1633

    
1634
    old_mask = env->interrupt_request;
1635
    env->interrupt_request |= mask;
1636

    
1637
#ifndef CONFIG_USER_ONLY
1638
    /*
1639
     * If called from iothread context, wake the target cpu in
1640
     * case its halted.
1641
     */
1642
    if (!qemu_cpu_self(env)) {
1643
        qemu_cpu_kick(env);
1644
        return;
1645
    }
1646
#endif
1647

    
1648
    if (use_icount) {
1649
        env->icount_decr.u16.high = 0xffff;
1650
#ifndef CONFIG_USER_ONLY
1651
        if (!can_do_io(env)
1652
            && (mask & ~old_mask) != 0) {
1653
            cpu_abort(env, "Raised interrupt while not in I/O function");
1654
        }
1655
#endif
1656
    } else {
1657
        cpu_unlink_tb(env);
1658
    }
1659
}
1660

    
1661
void cpu_reset_interrupt(CPUState *env, int mask)
1662
{
1663
    env->interrupt_request &= ~mask;
1664
}
1665

    
1666
void cpu_exit(CPUState *env)
1667
{
1668
    env->exit_request = 1;
1669
    cpu_unlink_tb(env);
1670
}
1671

    
1672
const CPULogItem cpu_log_items[] = {
1673
    { CPU_LOG_TB_OUT_ASM, "out_asm",
1674
      "show generated host assembly code for each compiled TB" },
1675
    { CPU_LOG_TB_IN_ASM, "in_asm",
1676
      "show target assembly code for each compiled TB" },
1677
    { CPU_LOG_TB_OP, "op",
1678
      "show micro ops for each compiled TB" },
1679
    { CPU_LOG_TB_OP_OPT, "op_opt",
1680
      "show micro ops "
1681
#ifdef TARGET_I386
1682
      "before eflags optimization and "
1683
#endif
1684
      "after liveness analysis" },
1685
    { CPU_LOG_INT, "int",
1686
      "show interrupts/exceptions in short format" },
1687
    { CPU_LOG_EXEC, "exec",
1688
      "show trace before each executed TB (lots of logs)" },
1689
    { CPU_LOG_TB_CPU, "cpu",
1690
      "show CPU state before block translation" },
1691
#ifdef TARGET_I386
1692
    { CPU_LOG_PCALL, "pcall",
1693
      "show protected mode far calls/returns/exceptions" },
1694
    { CPU_LOG_RESET, "cpu_reset",
1695
      "show CPU state before CPU resets" },
1696
#endif
1697
#ifdef DEBUG_IOPORT
1698
    { CPU_LOG_IOPORT, "ioport",
1699
      "show all i/o ports accesses" },
1700
#endif
1701
    { 0, NULL, NULL },
1702
};
1703

    
1704
#ifndef CONFIG_USER_ONLY
1705
static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
1706
    = QLIST_HEAD_INITIALIZER(memory_client_list);
1707

    
1708
static void cpu_notify_set_memory(target_phys_addr_t start_addr,
1709
                                  ram_addr_t size,
1710
                                  ram_addr_t phys_offset)
1711
{
1712
    CPUPhysMemoryClient *client;
1713
    QLIST_FOREACH(client, &memory_client_list, list) {
1714
        client->set_memory(client, start_addr, size, phys_offset);
1715
    }
1716
}
1717

    
1718
static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
1719
                                        target_phys_addr_t end)
1720
{
1721
    CPUPhysMemoryClient *client;
1722
    QLIST_FOREACH(client, &memory_client_list, list) {
1723
        int r = client->sync_dirty_bitmap(client, start, end);
1724
        if (r < 0)
1725
            return r;
1726
    }
1727
    return 0;
1728
}
1729

    
1730
static int cpu_notify_migration_log(int enable)
1731
{
1732
    CPUPhysMemoryClient *client;
1733
    QLIST_FOREACH(client, &memory_client_list, list) {
1734
        int r = client->migration_log(client, enable);
1735
        if (r < 0)
1736
            return r;
1737
    }
1738
    return 0;
1739
}
1740

    
1741
static void phys_page_for_each_1(CPUPhysMemoryClient *client,
1742
                                 int level, void **lp)
1743
{
1744
    int i;
1745

    
1746
    if (*lp == NULL) {
1747
        return;
1748
    }
1749
    if (level == 0) {
1750
        PhysPageDesc *pd = *lp;
1751
        for (i = 0; i < L2_SIZE; ++i) {
1752
            if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
1753
                client->set_memory(client, pd[i].region_offset,
1754
                                   TARGET_PAGE_SIZE, pd[i].phys_offset);
1755
            }
1756
        }
1757
    } else {
1758
        void **pp = *lp;
1759
        for (i = 0; i < L2_SIZE; ++i) {
1760
            phys_page_for_each_1(client, level - 1, pp + i);
1761
        }
1762
    }
1763
}
1764

    
1765
static void phys_page_for_each(CPUPhysMemoryClient *client)
1766
{
1767
    int i;
1768
    for (i = 0; i < P_L1_SIZE; ++i) {
1769
        phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
1770
                             l1_phys_map + 1);
1771
    }
1772
}
1773

    
1774
void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
1775
{
1776
    QLIST_INSERT_HEAD(&memory_client_list, client, list);
1777
    phys_page_for_each(client);
1778
}
1779

    
1780
void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
1781
{
1782
    QLIST_REMOVE(client, list);
1783
}
1784
#endif
1785

    
1786
static int cmp1(const char *s1, int n, const char *s2)
1787
{
1788
    if (strlen(s2) != n)
1789
        return 0;
1790
    return memcmp(s1, s2, n) == 0;
1791
}
1792

    
1793
/* takes a comma separated list of log masks. Return 0 if error. */
1794
int cpu_str_to_log_mask(const char *str)
1795
{
1796
    const CPULogItem *item;
1797
    int mask;
1798
    const char *p, *p1;
1799

    
1800
    p = str;
1801
    mask = 0;
1802
    for(;;) {
1803
        p1 = strchr(p, ',');
1804
        if (!p1)
1805
            p1 = p + strlen(p);
1806
        if(cmp1(p,p1-p,"all")) {
1807
            for(item = cpu_log_items; item->mask != 0; item++) {
1808
                mask |= item->mask;
1809
            }
1810
        } else {
1811
            for(item = cpu_log_items; item->mask != 0; item++) {
1812
                if (cmp1(p, p1 - p, item->name))
1813
                    goto found;
1814
            }
1815
            return 0;
1816
        }
1817
    found:
1818
        mask |= item->mask;
1819
        if (*p1 != ',')
1820
            break;
1821
        p = p1 + 1;
1822
    }
1823
    return mask;
1824
}
1825

    
1826
void cpu_abort(CPUState *env, const char *fmt, ...)
1827
{
1828
    va_list ap;
1829
    va_list ap2;
1830

    
1831
    va_start(ap, fmt);
1832
    va_copy(ap2, ap);
1833
    fprintf(stderr, "qemu: fatal: ");
1834
    vfprintf(stderr, fmt, ap);
1835
    fprintf(stderr, "\n");
1836
#ifdef TARGET_I386
1837
    cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1838
#else
1839
    cpu_dump_state(env, stderr, fprintf, 0);
1840
#endif
1841
    if (qemu_log_enabled()) {
1842
        qemu_log("qemu: fatal: ");
1843
        qemu_log_vprintf(fmt, ap2);
1844
        qemu_log("\n");
1845
#ifdef TARGET_I386
1846
        log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1847
#else
1848
        log_cpu_state(env, 0);
1849
#endif
1850
        qemu_log_flush();
1851
        qemu_log_close();
1852
    }
1853
    va_end(ap2);
1854
    va_end(ap);
1855
#if defined(CONFIG_USER_ONLY)
1856
    {
1857
        struct sigaction act;
1858
        sigfillset(&act.sa_mask);
1859
        act.sa_handler = SIG_DFL;
1860
        sigaction(SIGABRT, &act, NULL);
1861
    }
1862
#endif
1863
    abort();
1864
}
1865

    
1866
CPUState *cpu_copy(CPUState *env)
1867
{
1868
    CPUState *new_env = cpu_init(env->cpu_model_str);
1869
    CPUState *next_cpu = new_env->next_cpu;
1870
    int cpu_index = new_env->cpu_index;
1871
#if defined(TARGET_HAS_ICE)
1872
    CPUBreakpoint *bp;
1873
    CPUWatchpoint *wp;
1874
#endif
1875

    
1876
    memcpy(new_env, env, sizeof(CPUState));
1877

    
1878
    /* Preserve chaining and index. */
1879
    new_env->next_cpu = next_cpu;
1880
    new_env->cpu_index = cpu_index;
1881

    
1882
    /* Clone all break/watchpoints.
1883
       Note: Once we support ptrace with hw-debug register access, make sure
1884
       BP_CPU break/watchpoints are handled correctly on clone. */
1885
    QTAILQ_INIT(&env->breakpoints);
1886
    QTAILQ_INIT(&env->watchpoints);
1887
#if defined(TARGET_HAS_ICE)
1888
    QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1889
        cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1890
    }
1891
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1892
        cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1893
                              wp->flags, NULL);
1894
    }
1895
#endif
1896

    
1897
    return new_env;
1898
}
1899

    
1900
#if !defined(CONFIG_USER_ONLY)
1901

    
1902
static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1903
{
1904
    unsigned int i;
1905

    
1906
    /* Discard jump cache entries for any tb which might potentially
1907
       overlap the flushed page.  */
1908
    i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1909
    memset (&env->tb_jmp_cache[i], 0, 
1910
            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1911

    
1912
    i = tb_jmp_cache_hash_page(addr);
1913
    memset (&env->tb_jmp_cache[i], 0, 
1914
            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1915
}
1916

    
1917
static CPUTLBEntry s_cputlb_empty_entry = {
1918
    .addr_read  = -1,
1919
    .addr_write = -1,
1920
    .addr_code  = -1,
1921
    .addend     = -1,
1922
};
1923

    
1924
/* NOTE: if flush_global is true, also flush global entries (not
1925
   implemented yet) */
1926
void tlb_flush(CPUState *env, int flush_global)
1927
{
1928
    int i;
1929

    
1930
#if defined(DEBUG_TLB)
1931
    printf("tlb_flush:\n");
1932
#endif
1933
    /* must reset current TB so that interrupts cannot modify the
1934
       links while we are modifying them */
1935
    env->current_tb = NULL;
1936

    
1937
    for(i = 0; i < CPU_TLB_SIZE; i++) {
1938
        int mmu_idx;
1939
        for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1940
            env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1941
        }
1942
    }
1943

    
1944
    memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1945

    
1946
    env->tlb_flush_addr = -1;
1947
    env->tlb_flush_mask = 0;
1948
    tlb_flush_count++;
1949
}
1950

    
1951
static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1952
{
1953
    if (addr == (tlb_entry->addr_read &
1954
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1955
        addr == (tlb_entry->addr_write &
1956
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1957
        addr == (tlb_entry->addr_code &
1958
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1959
        *tlb_entry = s_cputlb_empty_entry;
1960
    }
1961
}
1962

    
1963
void tlb_flush_page(CPUState *env, target_ulong addr)
1964
{
1965
    int i;
1966
    int mmu_idx;
1967

    
1968
#if defined(DEBUG_TLB)
1969
    printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1970
#endif
1971
    /* Check if we need to flush due to large pages.  */
1972
    if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
1973
#if defined(DEBUG_TLB)
1974
        printf("tlb_flush_page: forced full flush ("
1975
               TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
1976
               env->tlb_flush_addr, env->tlb_flush_mask);
1977
#endif
1978
        tlb_flush(env, 1);
1979
        return;
1980
    }
1981
    /* must reset current TB so that interrupts cannot modify the
1982
       links while we are modifying them */
1983
    env->current_tb = NULL;
1984

    
1985
    addr &= TARGET_PAGE_MASK;
1986
    i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1987
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
1988
        tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
1989

    
1990
    tlb_flush_jmp_cache(env, addr);
1991
}
1992

    
1993
/* update the TLBs so that writes to code in the virtual page 'addr'
1994
   can be detected */
1995
static void tlb_protect_code(ram_addr_t ram_addr)
1996
{
1997
    cpu_physical_memory_reset_dirty(ram_addr,
1998
                                    ram_addr + TARGET_PAGE_SIZE,
1999
                                    CODE_DIRTY_FLAG);
2000
}
2001

    
2002
/* update the TLB so that writes in physical page 'phys_addr' are no longer
2003
   tested for self modifying code */
2004
static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
2005
                                    target_ulong vaddr)
2006
{
2007
    cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
2008
}
2009

    
2010
static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
2011
                                         unsigned long start, unsigned long length)
2012
{
2013
    unsigned long addr;
2014
    if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2015
        addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2016
        if ((addr - start) < length) {
2017
            tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
2018
        }
2019
    }
2020
}
2021

    
2022
/* Note: start and end must be within the same ram block.  */
2023
void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
2024
                                     int dirty_flags)
2025
{
2026
    CPUState *env;
2027
    unsigned long length, start1;
2028
    int i;
2029

    
2030
    start &= TARGET_PAGE_MASK;
2031
    end = TARGET_PAGE_ALIGN(end);
2032

    
2033
    length = end - start;
2034
    if (length == 0)
2035
        return;
2036
    cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
2037

    
2038
    /* we modify the TLB cache so that the dirty bit will be set again
2039
       when accessing the range */
2040
    start1 = (unsigned long)qemu_get_ram_ptr(start);
2041
    /* Chek that we don't span multiple blocks - this breaks the
2042
       address comparisons below.  */
2043
    if ((unsigned long)qemu_get_ram_ptr(end - 1) - start1
2044
            != (end - 1) - start) {
2045
        abort();
2046
    }
2047

    
2048
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
2049
        int mmu_idx;
2050
        for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2051
            for(i = 0; i < CPU_TLB_SIZE; i++)
2052
                tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
2053
                                      start1, length);
2054
        }
2055
    }
2056
}
2057

    
2058
int cpu_physical_memory_set_dirty_tracking(int enable)
2059
{
2060
    int ret = 0;
2061
    in_migration = enable;
2062
    ret = cpu_notify_migration_log(!!enable);
2063
    return ret;
2064
}
2065

    
2066
int cpu_physical_memory_get_dirty_tracking(void)
2067
{
2068
    return in_migration;
2069
}
2070

    
2071
int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2072
                                   target_phys_addr_t end_addr)
2073
{
2074
    int ret;
2075

    
2076
    ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
2077
    return ret;
2078
}
2079

    
2080
static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2081
{
2082
    ram_addr_t ram_addr;
2083
    void *p;
2084

    
2085
    if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2086
        p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2087
            + tlb_entry->addend);
2088
        ram_addr = qemu_ram_addr_from_host(p);
2089
        if (!cpu_physical_memory_is_dirty(ram_addr)) {
2090
            tlb_entry->addr_write |= TLB_NOTDIRTY;
2091
        }
2092
    }
2093
}
2094

    
2095
/* update the TLB according to the current state of the dirty bits */
2096
void cpu_tlb_update_dirty(CPUState *env)
2097
{
2098
    int i;
2099
    int mmu_idx;
2100
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2101
        for(i = 0; i < CPU_TLB_SIZE; i++)
2102
            tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2103
    }
2104
}
2105

    
2106
static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2107
{
2108
    if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
2109
        tlb_entry->addr_write = vaddr;
2110
}
2111

    
2112
/* update the TLB corresponding to virtual page vaddr
2113
   so that it is no longer dirty */
2114
static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2115
{
2116
    int i;
2117
    int mmu_idx;
2118

    
2119
    vaddr &= TARGET_PAGE_MASK;
2120
    i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2121
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2122
        tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2123
}
2124

    
2125
/* Our TLB does not support large pages, so remember the area covered by
2126
   large pages and trigger a full TLB flush if these are invalidated.  */
2127
static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
2128
                               target_ulong size)
2129
{
2130
    target_ulong mask = ~(size - 1);
2131

    
2132
    if (env->tlb_flush_addr == (target_ulong)-1) {
2133
        env->tlb_flush_addr = vaddr & mask;
2134
        env->tlb_flush_mask = mask;
2135
        return;
2136
    }
2137
    /* Extend the existing region to include the new page.
2138
       This is a compromise between unnecessary flushes and the cost
2139
       of maintaining a full variable size TLB.  */
2140
    mask &= env->tlb_flush_mask;
2141
    while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
2142
        mask <<= 1;
2143
    }
2144
    env->tlb_flush_addr &= mask;
2145
    env->tlb_flush_mask = mask;
2146
}
2147

    
2148
/* Add a new TLB entry. At most one entry for a given virtual address
2149
   is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2150
   supplied size is only used by tlb_flush_page.  */
2151
void tlb_set_page(CPUState *env, target_ulong vaddr,
2152
                  target_phys_addr_t paddr, int prot,
2153
                  int mmu_idx, target_ulong size)
2154
{
2155
    PhysPageDesc *p;
2156
    unsigned long pd;
2157
    unsigned int index;
2158
    target_ulong address;
2159
    target_ulong code_address;
2160
    unsigned long addend;
2161
    CPUTLBEntry *te;
2162
    CPUWatchpoint *wp;
2163
    target_phys_addr_t iotlb;
2164

    
2165
    assert(size >= TARGET_PAGE_SIZE);
2166
    if (size != TARGET_PAGE_SIZE) {
2167
        tlb_add_large_page(env, vaddr, size);
2168
    }
2169
    p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2170
    if (!p) {
2171
        pd = IO_MEM_UNASSIGNED;
2172
    } else {
2173
        pd = p->phys_offset;
2174
    }
2175
#if defined(DEBUG_TLB)
2176
    printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
2177
           " prot=%x idx=%d pd=0x%08lx\n",
2178
           vaddr, paddr, prot, mmu_idx, pd);
2179
#endif
2180

    
2181
    address = vaddr;
2182
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2183
        /* IO memory case (romd handled later) */
2184
        address |= TLB_MMIO;
2185
    }
2186
    addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2187
    if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2188
        /* Normal RAM.  */
2189
        iotlb = pd & TARGET_PAGE_MASK;
2190
        if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2191
            iotlb |= IO_MEM_NOTDIRTY;
2192
        else
2193
            iotlb |= IO_MEM_ROM;
2194
    } else {
2195
        /* IO handlers are currently passed a physical address.
2196
           It would be nice to pass an offset from the base address
2197
           of that region.  This would avoid having to special case RAM,
2198
           and avoid full address decoding in every device.
2199
           We can't use the high bits of pd for this because
2200
           IO_MEM_ROMD uses these as a ram address.  */
2201
        iotlb = (pd & ~TARGET_PAGE_MASK);
2202
        if (p) {
2203
            iotlb += p->region_offset;
2204
        } else {
2205
            iotlb += paddr;
2206
        }
2207
    }
2208

    
2209
    code_address = address;
2210
    /* Make accesses to pages with watchpoints go via the
2211
       watchpoint trap routines.  */
2212
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2213
        if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2214
            /* Avoid trapping reads of pages with a write breakpoint. */
2215
            if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
2216
                iotlb = io_mem_watch + paddr;
2217
                address |= TLB_MMIO;
2218
                break;
2219
            }
2220
        }
2221
    }
2222

    
2223
    index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2224
    env->iotlb[mmu_idx][index] = iotlb - vaddr;
2225
    te = &env->tlb_table[mmu_idx][index];
2226
    te->addend = addend - vaddr;
2227
    if (prot & PAGE_READ) {
2228
        te->addr_read = address;
2229
    } else {
2230
        te->addr_read = -1;
2231
    }
2232

    
2233
    if (prot & PAGE_EXEC) {
2234
        te->addr_code = code_address;
2235
    } else {
2236
        te->addr_code = -1;
2237
    }
2238
    if (prot & PAGE_WRITE) {
2239
        if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2240
            (pd & IO_MEM_ROMD)) {
2241
            /* Write access calls the I/O callback.  */
2242
            te->addr_write = address | TLB_MMIO;
2243
        } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2244
                   !cpu_physical_memory_is_dirty(pd)) {
2245
            te->addr_write = address | TLB_NOTDIRTY;
2246
        } else {
2247
            te->addr_write = address;
2248
        }
2249
    } else {
2250
        te->addr_write = -1;
2251
    }
2252
}
2253

    
2254
#else
2255

    
2256
void tlb_flush(CPUState *env, int flush_global)
2257
{
2258
}
2259

    
2260
void tlb_flush_page(CPUState *env, target_ulong addr)
2261
{
2262
}
2263

    
2264
/*
2265
 * Walks guest process memory "regions" one by one
2266
 * and calls callback function 'fn' for each region.
2267
 */
2268

    
2269
struct walk_memory_regions_data
2270
{
2271
    walk_memory_regions_fn fn;
2272
    void *priv;
2273
    unsigned long start;
2274
    int prot;
2275
};
2276

    
2277
static int walk_memory_regions_end(struct walk_memory_regions_data *data,
2278
                                   abi_ulong end, int new_prot)
2279
{
2280
    if (data->start != -1ul) {
2281
        int rc = data->fn(data->priv, data->start, end, data->prot);
2282
        if (rc != 0) {
2283
            return rc;
2284
        }
2285
    }
2286

    
2287
    data->start = (new_prot ? end : -1ul);
2288
    data->prot = new_prot;
2289

    
2290
    return 0;
2291
}
2292

    
2293
static int walk_memory_regions_1(struct walk_memory_regions_data *data,
2294
                                 abi_ulong base, int level, void **lp)
2295
{
2296
    abi_ulong pa;
2297
    int i, rc;
2298

    
2299
    if (*lp == NULL) {
2300
        return walk_memory_regions_end(data, base, 0);
2301
    }
2302

    
2303
    if (level == 0) {
2304
        PageDesc *pd = *lp;
2305
        for (i = 0; i < L2_SIZE; ++i) {
2306
            int prot = pd[i].flags;
2307

    
2308
            pa = base | (i << TARGET_PAGE_BITS);
2309
            if (prot != data->prot) {
2310
                rc = walk_memory_regions_end(data, pa, prot);
2311
                if (rc != 0) {
2312
                    return rc;
2313
                }
2314
            }
2315
        }
2316
    } else {
2317
        void **pp = *lp;
2318
        for (i = 0; i < L2_SIZE; ++i) {
2319
            pa = base | ((abi_ulong)i <<
2320
                (TARGET_PAGE_BITS + L2_BITS * level));
2321
            rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2322
            if (rc != 0) {
2323
                return rc;
2324
            }
2325
        }
2326
    }
2327

    
2328
    return 0;
2329
}
2330

    
2331
int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2332
{
2333
    struct walk_memory_regions_data data;
2334
    unsigned long i;
2335

    
2336
    data.fn = fn;
2337
    data.priv = priv;
2338
    data.start = -1ul;
2339
    data.prot = 0;
2340

    
2341
    for (i = 0; i < V_L1_SIZE; i++) {
2342
        int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
2343
                                       V_L1_SHIFT / L2_BITS - 1, l1_map + i);
2344
        if (rc != 0) {
2345
            return rc;
2346
        }
2347
    }
2348

    
2349
    return walk_memory_regions_end(&data, 0, 0);
2350
}
2351

    
2352
static int dump_region(void *priv, abi_ulong start,
2353
    abi_ulong end, unsigned long prot)
2354
{
2355
    FILE *f = (FILE *)priv;
2356

    
2357
    (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
2358
        " "TARGET_ABI_FMT_lx" %c%c%c\n",
2359
        start, end, end - start,
2360
        ((prot & PAGE_READ) ? 'r' : '-'),
2361
        ((prot & PAGE_WRITE) ? 'w' : '-'),
2362
        ((prot & PAGE_EXEC) ? 'x' : '-'));
2363

    
2364
    return (0);
2365
}
2366

    
2367
/* dump memory mappings */
2368
void page_dump(FILE *f)
2369
{
2370
    (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2371
            "start", "end", "size", "prot");
2372
    walk_memory_regions(f, dump_region);
2373
}
2374

    
2375
int page_get_flags(target_ulong address)
2376
{
2377
    PageDesc *p;
2378

    
2379
    p = page_find(address >> TARGET_PAGE_BITS);
2380
    if (!p)
2381
        return 0;
2382
    return p->flags;
2383
}
2384

    
2385
/* Modify the flags of a page and invalidate the code if necessary.
2386
   The flag PAGE_WRITE_ORG is positioned automatically depending
2387
   on PAGE_WRITE.  The mmap_lock should already be held.  */
2388
void page_set_flags(target_ulong start, target_ulong end, int flags)
2389
{
2390
    target_ulong addr, len;
2391

    
2392
    /* This function should never be called with addresses outside the
2393
       guest address space.  If this assert fires, it probably indicates
2394
       a missing call to h2g_valid.  */
2395
#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2396
    assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2397
#endif
2398
    assert(start < end);
2399

    
2400
    start = start & TARGET_PAGE_MASK;
2401
    end = TARGET_PAGE_ALIGN(end);
2402

    
2403
    if (flags & PAGE_WRITE) {
2404
        flags |= PAGE_WRITE_ORG;
2405
    }
2406

    
2407
    for (addr = start, len = end - start;
2408
         len != 0;
2409
         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2410
        PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2411

    
2412
        /* If the write protection bit is set, then we invalidate
2413
           the code inside.  */
2414
        if (!(p->flags & PAGE_WRITE) &&
2415
            (flags & PAGE_WRITE) &&
2416
            p->first_tb) {
2417
            tb_invalidate_phys_page(addr, 0, NULL);
2418
        }
2419
        p->flags = flags;
2420
    }
2421
}
2422

    
2423
int page_check_range(target_ulong start, target_ulong len, int flags)
2424
{
2425
    PageDesc *p;
2426
    target_ulong end;
2427
    target_ulong addr;
2428

    
2429
    /* This function should never be called with addresses outside the
2430
       guest address space.  If this assert fires, it probably indicates
2431
       a missing call to h2g_valid.  */
2432
#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2433
    assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2434
#endif
2435

    
2436
    if (len == 0) {
2437
        return 0;
2438
    }
2439
    if (start + len - 1 < start) {
2440
        /* We've wrapped around.  */
2441
        return -1;
2442
    }
2443

    
2444
    end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2445
    start = start & TARGET_PAGE_MASK;
2446

    
2447
    for (addr = start, len = end - start;
2448
         len != 0;
2449
         len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2450
        p = page_find(addr >> TARGET_PAGE_BITS);
2451
        if( !p )
2452
            return -1;
2453
        if( !(p->flags & PAGE_VALID) )
2454
            return -1;
2455

    
2456
        if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2457
            return -1;
2458
        if (flags & PAGE_WRITE) {
2459
            if (!(p->flags & PAGE_WRITE_ORG))
2460
                return -1;
2461
            /* unprotect the page if it was put read-only because it
2462
               contains translated code */
2463
            if (!(p->flags & PAGE_WRITE)) {
2464
                if (!page_unprotect(addr, 0, NULL))
2465
                    return -1;
2466
            }
2467
            return 0;
2468
        }
2469
    }
2470
    return 0;
2471
}
2472

    
2473
/* called from signal handler: invalidate the code and unprotect the
2474
   page. Return TRUE if the fault was successfully handled. */
2475
int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2476
{
2477
    unsigned int prot;
2478
    PageDesc *p;
2479
    target_ulong host_start, host_end, addr;
2480

    
2481
    /* Technically this isn't safe inside a signal handler.  However we
2482
       know this only ever happens in a synchronous SEGV handler, so in
2483
       practice it seems to be ok.  */
2484
    mmap_lock();
2485

    
2486
    p = page_find(address >> TARGET_PAGE_BITS);
2487
    if (!p) {
2488
        mmap_unlock();
2489
        return 0;
2490
    }
2491

    
2492
    /* if the page was really writable, then we change its
2493
       protection back to writable */
2494
    if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
2495
        host_start = address & qemu_host_page_mask;
2496
        host_end = host_start + qemu_host_page_size;
2497

    
2498
        prot = 0;
2499
        for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
2500
            p = page_find(addr >> TARGET_PAGE_BITS);
2501
            p->flags |= PAGE_WRITE;
2502
            prot |= p->flags;
2503

    
2504
            /* and since the content will be modified, we must invalidate
2505
               the corresponding translated code. */
2506
            tb_invalidate_phys_page(addr, pc, puc);
2507
#ifdef DEBUG_TB_CHECK
2508
            tb_invalidate_check(addr);
2509
#endif
2510
        }
2511
        mprotect((void *)g2h(host_start), qemu_host_page_size,
2512
                 prot & PAGE_BITS);
2513

    
2514
        mmap_unlock();
2515
        return 1;
2516
    }
2517
    mmap_unlock();
2518
    return 0;
2519
}
2520

    
2521
static inline void tlb_set_dirty(CPUState *env,
2522
                                 unsigned long addr, target_ulong vaddr)
2523
{
2524
}
2525
#endif /* defined(CONFIG_USER_ONLY) */
2526

    
2527
#if !defined(CONFIG_USER_ONLY)
2528

    
2529
#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2530
typedef struct subpage_t {
2531
    target_phys_addr_t base;
2532
    ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
2533
    ram_addr_t region_offset[TARGET_PAGE_SIZE];
2534
} subpage_t;
2535

    
2536
static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2537
                             ram_addr_t memory, ram_addr_t region_offset);
2538
static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2539
                                ram_addr_t orig_memory,
2540
                                ram_addr_t region_offset);
2541
#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2542
                      need_subpage)                                     \
2543
    do {                                                                \
2544
        if (addr > start_addr)                                          \
2545
            start_addr2 = 0;                                            \
2546
        else {                                                          \
2547
            start_addr2 = start_addr & ~TARGET_PAGE_MASK;               \
2548
            if (start_addr2 > 0)                                        \
2549
                need_subpage = 1;                                       \
2550
        }                                                               \
2551
                                                                        \
2552
        if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE)        \
2553
            end_addr2 = TARGET_PAGE_SIZE - 1;                           \
2554
        else {                                                          \
2555
            end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2556
            if (end_addr2 < TARGET_PAGE_SIZE - 1)                       \
2557
                need_subpage = 1;                                       \
2558
        }                                                               \
2559
    } while (0)
2560

    
2561
/* register physical memory.
2562
   For RAM, 'size' must be a multiple of the target page size.
2563
   If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2564
   io memory page.  The address used when calling the IO function is
2565
   the offset from the start of the region, plus region_offset.  Both
2566
   start_addr and region_offset are rounded down to a page boundary
2567
   before calculating this offset.  This should not be a problem unless
2568
   the low bits of start_addr and region_offset differ.  */
2569
void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2570
                                         ram_addr_t size,
2571
                                         ram_addr_t phys_offset,
2572
                                         ram_addr_t region_offset)
2573
{
2574
    target_phys_addr_t addr, end_addr;
2575
    PhysPageDesc *p;
2576
    CPUState *env;
2577
    ram_addr_t orig_size = size;
2578
    subpage_t *subpage;
2579

    
2580
    cpu_notify_set_memory(start_addr, size, phys_offset);
2581

    
2582
    if (phys_offset == IO_MEM_UNASSIGNED) {
2583
        region_offset = start_addr;
2584
    }
2585
    region_offset &= TARGET_PAGE_MASK;
2586
    size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2587
    end_addr = start_addr + (target_phys_addr_t)size;
2588
    for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2589
        p = phys_page_find(addr >> TARGET_PAGE_BITS);
2590
        if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2591
            ram_addr_t orig_memory = p->phys_offset;
2592
            target_phys_addr_t start_addr2, end_addr2;
2593
            int need_subpage = 0;
2594

    
2595
            CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2596
                          need_subpage);
2597
            if (need_subpage) {
2598
                if (!(orig_memory & IO_MEM_SUBPAGE)) {
2599
                    subpage = subpage_init((addr & TARGET_PAGE_MASK),
2600
                                           &p->phys_offset, orig_memory,
2601
                                           p->region_offset);
2602
                } else {
2603
                    subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2604
                                            >> IO_MEM_SHIFT];
2605
                }
2606
                subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2607
                                 region_offset);
2608
                p->region_offset = 0;
2609
            } else {
2610
                p->phys_offset = phys_offset;
2611
                if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2612
                    (phys_offset & IO_MEM_ROMD))
2613
                    phys_offset += TARGET_PAGE_SIZE;
2614
            }
2615
        } else {
2616
            p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2617
            p->phys_offset = phys_offset;
2618
            p->region_offset = region_offset;
2619
            if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2620
                (phys_offset & IO_MEM_ROMD)) {
2621
                phys_offset += TARGET_PAGE_SIZE;
2622
            } else {
2623
                target_phys_addr_t start_addr2, end_addr2;
2624
                int need_subpage = 0;
2625

    
2626
                CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2627
                              end_addr2, need_subpage);
2628

    
2629
                if (need_subpage) {
2630
                    subpage = subpage_init((addr & TARGET_PAGE_MASK),
2631
                                           &p->phys_offset, IO_MEM_UNASSIGNED,
2632
                                           addr & TARGET_PAGE_MASK);
2633
                    subpage_register(subpage, start_addr2, end_addr2,
2634
                                     phys_offset, region_offset);
2635
                    p->region_offset = 0;
2636
                }
2637
            }
2638
        }
2639
        region_offset += TARGET_PAGE_SIZE;
2640
    }
2641

    
2642
    /* since each CPU stores ram addresses in its TLB cache, we must
2643
       reset the modified entries */
2644
    /* XXX: slow ! */
2645
    for(env = first_cpu; env != NULL; env = env->next_cpu) {
2646
        tlb_flush(env, 1);
2647
    }
2648
}
2649

    
2650
/* XXX: temporary until new memory mapping API */
2651
ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2652
{
2653
    PhysPageDesc *p;
2654

    
2655
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
2656
    if (!p)
2657
        return IO_MEM_UNASSIGNED;
2658
    return p->phys_offset;
2659
}
2660

    
2661
void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2662
{
2663
    if (kvm_enabled())
2664
        kvm_coalesce_mmio_region(addr, size);
2665
}
2666

    
2667
void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2668
{
2669
    if (kvm_enabled())
2670
        kvm_uncoalesce_mmio_region(addr, size);
2671
}
2672

    
2673
void qemu_flush_coalesced_mmio_buffer(void)
2674
{
2675
    if (kvm_enabled())
2676
        kvm_flush_coalesced_mmio_buffer();
2677
}
2678

    
2679
#if defined(__linux__) && !defined(TARGET_S390X)
2680

    
2681
#include <sys/vfs.h>
2682

    
2683
#define HUGETLBFS_MAGIC       0x958458f6
2684

    
2685
static long gethugepagesize(const char *path)
2686
{
2687
    struct statfs fs;
2688
    int ret;
2689

    
2690
    do {
2691
        ret = statfs(path, &fs);
2692
    } while (ret != 0 && errno == EINTR);
2693

    
2694
    if (ret != 0) {
2695
        perror(path);
2696
        return 0;
2697
    }
2698

    
2699
    if (fs.f_type != HUGETLBFS_MAGIC)
2700
        fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
2701

    
2702
    return fs.f_bsize;
2703
}
2704

    
2705
static void *file_ram_alloc(RAMBlock *block,
2706
                            ram_addr_t memory,
2707
                            const char *path)
2708
{
2709
    char *filename;
2710
    void *area;
2711
    int fd;
2712
#ifdef MAP_POPULATE
2713
    int flags;
2714
#endif
2715
    unsigned long hpagesize;
2716

    
2717
    hpagesize = gethugepagesize(path);
2718
    if (!hpagesize) {
2719
        return NULL;
2720
    }
2721

    
2722
    if (memory < hpagesize) {
2723
        return NULL;
2724
    }
2725

    
2726
    if (kvm_enabled() && !kvm_has_sync_mmu()) {
2727
        fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2728
        return NULL;
2729
    }
2730

    
2731
    if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
2732
        return NULL;
2733
    }
2734

    
2735
    fd = mkstemp(filename);
2736
    if (fd < 0) {
2737
        perror("unable to create backing store for hugepages");
2738
        free(filename);
2739
        return NULL;
2740
    }
2741
    unlink(filename);
2742
    free(filename);
2743

    
2744
    memory = (memory+hpagesize-1) & ~(hpagesize-1);
2745

    
2746
    /*
2747
     * ftruncate is not supported by hugetlbfs in older
2748
     * hosts, so don't bother bailing out on errors.
2749
     * If anything goes wrong with it under other filesystems,
2750
     * mmap will fail.
2751
     */
2752
    if (ftruncate(fd, memory))
2753
        perror("ftruncate");
2754

    
2755
#ifdef MAP_POPULATE
2756
    /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2757
     * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
2758
     * to sidestep this quirk.
2759
     */
2760
    flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
2761
    area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
2762
#else
2763
    area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
2764
#endif
2765
    if (area == MAP_FAILED) {
2766
        perror("file_ram_alloc: can't mmap RAM pages");
2767
        close(fd);
2768
        return (NULL);
2769
    }
2770
    block->fd = fd;
2771
    return area;
2772
}
2773
#endif
2774

    
2775
static ram_addr_t find_ram_offset(ram_addr_t size)
2776
{
2777
    RAMBlock *block, *next_block;
2778
    ram_addr_t offset = 0, mingap = ULONG_MAX;
2779

    
2780
    if (QLIST_EMPTY(&ram_list.blocks))
2781
        return 0;
2782

    
2783
    QLIST_FOREACH(block, &ram_list.blocks, next) {
2784
        ram_addr_t end, next = ULONG_MAX;
2785

    
2786
        end = block->offset + block->length;
2787

    
2788
        QLIST_FOREACH(next_block, &ram_list.blocks, next) {
2789
            if (next_block->offset >= end) {
2790
                next = MIN(next, next_block->offset);
2791
            }
2792
        }
2793
        if (next - end >= size && next - end < mingap) {
2794
            offset =  end;
2795
            mingap = next - end;
2796
        }
2797
    }
2798
    return offset;
2799
}
2800

    
2801
static ram_addr_t last_ram_offset(void)
2802
{
2803
    RAMBlock *block;
2804
    ram_addr_t last = 0;
2805

    
2806
    QLIST_FOREACH(block, &ram_list.blocks, next)
2807
        last = MAX(last, block->offset + block->length);
2808

    
2809
    return last;
2810
}
2811

    
2812
ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
2813
                                   ram_addr_t size, void *host)
2814
{
2815
    RAMBlock *new_block, *block;
2816

    
2817
    size = TARGET_PAGE_ALIGN(size);
2818
    new_block = qemu_mallocz(sizeof(*new_block));
2819

    
2820
    if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
2821
        char *id = dev->parent_bus->info->get_dev_path(dev);
2822
        if (id) {
2823
            snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
2824
            qemu_free(id);
2825
        }
2826
    }
2827
    pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
2828

    
2829
    QLIST_FOREACH(block, &ram_list.blocks, next) {
2830
        if (!strcmp(block->idstr, new_block->idstr)) {
2831
            fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
2832
                    new_block->idstr);
2833
            abort();
2834
        }
2835
    }
2836

    
2837
    if (host) {
2838
        new_block->host = host;
2839
    } else {
2840
        if (mem_path) {
2841
#if defined (__linux__) && !defined(TARGET_S390X)
2842
            new_block->host = file_ram_alloc(new_block, size, mem_path);
2843
            if (!new_block->host) {
2844
                new_block->host = qemu_vmalloc(size);
2845
                qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2846
            }
2847
#else
2848
            fprintf(stderr, "-mem-path option unsupported\n");
2849
            exit(1);
2850
#endif
2851
        } else {
2852
#if defined(TARGET_S390X) && defined(CONFIG_KVM)
2853
            /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2854
            new_block->host = mmap((void*)0x1000000, size,
2855
                                   PROT_EXEC|PROT_READ|PROT_WRITE,
2856
                                   MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2857
#else
2858
            new_block->host = qemu_vmalloc(size);
2859
#endif
2860
            qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2861
        }
2862
    }
2863

    
2864
    new_block->offset = find_ram_offset(size);
2865
    new_block->length = size;
2866

    
2867
    QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
2868

    
2869
    ram_list.phys_dirty = qemu_realloc(ram_list.phys_dirty,
2870
                                       last_ram_offset() >> TARGET_PAGE_BITS);
2871
    memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
2872
           0xff, size >> TARGET_PAGE_BITS);
2873

    
2874
    if (kvm_enabled())
2875
        kvm_setup_guest_memory(new_block->host, size);
2876

    
2877
    return new_block->offset;
2878
}
2879

    
2880
ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size)
2881
{
2882
    return qemu_ram_alloc_from_ptr(dev, name, size, NULL);
2883
}
2884

    
2885
void qemu_ram_free(ram_addr_t addr)
2886
{
2887
    RAMBlock *block;
2888

    
2889
    QLIST_FOREACH(block, &ram_list.blocks, next) {
2890
        if (addr == block->offset) {
2891
            QLIST_REMOVE(block, next);
2892
            if (mem_path) {
2893
#if defined (__linux__) && !defined(TARGET_S390X)
2894
                if (block->fd) {
2895
                    munmap(block->host, block->length);
2896
                    close(block->fd);
2897
                } else {
2898
                    qemu_vfree(block->host);
2899
                }
2900
#endif
2901
            } else {
2902
#if defined(TARGET_S390X) && defined(CONFIG_KVM)
2903
                munmap(block->host, block->length);
2904
#else
2905
                qemu_vfree(block->host);
2906
#endif
2907
            }
2908
            qemu_free(block);
2909
            return;
2910
        }
2911
    }
2912

    
2913
}
2914

    
2915
/* Return a host pointer to ram allocated with qemu_ram_alloc.
2916
   With the exception of the softmmu code in this file, this should
2917
   only be used for local memory (e.g. video ram) that the device owns,
2918
   and knows it isn't going to access beyond the end of the block.
2919

2920
   It should not be used for general purpose DMA.
2921
   Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2922
 */
2923
void *qemu_get_ram_ptr(ram_addr_t addr)
2924
{
2925
    RAMBlock *block;
2926

    
2927
    QLIST_FOREACH(block, &ram_list.blocks, next) {
2928
        if (addr - block->offset < block->length) {
2929
            QLIST_REMOVE(block, next);
2930
            QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
2931
            return block->host + (addr - block->offset);
2932
        }
2933
    }
2934

    
2935
    fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
2936
    abort();
2937

    
2938
    return NULL;
2939
}
2940

    
2941
/* Some of the softmmu routines need to translate from a host pointer
2942
   (typically a TLB entry) back to a ram offset.  */
2943
ram_addr_t qemu_ram_addr_from_host(void *ptr)
2944
{
2945
    RAMBlock *block;
2946
    uint8_t *host = ptr;
2947

    
2948
    QLIST_FOREACH(block, &ram_list.blocks, next) {
2949
        if (host - block->host < block->length) {
2950
            return block->offset + (host - block->host);
2951
        }
2952
    }
2953

    
2954
    fprintf(stderr, "Bad ram pointer %p\n", ptr);
2955
    abort();
2956

    
2957
    return 0;
2958
}
2959

    
2960
static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2961
{
2962
#ifdef DEBUG_UNASSIGNED
2963
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2964
#endif
2965
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2966
    do_unassigned_access(addr, 0, 0, 0, 1);
2967
#endif
2968
    return 0;
2969
}
2970

    
2971
static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2972
{
2973
#ifdef DEBUG_UNASSIGNED
2974
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2975
#endif
2976
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2977
    do_unassigned_access(addr, 0, 0, 0, 2);
2978
#endif
2979
    return 0;
2980
}
2981

    
2982
static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2983
{
2984
#ifdef DEBUG_UNASSIGNED
2985
    printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2986
#endif
2987
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2988
    do_unassigned_access(addr, 0, 0, 0, 4);
2989
#endif
2990
    return 0;
2991
}
2992

    
2993
static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2994
{
2995
#ifdef DEBUG_UNASSIGNED
2996
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2997
#endif
2998
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2999
    do_unassigned_access(addr, 1, 0, 0, 1);
3000
#endif
3001
}
3002

    
3003
static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
3004
{
3005
#ifdef DEBUG_UNASSIGNED
3006
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3007
#endif
3008
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3009
    do_unassigned_access(addr, 1, 0, 0, 2);
3010
#endif
3011
}
3012

    
3013
static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
3014
{
3015
#ifdef DEBUG_UNASSIGNED
3016
    printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3017
#endif
3018
#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3019
    do_unassigned_access(addr, 1, 0, 0, 4);
3020
#endif
3021
}
3022

    
3023
static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
3024
    unassigned_mem_readb,
3025
    unassigned_mem_readw,
3026
    unassigned_mem_readl,
3027
};
3028

    
3029
static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
3030
    unassigned_mem_writeb,
3031
    unassigned_mem_writew,
3032
    unassigned_mem_writel,
3033
};
3034

    
3035
static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
3036
                                uint32_t val)
3037
{
3038
    int dirty_flags;
3039
    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3040
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3041
#if !defined(CONFIG_USER_ONLY)
3042
        tb_invalidate_phys_page_fast(ram_addr, 1);
3043
        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3044
#endif
3045
    }
3046
    stb_p(qemu_get_ram_ptr(ram_addr), val);
3047
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3048
    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3049
    /* we remove the notdirty callback only if the code has been
3050
       flushed */
3051
    if (dirty_flags == 0xff)
3052
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3053
}
3054

    
3055
static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
3056
                                uint32_t val)
3057
{
3058
    int dirty_flags;
3059
    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3060
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3061
#if !defined(CONFIG_USER_ONLY)
3062
        tb_invalidate_phys_page_fast(ram_addr, 2);
3063
        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3064
#endif
3065
    }
3066
    stw_p(qemu_get_ram_ptr(ram_addr), val);
3067
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3068
    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3069
    /* we remove the notdirty callback only if the code has been
3070
       flushed */
3071
    if (dirty_flags == 0xff)
3072
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3073
}
3074

    
3075
static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
3076
                                uint32_t val)
3077
{
3078
    int dirty_flags;
3079
    dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3080
    if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3081
#if !defined(CONFIG_USER_ONLY)
3082
        tb_invalidate_phys_page_fast(ram_addr, 4);
3083
        dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3084
#endif
3085
    }
3086
    stl_p(qemu_get_ram_ptr(ram_addr), val);
3087
    dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3088
    cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3089
    /* we remove the notdirty callback only if the code has been
3090
       flushed */
3091
    if (dirty_flags == 0xff)
3092
        tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3093
}
3094

    
3095
static CPUReadMemoryFunc * const error_mem_read[3] = {
3096
    NULL, /* never used */
3097
    NULL, /* never used */
3098
    NULL, /* never used */
3099
};
3100

    
3101
static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
3102
    notdirty_mem_writeb,
3103
    notdirty_mem_writew,
3104
    notdirty_mem_writel,
3105
};
3106

    
3107
/* Generate a debug exception if a watchpoint has been hit.  */
3108
static void check_watchpoint(int offset, int len_mask, int flags)
3109
{
3110
    CPUState *env = cpu_single_env;
3111
    target_ulong pc, cs_base;
3112
    TranslationBlock *tb;
3113
    target_ulong vaddr;
3114
    CPUWatchpoint *wp;
3115
    int cpu_flags;
3116

    
3117
    if (env->watchpoint_hit) {
3118
        /* We re-entered the check after replacing the TB. Now raise
3119
         * the debug interrupt so that is will trigger after the
3120
         * current instruction. */
3121
        cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
3122
        return;
3123
    }
3124
    vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
3125
    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
3126
        if ((vaddr == (wp->vaddr & len_mask) ||
3127
             (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
3128
            wp->flags |= BP_WATCHPOINT_HIT;
3129
            if (!env->watchpoint_hit) {
3130
                env->watchpoint_hit = wp;
3131
                tb = tb_find_pc(env->mem_io_pc);
3132
                if (!tb) {
3133
                    cpu_abort(env, "check_watchpoint: could not find TB for "
3134
                              "pc=%p", (void *)env->mem_io_pc);
3135
                }
3136
                cpu_restore_state(tb, env, env->mem_io_pc, NULL);
3137
                tb_phys_invalidate(tb, -1);
3138
                if (wp->flags & BP_STOP_BEFORE_ACCESS) {
3139
                    env->exception_index = EXCP_DEBUG;
3140
                } else {
3141
                    cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
3142
                    tb_gen_code(env, pc, cs_base, cpu_flags, 1);
3143
                }
3144
                cpu_resume_from_signal(env, NULL);
3145
            }
3146
        } else {
3147
            wp->flags &= ~BP_WATCHPOINT_HIT;
3148
        }
3149
    }
3150
}
3151

    
3152
/* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
3153
   so these check for a hit then pass through to the normal out-of-line
3154
   phys routines.  */
3155
static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
3156
{
3157
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
3158
    return ldub_phys(addr);
3159
}
3160

    
3161
static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
3162
{
3163
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
3164
    return lduw_phys(addr);
3165
}
3166

    
3167
static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
3168
{
3169
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
3170
    return ldl_phys(addr);
3171
}
3172

    
3173
static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
3174
                             uint32_t val)
3175
{
3176
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
3177
    stb_phys(addr, val);
3178
}
3179

    
3180
static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
3181
                             uint32_t val)
3182
{
3183
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
3184
    stw_phys(addr, val);
3185
}
3186

    
3187
static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
3188
                             uint32_t val)
3189
{
3190
    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
3191
    stl_phys(addr, val);
3192
}
3193

    
3194
static CPUReadMemoryFunc * const watch_mem_read[3] = {
3195
    watch_mem_readb,
3196
    watch_mem_readw,
3197
    watch_mem_readl,
3198
};
3199

    
3200
static CPUWriteMemoryFunc * const watch_mem_write[3] = {
3201
    watch_mem_writeb,
3202
    watch_mem_writew,
3203
    watch_mem_writel,
3204
};
3205

    
3206
static inline uint32_t subpage_readlen (subpage_t *mmio,
3207
                                        target_phys_addr_t addr,
3208
                                        unsigned int len)
3209
{
3210
    unsigned int idx = SUBPAGE_IDX(addr);
3211
#if defined(DEBUG_SUBPAGE)
3212
    printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
3213
           mmio, len, addr, idx);
3214
#endif
3215

    
3216
    addr += mmio->region_offset[idx];
3217
    idx = mmio->sub_io_index[idx];
3218
    return io_mem_read[idx][len](io_mem_opaque[idx], addr);
3219
}
3220

    
3221
static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
3222
                                     uint32_t value, unsigned int len)
3223
{
3224
    unsigned int idx = SUBPAGE_IDX(addr);
3225
#if defined(DEBUG_SUBPAGE)
3226
    printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n",
3227
           __func__, mmio, len, addr, idx, value);
3228
#endif
3229

    
3230
    addr += mmio->region_offset[idx];
3231
    idx = mmio->sub_io_index[idx];
3232
    io_mem_write[idx][len](io_mem_opaque[idx], addr, value);
3233
}
3234

    
3235
static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
3236
{
3237
    return subpage_readlen(opaque, addr, 0);
3238
}
3239

    
3240
static void subpage_writeb (void *opaque, target_phys_addr_t addr,
3241
                            uint32_t value)
3242
{
3243
    subpage_writelen(opaque, addr, value, 0);
3244
}
3245

    
3246
static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
3247
{
3248
    return subpage_readlen(opaque, addr, 1);
3249
}
3250

    
3251
static void subpage_writew (void *opaque, target_phys_addr_t addr,
3252
                            uint32_t value)
3253
{
3254
    subpage_writelen(opaque, addr, value, 1);
3255
}
3256

    
3257
static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
3258
{
3259
    return subpage_readlen(opaque, addr, 2);
3260
}
3261

    
3262
static void subpage_writel (void *opaque, target_phys_addr_t addr,
3263
                            uint32_t value)
3264
{
3265
    subpage_writelen(opaque, addr, value, 2);
3266
}
3267

    
3268
static CPUReadMemoryFunc * const subpage_read[] = {
3269
    &subpage_readb,
3270
    &subpage_readw,
3271
    &subpage_readl,
3272
};
3273

    
3274
static CPUWriteMemoryFunc * const subpage_write[] = {
3275
    &subpage_writeb,
3276
    &subpage_writew,
3277
    &subpage_writel,
3278
};
3279

    
3280
static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
3281
                             ram_addr_t memory, ram_addr_t region_offset)
3282
{
3283
    int idx, eidx;
3284

    
3285
    if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
3286
        return -1;
3287
    idx = SUBPAGE_IDX(start);
3288
    eidx = SUBPAGE_IDX(end);
3289
#if defined(DEBUG_SUBPAGE)
3290
    printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
3291
           mmio, start, end, idx, eidx, memory);
3292
#endif
3293
    if ((memory & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
3294
        memory = IO_MEM_UNASSIGNED;
3295
    memory = (memory >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3296
    for (; idx <= eidx; idx++) {
3297
        mmio->sub_io_index[idx] = memory;
3298
        mmio->region_offset[idx] = region_offset;
3299
    }
3300

    
3301
    return 0;
3302
}
3303

    
3304
static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3305
                                ram_addr_t orig_memory,
3306
                                ram_addr_t region_offset)
3307
{
3308
    subpage_t *mmio;
3309
    int subpage_memory;
3310

    
3311
    mmio = qemu_mallocz(sizeof(subpage_t));
3312

    
3313
    mmio->base = base;
3314
    subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio);
3315
#if defined(DEBUG_SUBPAGE)
3316
    printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3317
           mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3318
#endif
3319
    *phys = subpage_memory | IO_MEM_SUBPAGE;
3320
    subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
3321

    
3322
    return mmio;
3323
}
3324

    
3325
static int get_free_io_mem_idx(void)
3326
{
3327
    int i;
3328

    
3329
    for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3330
        if (!io_mem_used[i]) {
3331
            io_mem_used[i] = 1;
3332
            return i;
3333
        }
3334
    fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
3335
    return -1;
3336
}
3337

    
3338
/* mem_read and mem_write are arrays of functions containing the
3339
   function to access byte (index 0), word (index 1) and dword (index
3340
   2). Functions can be omitted with a NULL function pointer.
3341
   If io_index is non zero, the corresponding io zone is
3342
   modified. If it is zero, a new io zone is allocated. The return
3343
   value can be used with cpu_register_physical_memory(). (-1) is
3344
   returned if error. */
3345
static int cpu_register_io_memory_fixed(int io_index,
3346
                                        CPUReadMemoryFunc * const *mem_read,
3347
                                        CPUWriteMemoryFunc * const *mem_write,
3348
                                        void *opaque)
3349
{
3350
    int i;
3351

    
3352
    if (io_index <= 0) {
3353
        io_index = get_free_io_mem_idx();
3354
        if (io_index == -1)
3355
            return io_index;
3356
    } else {
3357
        io_index >>= IO_MEM_SHIFT;
3358
        if (io_index >= IO_MEM_NB_ENTRIES)
3359
            return -1;
3360
    }
3361

    
3362
    for (i = 0; i < 3; ++i) {
3363
        io_mem_read[io_index][i]
3364
            = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
3365
    }
3366
    for (i = 0; i < 3; ++i) {
3367
        io_mem_write[io_index][i]
3368
            = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
3369
    }
3370
    io_mem_opaque[io_index] = opaque;
3371

    
3372
    return (io_index << IO_MEM_SHIFT);
3373
}
3374

    
3375
int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
3376
                           CPUWriteMemoryFunc * const *mem_write,
3377
                           void *opaque)
3378
{
3379
    return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque);
3380
}
3381

    
3382
void cpu_unregister_io_memory(int io_table_address)
3383
{
3384
    int i;
3385
    int io_index = io_table_address >> IO_MEM_SHIFT;
3386

    
3387
    for (i=0;i < 3; i++) {
3388
        io_mem_read[io_index][i] = unassigned_mem_read[i];
3389
        io_mem_write[io_index][i] = unassigned_mem_write[i];
3390
    }
3391
    io_mem_opaque[io_index] = NULL;
3392
    io_mem_used[io_index] = 0;
3393
}
3394

    
3395
static void io_mem_init(void)
3396
{
3397
    int i;
3398

    
3399
    cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read, unassigned_mem_write, NULL);
3400
    cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read, unassigned_mem_write, NULL);
3401
    cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read, notdirty_mem_write, NULL);
3402
    for (i=0; i<5; i++)
3403
        io_mem_used[i] = 1;
3404

    
3405
    io_mem_watch = cpu_register_io_memory(watch_mem_read,
3406
                                          watch_mem_write, NULL);
3407
}
3408

    
3409
#endif /* !defined(CONFIG_USER_ONLY) */
3410

    
3411
/* physical memory access (slow version, mainly for debug) */
3412
#if defined(CONFIG_USER_ONLY)
3413
int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3414
                        uint8_t *buf, int len, int is_write)
3415
{
3416
    int l, flags;
3417
    target_ulong page;
3418
    void * p;
3419

    
3420
    while (len > 0) {
3421
        page = addr & TARGET_PAGE_MASK;
3422
        l = (page + TARGET_PAGE_SIZE) - addr;
3423
        if (l > len)
3424
            l = len;
3425
        flags = page_get_flags(page);
3426
        if (!(flags & PAGE_VALID))
3427
            return -1;
3428
        if (is_write) {
3429
            if (!(flags & PAGE_WRITE))
3430
                return -1;
3431
            /* XXX: this code should not depend on lock_user */
3432
            if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3433
                return -1;
3434
            memcpy(p, buf, l);
3435
            unlock_user(p, addr, l);
3436
        } else {
3437
            if (!(flags & PAGE_READ))
3438
                return -1;
3439
            /* XXX: this code should not depend on lock_user */
3440
            if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3441
                return -1;
3442
            memcpy(buf, p, l);
3443
            unlock_user(p, addr, 0);
3444
        }
3445
        len -= l;
3446
        buf += l;
3447
        addr += l;
3448
    }
3449
    return 0;
3450
}
3451

    
3452
#else
3453
void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3454
                            int len, int is_write)
3455
{
3456
    int l, io_index;
3457
    uint8_t *ptr;
3458
    uint32_t val;
3459
    target_phys_addr_t page;
3460
    unsigned long pd;
3461
    PhysPageDesc *p;
3462

    
3463
    while (len > 0) {
3464
        page = addr & TARGET_PAGE_MASK;
3465
        l = (page + TARGET_PAGE_SIZE) - addr;
3466
        if (l > len)
3467
            l = len;
3468
        p = phys_page_find(page >> TARGET_PAGE_BITS);
3469
        if (!p) {
3470
            pd = IO_MEM_UNASSIGNED;
3471
        } else {
3472
            pd = p->phys_offset;
3473
        }
3474

    
3475
        if (is_write) {
3476
            if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3477
                target_phys_addr_t addr1 = addr;
3478
                io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3479
                if (p)
3480
                    addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3481
                /* XXX: could force cpu_single_env to NULL to avoid
3482
                   potential bugs */
3483
                if (l >= 4 && ((addr1 & 3) == 0)) {
3484
                    /* 32 bit write access */
3485
                    val = ldl_p(buf);
3486
                    io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3487
                    l = 4;
3488
                } else if (l >= 2 && ((addr1 & 1) == 0)) {
3489
                    /* 16 bit write access */
3490
                    val = lduw_p(buf);
3491
                    io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3492
                    l = 2;
3493
                } else {
3494
                    /* 8 bit write access */
3495
                    val = ldub_p(buf);
3496
                    io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3497
                    l = 1;
3498
                }
3499
            } else {
3500
                unsigned long addr1;
3501
                addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3502
                /* RAM case */
3503
                ptr = qemu_get_ram_ptr(addr1);
3504
                memcpy(ptr, buf, l);
3505
                if (!cpu_physical_memory_is_dirty(addr1)) {
3506
                    /* invalidate code */
3507
                    tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3508
                    /* set dirty bit */
3509
                    cpu_physical_memory_set_dirty_flags(
3510
                        addr1, (0xff & ~CODE_DIRTY_FLAG));
3511
                }
3512
            }
3513
        } else {
3514
            if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3515
                !(pd & IO_MEM_ROMD)) {
3516
                target_phys_addr_t addr1 = addr;
3517
                /* I/O case */
3518
                io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3519
                if (p)
3520
                    addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3521
                if (l >= 4 && ((addr1 & 3) == 0)) {
3522
                    /* 32 bit read access */
3523
                    val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3524
                    stl_p(buf, val);
3525
                    l = 4;
3526
                } else if (l >= 2 && ((addr1 & 1) == 0)) {
3527
                    /* 16 bit read access */
3528
                    val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3529
                    stw_p(buf, val);
3530
                    l = 2;
3531
                } else {
3532
                    /* 8 bit read access */
3533
                    val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3534
                    stb_p(buf, val);
3535
                    l = 1;
3536
                }
3537
            } else {
3538
                /* RAM case */
3539
                ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3540
                    (addr & ~TARGET_PAGE_MASK);
3541
                memcpy(buf, ptr, l);
3542
            }
3543
        }
3544
        len -= l;
3545
        buf += l;
3546
        addr += l;
3547
    }
3548
}
3549

    
3550
/* used for ROM loading : can write in RAM and ROM */
3551
void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3552
                                   const uint8_t *buf, int len)
3553
{
3554
    int l;
3555
    uint8_t *ptr;
3556
    target_phys_addr_t page;
3557
    unsigned long pd;
3558
    PhysPageDesc *p;
3559

    
3560
    while (len > 0) {
3561
        page = addr & TARGET_PAGE_MASK;
3562
        l = (page + TARGET_PAGE_SIZE) - addr;
3563
        if (l > len)
3564
            l = len;
3565
        p = phys_page_find(page >> TARGET_PAGE_BITS);
3566
        if (!p) {
3567
            pd = IO_MEM_UNASSIGNED;
3568
        } else {
3569
            pd = p->phys_offset;
3570
        }
3571

    
3572
        if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3573
            (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3574
            !(pd & IO_MEM_ROMD)) {
3575
            /* do nothing */
3576
        } else {
3577
            unsigned long addr1;
3578
            addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3579
            /* ROM/RAM case */
3580
            ptr = qemu_get_ram_ptr(addr1);
3581
            memcpy(ptr, buf, l);
3582
        }
3583
        len -= l;
3584
        buf += l;
3585
        addr += l;
3586
    }
3587
}
3588

    
3589
typedef struct {
3590
    void *buffer;
3591
    target_phys_addr_t addr;
3592
    target_phys_addr_t len;
3593
} BounceBuffer;
3594

    
3595
static BounceBuffer bounce;
3596

    
3597
typedef struct MapClient {
3598
    void *opaque;
3599
    void (*callback)(void *opaque);
3600
    QLIST_ENTRY(MapClient) link;
3601
} MapClient;
3602

    
3603
static QLIST_HEAD(map_client_list, MapClient) map_client_list
3604
    = QLIST_HEAD_INITIALIZER(map_client_list);
3605

    
3606
void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3607
{
3608
    MapClient *client = qemu_malloc(sizeof(*client));
3609

    
3610
    client->opaque = opaque;
3611
    client->callback = callback;
3612
    QLIST_INSERT_HEAD(&map_client_list, client, link);
3613
    return client;
3614
}
3615

    
3616
void cpu_unregister_map_client(void *_client)
3617
{
3618
    MapClient *client = (MapClient *)_client;
3619

    
3620
    QLIST_REMOVE(client, link);
3621
    qemu_free(client);
3622
}
3623

    
3624
static void cpu_notify_map_clients(void)
3625
{
3626
    MapClient *client;
3627

    
3628
    while (!QLIST_EMPTY(&map_client_list)) {
3629
        client = QLIST_FIRST(&map_client_list);
3630
        client->callback(client->opaque);
3631
        cpu_unregister_map_client(client);
3632
    }
3633
}
3634

    
3635
/* Map a physical memory region into a host virtual address.
3636
 * May map a subset of the requested range, given by and returned in *plen.
3637
 * May return NULL if resources needed to perform the mapping are exhausted.
3638
 * Use only for reads OR writes - not for read-modify-write operations.
3639
 * Use cpu_register_map_client() to know when retrying the map operation is
3640
 * likely to succeed.
3641
 */
3642
void *cpu_physical_memory_map(target_phys_addr_t addr,
3643
                              target_phys_addr_t *plen,
3644
                              int is_write)
3645
{
3646
    target_phys_addr_t len = *plen;
3647
    target_phys_addr_t done = 0;
3648
    int l;
3649
    uint8_t *ret = NULL;
3650
    uint8_t *ptr;
3651
    target_phys_addr_t page;
3652
    unsigned long pd;
3653
    PhysPageDesc *p;
3654
    unsigned long addr1;
3655

    
3656
    while (len > 0) {
3657
        page = addr & TARGET_PAGE_MASK;
3658
        l = (page + TARGET_PAGE_SIZE) - addr;
3659
        if (l > len)
3660
            l = len;
3661
        p = phys_page_find(page >> TARGET_PAGE_BITS);
3662
        if (!p) {
3663
            pd = IO_MEM_UNASSIGNED;
3664
        } else {
3665
            pd = p->phys_offset;
3666
        }
3667

    
3668
        if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3669
            if (done || bounce.buffer) {
3670
                break;
3671
            }
3672
            bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3673
            bounce.addr = addr;
3674
            bounce.len = l;
3675
            if (!is_write) {
3676
                cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3677
            }
3678
            ptr = bounce.buffer;
3679
        } else {
3680
            addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3681
            ptr = qemu_get_ram_ptr(addr1);
3682
        }
3683
        if (!done) {
3684
            ret = ptr;
3685
        } else if (ret + done != ptr) {
3686
            break;
3687
        }
3688

    
3689
        len -= l;
3690
        addr += l;
3691
        done += l;
3692
    }
3693
    *plen = done;
3694
    return ret;
3695
}
3696

    
3697
/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3698
 * Will also mark the memory as dirty if is_write == 1.  access_len gives
3699
 * the amount of memory that was actually read or written by the caller.
3700
 */
3701
void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3702
                               int is_write, target_phys_addr_t access_len)
3703
{
3704
    if (buffer != bounce.buffer) {
3705
        if (is_write) {
3706
            ram_addr_t addr1 = qemu_ram_addr_from_host(buffer);
3707
            while (access_len) {
3708
                unsigned l;
3709
                l = TARGET_PAGE_SIZE;
3710
                if (l > access_len)
3711
                    l = access_len;
3712
                if (!cpu_physical_memory_is_dirty(addr1)) {
3713
                    /* invalidate code */
3714
                    tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3715
                    /* set dirty bit */
3716
                    cpu_physical_memory_set_dirty_flags(
3717
                        addr1, (0xff & ~CODE_DIRTY_FLAG));
3718
                }
3719
                addr1 += l;
3720
                access_len -= l;
3721
            }
3722
        }
3723
        return;
3724
    }
3725
    if (is_write) {
3726
        cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3727
    }
3728
    qemu_vfree(bounce.buffer);
3729
    bounce.buffer = NULL;
3730
    cpu_notify_map_clients();
3731
}
3732

    
3733
/* warning: addr must be aligned */
3734
uint32_t ldl_phys(target_phys_addr_t addr)
3735
{
3736
    int io_index;
3737
    uint8_t *ptr;
3738
    uint32_t val;
3739
    unsigned long pd;
3740
    PhysPageDesc *p;
3741

    
3742
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
3743
    if (!p) {
3744
        pd = IO_MEM_UNASSIGNED;
3745
    } else {
3746
        pd = p->phys_offset;
3747
    }
3748

    
3749
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3750
        !(pd & IO_MEM_ROMD)) {
3751
        /* I/O case */
3752
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3753
        if (p)
3754
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3755
        val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3756
    } else {
3757
        /* RAM case */
3758
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3759
            (addr & ~TARGET_PAGE_MASK);
3760
        val = ldl_p(ptr);
3761
    }
3762
    return val;
3763
}
3764

    
3765
/* warning: addr must be aligned */
3766
uint64_t ldq_phys(target_phys_addr_t addr)
3767
{
3768
    int io_index;
3769
    uint8_t *ptr;
3770
    uint64_t val;
3771
    unsigned long pd;
3772
    PhysPageDesc *p;
3773

    
3774
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
3775
    if (!p) {
3776
        pd = IO_MEM_UNASSIGNED;
3777
    } else {
3778
        pd = p->phys_offset;
3779
    }
3780

    
3781
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3782
        !(pd & IO_MEM_ROMD)) {
3783
        /* I/O case */
3784
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3785
        if (p)
3786
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3787
#ifdef TARGET_WORDS_BIGENDIAN
3788
        val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3789
        val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3790
#else
3791
        val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3792
        val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3793
#endif
3794
    } else {
3795
        /* RAM case */
3796
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3797
            (addr & ~TARGET_PAGE_MASK);
3798
        val = ldq_p(ptr);
3799
    }
3800
    return val;
3801
}
3802

    
3803
/* XXX: optimize */
3804
uint32_t ldub_phys(target_phys_addr_t addr)
3805
{
3806
    uint8_t val;
3807
    cpu_physical_memory_read(addr, &val, 1);
3808
    return val;
3809
}
3810

    
3811
/* warning: addr must be aligned */
3812
uint32_t lduw_phys(target_phys_addr_t addr)
3813
{
3814
    int io_index;
3815
    uint8_t *ptr;
3816
    uint64_t val;
3817
    unsigned long pd;
3818
    PhysPageDesc *p;
3819

    
3820
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
3821
    if (!p) {
3822
        pd = IO_MEM_UNASSIGNED;
3823
    } else {
3824
        pd = p->phys_offset;
3825
    }
3826

    
3827
    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3828
        !(pd & IO_MEM_ROMD)) {
3829
        /* I/O case */
3830
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3831
        if (p)
3832
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3833
        val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
3834
    } else {
3835
        /* RAM case */
3836
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3837
            (addr & ~TARGET_PAGE_MASK);
3838
        val = lduw_p(ptr);
3839
    }
3840
    return val;
3841
}
3842

    
3843
/* warning: addr must be aligned. The ram page is not masked as dirty
3844
   and the code inside is not invalidated. It is useful if the dirty
3845
   bits are used to track modified PTEs */
3846
void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3847
{
3848
    int io_index;
3849
    uint8_t *ptr;
3850
    unsigned long pd;
3851
    PhysPageDesc *p;
3852

    
3853
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
3854
    if (!p) {
3855
        pd = IO_MEM_UNASSIGNED;
3856
    } else {
3857
        pd = p->phys_offset;
3858
    }
3859

    
3860
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3861
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3862
        if (p)
3863
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3864
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3865
    } else {
3866
        unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3867
        ptr = qemu_get_ram_ptr(addr1);
3868
        stl_p(ptr, val);
3869

    
3870
        if (unlikely(in_migration)) {
3871
            if (!cpu_physical_memory_is_dirty(addr1)) {
3872
                /* invalidate code */
3873
                tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3874
                /* set dirty bit */
3875
                cpu_physical_memory_set_dirty_flags(
3876
                    addr1, (0xff & ~CODE_DIRTY_FLAG));
3877
            }
3878
        }
3879
    }
3880
}
3881

    
3882
void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3883
{
3884
    int io_index;
3885
    uint8_t *ptr;
3886
    unsigned long pd;
3887
    PhysPageDesc *p;
3888

    
3889
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
3890
    if (!p) {
3891
        pd = IO_MEM_UNASSIGNED;
3892
    } else {
3893
        pd = p->phys_offset;
3894
    }
3895

    
3896
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3897
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3898
        if (p)
3899
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3900
#ifdef TARGET_WORDS_BIGENDIAN
3901
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3902
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3903
#else
3904
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3905
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3906
#endif
3907
    } else {
3908
        ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3909
            (addr & ~TARGET_PAGE_MASK);
3910
        stq_p(ptr, val);
3911
    }
3912
}
3913

    
3914
/* warning: addr must be aligned */
3915
void stl_phys(target_phys_addr_t addr, uint32_t val)
3916
{
3917
    int io_index;
3918
    uint8_t *ptr;
3919
    unsigned long pd;
3920
    PhysPageDesc *p;
3921

    
3922
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
3923
    if (!p) {
3924
        pd = IO_MEM_UNASSIGNED;
3925
    } else {
3926
        pd = p->phys_offset;
3927
    }
3928

    
3929
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3930
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3931
        if (p)
3932
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3933
        io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3934
    } else {
3935
        unsigned long addr1;
3936
        addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3937
        /* RAM case */
3938
        ptr = qemu_get_ram_ptr(addr1);
3939
        stl_p(ptr, val);
3940
        if (!cpu_physical_memory_is_dirty(addr1)) {
3941
            /* invalidate code */
3942
            tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3943
            /* set dirty bit */
3944
            cpu_physical_memory_set_dirty_flags(addr1,
3945
                (0xff & ~CODE_DIRTY_FLAG));
3946
        }
3947
    }
3948
}
3949

    
3950
/* XXX: optimize */
3951
void stb_phys(target_phys_addr_t addr, uint32_t val)
3952
{
3953
    uint8_t v = val;
3954
    cpu_physical_memory_write(addr, &v, 1);
3955
}
3956

    
3957
/* warning: addr must be aligned */
3958
void stw_phys(target_phys_addr_t addr, uint32_t val)
3959
{
3960
    int io_index;
3961
    uint8_t *ptr;
3962
    unsigned long pd;
3963
    PhysPageDesc *p;
3964

    
3965
    p = phys_page_find(addr >> TARGET_PAGE_BITS);
3966
    if (!p) {
3967
        pd = IO_MEM_UNASSIGNED;
3968
    } else {
3969
        pd = p->phys_offset;
3970
    }
3971

    
3972
    if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3973
        io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3974
        if (p)
3975
            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3976
        io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
3977
    } else {
3978
        unsigned long addr1;
3979
        addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3980
        /* RAM case */
3981
        ptr = qemu_get_ram_ptr(addr1);
3982
        stw_p(ptr, val);
3983
        if (!cpu_physical_memory_is_dirty(addr1)) {
3984
            /* invalidate code */
3985
            tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
3986
            /* set dirty bit */
3987
            cpu_physical_memory_set_dirty_flags(addr1,
3988
                (0xff & ~CODE_DIRTY_FLAG));
3989
        }
3990
    }
3991
}
3992

    
3993
/* XXX: optimize */
3994
void stq_phys(target_phys_addr_t addr, uint64_t val)
3995
{
3996
    val = tswap64(val);
3997
    cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3998
}
3999

    
4000
/* virtual memory access for debug (includes writing to ROM) */
4001
int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
4002
                        uint8_t *buf, int len, int is_write)
4003
{
4004
    int l;
4005
    target_phys_addr_t phys_addr;
4006
    target_ulong page;
4007

    
4008
    while (len > 0) {
4009
        page = addr & TARGET_PAGE_MASK;
4010
        phys_addr = cpu_get_phys_page_debug(env, page);
4011
        /* if no physical page mapped, return an error */
4012
        if (phys_addr == -1)
4013
            return -1;
4014
        l = (page + TARGET_PAGE_SIZE) - addr;
4015
        if (l > len)
4016
            l = len;
4017
        phys_addr += (addr & ~TARGET_PAGE_MASK);
4018
        if (is_write)
4019
            cpu_physical_memory_write_rom(phys_addr, buf, l);
4020
        else
4021
            cpu_physical_memory_rw(phys_addr, buf, l, is_write);
4022
        len -= l;
4023
        buf += l;
4024
        addr += l;
4025
    }
4026
    return 0;
4027
}
4028
#endif
4029

    
4030
/* in deterministic execution mode, instructions doing device I/Os
4031
   must be at the end of the TB */
4032
void cpu_io_recompile(CPUState *env, void *retaddr)
4033
{
4034
    TranslationBlock *tb;
4035
    uint32_t n, cflags;
4036
    target_ulong pc, cs_base;
4037
    uint64_t flags;
4038

    
4039
    tb = tb_find_pc((unsigned long)retaddr);
4040
    if (!tb) {
4041
        cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p", 
4042
                  retaddr);
4043
    }
4044
    n = env->icount_decr.u16.low + tb->icount;
4045
    cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
4046
    /* Calculate how many instructions had been executed before the fault
4047
       occurred.  */
4048
    n = n - env->icount_decr.u16.low;
4049
    /* Generate a new TB ending on the I/O insn.  */
4050
    n++;
4051
    /* On MIPS and SH, delay slot instructions can only be restarted if
4052
       they were already the first instruction in the TB.  If this is not
4053
       the first instruction in a TB then re-execute the preceding
4054
       branch.  */
4055
#if defined(TARGET_MIPS)
4056
    if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
4057
        env->active_tc.PC -= 4;
4058
        env->icount_decr.u16.low++;
4059
        env->hflags &= ~MIPS_HFLAG_BMASK;
4060
    }
4061
#elif defined(TARGET_SH4)
4062
    if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
4063
            && n > 1) {
4064
        env->pc -= 2;
4065
        env->icount_decr.u16.low++;
4066
        env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
4067
    }
4068
#endif
4069
    /* This should never happen.  */
4070
    if (n > CF_COUNT_MASK)
4071
        cpu_abort(env, "TB too big during recompile");
4072

    
4073
    cflags = n | CF_LAST_IO;
4074
    pc = tb->pc;
4075
    cs_base = tb->cs_base;
4076
    flags = tb->flags;
4077
    tb_phys_invalidate(tb, -1);
4078
    /* FIXME: In theory this could raise an exception.  In practice
4079
       we have already translated the block once so it's probably ok.  */
4080
    tb_gen_code(env, pc, cs_base, flags, cflags);
4081
    /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4082
       the first in the TB) then we end up generating a whole new TB and
4083
       repeating the fault, which is horribly inefficient.
4084
       Better would be to execute just this insn uncached, or generate a
4085
       second new TB.  */
4086
    cpu_resume_from_signal(env, NULL);
4087
}
4088

    
4089
#if !defined(CONFIG_USER_ONLY)
4090

    
4091
void dump_exec_info(FILE *f,
4092
                    int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
4093
{
4094
    int i, target_code_size, max_target_code_size;
4095
    int direct_jmp_count, direct_jmp2_count, cross_page;
4096
    TranslationBlock *tb;
4097

    
4098
    target_code_size = 0;
4099
    max_target_code_size = 0;
4100
    cross_page = 0;
4101
    direct_jmp_count = 0;
4102
    direct_jmp2_count = 0;
4103
    for(i = 0; i < nb_tbs; i++) {
4104
        tb = &tbs[i];
4105
        target_code_size += tb->size;
4106
        if (tb->size > max_target_code_size)
4107
            max_target_code_size = tb->size;
4108
        if (tb->page_addr[1] != -1)
4109
            cross_page++;
4110
        if (tb->tb_next_offset[0] != 0xffff) {
4111
            direct_jmp_count++;
4112
            if (tb->tb_next_offset[1] != 0xffff) {
4113
                direct_jmp2_count++;
4114
            }
4115
        }
4116
    }
4117
    /* XXX: avoid using doubles ? */
4118
    cpu_fprintf(f, "Translation buffer state:\n");
4119
    cpu_fprintf(f, "gen code size       %ld/%ld\n",
4120
                code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
4121
    cpu_fprintf(f, "TB count            %d/%d\n", 
4122
                nb_tbs, code_gen_max_blocks);
4123
    cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
4124
                nb_tbs ? target_code_size / nb_tbs : 0,
4125
                max_target_code_size);
4126
    cpu_fprintf(f, "TB avg host size    %d bytes (expansion ratio: %0.1f)\n",
4127
                nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
4128
                target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
4129
    cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
4130
            cross_page,
4131
            nb_tbs ? (cross_page * 100) / nb_tbs : 0);
4132
    cpu_fprintf(f, "direct jump count   %d (%d%%) (2 jumps=%d %d%%)\n",
4133
                direct_jmp_count,
4134
                nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
4135
                direct_jmp2_count,
4136
                nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
4137
    cpu_fprintf(f, "\nStatistics:\n");
4138
    cpu_fprintf(f, "TB flush count      %d\n", tb_flush_count);
4139
    cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
4140
    cpu_fprintf(f, "TLB flush count     %d\n", tlb_flush_count);
4141
    tcg_dump_info(f, cpu_fprintf);
4142
}
4143

    
4144
#define MMUSUFFIX _cmmu
4145
#define GETPC() NULL
4146
#define env cpu_single_env
4147
#define SOFTMMU_CODE_ACCESS
4148

    
4149
#define SHIFT 0
4150
#include "softmmu_template.h"
4151

    
4152
#define SHIFT 1
4153
#include "softmmu_template.h"
4154

    
4155
#define SHIFT 2
4156
#include "softmmu_template.h"
4157

    
4158
#define SHIFT 3
4159
#include "softmmu_template.h"
4160

    
4161
#undef env
4162

    
4163
#endif