Archipelago » qemu

/*

 *  virtual page mapping and translated block handling

 * 

 *  Copyright (c) 2003 Fabrice Bellard

 *

 * This library is free software; you can redistribute it and/or

 * modify it under the terms of the GNU Lesser General Public

 * License as published by the Free Software Foundation; either

 * version 2 of the License, or (at your option) any later version.

 *

 * This library is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * Lesser General Public License for more details.

 *

 * You should have received a copy of the GNU Lesser General Public

 * License along with this library; if not, write to the Free Software

 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

#include "config.h"

#ifdef _WIN32

#include <windows.h>

#else

#include <sys/types.h>

#include <sys/mman.h>

#endif

#include <stdlib.h>

#include <stdio.h>

#include <stdarg.h>

#include <string.h>

#include <errno.h>

#include <unistd.h>

#include <inttypes.h>

#include "cpu.h"

#include "exec-all.h"

//#define DEBUG_TB_INVALIDATE

//#define DEBUG_FLUSH

//#define DEBUG_TLB

/* make various TB consistency checks */

//#define DEBUG_TB_CHECK 

//#define DEBUG_TLB_CHECK 

/* threshold to flush the translated code buffer */

#define CODE_GEN_BUFFER_MAX_SIZE (CODE_GEN_BUFFER_SIZE - CODE_GEN_MAX_SIZE)

#define SMC_BITMAP_USE_THRESHOLD 10

#define MMAP_AREA_START        0x00000000

#define MMAP_AREA_END          0xa8000000

#if defined(TARGET_SPARC64)

#define TARGET_PHYS_ADDR_SPACE_BITS 41

#elif defined(TARGET_PPC64)

#define TARGET_PHYS_ADDR_SPACE_BITS 42

#else

/* Note: for compatibility with kqemu, we use 32 bits for x86_64 */

#define TARGET_PHYS_ADDR_SPACE_BITS 32

#endif

TranslationBlock tbs[CODE_GEN_MAX_BLOCKS];

TranslationBlock *tb_hash[CODE_GEN_HASH_SIZE];

TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];

int nb_tbs;

/* any access to the tbs or the page table must use this lock */

spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;

uint8_t code_gen_buffer[CODE_GEN_BUFFER_SIZE] __attribute__((aligned (32)));

uint8_t *code_gen_ptr;

int phys_ram_size;

int phys_ram_fd;

uint8_t *phys_ram_base;

uint8_t *phys_ram_dirty;

typedef struct PageDesc {

    /* list of TBs intersecting this ram page */

    TranslationBlock *first_tb;

    /* in order to optimize self modifying code, we count the number

       of lookups we do to a given page to use a bitmap */

    unsigned int code_write_count;

    uint8_t *code_bitmap;

#if defined(CONFIG_USER_ONLY)

    unsigned long flags;

#endif

} PageDesc;

typedef struct PhysPageDesc {

    /* offset in host memory of the page + io_index in the low 12 bits */

    uint32_t phys_offset;

} PhysPageDesc;

/* Note: the VirtPage handling is absolete and will be suppressed

   ASAP */

typedef struct VirtPageDesc {

    /* physical address of code page. It is valid only if 'valid_tag'

       matches 'virt_valid_tag' */ 

    target_ulong phys_addr; 

    unsigned int valid_tag;

#if !defined(CONFIG_SOFTMMU)

    /* original page access rights. It is valid only if 'valid_tag'

       matches 'virt_valid_tag' */

    unsigned int prot;

#endif

} VirtPageDesc;

#define L2_BITS 10

#define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)

#define L1_SIZE (1 << L1_BITS)

#define L2_SIZE (1 << L2_BITS)

static void io_mem_init(void);

unsigned long qemu_real_host_page_size;

unsigned long qemu_host_page_bits;

unsigned long qemu_host_page_size;

unsigned long qemu_host_page_mask;

/* XXX: for system emulation, it could just be an array */

static PageDesc *l1_map[L1_SIZE];

PhysPageDesc **l1_phys_map;

#if !defined(CONFIG_USER_ONLY)

#if TARGET_LONG_BITS > 32

#define VIRT_L_BITS 9

#define VIRT_L_SIZE (1 << VIRT_L_BITS)

static void *l1_virt_map[VIRT_L_SIZE];

#else

static VirtPageDesc *l1_virt_map[L1_SIZE];

#endif

static unsigned int virt_valid_tag;

#endif

/* io memory support */

CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];

CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];

void *io_mem_opaque[IO_MEM_NB_ENTRIES];

static int io_mem_nb;

/* log support */

char *logfilename = "/tmp/qemu.log";

FILE *logfile;

int loglevel;

/* statistics */

static int tlb_flush_count;

static int tb_flush_count;

static int tb_phys_invalidate_count;

static void page_init(void)

{

    /* NOTE: we can always suppose that qemu_host_page_size >=

       TARGET_PAGE_SIZE */

#ifdef _WIN32

    {

        SYSTEM_INFO system_info;

        DWORD old_protect;

        GetSystemInfo(&system_info);

        qemu_real_host_page_size = system_info.dwPageSize;

        VirtualProtect(code_gen_buffer, sizeof(code_gen_buffer),

                       PAGE_EXECUTE_READWRITE, &old_protect);

    }

#else

    qemu_real_host_page_size = getpagesize();

    {

        unsigned long start, end;

        start = (unsigned long)code_gen_buffer;

        start &= ~(qemu_real_host_page_size - 1);

        end = (unsigned long)code_gen_buffer + sizeof(code_gen_buffer);

        end += qemu_real_host_page_size - 1;

        end &= ~(qemu_real_host_page_size - 1);

        mprotect((void *)start, end - start, 

                 PROT_READ | PROT_WRITE | PROT_EXEC);

    }

#endif

    if (qemu_host_page_size == 0)

        qemu_host_page_size = qemu_real_host_page_size;

    if (qemu_host_page_size < TARGET_PAGE_SIZE)

        qemu_host_page_size = TARGET_PAGE_SIZE;

    qemu_host_page_bits = 0;

    while ((1 << qemu_host_page_bits) < qemu_host_page_size)

        qemu_host_page_bits++;

    qemu_host_page_mask = ~(qemu_host_page_size - 1);

#if !defined(CONFIG_USER_ONLY)

    virt_valid_tag = 1;

#endif

    l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));

    memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));

}

static inline PageDesc *page_find_alloc(unsigned int index)

{

    PageDesc **lp, *p;

    lp = &l1_map[index >> L2_BITS];

    p = *lp;

    if (!p) {

        /* allocate if not found */

        p = qemu_malloc(sizeof(PageDesc) * L2_SIZE);

        memset(p, 0, sizeof(PageDesc) * L2_SIZE);

        *lp = p;

    }

    return p + (index & (L2_SIZE - 1));

}

static inline PageDesc *page_find(unsigned int index)

{

    PageDesc *p;

    p = l1_map[index >> L2_BITS];

    if (!p)

        return 0;

    return p + (index & (L2_SIZE - 1));

}

static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)

{

    void **lp, **p;

    p = (void **)l1_phys_map;

#if TARGET_PHYS_ADDR_SPACE_BITS > 32

#if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)

#error unsupported TARGET_PHYS_ADDR_SPACE_BITS

#endif

    lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));

    p = *lp;

    if (!p) {

        /* allocate if not found */

        if (!alloc)

            return NULL;

        p = qemu_vmalloc(sizeof(void *) * L1_SIZE);

        memset(p, 0, sizeof(void *) * L1_SIZE);

        *lp = p;

    }

#endif

    lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));

    p = *lp;

    if (!p) {

        /* allocate if not found */

        if (!alloc)

            return NULL;

        p = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);

        memset(p, 0, sizeof(PhysPageDesc) * L2_SIZE);

        *lp = p;

    }

    return ((PhysPageDesc *)p) + (index & (L2_SIZE - 1));

}

static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)

{

    return phys_page_find_alloc(index, 0);

}

#if !defined(CONFIG_USER_ONLY)

static void tlb_protect_code(CPUState *env, target_ulong addr);

static void tlb_unprotect_code_phys(CPUState *env, unsigned long phys_addr, target_ulong vaddr);

static VirtPageDesc *virt_page_find_alloc(target_ulong index, int alloc)

{

#if TARGET_LONG_BITS > 32

    void **p, **lp;

    p = l1_virt_map;

    lp = p + ((index >> (5 * VIRT_L_BITS)) & (VIRT_L_SIZE - 1));

    p = *lp;

    if (!p) {

        if (!alloc)

            return NULL;

        p = qemu_mallocz(sizeof(void *) * VIRT_L_SIZE);

        *lp = p;

    }

    lp = p + ((index >> (4 * VIRT_L_BITS)) & (VIRT_L_SIZE - 1));

    p = *lp;

    if (!p) {

        if (!alloc)

            return NULL;

        p = qemu_mallocz(sizeof(void *) * VIRT_L_SIZE);

        *lp = p;

    }

    lp = p + ((index >> (3 * VIRT_L_BITS)) & (VIRT_L_SIZE - 1));

    p = *lp;

    if (!p) {

        if (!alloc)

            return NULL;

        p = qemu_mallocz(sizeof(void *) * VIRT_L_SIZE);

        *lp = p;

    }

    lp = p + ((index >> (2 * VIRT_L_BITS)) & (VIRT_L_SIZE - 1));

    p = *lp;

    if (!p) {

        if (!alloc)

            return NULL;

        p = qemu_mallocz(sizeof(void *) * VIRT_L_SIZE);

        *lp = p;

    }

    lp = p + ((index >> (1 * VIRT_L_BITS)) & (VIRT_L_SIZE - 1));

    p = *lp;

    if (!p) {

        if (!alloc)

            return NULL;

        p = qemu_mallocz(sizeof(VirtPageDesc) * VIRT_L_SIZE);

        *lp = p;

    }

    return ((VirtPageDesc *)p) + (index & (VIRT_L_SIZE - 1));

#else

    VirtPageDesc *p, **lp;

    lp = &l1_virt_map[index >> L2_BITS];

    p = *lp;

    if (!p) {

        /* allocate if not found */

        if (!alloc)

            return NULL;

        p = qemu_mallocz(sizeof(VirtPageDesc) * L2_SIZE);

        *lp = p;

    }

    return p + (index & (L2_SIZE - 1));

#endif

}

static inline VirtPageDesc *virt_page_find(target_ulong index)

{

    return virt_page_find_alloc(index, 0);

}

#if TARGET_LONG_BITS > 32

static void virt_page_flush_internal(void **p, int level)

{

    int i; 

    if (level == 0) {

        VirtPageDesc *q = (VirtPageDesc *)p;

        for(i = 0; i < VIRT_L_SIZE; i++)

            q[i].valid_tag = 0;

    } else {

        level--;

        for(i = 0; i < VIRT_L_SIZE; i++) {

            if (p[i])

                virt_page_flush_internal(p[i], level);

        }

    }

}

#endif

static void virt_page_flush(void)

{

    virt_valid_tag++;

    if (virt_valid_tag == 0) {

        virt_valid_tag = 1;

#if TARGET_LONG_BITS > 32

        virt_page_flush_internal(l1_virt_map, 5);

#else

        {

            int i, j;

            VirtPageDesc *p;

            for(i = 0; i < L1_SIZE; i++) {

                p = l1_virt_map[i];

                if (p) {

                    for(j = 0; j < L2_SIZE; j++)

                        p[j].valid_tag = 0;

                }

            }

        }

#endif

    }

}

#else

static void virt_page_flush(void)

{

}

#endif

void cpu_exec_init(void)

{

    if (!code_gen_ptr) {

        code_gen_ptr = code_gen_buffer;

        page_init();

        io_mem_init();

    }

}

static inline void invalidate_page_bitmap(PageDesc *p)

{

    if (p->code_bitmap) {

        qemu_free(p->code_bitmap);

        p->code_bitmap = NULL;

    }

    p->code_write_count = 0;

}

/* set to NULL all the 'first_tb' fields in all PageDescs */

static void page_flush_tb(void)

{

    int i, j;

    PageDesc *p;

    for(i = 0; i < L1_SIZE; i++) {

        p = l1_map[i];

        if (p) {

            for(j = 0; j < L2_SIZE; j++) {

                p->first_tb = NULL;

                invalidate_page_bitmap(p);

                p++;

            }

        }

    }

}

/* flush all the translation blocks */

/* XXX: tb_flush is currently not thread safe */

void tb_flush(CPUState *env)

{

#if defined(DEBUG_FLUSH)

    printf("qemu: flush code_size=%d nb_tbs=%d avg_tb_size=%d\n", 

           code_gen_ptr - code_gen_buffer, 

           nb_tbs, 

           nb_tbs > 0 ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0);

#endif

    nb_tbs = 0;

    memset (tb_hash, 0, CODE_GEN_HASH_SIZE * sizeof (void *));

    virt_page_flush();

    memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));

    page_flush_tb();

    code_gen_ptr = code_gen_buffer;

    /* XXX: flush processor icache at this point if cache flush is

       expensive */

    tb_flush_count++;

}

#ifdef DEBUG_TB_CHECK

static void tb_invalidate_check(unsigned long address)

{

    TranslationBlock *tb;

    int i;

    address &= TARGET_PAGE_MASK;

    for(i = 0;i < CODE_GEN_HASH_SIZE; i++) {

        for(tb = tb_hash[i]; tb != NULL; tb = tb->hash_next) {

            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||

                  address >= tb->pc + tb->size)) {

                printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",

                       address, tb->pc, tb->size);

            }

        }

    }

}

/* verify that all the pages have correct rights for code */

static void tb_page_check(void)

{

    TranslationBlock *tb;

    int i, flags1, flags2;

    for(i = 0;i < CODE_GEN_HASH_SIZE; i++) {

        for(tb = tb_hash[i]; tb != NULL; tb = tb->hash_next) {

            flags1 = page_get_flags(tb->pc);

            flags2 = page_get_flags(tb->pc + tb->size - 1);

            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {

                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",

                       tb->pc, tb->size, flags1, flags2);

            }

        }

    }

}

void tb_jmp_check(TranslationBlock *tb)

{

    TranslationBlock *tb1;

    unsigned int n1;

    /* suppress any remaining jumps to this TB */

    tb1 = tb->jmp_first;

    for(;;) {

        n1 = (long)tb1 & 3;

        tb1 = (TranslationBlock *)((long)tb1 & ~3);

        if (n1 == 2)

            break;

        tb1 = tb1->jmp_next[n1];

    }

    /* check end of list */

    if (tb1 != tb) {

        printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);

    }

}

#endif

/* invalidate one TB */

static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,

                             int next_offset)

{

    TranslationBlock *tb1;

    for(;;) {

        tb1 = *ptb;

        if (tb1 == tb) {

            *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);

            break;

        }

        ptb = (TranslationBlock **)((char *)tb1 + next_offset);

    }

}

static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)

{

    TranslationBlock *tb1;

    unsigned int n1;

    for(;;) {

        tb1 = *ptb;

        n1 = (long)tb1 & 3;

        tb1 = (TranslationBlock *)((long)tb1 & ~3);

        if (tb1 == tb) {

            *ptb = tb1->page_next[n1];

            break;

        }

        ptb = &tb1->page_next[n1];

    }

}

static inline void tb_jmp_remove(TranslationBlock *tb, int n)

{

    TranslationBlock *tb1, **ptb;

    unsigned int n1;

    ptb = &tb->jmp_next[n];

    tb1 = *ptb;

    if (tb1) {

        /* find tb(n) in circular list */

        for(;;) {

            tb1 = *ptb;

            n1 = (long)tb1 & 3;

            tb1 = (TranslationBlock *)((long)tb1 & ~3);

            if (n1 == n && tb1 == tb)

                break;

            if (n1 == 2) {

                ptb = &tb1->jmp_first;

            } else {

                ptb = &tb1->jmp_next[n1];

            }

        }

        /* now we can suppress tb(n) from the list */

        *ptb = tb->jmp_next[n];

        tb->jmp_next[n] = NULL;

    }

}

/* reset the jump entry 'n' of a TB so that it is not chained to

   another TB */

static inline void tb_reset_jump(TranslationBlock *tb, int n)

{

    tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));

}

static inline void tb_invalidate(TranslationBlock *tb)

{

    unsigned int h, n1;

    TranslationBlock *tb1, *tb2, **ptb;

    tb_invalidated_flag = 1;

    /* remove the TB from the hash list */

    h = tb_hash_func(tb->pc);

    ptb = &tb_hash[h];

    for(;;) {

        tb1 = *ptb;

        /* NOTE: the TB is not necessarily linked in the hash. It

           indicates that it is not currently used */

        if (tb1 == NULL)

            return;

        if (tb1 == tb) {

            *ptb = tb1->hash_next;

            break;

        }

        ptb = &tb1->hash_next;

    }

    /* suppress this TB from the two jump lists */

    tb_jmp_remove(tb, 0);

    tb_jmp_remove(tb, 1);

    /* suppress any remaining jumps to this TB */

    tb1 = tb->jmp_first;

    for(;;) {

        n1 = (long)tb1 & 3;

        if (n1 == 2)

            break;

        tb1 = (TranslationBlock *)((long)tb1 & ~3);

        tb2 = tb1->jmp_next[n1];

        tb_reset_jump(tb1, n1);

        tb1->jmp_next[n1] = NULL;

        tb1 = tb2;

    }

    tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */

}

static inline void tb_phys_invalidate(TranslationBlock *tb, unsigned int page_addr)

{

    PageDesc *p;

    unsigned int h;

    target_ulong phys_pc;

    /* remove the TB from the hash list */

    phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);

    h = tb_phys_hash_func(phys_pc);

    tb_remove(&tb_phys_hash[h], tb, 

              offsetof(TranslationBlock, phys_hash_next));

    /* remove the TB from the page list */

    if (tb->page_addr[0] != page_addr) {

        p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);

        tb_page_remove(&p->first_tb, tb);

        invalidate_page_bitmap(p);

    }

    if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {

        p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);

        tb_page_remove(&p->first_tb, tb);

        invalidate_page_bitmap(p);

    }

    tb_invalidate(tb);

    tb_phys_invalidate_count++;

}

static inline void set_bits(uint8_t *tab, int start, int len)

{

    int end, mask, end1;

    end = start + len;

    tab += start >> 3;

    mask = 0xff << (start & 7);

    if ((start & ~7) == (end & ~7)) {

        if (start < end) {

            mask &= ~(0xff << (end & 7));

            *tab |= mask;

        }

    } else {

        *tab++ |= mask;

        start = (start + 8) & ~7;

        end1 = end & ~7;

        while (start < end1) {

            *tab++ = 0xff;

            start += 8;

        }

        if (start < end) {

            mask = ~(0xff << (end & 7));

            *tab |= mask;

        }

    }

}

static void build_page_bitmap(PageDesc *p)

{

    int n, tb_start, tb_end;

    TranslationBlock *tb;

    p->code_bitmap = qemu_malloc(TARGET_PAGE_SIZE / 8);

    if (!p->code_bitmap)

        return;

    memset(p->code_bitmap, 0, TARGET_PAGE_SIZE / 8);

    tb = p->first_tb;

    while (tb != NULL) {

        n = (long)tb & 3;

        tb = (TranslationBlock *)((long)tb & ~3);

        /* NOTE: this is subtle as a TB may span two physical pages */

        if (n == 0) {

            /* NOTE: tb_end may be after the end of the page, but

               it is not a problem */

            tb_start = tb->pc & ~TARGET_PAGE_MASK;

            tb_end = tb_start + tb->size;

            if (tb_end > TARGET_PAGE_SIZE)

                tb_end = TARGET_PAGE_SIZE;

        } else {

            tb_start = 0;

            tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);

        }

        set_bits(p->code_bitmap, tb_start, tb_end - tb_start);

        tb = tb->page_next[n];

    }

}

#ifdef TARGET_HAS_PRECISE_SMC

static void tb_gen_code(CPUState *env, 

                        target_ulong pc, target_ulong cs_base, int flags,

                        int cflags)

{

    TranslationBlock *tb;

    uint8_t *tc_ptr;

    target_ulong phys_pc, phys_page2, virt_page2;

    int code_gen_size;

    phys_pc = get_phys_addr_code(env, pc);

    tb = tb_alloc(pc);

    if (!tb) {

        /* flush must be done */

        tb_flush(env);

        /* cannot fail at this point */

        tb = tb_alloc(pc);

    }

    tc_ptr = code_gen_ptr;

    tb->tc_ptr = tc_ptr;

    tb->cs_base = cs_base;

    tb->flags = flags;

    tb->cflags = cflags;

    cpu_gen_code(env, tb, CODE_GEN_MAX_SIZE, &code_gen_size);

    code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));

    /* check next page if needed */

    virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;

    phys_page2 = -1;

    if ((pc & TARGET_PAGE_MASK) != virt_page2) {

        phys_page2 = get_phys_addr_code(env, virt_page2);

    }

    tb_link_phys(tb, phys_pc, phys_page2);

}

#endif

/* invalidate all TBs which intersect with the target physical page

   starting in range [start;end[. NOTE: start and end must refer to

   the same physical page. 'is_cpu_write_access' should be true if called

   from a real cpu write access: the virtual CPU will exit the current

   TB if code is modified inside this TB. */

void tb_invalidate_phys_page_range(target_ulong start, target_ulong end, 

                                   int is_cpu_write_access)

{

    int n, current_tb_modified, current_tb_not_found, current_flags;

    CPUState *env = cpu_single_env;

    PageDesc *p;

    TranslationBlock *tb, *tb_next, *current_tb, *saved_tb;

    target_ulong tb_start, tb_end;

    target_ulong current_pc, current_cs_base;

    p = page_find(start >> TARGET_PAGE_BITS);

    if (!p) 

        return;

    if (!p->code_bitmap && 

        ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&

        is_cpu_write_access) {

        /* build code bitmap */

        build_page_bitmap(p);

    }

    /* we remove all the TBs in the range [start, end[ */

    /* XXX: see if in some cases it could be faster to invalidate all the code */

    current_tb_not_found = is_cpu_write_access;

    current_tb_modified = 0;

    current_tb = NULL; /* avoid warning */

    current_pc = 0; /* avoid warning */

    current_cs_base = 0; /* avoid warning */

    current_flags = 0; /* avoid warning */

    tb = p->first_tb;

    while (tb != NULL) {

        n = (long)tb & 3;

        tb = (TranslationBlock *)((long)tb & ~3);

        tb_next = tb->page_next[n];

        /* NOTE: this is subtle as a TB may span two physical pages */

        if (n == 0) {

            /* NOTE: tb_end may be after the end of the page, but

               it is not a problem */

            tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);

            tb_end = tb_start + tb->size;

        } else {

            tb_start = tb->page_addr[1];

            tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);

        }

        if (!(tb_end <= start || tb_start >= end)) {

#ifdef TARGET_HAS_PRECISE_SMC

            if (current_tb_not_found) {

                current_tb_not_found = 0;

                current_tb = NULL;

                if (env->mem_write_pc) {

                    /* now we have a real cpu fault */

                    current_tb = tb_find_pc(env->mem_write_pc);

                }

            }

            if (current_tb == tb &&

                !(current_tb->cflags & CF_SINGLE_INSN)) {

                /* If we are modifying the current TB, we must stop

                its execution. We could be more precise by checking

                that the modification is after the current PC, but it

                would require a specialized function to partially

                restore the CPU state */

                current_tb_modified = 1;

                cpu_restore_state(current_tb, env, 

                                  env->mem_write_pc, NULL);

#if defined(TARGET_I386)

                current_flags = env->hflags;

                current_flags |= (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK));

                current_cs_base = (target_ulong)env->segs[R_CS].base;

                current_pc = current_cs_base + env->eip;

#else

#error unsupported CPU

#endif

            }

#endif /* TARGET_HAS_PRECISE_SMC */

            saved_tb = env->current_tb;

            env->current_tb = NULL;

            tb_phys_invalidate(tb, -1);

            env->current_tb = saved_tb;

            if (env->interrupt_request && env->current_tb)

                cpu_interrupt(env, env->interrupt_request);

        }

        tb = tb_next;

    }

#if !defined(CONFIG_USER_ONLY)

    /* if no code remaining, no need to continue to use slow writes */

    if (!p->first_tb) {

        invalidate_page_bitmap(p);

        if (is_cpu_write_access) {

            tlb_unprotect_code_phys(env, start, env->mem_write_vaddr);

        }

    }

#endif

#ifdef TARGET_HAS_PRECISE_SMC

    if (current_tb_modified) {

        /* we generate a block containing just the instruction

           modifying the memory. It will ensure that it cannot modify

           itself */

        env->current_tb = NULL;

        tb_gen_code(env, current_pc, current_cs_base, current_flags, 

                    CF_SINGLE_INSN);

        cpu_resume_from_signal(env, NULL);

    }

#endif

}

/* len must be <= 8 and start must be a multiple of len */

static inline void tb_invalidate_phys_page_fast(target_ulong start, int len)

{

    PageDesc *p;

    int offset, b;

#if 0

    if (1) {

        if (loglevel) {

            fprintf(logfile, "modifying code at 0x%x size=%d EIP=%x PC=%08x\n", 

                   cpu_single_env->mem_write_vaddr, len, 

                   cpu_single_env->eip, 

                   cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);

        }

    }

#endif

    p = page_find(start >> TARGET_PAGE_BITS);

    if (!p) 

        return;

    if (p->code_bitmap) {

        offset = start & ~TARGET_PAGE_MASK;

        b = p->code_bitmap[offset >> 3] >> (offset & 7);

        if (b & ((1 << len) - 1))

            goto do_invalidate;

    } else {

    do_invalidate:

        tb_invalidate_phys_page_range(start, start + len, 1);

    }

}

#if !defined(CONFIG_SOFTMMU)

static void tb_invalidate_phys_page(target_ulong addr, 

                                    unsigned long pc, void *puc)

{

    int n, current_flags, current_tb_modified;

    target_ulong current_pc, current_cs_base;

    PageDesc *p;

    TranslationBlock *tb, *current_tb;

#ifdef TARGET_HAS_PRECISE_SMC

    CPUState *env = cpu_single_env;

#endif

    addr &= TARGET_PAGE_MASK;

    p = page_find(addr >> TARGET_PAGE_BITS);

    if (!p) 

        return;

    tb = p->first_tb;

    current_tb_modified = 0;

    current_tb = NULL;

    current_pc = 0; /* avoid warning */

    current_cs_base = 0; /* avoid warning */

    current_flags = 0; /* avoid warning */

#ifdef TARGET_HAS_PRECISE_SMC

    if (tb && pc != 0) {

        current_tb = tb_find_pc(pc);

    }

#endif

    while (tb != NULL) {