Archipelago » qemu

/*

 *  virtual page mapping and translated block handling

 *

 *  Copyright (c) 2003 Fabrice Bellard

 *

 * This library is free software; you can redistribute it and/or

 * modify it under the terms of the GNU Lesser General Public

 * License as published by the Free Software Foundation; either

 * version 2 of the License, or (at your option) any later version.

 *

 * This library is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * Lesser General Public License for more details.

 *

 * You should have received a copy of the GNU Lesser General Public

 * License along with this library; if not, see <http://www.gnu.org/licenses/>.

 */

#include "config.h"

#ifdef _WIN32

#include <windows.h>

#else

#include <sys/types.h>

#include <sys/mman.h>

#endif

#include <stdlib.h>

#include <stdio.h>

#include <stdarg.h>

#include <string.h>

#include <errno.h>

#include <unistd.h>

#include <inttypes.h>

#include "cpu.h"

#include "exec-all.h"

#include "qemu-common.h"

#include "tcg.h"

#include "hw/hw.h"

#include "osdep.h"

#include "kvm.h"

#if defined(CONFIG_USER_ONLY)

#include <qemu.h>

#endif

//#define DEBUG_TB_INVALIDATE

//#define DEBUG_FLUSH

//#define DEBUG_TLB

//#define DEBUG_UNASSIGNED

/* make various TB consistency checks */

//#define DEBUG_TB_CHECK

//#define DEBUG_TLB_CHECK

//#define DEBUG_IOPORT

//#define DEBUG_SUBPAGE

#if !defined(CONFIG_USER_ONLY)

/* TB consistency checks only implemented for usermode emulation.  */

#undef DEBUG_TB_CHECK

#endif

#define SMC_BITMAP_USE_THRESHOLD 10

#if defined(TARGET_SPARC64)

#define TARGET_PHYS_ADDR_SPACE_BITS 41

#elif defined(TARGET_SPARC)

#define TARGET_PHYS_ADDR_SPACE_BITS 36

#elif defined(TARGET_ALPHA)

#define TARGET_PHYS_ADDR_SPACE_BITS 42

#define TARGET_VIRT_ADDR_SPACE_BITS 42

#elif defined(TARGET_PPC64)

#define TARGET_PHYS_ADDR_SPACE_BITS 42

#elif defined(TARGET_X86_64)

#define TARGET_PHYS_ADDR_SPACE_BITS 42

#elif defined(TARGET_I386)

#define TARGET_PHYS_ADDR_SPACE_BITS 36

#else

#define TARGET_PHYS_ADDR_SPACE_BITS 32

#endif

static TranslationBlock *tbs;

int code_gen_max_blocks;

TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];

static int nb_tbs;

/* any access to the tbs or the page table must use this lock */

spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;

#if defined(__arm__) || defined(__sparc_v9__)

/* The prologue must be reachable with a direct jump. ARM and Sparc64

 have limited branch ranges (possibly also PPC) so place it in a

 section close to code segment. */

#define code_gen_section                                \

    __attribute__((__section__(".gen_code")))           \

    __attribute__((aligned (32)))

#elif defined(_WIN32)

/* Maximum alignment for Win32 is 16. */

#define code_gen_section                                \

    __attribute__((aligned (16)))

#else

#define code_gen_section                                \

    __attribute__((aligned (32)))

#endif

uint8_t code_gen_prologue[1024] code_gen_section;

static uint8_t *code_gen_buffer;

static unsigned long code_gen_buffer_size;

/* threshold to flush the translated code buffer */

static unsigned long code_gen_buffer_max_size;

uint8_t *code_gen_ptr;

#if !defined(CONFIG_USER_ONLY)

int phys_ram_fd;

uint8_t *phys_ram_dirty;

static int in_migration;

typedef struct RAMBlock {

    uint8_t *host;

    ram_addr_t offset;

    ram_addr_t length;

    struct RAMBlock *next;

} RAMBlock;

static RAMBlock *ram_blocks;

/* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)

   then we can no longer assume contiguous ram offsets, and external uses

   of this variable will break.  */

ram_addr_t last_ram_offset;

#endif

CPUState *first_cpu;

/* current CPU in the current thread. It is only valid inside

   cpu_exec() */

CPUState *cpu_single_env;

/* 0 = Do not count executed instructions.

   1 = Precise instruction counting.

   2 = Adaptive rate instruction counting.  */

int use_icount = 0;

/* Current instruction counter.  While executing translated code this may

   include some instructions that have not yet been executed.  */

int64_t qemu_icount;

typedef struct PageDesc {

    /* list of TBs intersecting this ram page */

    TranslationBlock *first_tb;

    /* in order to optimize self modifying code, we count the number

       of lookups we do to a given page to use a bitmap */

    unsigned int code_write_count;

    uint8_t *code_bitmap;

#if defined(CONFIG_USER_ONLY)

    unsigned long flags;

#endif

} PageDesc;

typedef struct PhysPageDesc {

    /* offset in host memory of the page + io_index in the low bits */

    ram_addr_t phys_offset;

    ram_addr_t region_offset;

} PhysPageDesc;

#define L2_BITS 10

#if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)

/* XXX: this is a temporary hack for alpha target.

 *      In the future, this is to be replaced by a multi-level table

 *      to actually be able to handle the complete 64 bits address space.

 */

#define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)

#else

#define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)

#endif

#define L1_SIZE (1 << L1_BITS)

#define L2_SIZE (1 << L2_BITS)

unsigned long qemu_real_host_page_size;

unsigned long qemu_host_page_bits;

unsigned long qemu_host_page_size;

unsigned long qemu_host_page_mask;

/* XXX: for system emulation, it could just be an array */

static PageDesc *l1_map[L1_SIZE];

static PhysPageDesc **l1_phys_map;

#if !defined(CONFIG_USER_ONLY)

static void io_mem_init(void);

/* io memory support */

CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];

CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];

void *io_mem_opaque[IO_MEM_NB_ENTRIES];

static char io_mem_used[IO_MEM_NB_ENTRIES];

static int io_mem_watch;

#endif

/* log support */

static const char *logfilename = "/tmp/qemu.log";

FILE *logfile;

int loglevel;

static int log_append = 0;

/* statistics */

static int tlb_flush_count;

static int tb_flush_count;

static int tb_phys_invalidate_count;

#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)

typedef struct subpage_t {

    target_phys_addr_t base;

    CPUReadMemoryFunc * const *mem_read[TARGET_PAGE_SIZE][4];

    CPUWriteMemoryFunc * const *mem_write[TARGET_PAGE_SIZE][4];

    void *opaque[TARGET_PAGE_SIZE][2][4];

    ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];

} subpage_t;

#ifdef _WIN32

static void map_exec(void *addr, long size)

{

    DWORD old_protect;

    VirtualProtect(addr, size,

                   PAGE_EXECUTE_READWRITE, &old_protect);

}

#else

static void map_exec(void *addr, long size)

{

    unsigned long start, end, page_size;

    page_size = getpagesize();

    start = (unsigned long)addr;

    start &= ~(page_size - 1);

    end = (unsigned long)addr + size;

    end += page_size - 1;

    end &= ~(page_size - 1);

    mprotect((void *)start, end - start,

             PROT_READ | PROT_WRITE | PROT_EXEC);

}

#endif

static void page_init(void)

{

    /* NOTE: we can always suppose that qemu_host_page_size >=

       TARGET_PAGE_SIZE */

#ifdef _WIN32

    {

        SYSTEM_INFO system_info;

        GetSystemInfo(&system_info);

        qemu_real_host_page_size = system_info.dwPageSize;

    }

#else

    qemu_real_host_page_size = getpagesize();

#endif

    if (qemu_host_page_size == 0)

        qemu_host_page_size = qemu_real_host_page_size;

    if (qemu_host_page_size < TARGET_PAGE_SIZE)

        qemu_host_page_size = TARGET_PAGE_SIZE;

    qemu_host_page_bits = 0;

    while ((1 << qemu_host_page_bits) < qemu_host_page_size)

        qemu_host_page_bits++;

    qemu_host_page_mask = ~(qemu_host_page_size - 1);

    l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));

    memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));

#if !defined(_WIN32) && defined(CONFIG_USER_ONLY)

    {

        long long startaddr, endaddr;

        FILE *f;

        int n;

        mmap_lock();

        last_brk = (unsigned long)sbrk(0);

        f = fopen("/proc/self/maps", "r");

        if (f) {

            do {

                n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);

                if (n == 2) {

                    startaddr = MIN(startaddr,

                                    (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);

                    endaddr = MIN(endaddr,

                                    (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);

                    page_set_flags(startaddr & TARGET_PAGE_MASK,

                                   TARGET_PAGE_ALIGN(endaddr),

                                   PAGE_RESERVED); 

                }

            } while (!feof(f));

            fclose(f);

        }

        mmap_unlock();

    }

#endif

}

static inline PageDesc **page_l1_map(target_ulong index)

{

#if TARGET_LONG_BITS > 32

    /* Host memory outside guest VM.  For 32-bit targets we have already

       excluded high addresses.  */

    if (index > ((target_ulong)L2_SIZE * L1_SIZE))

        return NULL;

#endif

    return &l1_map[index >> L2_BITS];

}

static inline PageDesc *page_find_alloc(target_ulong index)

{

    PageDesc **lp, *p;

    lp = page_l1_map(index);

    if (!lp)

        return NULL;

    p = *lp;

    if (!p) {

        /* allocate if not found */

#if defined(CONFIG_USER_ONLY)

        size_t len = sizeof(PageDesc) * L2_SIZE;

        /* Don't use qemu_malloc because it may recurse.  */

        p = mmap(NULL, len, PROT_READ | PROT_WRITE,

                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);

        *lp = p;

        if (h2g_valid(p)) {

            unsigned long addr = h2g(p);

            page_set_flags(addr & TARGET_PAGE_MASK,

                           TARGET_PAGE_ALIGN(addr + len),

                           PAGE_RESERVED); 

        }

#else

        p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);

        *lp = p;

#endif

    }

    return p + (index & (L2_SIZE - 1));

}

static inline PageDesc *page_find(target_ulong index)

{

    PageDesc **lp, *p;

    lp = page_l1_map(index);

    if (!lp)

        return NULL;

    p = *lp;

    if (!p) {

        return NULL;

    }

    return p + (index & (L2_SIZE - 1));

}

static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)

{

    void **lp, **p;

    PhysPageDesc *pd;

    p = (void **)l1_phys_map;

#if TARGET_PHYS_ADDR_SPACE_BITS > 32

#if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)

#error unsupported TARGET_PHYS_ADDR_SPACE_BITS

#endif

    lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));

    p = *lp;

    if (!p) {

        /* allocate if not found */

        if (!alloc)

            return NULL;

        p = qemu_vmalloc(sizeof(void *) * L1_SIZE);

        memset(p, 0, sizeof(void *) * L1_SIZE);

        *lp = p;

    }

#endif

    lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));

    pd = *lp;

    if (!pd) {

        int i;

        /* allocate if not found */

        if (!alloc)

            return NULL;

        pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);

        *lp = pd;

        for (i = 0; i < L2_SIZE; i++) {

          pd[i].phys_offset = IO_MEM_UNASSIGNED;

          pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;

        }

    }

    return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));

}

static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)

{

    return phys_page_find_alloc(index, 0);

}

#if !defined(CONFIG_USER_ONLY)

static void tlb_protect_code(ram_addr_t ram_addr);

static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,

                                    target_ulong vaddr);

#define mmap_lock() do { } while(0)

#define mmap_unlock() do { } while(0)

#endif

#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)

#if defined(CONFIG_USER_ONLY)

/* Currently it is not recommended to allocate big chunks of data in

   user mode. It will change when a dedicated libc will be used */

#define USE_STATIC_CODE_GEN_BUFFER

#endif

#ifdef USE_STATIC_CODE_GEN_BUFFER

static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];

#endif

static void code_gen_alloc(unsigned long tb_size)

{

#ifdef USE_STATIC_CODE_GEN_BUFFER

    code_gen_buffer = static_code_gen_buffer;

    code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;

    map_exec(code_gen_buffer, code_gen_buffer_size);

#else

    code_gen_buffer_size = tb_size;

    if (code_gen_buffer_size == 0) {

#if defined(CONFIG_USER_ONLY)

        /* in user mode, phys_ram_size is not meaningful */

        code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;

#else

        /* XXX: needs adjustments */

        code_gen_buffer_size = (unsigned long)(ram_size / 4);

#endif

    }

    if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)

        code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;

    /* The code gen buffer location may have constraints depending on

       the host cpu and OS */

#if defined(__linux__) 

    {

        int flags;

        void *start = NULL;

        flags = MAP_PRIVATE | MAP_ANONYMOUS;

#if defined(__x86_64__)

        flags |= MAP_32BIT;

        /* Cannot map more than that */

        if (code_gen_buffer_size > (800 * 1024 * 1024))

            code_gen_buffer_size = (800 * 1024 * 1024);

#elif defined(__sparc_v9__)

        // Map the buffer below 2G, so we can use direct calls and branches

        flags |= MAP_FIXED;

        start = (void *) 0x60000000UL;

        if (code_gen_buffer_size > (512 * 1024 * 1024))

            code_gen_buffer_size = (512 * 1024 * 1024);

#elif defined(__arm__)

        /* Map the buffer below 32M, so we can use direct calls and branches */

        flags |= MAP_FIXED;

        start = (void *) 0x01000000UL;

        if (code_gen_buffer_size > 16 * 1024 * 1024)

            code_gen_buffer_size = 16 * 1024 * 1024;

#endif

        code_gen_buffer = mmap(start, code_gen_buffer_size,

                               PROT_WRITE | PROT_READ | PROT_EXEC,

                               flags, -1, 0);

        if (code_gen_buffer == MAP_FAILED) {

            fprintf(stderr, "Could not allocate dynamic translator buffer\n");

            exit(1);

        }

    }

#elif defined(__FreeBSD__) || defined(__DragonFly__)

    {

        int flags;

        void *addr = NULL;

        flags = MAP_PRIVATE | MAP_ANONYMOUS;

#if defined(__x86_64__)

        /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume

         * 0x40000000 is free */

        flags |= MAP_FIXED;

        addr = (void *)0x40000000;

        /* Cannot map more than that */

        if (code_gen_buffer_size > (800 * 1024 * 1024))

            code_gen_buffer_size = (800 * 1024 * 1024);

#endif

        code_gen_buffer = mmap(addr, code_gen_buffer_size,

                               PROT_WRITE | PROT_READ | PROT_EXEC, 

                               flags, -1, 0);

        if (code_gen_buffer == MAP_FAILED) {

            fprintf(stderr, "Could not allocate dynamic translator buffer\n");

            exit(1);

        }

    }

#else

    code_gen_buffer = qemu_malloc(code_gen_buffer_size);

    map_exec(code_gen_buffer, code_gen_buffer_size);

#endif

#endif /* !USE_STATIC_CODE_GEN_BUFFER */

    map_exec(code_gen_prologue, sizeof(code_gen_prologue));

    code_gen_buffer_max_size = code_gen_buffer_size - 

        code_gen_max_block_size();

    code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;

    tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));

}

/* Must be called before using the QEMU cpus. 'tb_size' is the size

   (in bytes) allocated to the translation buffer. Zero means default

   size. */

void cpu_exec_init_all(unsigned long tb_size)

{

    cpu_gen_init();

    code_gen_alloc(tb_size);

    code_gen_ptr = code_gen_buffer;

    page_init();

#if !defined(CONFIG_USER_ONLY)

    io_mem_init();

#endif

}

#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)

static void cpu_common_pre_save(const void *opaque)

{

    CPUState *env = (void *)opaque;

    cpu_synchronize_state(env);

}

static int cpu_common_pre_load(void *opaque)

{

    CPUState *env = opaque;

    cpu_synchronize_state(env);

    return 0;

}

static int cpu_common_post_load(void *opaque)

{

    CPUState *env = opaque;

    /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the

       version_id is increased. */

    env->interrupt_request &= ~0x01;

    tlb_flush(env, 1);

    return 0;

}

static const VMStateDescription vmstate_cpu_common = {

    .name = "cpu_common",

    .version_id = 1,

    .minimum_version_id = 1,

    .minimum_version_id_old = 1,

    .pre_save = cpu_common_pre_save,

    .pre_load = cpu_common_pre_load,

    .post_load = cpu_common_post_load,

    .fields      = (VMStateField []) {

        VMSTATE_UINT32(halted, CPUState),

        VMSTATE_UINT32(interrupt_request, CPUState),

        VMSTATE_END_OF_LIST()

    }

};

#endif

CPUState *qemu_get_cpu(int cpu)

{

    CPUState *env = first_cpu;

    while (env) {

        if (env->cpu_index == cpu)

            break;

        env = env->next_cpu;

    }

    return env;

}

void cpu_exec_init(CPUState *env)

{

    CPUState **penv;

    int cpu_index;

#if defined(CONFIG_USER_ONLY)

    cpu_list_lock();

#endif

    env->next_cpu = NULL;

    penv = &first_cpu;

    cpu_index = 0;

    while (*penv != NULL) {

        penv = &(*penv)->next_cpu;

        cpu_index++;

    }

    env->cpu_index = cpu_index;

    env->numa_node = 0;

    QTAILQ_INIT(&env->breakpoints);

    QTAILQ_INIT(&env->watchpoints);

    *penv = env;

#if defined(CONFIG_USER_ONLY)

    cpu_list_unlock();

#endif

#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)

    vmstate_register(cpu_index, &vmstate_cpu_common, env);

    register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,

                    cpu_save, cpu_load, env);

#endif

}

static inline void invalidate_page_bitmap(PageDesc *p)

{

    if (p->code_bitmap) {

        qemu_free(p->code_bitmap);

        p->code_bitmap = NULL;

    }

    p->code_write_count = 0;

}

/* set to NULL all the 'first_tb' fields in all PageDescs */

static void page_flush_tb(void)

{

    int i, j;

    PageDesc *p;

    for(i = 0; i < L1_SIZE; i++) {

        p = l1_map[i];

        if (p) {

            for(j = 0; j < L2_SIZE; j++) {

                p->first_tb = NULL;

                invalidate_page_bitmap(p);

                p++;

            }

        }

    }

}

/* flush all the translation blocks */

/* XXX: tb_flush is currently not thread safe */

void tb_flush(CPUState *env1)

{

    CPUState *env;

#if defined(DEBUG_FLUSH)

    printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",

           (unsigned long)(code_gen_ptr - code_gen_buffer),

           nb_tbs, nb_tbs > 0 ?

           ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);

#endif

    if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)

        cpu_abort(env1, "Internal error: code buffer overflow\n");

    nb_tbs = 0;

    for(env = first_cpu; env != NULL; env = env->next_cpu) {

        memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));

    }

    memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));

    page_flush_tb();

    code_gen_ptr = code_gen_buffer;

    /* XXX: flush processor icache at this point if cache flush is

       expensive */

    tb_flush_count++;

}

#ifdef DEBUG_TB_CHECK

static void tb_invalidate_check(target_ulong address)

{

    TranslationBlock *tb;

    int i;

    address &= TARGET_PAGE_MASK;

    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {

        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {

            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||

                  address >= tb->pc + tb->size)) {

                printf("ERROR invalidate: address=" TARGET_FMT_lx

                       " PC=%08lx size=%04x\n",

                       address, (long)tb->pc, tb->size);

            }

        }

    }

}

/* verify that all the pages have correct rights for code */

static void tb_page_check(void)

{

    TranslationBlock *tb;

    int i, flags1, flags2;

    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {

        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {

            flags1 = page_get_flags(tb->pc);

            flags2 = page_get_flags(tb->pc + tb->size - 1);

            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {

                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",

                       (long)tb->pc, tb->size, flags1, flags2);

            }

        }

    }

}

#endif

/* invalidate one TB */

static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,

                             int next_offset)

{

    TranslationBlock *tb1;

    for(;;) {

        tb1 = *ptb;

        if (tb1 == tb) {

            *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);

            break;

        }

        ptb = (TranslationBlock **)((char *)tb1 + next_offset);

    }

}

static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)

{

    TranslationBlock *tb1;

    unsigned int n1;

    for(;;) {

        tb1 = *ptb;

        n1 = (long)tb1 & 3;

        tb1 = (TranslationBlock *)((long)tb1 & ~3);

        if (tb1 == tb) {

            *ptb = tb1->page_next[n1];

            break;

        }

        ptb = &tb1->page_next[n1];

    }

}

static inline void tb_jmp_remove(TranslationBlock *tb, int n)

{

    TranslationBlock *tb1, **ptb;

    unsigned int n1;

    ptb = &tb->jmp_next[n];

    tb1 = *ptb;

    if (tb1) {

        /* find tb(n) in circular list */

        for(;;) {

            tb1 = *ptb;

            n1 = (long)tb1 & 3;

            tb1 = (TranslationBlock *)((long)tb1 & ~3);

            if (n1 == n && tb1 == tb)

                break;

            if (n1 == 2) {

                ptb = &tb1->jmp_first;

            } else {

                ptb = &tb1->jmp_next[n1];

            }

        }

        /* now we can suppress tb(n) from the list */

        *ptb = tb->jmp_next[n];

        tb->jmp_next[n] = NULL;

    }

}

/* reset the jump entry 'n' of a TB so that it is not chained to

   another TB */

static inline void tb_reset_jump(TranslationBlock *tb, int n)

{

    tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));

}

void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)

{

    CPUState *env;

    PageDesc *p;

    unsigned int h, n1;

    target_phys_addr_t phys_pc;

    TranslationBlock *tb1, *tb2;

    /* remove the TB from the hash list */

    phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);

    h = tb_phys_hash_func(phys_pc);

    tb_remove(&tb_phys_hash[h], tb,

              offsetof(TranslationBlock, phys_hash_next));

    /* remove the TB from the page list */

    if (tb->page_addr[0] != page_addr) {

        p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);

        tb_page_remove(&p->first_tb, tb);

        invalidate_page_bitmap(p);

    }

    if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {

        p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);

        tb_page_remove(&p->first_tb, tb);

        invalidate_page_bitmap(p);

    }

    tb_invalidated_flag = 1;

    /* remove the TB from the hash list */

    h = tb_jmp_cache_hash_func(tb->pc);

    for(env = first_cpu; env != NULL; env = env->next_cpu) {

        if (env->tb_jmp_cache[h] == tb)

            env->tb_jmp_cache[h] = NULL;

    }

    /* suppress this TB from the two jump lists */

    tb_jmp_remove(tb, 0);

    tb_jmp_remove(tb, 1);

    /* suppress any remaining jumps to this TB */

    tb1 = tb->jmp_first;

    for(;;) {

        n1 = (long)tb1 & 3;

        if (n1 == 2)

            break;

        tb1 = (TranslationBlock *)((long)tb1 & ~3);

        tb2 = tb1->jmp_next[n1];

        tb_reset_jump(tb1, n1);

        tb1->jmp_next[n1] = NULL;

        tb1 = tb2;

    }

    tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */

    tb_phys_invalidate_count++;

}

static inline void set_bits(uint8_t *tab, int start, int len)

{

    int end, mask, end1;

    end = start + len;

    tab += start >> 3;

    mask = 0xff << (start & 7);

    if ((start & ~7) == (end & ~7)) {

        if (start < end) {

            mask &= ~(0xff << (end & 7));

            *tab |= mask;

        }

    } else {

        *tab++ |= mask;

        start = (start + 8) & ~7;

        end1 = end & ~7;

        while (start < end1) {

            *tab++ = 0xff;

            start += 8;

        }

        if (start < end) {

            mask = ~(0xff << (end & 7));

            *tab |= mask;

        }

    }

}

static void build_page_bitmap(PageDesc *p)

{

    int n, tb_start, tb_end;

    TranslationBlock *tb;

    p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);

    tb = p->first_tb;

    while (tb != NULL) {

        n = (long)tb & 3;

        tb = (TranslationBlock *)((long)tb & ~3);

        /* NOTE: this is subtle as a TB may span two physical pages */

        if (n == 0) {

            /* NOTE: tb_end may be after the end of the page, but

               it is not a problem */

            tb_start = tb->pc & ~TARGET_PAGE_MASK;

            tb_end = tb_start + tb->size;

            if (tb_end > TARGET_PAGE_SIZE)

                tb_end = TARGET_PAGE_SIZE;

        } else {

            tb_start = 0;

            tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);

        }

        set_bits(p->code_bitmap, tb_start, tb_end - tb_start);

        tb = tb->page_next[n];

    }

}

TranslationBlock *tb_gen_code(CPUState *env,

                              target_ulong pc, target_ulong cs_base,

                              int flags, int cflags)

{

    TranslationBlock *tb;

    uint8_t *tc_ptr;

    target_ulong phys_pc, phys_page2, virt_page2;

    int code_gen_size;

    phys_pc = get_phys_addr_code(env, pc);

    tb = tb_alloc(pc);

    if (!tb) {

        /* flush must be done */

        tb_flush(env);

        /* cannot fail at this point */

        tb = tb_alloc(pc);

        /* Don't forget to invalidate previous TB info.  */