Archipelago » qemu

/*

 *  virtual page mapping and translated block handling

 *

 *  Copyright (c) 2003 Fabrice Bellard

 *

 * This library is free software; you can redistribute it and/or

 * modify it under the terms of the GNU Lesser General Public

 * License as published by the Free Software Foundation; either

 * version 2 of the License, or (at your option) any later version.

 *

 * This library is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * Lesser General Public License for more details.

 *

 * You should have received a copy of the GNU Lesser General Public

 * License along with this library; if not, see <http://www.gnu.org/licenses/>.

 */

#include "config.h"

#ifdef _WIN32

#include <windows.h>

#else

#include <sys/types.h>

#include <sys/mman.h>

#endif

#include "qemu-common.h"

#include "cpu.h"

#include "tcg.h"

#include "hw/hw.h"

#include "hw/qdev.h"

#include "osdep.h"

#include "kvm.h"

#include "hw/xen.h"

#include "qemu-timer.h"

#include "memory.h"

#include "exec-memory.h"

#if defined(CONFIG_USER_ONLY)

#include <qemu.h>

#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)

#include <sys/param.h>

#if __FreeBSD_version >= 700104

#define HAVE_KINFO_GETVMMAP

#define sigqueue sigqueue_freebsd  /* avoid redefinition */

#include <sys/time.h>

#include <sys/proc.h>

#include <machine/profile.h>

#define _KERNEL

#include <sys/user.h>

#undef _KERNEL

#undef sigqueue

#include <libutil.h>

#endif

#endif

#else /* !CONFIG_USER_ONLY */

#include "xen-mapcache.h"

#include "trace.h"

#endif

#include "cputlb.h"

#define WANT_EXEC_OBSOLETE

#include "exec-obsolete.h"

//#define DEBUG_TB_INVALIDATE

//#define DEBUG_FLUSH

//#define DEBUG_UNASSIGNED

/* make various TB consistency checks */

//#define DEBUG_TB_CHECK

//#define DEBUG_IOPORT

//#define DEBUG_SUBPAGE

#if !defined(CONFIG_USER_ONLY)

/* TB consistency checks only implemented for usermode emulation.  */

#undef DEBUG_TB_CHECK

#endif

#define SMC_BITMAP_USE_THRESHOLD 10

static TranslationBlock *tbs;

static int code_gen_max_blocks;

TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];

static int nb_tbs;

/* any access to the tbs or the page table must use this lock */

spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;

#if defined(__arm__) || defined(__sparc_v9__)

/* The prologue must be reachable with a direct jump. ARM and Sparc64

 have limited branch ranges (possibly also PPC) so place it in a

 section close to code segment. */

#define code_gen_section                                \

    __attribute__((__section__(".gen_code")))           \

    __attribute__((aligned (32)))

#elif defined(_WIN32) && !defined(_WIN64)

#define code_gen_section                                \

    __attribute__((aligned (16)))

#else

#define code_gen_section                                \

    __attribute__((aligned (32)))

#endif

uint8_t code_gen_prologue[1024] code_gen_section;

static uint8_t *code_gen_buffer;

static unsigned long code_gen_buffer_size;

/* threshold to flush the translated code buffer */

static unsigned long code_gen_buffer_max_size;

static uint8_t *code_gen_ptr;

#if !defined(CONFIG_USER_ONLY)

int phys_ram_fd;

static int in_migration;

RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };

static MemoryRegion *system_memory;

static MemoryRegion *system_io;

MemoryRegion io_mem_ram, io_mem_rom, io_mem_unassigned, io_mem_notdirty;

static MemoryRegion io_mem_subpage_ram;

#endif

CPUArchState *first_cpu;

/* current CPU in the current thread. It is only valid inside

   cpu_exec() */

DEFINE_TLS(CPUArchState *,cpu_single_env);

/* 0 = Do not count executed instructions.

   1 = Precise instruction counting.

   2 = Adaptive rate instruction counting.  */

int use_icount = 0;

typedef struct PageDesc {

    /* list of TBs intersecting this ram page */

    TranslationBlock *first_tb;

    /* in order to optimize self modifying code, we count the number

       of lookups we do to a given page to use a bitmap */

    unsigned int code_write_count;

    uint8_t *code_bitmap;

#if defined(CONFIG_USER_ONLY)

    unsigned long flags;

#endif

} PageDesc;

/* In system mode we want L1_MAP to be based on ram offsets,

   while in user mode we want it to be based on virtual addresses.  */

#if !defined(CONFIG_USER_ONLY)

#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS

# define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS

#else

# define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS

#endif

#else

# define L1_MAP_ADDR_SPACE_BITS  TARGET_VIRT_ADDR_SPACE_BITS

#endif

/* Size of the L2 (and L3, etc) page tables.  */

#define L2_BITS 10

#define L2_SIZE (1 << L2_BITS)

#define P_L2_LEVELS \

    (((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / L2_BITS) + 1)

/* The bits remaining after N lower levels of page tables.  */

#define V_L1_BITS_REM \

    ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)

#if V_L1_BITS_REM < 4

#define V_L1_BITS  (V_L1_BITS_REM + L2_BITS)

#else

#define V_L1_BITS  V_L1_BITS_REM

#endif

#define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)

#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)

uintptr_t qemu_real_host_page_size;

uintptr_t qemu_host_page_size;

uintptr_t qemu_host_page_mask;

/* This is a multi-level map on the virtual address space.

   The bottom level has pointers to PageDesc.  */

static void *l1_map[V_L1_SIZE];

#if !defined(CONFIG_USER_ONLY)

typedef struct PhysPageEntry PhysPageEntry;

static MemoryRegionSection *phys_sections;

static unsigned phys_sections_nb, phys_sections_nb_alloc;

static uint16_t phys_section_unassigned;

static uint16_t phys_section_notdirty;

static uint16_t phys_section_rom;

static uint16_t phys_section_watch;

struct PhysPageEntry {

    uint16_t is_leaf : 1;

     /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */

    uint16_t ptr : 15;

};

/* Simple allocator for PhysPageEntry nodes */

static PhysPageEntry (*phys_map_nodes)[L2_SIZE];

static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;

#define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)

/* This is a multi-level map on the physical address space.

   The bottom level has pointers to MemoryRegionSections.  */

static PhysPageEntry phys_map = { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };

static void io_mem_init(void);

static void memory_map_init(void);

static MemoryRegion io_mem_watch;

#endif

/* log support */

#ifdef WIN32

static const char *logfilename = "qemu.log";

#else

static const char *logfilename = "/tmp/qemu.log";

#endif

FILE *logfile;

int loglevel;

static int log_append = 0;

/* statistics */

static int tb_flush_count;

static int tb_phys_invalidate_count;

#ifdef _WIN32

static void map_exec(void *addr, long size)

{

    DWORD old_protect;

    VirtualProtect(addr, size,

                   PAGE_EXECUTE_READWRITE, &old_protect);

}

#else

static void map_exec(void *addr, long size)

{

    unsigned long start, end, page_size;

    page_size = getpagesize();

    start = (unsigned long)addr;

    start &= ~(page_size - 1);

    end = (unsigned long)addr + size;

    end += page_size - 1;

    end &= ~(page_size - 1);

    mprotect((void *)start, end - start,

             PROT_READ | PROT_WRITE | PROT_EXEC);

}

#endif

static void page_init(void)

{

    /* NOTE: we can always suppose that qemu_host_page_size >=

       TARGET_PAGE_SIZE */

#ifdef _WIN32

    {

        SYSTEM_INFO system_info;

        GetSystemInfo(&system_info);

        qemu_real_host_page_size = system_info.dwPageSize;

    }

#else

    qemu_real_host_page_size = getpagesize();

#endif

    if (qemu_host_page_size == 0)

        qemu_host_page_size = qemu_real_host_page_size;

    if (qemu_host_page_size < TARGET_PAGE_SIZE)

        qemu_host_page_size = TARGET_PAGE_SIZE;

    qemu_host_page_mask = ~(qemu_host_page_size - 1);

#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)

    {

#ifdef HAVE_KINFO_GETVMMAP

        struct kinfo_vmentry *freep;

        int i, cnt;

        freep = kinfo_getvmmap(getpid(), &cnt);

        if (freep) {

            mmap_lock();

            for (i = 0; i < cnt; i++) {

                unsigned long startaddr, endaddr;

                startaddr = freep[i].kve_start;

                endaddr = freep[i].kve_end;

                if (h2g_valid(startaddr)) {

                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;

                    if (h2g_valid(endaddr)) {

                        endaddr = h2g(endaddr);

                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);

                    } else {

#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS

                        endaddr = ~0ul;

                        page_set_flags(startaddr, endaddr, PAGE_RESERVED);

#endif

                    }

                }

            }

            free(freep);

            mmap_unlock();

        }

#else

        FILE *f;

        last_brk = (unsigned long)sbrk(0);

        f = fopen("/compat/linux/proc/self/maps", "r");

        if (f) {

            mmap_lock();

            do {

                unsigned long startaddr, endaddr;

                int n;

                n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);

                if (n == 2 && h2g_valid(startaddr)) {

                    startaddr = h2g(startaddr) & TARGET_PAGE_MASK;

                    if (h2g_valid(endaddr)) {

                        endaddr = h2g(endaddr);

                    } else {

                        endaddr = ~0ul;

                    }

                    page_set_flags(startaddr, endaddr, PAGE_RESERVED);

                }

            } while (!feof(f));

            fclose(f);

            mmap_unlock();

        }

#endif

    }

#endif

}

static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)

{

    PageDesc *pd;

    void **lp;

    int i;

#if defined(CONFIG_USER_ONLY)

    /* We can't use g_malloc because it may recurse into a locked mutex. */

# define ALLOC(P, SIZE)                                 \

    do {                                                \

        P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,    \

                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);   \

    } while (0)

#else

# define ALLOC(P, SIZE) \

    do { P = g_malloc0(SIZE); } while (0)

#endif

    /* Level 1.  Always allocated.  */

    lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));

    /* Level 2..N-1.  */

    for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {

        void **p = *lp;

        if (p == NULL) {

            if (!alloc) {

                return NULL;

            }

            ALLOC(p, sizeof(void *) * L2_SIZE);

            *lp = p;

        }

        lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));

    }

    pd = *lp;

    if (pd == NULL) {

        if (!alloc) {

            return NULL;

        }

        ALLOC(pd, sizeof(PageDesc) * L2_SIZE);

        *lp = pd;

    }

#undef ALLOC

    return pd + (index & (L2_SIZE - 1));

}

static inline PageDesc *page_find(tb_page_addr_t index)

{

    return page_find_alloc(index, 0);

}

#if !defined(CONFIG_USER_ONLY)

static void phys_map_node_reserve(unsigned nodes)

{

    if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {

        typedef PhysPageEntry Node[L2_SIZE];

        phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);

        phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,

                                      phys_map_nodes_nb + nodes);

        phys_map_nodes = g_renew(Node, phys_map_nodes,

                                 phys_map_nodes_nb_alloc);

    }

}

static uint16_t phys_map_node_alloc(void)

{

    unsigned i;

    uint16_t ret;

    ret = phys_map_nodes_nb++;

    assert(ret != PHYS_MAP_NODE_NIL);

    assert(ret != phys_map_nodes_nb_alloc);

    for (i = 0; i < L2_SIZE; ++i) {

        phys_map_nodes[ret][i].is_leaf = 0;

        phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;

    }

    return ret;

}

static void phys_map_nodes_reset(void)

{

    phys_map_nodes_nb = 0;

}

static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index,

                                target_phys_addr_t *nb, uint16_t leaf,

                                int level)

{

    PhysPageEntry *p;

    int i;

    target_phys_addr_t step = (target_phys_addr_t)1 << (level * L2_BITS);

    if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {

        lp->ptr = phys_map_node_alloc();

        p = phys_map_nodes[lp->ptr];

        if (level == 0) {

            for (i = 0; i < L2_SIZE; i++) {

                p[i].is_leaf = 1;

                p[i].ptr = phys_section_unassigned;

            }

        }

    } else {

        p = phys_map_nodes[lp->ptr];

    }

    lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];

    while (*nb && lp < &p[L2_SIZE]) {

        if ((*index & (step - 1)) == 0 && *nb >= step) {

            lp->is_leaf = true;

            lp->ptr = leaf;

            *index += step;

            *nb -= step;

        } else {

            phys_page_set_level(lp, index, nb, leaf, level - 1);

        }

        ++lp;

    }

}

static void phys_page_set(target_phys_addr_t index, target_phys_addr_t nb,

                          uint16_t leaf)

{

    /* Wildly overreserve - it doesn't matter much. */

    phys_map_node_reserve(3 * P_L2_LEVELS);

    phys_page_set_level(&phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);

}

MemoryRegionSection *phys_page_find(target_phys_addr_t index)

{

    PhysPageEntry lp = phys_map;

    PhysPageEntry *p;

    int i;

    uint16_t s_index = phys_section_unassigned;

    for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {

        if (lp.ptr == PHYS_MAP_NODE_NIL) {

            goto not_found;

        }

        p = phys_map_nodes[lp.ptr];

        lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];

    }

    s_index = lp.ptr;

not_found:

    return &phys_sections[s_index];

}

bool memory_region_is_unassigned(MemoryRegion *mr)

{

    return mr != &io_mem_ram && mr != &io_mem_rom

        && mr != &io_mem_notdirty && !mr->rom_device

        && mr != &io_mem_watch;

}

#define mmap_lock() do { } while(0)

#define mmap_unlock() do { } while(0)

#endif

#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)

#if defined(CONFIG_USER_ONLY)

/* Currently it is not recommended to allocate big chunks of data in

   user mode. It will change when a dedicated libc will be used */

#define USE_STATIC_CODE_GEN_BUFFER

#endif

#ifdef USE_STATIC_CODE_GEN_BUFFER

static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]

               __attribute__((aligned (CODE_GEN_ALIGN)));

#endif

static void code_gen_alloc(unsigned long tb_size)

{

#ifdef USE_STATIC_CODE_GEN_BUFFER

    code_gen_buffer = static_code_gen_buffer;

    code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;

    map_exec(code_gen_buffer, code_gen_buffer_size);

#else

    code_gen_buffer_size = tb_size;

    if (code_gen_buffer_size == 0) {

#if defined(CONFIG_USER_ONLY)

        code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;

#else

        /* XXX: needs adjustments */

        code_gen_buffer_size = (unsigned long)(ram_size / 4);

#endif

    }

    if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)

        code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;

    /* The code gen buffer location may have constraints depending on

       the host cpu and OS */

#if defined(__linux__) 

    {

        int flags;

        void *start = NULL;

        flags = MAP_PRIVATE | MAP_ANONYMOUS;

#if defined(__x86_64__)

        flags |= MAP_32BIT;

        /* Cannot map more than that */

        if (code_gen_buffer_size > (800 * 1024 * 1024))

            code_gen_buffer_size = (800 * 1024 * 1024);

#elif defined(__sparc_v9__)

        // Map the buffer below 2G, so we can use direct calls and branches

        flags |= MAP_FIXED;

        start = (void *) 0x60000000UL;

        if (code_gen_buffer_size > (512 * 1024 * 1024))

            code_gen_buffer_size = (512 * 1024 * 1024);

#elif defined(__arm__)

        /* Keep the buffer no bigger than 16MB to branch between blocks */

        if (code_gen_buffer_size > 16 * 1024 * 1024)

            code_gen_buffer_size = 16 * 1024 * 1024;

#elif defined(__s390x__)

        /* Map the buffer so that we can use direct calls and branches.  */

        /* We have a +- 4GB range on the branches; leave some slop.  */

        if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {

            code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;

        }

        start = (void *)0x90000000UL;

#endif

        code_gen_buffer = mmap(start, code_gen_buffer_size,

                               PROT_WRITE | PROT_READ | PROT_EXEC,

                               flags, -1, 0);

        if (code_gen_buffer == MAP_FAILED) {

            fprintf(stderr, "Could not allocate dynamic translator buffer\n");

            exit(1);

        }

    }

#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \

    || defined(__DragonFly__) || defined(__OpenBSD__) \

    || defined(__NetBSD__)

    {

        int flags;

        void *addr = NULL;

        flags = MAP_PRIVATE | MAP_ANONYMOUS;

#if defined(__x86_64__)

        /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume

         * 0x40000000 is free */

        flags |= MAP_FIXED;

        addr = (void *)0x40000000;

        /* Cannot map more than that */

        if (code_gen_buffer_size > (800 * 1024 * 1024))

            code_gen_buffer_size = (800 * 1024 * 1024);

#elif defined(__sparc_v9__)

        // Map the buffer below 2G, so we can use direct calls and branches

        flags |= MAP_FIXED;

        addr = (void *) 0x60000000UL;

        if (code_gen_buffer_size > (512 * 1024 * 1024)) {

            code_gen_buffer_size = (512 * 1024 * 1024);

        }

#endif

        code_gen_buffer = mmap(addr, code_gen_buffer_size,

                               PROT_WRITE | PROT_READ | PROT_EXEC, 

                               flags, -1, 0);

        if (code_gen_buffer == MAP_FAILED) {

            fprintf(stderr, "Could not allocate dynamic translator buffer\n");

            exit(1);

        }

    }

#else

    code_gen_buffer = g_malloc(code_gen_buffer_size);

    map_exec(code_gen_buffer, code_gen_buffer_size);

#endif

#endif /* !USE_STATIC_CODE_GEN_BUFFER */

    map_exec(code_gen_prologue, sizeof(code_gen_prologue));

    code_gen_buffer_max_size = code_gen_buffer_size -

        (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);

    code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;

    tbs = g_malloc(code_gen_max_blocks * sizeof(TranslationBlock));

}

/* Must be called before using the QEMU cpus. 'tb_size' is the size

   (in bytes) allocated to the translation buffer. Zero means default

   size. */

void tcg_exec_init(unsigned long tb_size)

{

    cpu_gen_init();

    code_gen_alloc(tb_size);

    code_gen_ptr = code_gen_buffer;

    tcg_register_jit(code_gen_buffer, code_gen_buffer_size);

    page_init();

#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)

    /* There's no guest base to take into account, so go ahead and

       initialize the prologue now.  */

    tcg_prologue_init(&tcg_ctx);

#endif

}

bool tcg_enabled(void)

{

    return code_gen_buffer != NULL;

}

void cpu_exec_init_all(void)

{

#if !defined(CONFIG_USER_ONLY)

    memory_map_init();

    io_mem_init();

#endif

}

#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)

static int cpu_common_post_load(void *opaque, int version_id)

{

    CPUArchState *env = opaque;

    /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the

       version_id is increased. */

    env->interrupt_request &= ~0x01;

    tlb_flush(env, 1);

    return 0;

}

static const VMStateDescription vmstate_cpu_common = {

    .name = "cpu_common",

    .version_id = 1,

    .minimum_version_id = 1,

    .minimum_version_id_old = 1,

    .post_load = cpu_common_post_load,

    .fields      = (VMStateField []) {

        VMSTATE_UINT32(halted, CPUArchState),

        VMSTATE_UINT32(interrupt_request, CPUArchState),

        VMSTATE_END_OF_LIST()

    }

};

#endif

CPUArchState *qemu_get_cpu(int cpu)

{

    CPUArchState *env = first_cpu;

    while (env) {

        if (env->cpu_index == cpu)

            break;

        env = env->next_cpu;

    }

    return env;

}

void cpu_exec_init(CPUArchState *env)

{

    CPUArchState **penv;

    int cpu_index;

#if defined(CONFIG_USER_ONLY)

    cpu_list_lock();

#endif

    env->next_cpu = NULL;

    penv = &first_cpu;

    cpu_index = 0;

    while (*penv != NULL) {

        penv = &(*penv)->next_cpu;

        cpu_index++;

    }

    env->cpu_index = cpu_index;

    env->numa_node = 0;

    QTAILQ_INIT(&env->breakpoints);

    QTAILQ_INIT(&env->watchpoints);

#ifndef CONFIG_USER_ONLY

    env->thread_id = qemu_get_thread_id();

#endif

    *penv = env;

#if defined(CONFIG_USER_ONLY)

    cpu_list_unlock();

#endif

#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)

    vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);

    register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,

                    cpu_save, cpu_load, env);

#endif

}

/* Allocate a new translation block. Flush the translation buffer if

   too many translation blocks or too much generated code. */

static TranslationBlock *tb_alloc(target_ulong pc)

{

    TranslationBlock *tb;

    if (nb_tbs >= code_gen_max_blocks ||

        (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)

        return NULL;

    tb = &tbs[nb_tbs++];

    tb->pc = pc;

    tb->cflags = 0;

    return tb;

}

void tb_free(TranslationBlock *tb)

{

    /* In practice this is mostly used for single use temporary TB

       Ignore the hard cases and just back up if this TB happens to

       be the last one generated.  */

    if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {

        code_gen_ptr = tb->tc_ptr;

        nb_tbs--;

    }

}

static inline void invalidate_page_bitmap(PageDesc *p)

{

    if (p->code_bitmap) {

        g_free(p->code_bitmap);

        p->code_bitmap = NULL;

    }

    p->code_write_count = 0;

}

/* Set to NULL all the 'first_tb' fields in all PageDescs. */

static void page_flush_tb_1 (int level, void **lp)

{

    int i;

    if (*lp == NULL) {

        return;

    }

    if (level == 0) {

        PageDesc *pd = *lp;

        for (i = 0; i < L2_SIZE; ++i) {

            pd[i].first_tb = NULL;

            invalidate_page_bitmap(pd + i);

        }

    } else {

        void **pp = *lp;

        for (i = 0; i < L2_SIZE; ++i) {

            page_flush_tb_1 (level - 1, pp + i);

        }

    }

}

static void page_flush_tb(void)

{

    int i;

    for (i = 0; i < V_L1_SIZE; i++) {

        page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);

    }

}

/* flush all the translation blocks */

/* XXX: tb_flush is currently not thread safe */

void tb_flush(CPUArchState *env1)

{

    CPUArchState *env;

#if defined(DEBUG_FLUSH)

    printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",

           (unsigned long)(code_gen_ptr - code_gen_buffer),

           nb_tbs, nb_tbs > 0 ?

           ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);

#endif

    if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)

        cpu_abort(env1, "Internal error: code buffer overflow\n");

    nb_tbs = 0;

    for(env = first_cpu; env != NULL; env = env->next_cpu) {

        memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));

    }

    memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));

    page_flush_tb();

    code_gen_ptr = code_gen_buffer;

    /* XXX: flush processor icache at this point if cache flush is

       expensive */

    tb_flush_count++;

}

#ifdef DEBUG_TB_CHECK

static void tb_invalidate_check(target_ulong address)

{

    TranslationBlock *tb;

    int i;

    address &= TARGET_PAGE_MASK;

    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {

        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {

            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||

                  address >= tb->pc + tb->size)) {

                printf("ERROR invalidate: address=" TARGET_FMT_lx

                       " PC=%08lx size=%04x\n",

                       address, (long)tb->pc, tb->size);

            }

        }

    }

}

/* verify that all the pages have correct rights for code */

static void tb_page_check(void)

{

    TranslationBlock *tb;

    int i, flags1, flags2;

    for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {

        for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {

            flags1 = page_get_flags(tb->pc);

            flags2 = page_get_flags(tb->pc + tb->size - 1);

            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {

                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",

                       (long)tb->pc, tb->size, flags1, flags2);

            }

        }

    }

}

#endif

/* invalidate one TB */

static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,

                             int next_offset)

{

    TranslationBlock *tb1;

    for(;;) {

        tb1 = *ptb;

        if (tb1 == tb) {

            *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);

            break;

        }

        ptb = (TranslationBlock **)((char *)tb1 + next_offset);

    }

}

static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)

{

    TranslationBlock *tb1;

    unsigned int n1;

    for(;;) {

        tb1 = *ptb;

        n1 = (uintptr_t)tb1 & 3;

        tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);

        if (tb1 == tb) {

            *ptb = tb1->page_next[n1];

            break;

        }

        ptb = &tb1->page_next[n1];

    }

}

static inline void tb_jmp_remove(TranslationBlock *tb, int n)