Archipelago » qemu

/*

 *  Virtual page mapping

 *

 *  Copyright (c) 2003 Fabrice Bellard

 *

 * This library is free software; you can redistribute it and/or

 * modify it under the terms of the GNU Lesser General Public

 * License as published by the Free Software Foundation; either

 * version 2 of the License, or (at your option) any later version.

 *

 * This library is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * Lesser General Public License for more details.

 *

 * You should have received a copy of the GNU Lesser General Public

 * License along with this library; if not, see <http://www.gnu.org/licenses/>.

 */

#include "config.h"

#ifdef _WIN32

#include <windows.h>

#else

#include <sys/types.h>

#include <sys/mman.h>

#endif

#include "qemu-common.h"

#include "cpu.h"

#include "tcg.h"

#include "hw/hw.h"

#include "hw/qdev.h"

#include "qemu/osdep.h"

#include "sysemu/kvm.h"

#include "sysemu/sysemu.h"

#include "hw/xen/xen.h"

#include "qemu/timer.h"

#include "qemu/config-file.h"

#include "exec/memory.h"

#include "sysemu/dma.h"

#include "exec/address-spaces.h"

#if defined(CONFIG_USER_ONLY)

#include <qemu.h>

#else /* !CONFIG_USER_ONLY */

#include "sysemu/xen-mapcache.h"

#include "trace.h"

#endif

#include "exec/cpu-all.h"

#include "exec/cputlb.h"

#include "translate-all.h"

#include "exec/memory-internal.h"

#include "qemu/cache-utils.h"

#include "qemu/range.h"

//#define DEBUG_SUBPAGE

#if !defined(CONFIG_USER_ONLY)

static bool in_migration;

RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };

static MemoryRegion *system_memory;

static MemoryRegion *system_io;

AddressSpace address_space_io;

AddressSpace address_space_memory;

MemoryRegion io_mem_rom, io_mem_notdirty;

static MemoryRegion io_mem_unassigned;

#endif

struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);

/* current CPU in the current thread. It is only valid inside

   cpu_exec() */

DEFINE_TLS(CPUState *, current_cpu);

/* 0 = Do not count executed instructions.

   1 = Precise instruction counting.

   2 = Adaptive rate instruction counting.  */

int use_icount;

#if !defined(CONFIG_USER_ONLY)

typedef struct PhysPageEntry PhysPageEntry;

struct PhysPageEntry {

    /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */

    uint32_t skip : 6;

     /* index into phys_sections (!skip) or phys_map_nodes (skip) */

    uint32_t ptr : 26;

};

#define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)

/* Size of the L2 (and L3, etc) page tables.  */

#define ADDR_SPACE_BITS 64

#define P_L2_BITS 9

#define P_L2_SIZE (1 << P_L2_BITS)

#define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)

typedef PhysPageEntry Node[P_L2_SIZE];

typedef struct PhysPageMap {

    unsigned sections_nb;

    unsigned sections_nb_alloc;

    unsigned nodes_nb;

    unsigned nodes_nb_alloc;

    Node *nodes;

    MemoryRegionSection *sections;

} PhysPageMap;

struct AddressSpaceDispatch {

    /* This is a multi-level map on the physical address space.

     * The bottom level has pointers to MemoryRegionSections.

     */

    PhysPageEntry phys_map;

    PhysPageMap map;

    AddressSpace *as;

};

#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)

typedef struct subpage_t {

    MemoryRegion iomem;

    AddressSpace *as;

    hwaddr base;

    uint16_t sub_section[TARGET_PAGE_SIZE];

} subpage_t;

#define PHYS_SECTION_UNASSIGNED 0

#define PHYS_SECTION_NOTDIRTY 1

#define PHYS_SECTION_ROM 2

#define PHYS_SECTION_WATCH 3

static void io_mem_init(void);

static void memory_map_init(void);

static MemoryRegion io_mem_watch;

#endif

#if !defined(CONFIG_USER_ONLY)

static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)

{

    if (map->nodes_nb + nodes > map->nodes_nb_alloc) {

        map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);

        map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);

        map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);

    }

}

static uint32_t phys_map_node_alloc(PhysPageMap *map)

{

    unsigned i;

    uint32_t ret;

    ret = map->nodes_nb++;

    assert(ret != PHYS_MAP_NODE_NIL);

    assert(ret != map->nodes_nb_alloc);

    for (i = 0; i < P_L2_SIZE; ++i) {

        map->nodes[ret][i].skip = 1;

        map->nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;

    }

    return ret;

}

static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,

                                hwaddr *index, hwaddr *nb, uint16_t leaf,

                                int level)

{

    PhysPageEntry *p;

    int i;

    hwaddr step = (hwaddr)1 << (level * P_L2_BITS);

    if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {

        lp->ptr = phys_map_node_alloc(map);

        p = map->nodes[lp->ptr];

        if (level == 0) {

            for (i = 0; i < P_L2_SIZE; i++) {

                p[i].skip = 0;

                p[i].ptr = PHYS_SECTION_UNASSIGNED;

            }

        }

    } else {

        p = map->nodes[lp->ptr];

    }

    lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];

    while (*nb && lp < &p[P_L2_SIZE]) {

        if ((*index & (step - 1)) == 0 && *nb >= step) {

            lp->skip = 0;

            lp->ptr = leaf;

            *index += step;

            *nb -= step;

        } else {

            phys_page_set_level(map, lp, index, nb, leaf, level - 1);

        }

        ++lp;

    }

}

static void phys_page_set(AddressSpaceDispatch *d,

                          hwaddr index, hwaddr nb,

                          uint16_t leaf)

{

    /* Wildly overreserve - it doesn't matter much. */

    phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);

    phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);

}

/* Compact a non leaf page entry. Simply detect that the entry has a single child,

 * and update our entry so we can skip it and go directly to the destination.

 */

static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)

{

    unsigned valid_ptr = P_L2_SIZE;

    int valid = 0;

    PhysPageEntry *p;

    int i;

    if (lp->ptr == PHYS_MAP_NODE_NIL) {

        return;

    }

    p = nodes[lp->ptr];

    for (i = 0; i < P_L2_SIZE; i++) {

        if (p[i].ptr == PHYS_MAP_NODE_NIL) {

            continue;

        }

        valid_ptr = i;

        valid++;

        if (p[i].skip) {

            phys_page_compact(&p[i], nodes, compacted);

        }

    }

    /* We can only compress if there's only one child. */

    if (valid != 1) {

        return;

    }

    assert(valid_ptr < P_L2_SIZE);

    /* Don't compress if it won't fit in the # of bits we have. */

    if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {

        return;

    }

    lp->ptr = p[valid_ptr].ptr;

    if (!p[valid_ptr].skip) {

        /* If our only child is a leaf, make this a leaf. */

        /* By design, we should have made this node a leaf to begin with so we

         * should never reach here.

         * But since it's so simple to handle this, let's do it just in case we

         * change this rule.

         */

        lp->skip = 0;

    } else {

        lp->skip += p[valid_ptr].skip;

    }

}

static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)

{

    DECLARE_BITMAP(compacted, nodes_nb);

    if (d->phys_map.skip) {

        phys_page_compact(&d->phys_map, d->map.nodes, compacted);

    }

}

static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,

                                           Node *nodes, MemoryRegionSection *sections)

{

    PhysPageEntry *p;

    hwaddr index = addr >> TARGET_PAGE_BITS;

    int i;

    for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {

        if (lp.ptr == PHYS_MAP_NODE_NIL) {

            return &sections[PHYS_SECTION_UNASSIGNED];

        }

        p = nodes[lp.ptr];

        lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];

    }

    if (sections[lp.ptr].size.hi ||

        range_covers_byte(sections[lp.ptr].offset_within_address_space,

                          sections[lp.ptr].size.lo, addr)) {

        return &sections[lp.ptr];

    } else {

        return &sections[PHYS_SECTION_UNASSIGNED];

    }

}

bool memory_region_is_unassigned(MemoryRegion *mr)

{

    return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device

        && mr != &io_mem_watch;

}

static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,

                                                        hwaddr addr,

                                                        bool resolve_subpage)

{

    MemoryRegionSection *section;

    subpage_t *subpage;

    section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);

    if (resolve_subpage && section->mr->subpage) {

        subpage = container_of(section->mr, subpage_t, iomem);

        section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];

    }

    return section;

}

static MemoryRegionSection *

address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,

                                 hwaddr *plen, bool resolve_subpage)

{

    MemoryRegionSection *section;

    Int128 diff;

    section = address_space_lookup_region(d, addr, resolve_subpage);

    /* Compute offset within MemoryRegionSection */

    addr -= section->offset_within_address_space;

    /* Compute offset within MemoryRegion */

    *xlat = addr + section->offset_within_region;

    diff = int128_sub(section->mr->size, int128_make64(addr));

    *plen = int128_get64(int128_min(diff, int128_make64(*plen)));

    return section;

}

MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,

                                      hwaddr *xlat, hwaddr *plen,

                                      bool is_write)

{

    IOMMUTLBEntry iotlb;

    MemoryRegionSection *section;

    MemoryRegion *mr;

    hwaddr len = *plen;

    for (;;) {

        section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);

        mr = section->mr;

        if (!mr->iommu_ops) {

            break;

        }

        iotlb = mr->iommu_ops->translate(mr, addr);

        addr = ((iotlb.translated_addr & ~iotlb.addr_mask)

                | (addr & iotlb.addr_mask));

        len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);

        if (!(iotlb.perm & (1 << is_write))) {

            mr = &io_mem_unassigned;

            break;

        }

        as = iotlb.target_as;

    }

    *plen = len;

    *xlat = addr;

    return mr;

}

MemoryRegionSection *

address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,

                                  hwaddr *plen)

{

    MemoryRegionSection *section;

    section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);

    assert(!section->mr->iommu_ops);

    return section;

}

#endif

void cpu_exec_init_all(void)

{

#if !defined(CONFIG_USER_ONLY)

    qemu_mutex_init(&ram_list.mutex);

    memory_map_init();

    io_mem_init();

#endif

}

#if !defined(CONFIG_USER_ONLY)

static int cpu_common_post_load(void *opaque, int version_id)

{

    CPUState *cpu = opaque;

    /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the

       version_id is increased. */

    cpu->interrupt_request &= ~0x01;

    tlb_flush(cpu->env_ptr, 1);

    return 0;

}

const VMStateDescription vmstate_cpu_common = {

    .name = "cpu_common",

    .version_id = 1,

    .minimum_version_id = 1,

    .minimum_version_id_old = 1,

    .post_load = cpu_common_post_load,

    .fields      = (VMStateField []) {

        VMSTATE_UINT32(halted, CPUState),

        VMSTATE_UINT32(interrupt_request, CPUState),

        VMSTATE_END_OF_LIST()

    }

};

#endif

CPUState *qemu_get_cpu(int index)

{

    CPUState *cpu;

    CPU_FOREACH(cpu) {

        if (cpu->cpu_index == index) {

            return cpu;

        }

    }

    return NULL;

}

void cpu_exec_init(CPUArchState *env)

{

    CPUState *cpu = ENV_GET_CPU(env);

    CPUClass *cc = CPU_GET_CLASS(cpu);

    CPUState *some_cpu;

    int cpu_index;

#if defined(CONFIG_USER_ONLY)

    cpu_list_lock();

#endif

    cpu_index = 0;

    CPU_FOREACH(some_cpu) {

        cpu_index++;

    }

    cpu->cpu_index = cpu_index;

    cpu->numa_node = 0;

    QTAILQ_INIT(&env->breakpoints);

    QTAILQ_INIT(&env->watchpoints);

#ifndef CONFIG_USER_ONLY

    cpu->thread_id = qemu_get_thread_id();

#endif

    QTAILQ_INSERT_TAIL(&cpus, cpu, node);

#if defined(CONFIG_USER_ONLY)

    cpu_list_unlock();

#endif

    if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {

        vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);

    }

#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)

    register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,

                    cpu_save, cpu_load, env);

    assert(cc->vmsd == NULL);

    assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);

#endif

    if (cc->vmsd != NULL) {

        vmstate_register(NULL, cpu_index, cc->vmsd, cpu);

    }

}

#if defined(TARGET_HAS_ICE)

#if defined(CONFIG_USER_ONLY)

static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)

{

    tb_invalidate_phys_page_range(pc, pc + 1, 0);

}

#else

static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)

{

    hwaddr phys = cpu_get_phys_page_debug(cpu, pc);

    if (phys != -1) {

        tb_invalidate_phys_addr(phys | (pc & ~TARGET_PAGE_MASK));

    }

}

#endif

#endif /* TARGET_HAS_ICE */

#if defined(CONFIG_USER_ONLY)

void cpu_watchpoint_remove_all(CPUArchState *env, int mask)

{

}

int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,

                          int flags, CPUWatchpoint **watchpoint)

{

    return -ENOSYS;

}

#else

/* Add a watchpoint.  */

int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,

                          int flags, CPUWatchpoint **watchpoint)

{

    target_ulong len_mask = ~(len - 1);

    CPUWatchpoint *wp;

    /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */

    if ((len & (len - 1)) || (addr & ~len_mask) ||

            len == 0 || len > TARGET_PAGE_SIZE) {

        fprintf(stderr, "qemu: tried to set invalid watchpoint at "

                TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);

        return -EINVAL;

    }

    wp = g_malloc(sizeof(*wp));

    wp->vaddr = addr;

    wp->len_mask = len_mask;

    wp->flags = flags;

    /* keep all GDB-injected watchpoints in front */

    if (flags & BP_GDB)

        QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);

    else

        QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);

    tlb_flush_page(env, addr);

    if (watchpoint)

        *watchpoint = wp;

    return 0;

}

/* Remove a specific watchpoint.  */

int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,

                          int flags)

{

    target_ulong len_mask = ~(len - 1);

    CPUWatchpoint *wp;

    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {

        if (addr == wp->vaddr && len_mask == wp->len_mask

                && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {

            cpu_watchpoint_remove_by_ref(env, wp);

            return 0;

        }

    }

    return -ENOENT;

}

/* Remove a specific watchpoint by reference.  */

void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)

{

    QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);

    tlb_flush_page(env, watchpoint->vaddr);

    g_free(watchpoint);

}

/* Remove all matching watchpoints.  */

void cpu_watchpoint_remove_all(CPUArchState *env, int mask)

{

    CPUWatchpoint *wp, *next;

    QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {

        if (wp->flags & mask)

            cpu_watchpoint_remove_by_ref(env, wp);

    }

}

#endif

/* Add a breakpoint.  */

int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,

                          CPUBreakpoint **breakpoint)

{

#if defined(TARGET_HAS_ICE)

    CPUBreakpoint *bp;

    bp = g_malloc(sizeof(*bp));

    bp->pc = pc;

    bp->flags = flags;

    /* keep all GDB-injected breakpoints in front */

    if (flags & BP_GDB) {

        QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);

    } else {

        QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);

    }

    breakpoint_invalidate(ENV_GET_CPU(env), pc);

    if (breakpoint) {

        *breakpoint = bp;

    }

    return 0;

#else

    return -ENOSYS;

#endif

}

/* Remove a specific breakpoint.  */

int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)

{

#if defined(TARGET_HAS_ICE)

    CPUBreakpoint *bp;

    QTAILQ_FOREACH(bp, &env->breakpoints, entry) {

        if (bp->pc == pc && bp->flags == flags) {

            cpu_breakpoint_remove_by_ref(env, bp);

            return 0;

        }

    }

    return -ENOENT;

#else

    return -ENOSYS;

#endif

}

/* Remove a specific breakpoint by reference.  */

void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)

{

#if defined(TARGET_HAS_ICE)

    QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);

    breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);

    g_free(breakpoint);

#endif

}

/* Remove all matching breakpoints. */

void cpu_breakpoint_remove_all(CPUArchState *env, int mask)

{

#if defined(TARGET_HAS_ICE)

    CPUBreakpoint *bp, *next;

    QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {

        if (bp->flags & mask)

            cpu_breakpoint_remove_by_ref(env, bp);

    }

#endif

}

/* enable or disable single step mode. EXCP_DEBUG is returned by the

   CPU loop after each instruction */

void cpu_single_step(CPUState *cpu, int enabled)

{

#if defined(TARGET_HAS_ICE)

    if (cpu->singlestep_enabled != enabled) {

        cpu->singlestep_enabled = enabled;

        if (kvm_enabled()) {

            kvm_update_guest_debug(cpu, 0);

        } else {

            /* must flush all the translated code to avoid inconsistencies */

            /* XXX: only flush what is necessary */

            CPUArchState *env = cpu->env_ptr;

            tb_flush(env);

        }

    }

#endif

}

void cpu_abort(CPUArchState *env, const char *fmt, ...)

{

    CPUState *cpu = ENV_GET_CPU(env);

    va_list ap;

    va_list ap2;

    va_start(ap, fmt);

    va_copy(ap2, ap);

    fprintf(stderr, "qemu: fatal: ");

    vfprintf(stderr, fmt, ap);

    fprintf(stderr, "\n");

    cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);

    if (qemu_log_enabled()) {

        qemu_log("qemu: fatal: ");

        qemu_log_vprintf(fmt, ap2);

        qemu_log("\n");

        log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);

        qemu_log_flush();

        qemu_log_close();

    }

    va_end(ap2);

    va_end(ap);

#if defined(CONFIG_USER_ONLY)

    {

        struct sigaction act;

        sigfillset(&act.sa_mask);

        act.sa_handler = SIG_DFL;

        sigaction(SIGABRT, &act, NULL);

    }

#endif

    abort();

}

#if !defined(CONFIG_USER_ONLY)

static RAMBlock *qemu_get_ram_block(ram_addr_t addr)

{

    RAMBlock *block;

    /* The list is protected by the iothread lock here.  */

    block = ram_list.mru_block;

    if (block && addr - block->offset < block->length) {

        goto found;

    }

    QTAILQ_FOREACH(block, &ram_list.blocks, next) {

        if (addr - block->offset < block->length) {

            goto found;

        }

    }

    fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);

    abort();

found:

    ram_list.mru_block = block;

    return block;

}

static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)

{

    ram_addr_t start1;

    RAMBlock *block;

    ram_addr_t end;

    end = TARGET_PAGE_ALIGN(start + length);

    start &= TARGET_PAGE_MASK;

    block = qemu_get_ram_block(start);

    assert(block == qemu_get_ram_block(end - 1));

    start1 = (uintptr_t)block->host + (start - block->offset);

    cpu_tlb_reset_dirty_all(start1, length);

}

/* Note: start and end must be within the same ram block.  */

void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length,

                                     unsigned client)

{

    if (length == 0)

        return;

    cpu_physical_memory_clear_dirty_range(start, length, client);

    if (tcg_enabled()) {

        tlb_reset_dirty_range_all(start, length);

    }

}

static void cpu_physical_memory_set_dirty_tracking(bool enable)

{

    in_migration = enable;

}

hwaddr memory_region_section_get_iotlb(CPUArchState *env,

                                       MemoryRegionSection *section,

                                       target_ulong vaddr,

                                       hwaddr paddr, hwaddr xlat,

                                       int prot,

                                       target_ulong *address)

{

    hwaddr iotlb;

    CPUWatchpoint *wp;

    if (memory_region_is_ram(section->mr)) {

        /* Normal RAM.  */

        iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)

            + xlat;

        if (!section->readonly) {

            iotlb |= PHYS_SECTION_NOTDIRTY;

        } else {

            iotlb |= PHYS_SECTION_ROM;

        }

    } else {

        iotlb = section - address_space_memory.dispatch->map.sections;

        iotlb += xlat;

    }

    /* Make accesses to pages with watchpoints go via the

       watchpoint trap routines.  */

    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {

        if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {

            /* Avoid trapping reads of pages with a write breakpoint. */

            if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {

                iotlb = PHYS_SECTION_WATCH + paddr;

                *address |= TLB_MMIO;

                break;

            }

        }

    }

    return iotlb;

}

#endif /* defined(CONFIG_USER_ONLY) */

#if !defined(CONFIG_USER_ONLY)

static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,

                             uint16_t section);

static subpage_t *subpage_init(AddressSpace *as, hwaddr base);

static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;

/*

 * Set a custom physical guest memory alloator.

 * Accelerators with unusual needs may need this.  Hopefully, we can

 * get rid of it eventually.

 */

void phys_mem_set_alloc(void *(*alloc)(size_t))

{

    phys_mem_alloc = alloc;

}

static uint16_t phys_section_add(PhysPageMap *map,

                                 MemoryRegionSection *section)

{

    /* The physical section number is ORed with a page-aligned

     * pointer to produce the iotlb entries.  Thus it should

     * never overflow into the page-aligned value.

     */

    assert(map->sections_nb < TARGET_PAGE_SIZE);

    if (map->sections_nb == map->sections_nb_alloc) {

        map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);

        map->sections = g_renew(MemoryRegionSection, map->sections,

                                map->sections_nb_alloc);

    }

    map->sections[map->sections_nb] = *section;

    memory_region_ref(section->mr);

    return map->sections_nb++;

}

static void phys_section_destroy(MemoryRegion *mr)

{

    memory_region_unref(mr);

    if (mr->subpage) {

        subpage_t *subpage = container_of(mr, subpage_t, iomem);

        memory_region_destroy(&subpage->iomem);

        g_free(subpage);

    }

}

static void phys_sections_free(PhysPageMap *map)

{

    while (map->sections_nb > 0) {

        MemoryRegionSection *section = &map->sections[--map->sections_nb];

        phys_section_destroy(section->mr);

    }

    g_free(map->sections);

    g_free(map->nodes);

}

static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)

{

    subpage_t *subpage;

    hwaddr base = section->offset_within_address_space

        & TARGET_PAGE_MASK;

    MemoryRegionSection *existing = phys_page_find(d->phys_map, base,

                                                   d->map.nodes, d->map.sections);

    MemoryRegionSection subsection = {

        .offset_within_address_space = base,

        .size = int128_make64(TARGET_PAGE_SIZE),

    };

    hwaddr start, end;

    assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);

    if (!(existing->mr->subpage)) {

        subpage = subpage_init(d->as, base);

        subsection.mr = &subpage->iomem;

        phys_page_set(d, base >> TARGET_PAGE_BITS, 1,

                      phys_section_add(&d->map, &subsection));

    } else {

        subpage = container_of(existing->mr, subpage_t, iomem);

    }

    start = section->offset_within_address_space & ~TARGET_PAGE_MASK;

    end = start + int128_get64(section->size) - 1;

    subpage_register(subpage, start, end,

                     phys_section_add(&d->map, section));

}

static void register_multipage(AddressSpaceDispatch *d,

                               MemoryRegionSection *section)

{

    hwaddr start_addr = section->offset_within_address_space;

    uint16_t section_index = phys_section_add(&d->map, section);

    uint64_t num_pages = int128_get64(int128_rshift(section->size,

                                                    TARGET_PAGE_BITS));

    assert(num_pages);

    phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);

}

static void mem_add(MemoryListener *listener, MemoryRegionSection *section)

{

    AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);

    AddressSpaceDispatch *d = as->next_dispatch;

    MemoryRegionSection now = *section, remain = *section;

    Int128 page_size = int128_make64(TARGET_PAGE_SIZE);

    if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {

        uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)

                       - now.offset_within_address_space;

        now.size = int128_min(int128_make64(left), now.size);

        register_subpage(d, &now);

    } else {

        now.size = int128_zero();

    }

    while (int128_ne(remain.size, now.size)) {

        remain.size = int128_sub(remain.size, now.size);

        remain.offset_within_address_space += int128_get64(now.size);

        remain.offset_within_region += int128_get64(now.size);

        now = remain;

        if (int128_lt(remain.size, page_size)) {

            register_subpage(d, &now);

        } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {