Archipelago » qemu

/*

 *  Virtual page mapping

 *

 *  Copyright (c) 2003 Fabrice Bellard

 *

 * This library is free software; you can redistribute it and/or

 * modify it under the terms of the GNU Lesser General Public

 * License as published by the Free Software Foundation; either

 * version 2 of the License, or (at your option) any later version.

 *

 * This library is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * Lesser General Public License for more details.

 *

 * You should have received a copy of the GNU Lesser General Public

 * License along with this library; if not, see <http://www.gnu.org/licenses/>.

 */

#include "config.h"

#ifdef _WIN32

#include <windows.h>

#else

#include <sys/types.h>

#include <sys/mman.h>

#endif

#include "qemu-common.h"

#include "cpu.h"

#include "tcg.h"

#include "hw/hw.h"

#include "hw/qdev.h"

#include "qemu/osdep.h"

#include "sysemu/kvm.h"

#include "sysemu/sysemu.h"

#include "hw/xen/xen.h"

#include "qemu/timer.h"

#include "qemu/config-file.h"

#include "exec/memory.h"

#include "sysemu/dma.h"

#include "exec/address-spaces.h"

#if defined(CONFIG_USER_ONLY)

#include <qemu.h>

#else /* !CONFIG_USER_ONLY */

#include "sysemu/xen-mapcache.h"

#include "trace.h"

#endif

#include "exec/cpu-all.h"

#include "exec/cputlb.h"

#include "translate-all.h"

#include "exec/memory-internal.h"

#include "exec/ram_addr.h"

#include "qemu/cache-utils.h"

#include "qemu/range.h"

//#define DEBUG_SUBPAGE

#if !defined(CONFIG_USER_ONLY)

static bool in_migration;

RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };

static MemoryRegion *system_memory;

static MemoryRegion *system_io;

AddressSpace address_space_io;

AddressSpace address_space_memory;

MemoryRegion io_mem_rom, io_mem_notdirty;

static MemoryRegion io_mem_unassigned;

#endif

struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);

/* current CPU in the current thread. It is only valid inside

   cpu_exec() */

DEFINE_TLS(CPUState *, current_cpu);

/* 0 = Do not count executed instructions.

   1 = Precise instruction counting.

   2 = Adaptive rate instruction counting.  */

int use_icount;

#if !defined(CONFIG_USER_ONLY)

typedef struct PhysPageEntry PhysPageEntry;

struct PhysPageEntry {

    /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */

    uint32_t skip : 6;

     /* index into phys_sections (!skip) or phys_map_nodes (skip) */

    uint32_t ptr : 26;

};

#define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)

/* Size of the L2 (and L3, etc) page tables.  */

#define ADDR_SPACE_BITS 64

#define P_L2_BITS 9

#define P_L2_SIZE (1 << P_L2_BITS)

#define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)

typedef PhysPageEntry Node[P_L2_SIZE];

typedef struct PhysPageMap {

    unsigned sections_nb;

    unsigned sections_nb_alloc;

    unsigned nodes_nb;

    unsigned nodes_nb_alloc;

    Node *nodes;

    MemoryRegionSection *sections;

} PhysPageMap;

struct AddressSpaceDispatch {

    /* This is a multi-level map on the physical address space.

     * The bottom level has pointers to MemoryRegionSections.

     */

    PhysPageEntry phys_map;

    PhysPageMap map;

    AddressSpace *as;

};

#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)

typedef struct subpage_t {

    MemoryRegion iomem;

    AddressSpace *as;

    hwaddr base;

    uint16_t sub_section[TARGET_PAGE_SIZE];

} subpage_t;

#define PHYS_SECTION_UNASSIGNED 0

#define PHYS_SECTION_NOTDIRTY 1

#define PHYS_SECTION_ROM 2

#define PHYS_SECTION_WATCH 3

static void io_mem_init(void);

static void memory_map_init(void);

static void tcg_commit(MemoryListener *listener);

static MemoryRegion io_mem_watch;

#endif

#if !defined(CONFIG_USER_ONLY)

static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)

{

    if (map->nodes_nb + nodes > map->nodes_nb_alloc) {

        map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);

        map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);

        map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);

    }

}

static uint32_t phys_map_node_alloc(PhysPageMap *map)

{

    unsigned i;

    uint32_t ret;

    ret = map->nodes_nb++;

    assert(ret != PHYS_MAP_NODE_NIL);

    assert(ret != map->nodes_nb_alloc);

    for (i = 0; i < P_L2_SIZE; ++i) {

        map->nodes[ret][i].skip = 1;

        map->nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;

    }

    return ret;

}

static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,

                                hwaddr *index, hwaddr *nb, uint16_t leaf,

                                int level)

{

    PhysPageEntry *p;

    int i;

    hwaddr step = (hwaddr)1 << (level * P_L2_BITS);

    if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {

        lp->ptr = phys_map_node_alloc(map);

        p = map->nodes[lp->ptr];

        if (level == 0) {

            for (i = 0; i < P_L2_SIZE; i++) {

                p[i].skip = 0;

                p[i].ptr = PHYS_SECTION_UNASSIGNED;

            }

        }

    } else {

        p = map->nodes[lp->ptr];

    }

    lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];

    while (*nb && lp < &p[P_L2_SIZE]) {

        if ((*index & (step - 1)) == 0 && *nb >= step) {

            lp->skip = 0;

            lp->ptr = leaf;

            *index += step;

            *nb -= step;

        } else {

            phys_page_set_level(map, lp, index, nb, leaf, level - 1);

        }

        ++lp;

    }

}

static void phys_page_set(AddressSpaceDispatch *d,

                          hwaddr index, hwaddr nb,

                          uint16_t leaf)

{

    /* Wildly overreserve - it doesn't matter much. */

    phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);

    phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);

}

/* Compact a non leaf page entry. Simply detect that the entry has a single child,

 * and update our entry so we can skip it and go directly to the destination.

 */

static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)

{

    unsigned valid_ptr = P_L2_SIZE;

    int valid = 0;

    PhysPageEntry *p;

    int i;

    if (lp->ptr == PHYS_MAP_NODE_NIL) {

        return;

    }

    p = nodes[lp->ptr];

    for (i = 0; i < P_L2_SIZE; i++) {

        if (p[i].ptr == PHYS_MAP_NODE_NIL) {

            continue;

        }

        valid_ptr = i;

        valid++;

        if (p[i].skip) {

            phys_page_compact(&p[i], nodes, compacted);

        }

    }

    /* We can only compress if there's only one child. */

    if (valid != 1) {

        return;

    }

    assert(valid_ptr < P_L2_SIZE);

    /* Don't compress if it won't fit in the # of bits we have. */

    if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {

        return;

    }

    lp->ptr = p[valid_ptr].ptr;

    if (!p[valid_ptr].skip) {

        /* If our only child is a leaf, make this a leaf. */

        /* By design, we should have made this node a leaf to begin with so we

         * should never reach here.

         * But since it's so simple to handle this, let's do it just in case we

         * change this rule.

         */

        lp->skip = 0;

    } else {

        lp->skip += p[valid_ptr].skip;

    }

}

static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)

{

    DECLARE_BITMAP(compacted, nodes_nb);

    if (d->phys_map.skip) {

        phys_page_compact(&d->phys_map, d->map.nodes, compacted);

    }

}

static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,

                                           Node *nodes, MemoryRegionSection *sections)

{

    PhysPageEntry *p;

    hwaddr index = addr >> TARGET_PAGE_BITS;

    int i;

    for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {

        if (lp.ptr == PHYS_MAP_NODE_NIL) {

            return &sections[PHYS_SECTION_UNASSIGNED];

        }

        p = nodes[lp.ptr];

        lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];

    }

    if (sections[lp.ptr].size.hi ||

        range_covers_byte(sections[lp.ptr].offset_within_address_space,

                          sections[lp.ptr].size.lo, addr)) {

        return &sections[lp.ptr];

    } else {

        return &sections[PHYS_SECTION_UNASSIGNED];

    }

}

bool memory_region_is_unassigned(MemoryRegion *mr)

{

    return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device

        && mr != &io_mem_watch;

}

static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,

                                                        hwaddr addr,

                                                        bool resolve_subpage)

{

    MemoryRegionSection *section;

    subpage_t *subpage;

    section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);

    if (resolve_subpage && section->mr->subpage) {

        subpage = container_of(section->mr, subpage_t, iomem);

        section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];

    }

    return section;

}

static MemoryRegionSection *

address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,

                                 hwaddr *plen, bool resolve_subpage)

{

    MemoryRegionSection *section;

    Int128 diff;

    section = address_space_lookup_region(d, addr, resolve_subpage);

    /* Compute offset within MemoryRegionSection */

    addr -= section->offset_within_address_space;

    /* Compute offset within MemoryRegion */

    *xlat = addr + section->offset_within_region;

    diff = int128_sub(section->mr->size, int128_make64(addr));

    *plen = int128_get64(int128_min(diff, int128_make64(*plen)));

    return section;

}

static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)

{

    if (memory_region_is_ram(mr)) {

        return !(is_write && mr->readonly);

    }

    if (memory_region_is_romd(mr)) {

        return !is_write;

    }

    return false;

}

MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,

                                      hwaddr *xlat, hwaddr *plen,

                                      bool is_write)

{

    IOMMUTLBEntry iotlb;

    MemoryRegionSection *section;

    MemoryRegion *mr;

    hwaddr len = *plen;

    for (;;) {

        section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);

        mr = section->mr;

        if (!mr->iommu_ops) {

            break;

        }

        iotlb = mr->iommu_ops->translate(mr, addr);

        addr = ((iotlb.translated_addr & ~iotlb.addr_mask)

                | (addr & iotlb.addr_mask));

        len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);

        if (!(iotlb.perm & (1 << is_write))) {

            mr = &io_mem_unassigned;

            break;

        }

        as = iotlb.target_as;

    }

    if (memory_access_is_direct(mr, is_write)) {

        hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;

        len = MIN(page, len);

    }

    *plen = len;

    *xlat = addr;

    return mr;

}

MemoryRegionSection *

address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,

                                  hwaddr *plen)

{

    MemoryRegionSection *section;

    section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);

    assert(!section->mr->iommu_ops);

    return section;

}

#endif

void cpu_exec_init_all(void)

{

#if !defined(CONFIG_USER_ONLY)

    qemu_mutex_init(&ram_list.mutex);

    memory_map_init();

    io_mem_init();

#endif

}

#if !defined(CONFIG_USER_ONLY)

static int cpu_common_post_load(void *opaque, int version_id)

{

    CPUState *cpu = opaque;

    /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the

       version_id is increased. */

    cpu->interrupt_request &= ~0x01;

    tlb_flush(cpu->env_ptr, 1);

    return 0;

}

const VMStateDescription vmstate_cpu_common = {

    .name = "cpu_common",

    .version_id = 1,

    .minimum_version_id = 1,

    .minimum_version_id_old = 1,

    .post_load = cpu_common_post_load,

    .fields      = (VMStateField []) {

        VMSTATE_UINT32(halted, CPUState),

        VMSTATE_UINT32(interrupt_request, CPUState),

        VMSTATE_END_OF_LIST()

    }

};

#endif

CPUState *qemu_get_cpu(int index)

{

    CPUState *cpu;

    CPU_FOREACH(cpu) {

        if (cpu->cpu_index == index) {

            return cpu;

        }

    }

    return NULL;

}

#if !defined(CONFIG_USER_ONLY)

void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as)

{

    /* We only support one address space per cpu at the moment.  */

    assert(cpu->as == as);

    if (cpu->tcg_as_listener) {

        memory_listener_unregister(cpu->tcg_as_listener);

    } else {

        cpu->tcg_as_listener = g_new0(MemoryListener, 1);

    }

    cpu->tcg_as_listener->commit = tcg_commit;

    memory_listener_register(cpu->tcg_as_listener, as);

}

#endif

void cpu_exec_init(CPUArchState *env)

{

    CPUState *cpu = ENV_GET_CPU(env);

    CPUClass *cc = CPU_GET_CLASS(cpu);

    CPUState *some_cpu;

    int cpu_index;

#if defined(CONFIG_USER_ONLY)

    cpu_list_lock();

#endif

    cpu_index = 0;

    CPU_FOREACH(some_cpu) {

        cpu_index++;

    }

    cpu->cpu_index = cpu_index;

    cpu->numa_node = 0;

    QTAILQ_INIT(&env->breakpoints);

    QTAILQ_INIT(&env->watchpoints);

#ifndef CONFIG_USER_ONLY

    cpu->as = &address_space_memory;

    cpu->thread_id = qemu_get_thread_id();

#endif

    QTAILQ_INSERT_TAIL(&cpus, cpu, node);

#if defined(CONFIG_USER_ONLY)

    cpu_list_unlock();

#endif

    if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {

        vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);

    }

#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)

    register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,

                    cpu_save, cpu_load, env);

    assert(cc->vmsd == NULL);

    assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);

#endif

    if (cc->vmsd != NULL) {

        vmstate_register(NULL, cpu_index, cc->vmsd, cpu);

    }

}

#if defined(TARGET_HAS_ICE)

#if defined(CONFIG_USER_ONLY)

static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)

{

    tb_invalidate_phys_page_range(pc, pc + 1, 0);

}

#else

static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)

{

    hwaddr phys = cpu_get_phys_page_debug(cpu, pc);

    if (phys != -1) {

        tb_invalidate_phys_addr(cpu->as,

                                phys | (pc & ~TARGET_PAGE_MASK));

    }

}

#endif

#endif /* TARGET_HAS_ICE */

#if defined(CONFIG_USER_ONLY)

void cpu_watchpoint_remove_all(CPUArchState *env, int mask)

{

}

int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,

                          int flags, CPUWatchpoint **watchpoint)

{

    return -ENOSYS;

}

#else

/* Add a watchpoint.  */

int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,

                          int flags, CPUWatchpoint **watchpoint)

{

    target_ulong len_mask = ~(len - 1);

    CPUWatchpoint *wp;

    /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */

    if ((len & (len - 1)) || (addr & ~len_mask) ||

            len == 0 || len > TARGET_PAGE_SIZE) {

        fprintf(stderr, "qemu: tried to set invalid watchpoint at "

                TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);

        return -EINVAL;

    }

    wp = g_malloc(sizeof(*wp));

    wp->vaddr = addr;

    wp->len_mask = len_mask;

    wp->flags = flags;

    /* keep all GDB-injected watchpoints in front */

    if (flags & BP_GDB)

        QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);

    else

        QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);

    tlb_flush_page(env, addr);

    if (watchpoint)

        *watchpoint = wp;

    return 0;

}

/* Remove a specific watchpoint.  */

int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,

                          int flags)

{

    target_ulong len_mask = ~(len - 1);

    CPUWatchpoint *wp;

    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {

        if (addr == wp->vaddr && len_mask == wp->len_mask

                && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {

            cpu_watchpoint_remove_by_ref(env, wp);

            return 0;

        }

    }

    return -ENOENT;

}

/* Remove a specific watchpoint by reference.  */

void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)

{

    QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);

    tlb_flush_page(env, watchpoint->vaddr);

    g_free(watchpoint);

}

/* Remove all matching watchpoints.  */

void cpu_watchpoint_remove_all(CPUArchState *env, int mask)

{

    CPUWatchpoint *wp, *next;

    QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {

        if (wp->flags & mask)

            cpu_watchpoint_remove_by_ref(env, wp);

    }

}

#endif

/* Add a breakpoint.  */

int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,

                          CPUBreakpoint **breakpoint)

{

#if defined(TARGET_HAS_ICE)

    CPUBreakpoint *bp;

    bp = g_malloc(sizeof(*bp));

    bp->pc = pc;

    bp->flags = flags;

    /* keep all GDB-injected breakpoints in front */

    if (flags & BP_GDB) {

        QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);

    } else {

        QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);

    }

    breakpoint_invalidate(ENV_GET_CPU(env), pc);

    if (breakpoint) {

        *breakpoint = bp;

    }

    return 0;

#else

    return -ENOSYS;

#endif

}

/* Remove a specific breakpoint.  */

int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)

{

#if defined(TARGET_HAS_ICE)

    CPUBreakpoint *bp;

    QTAILQ_FOREACH(bp, &env->breakpoints, entry) {

        if (bp->pc == pc && bp->flags == flags) {

            cpu_breakpoint_remove_by_ref(env, bp);

            return 0;

        }

    }

    return -ENOENT;

#else

    return -ENOSYS;

#endif

}

/* Remove a specific breakpoint by reference.  */

void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)

{

#if defined(TARGET_HAS_ICE)

    QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);

    breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);

    g_free(breakpoint);

#endif

}

/* Remove all matching breakpoints. */

void cpu_breakpoint_remove_all(CPUArchState *env, int mask)

{

#if defined(TARGET_HAS_ICE)

    CPUBreakpoint *bp, *next;

    QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {

        if (bp->flags & mask)

            cpu_breakpoint_remove_by_ref(env, bp);

    }

#endif

}

/* enable or disable single step mode. EXCP_DEBUG is returned by the

   CPU loop after each instruction */

void cpu_single_step(CPUState *cpu, int enabled)

{

#if defined(TARGET_HAS_ICE)

    if (cpu->singlestep_enabled != enabled) {

        cpu->singlestep_enabled = enabled;

        if (kvm_enabled()) {

            kvm_update_guest_debug(cpu, 0);

        } else {

            /* must flush all the translated code to avoid inconsistencies */

            /* XXX: only flush what is necessary */

            CPUArchState *env = cpu->env_ptr;

            tb_flush(env);

        }

    }

#endif

}

void cpu_abort(CPUArchState *env, const char *fmt, ...)

{

    CPUState *cpu = ENV_GET_CPU(env);

    va_list ap;

    va_list ap2;

    va_start(ap, fmt);

    va_copy(ap2, ap);

    fprintf(stderr, "qemu: fatal: ");

    vfprintf(stderr, fmt, ap);

    fprintf(stderr, "\n");

    cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);

    if (qemu_log_enabled()) {

        qemu_log("qemu: fatal: ");

        qemu_log_vprintf(fmt, ap2);

        qemu_log("\n");

        log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);

        qemu_log_flush();

        qemu_log_close();

    }

    va_end(ap2);

    va_end(ap);

#if defined(CONFIG_USER_ONLY)

    {

        struct sigaction act;

        sigfillset(&act.sa_mask);

        act.sa_handler = SIG_DFL;

        sigaction(SIGABRT, &act, NULL);

    }

#endif

    abort();

}

#if !defined(CONFIG_USER_ONLY)

static RAMBlock *qemu_get_ram_block(ram_addr_t addr)

{

    RAMBlock *block;

    /* The list is protected by the iothread lock here.  */

    block = ram_list.mru_block;

    if (block && addr - block->offset < block->length) {

        goto found;

    }

    QTAILQ_FOREACH(block, &ram_list.blocks, next) {

        if (addr - block->offset < block->length) {

            goto found;

        }

    }

    fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);

    abort();

found:

    ram_list.mru_block = block;

    return block;

}

static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)

{

    ram_addr_t start1;

    RAMBlock *block;

    ram_addr_t end;

    end = TARGET_PAGE_ALIGN(start + length);

    start &= TARGET_PAGE_MASK;

    block = qemu_get_ram_block(start);

    assert(block == qemu_get_ram_block(end - 1));

    start1 = (uintptr_t)block->host + (start - block->offset);

    cpu_tlb_reset_dirty_all(start1, length);

}

/* Note: start and end must be within the same ram block.  */

void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length,

                                     unsigned client)

{

    if (length == 0)

        return;

    cpu_physical_memory_clear_dirty_range(start, length, client);

    if (tcg_enabled()) {

        tlb_reset_dirty_range_all(start, length);

    }

}

static void cpu_physical_memory_set_dirty_tracking(bool enable)

{

    in_migration = enable;

}

hwaddr memory_region_section_get_iotlb(CPUArchState *env,

                                       MemoryRegionSection *section,

                                       target_ulong vaddr,

                                       hwaddr paddr, hwaddr xlat,

                                       int prot,

                                       target_ulong *address)

{

    hwaddr iotlb;

    CPUWatchpoint *wp;

    if (memory_region_is_ram(section->mr)) {

        /* Normal RAM.  */

        iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)

            + xlat;

        if (!section->readonly) {

            iotlb |= PHYS_SECTION_NOTDIRTY;

        } else {

            iotlb |= PHYS_SECTION_ROM;

        }

    } else {

        iotlb = section - section->address_space->dispatch->map.sections;

        iotlb += xlat;

    }

    /* Make accesses to pages with watchpoints go via the

       watchpoint trap routines.  */

    QTAILQ_FOREACH(wp, &env->watchpoints, entry) {

        if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {

            /* Avoid trapping reads of pages with a write breakpoint. */

            if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {

                iotlb = PHYS_SECTION_WATCH + paddr;

                *address |= TLB_MMIO;

                break;

            }

        }

    }

    return iotlb;

}

#endif /* defined(CONFIG_USER_ONLY) */

#if !defined(CONFIG_USER_ONLY)

static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,

                             uint16_t section);

static subpage_t *subpage_init(AddressSpace *as, hwaddr base);

static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;

/*

 * Set a custom physical guest memory alloator.

 * Accelerators with unusual needs may need this.  Hopefully, we can

 * get rid of it eventually.

 */

void phys_mem_set_alloc(void *(*alloc)(size_t))

{

    phys_mem_alloc = alloc;

}

static uint16_t phys_section_add(PhysPageMap *map,

                                 MemoryRegionSection *section)

{

    /* The physical section number is ORed with a page-aligned

     * pointer to produce the iotlb entries.  Thus it should

     * never overflow into the page-aligned value.

     */

    assert(map->sections_nb < TARGET_PAGE_SIZE);

    if (map->sections_nb == map->sections_nb_alloc) {

        map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);

        map->sections = g_renew(MemoryRegionSection, map->sections,

                                map->sections_nb_alloc);

    }

    map->sections[map->sections_nb] = *section;

    memory_region_ref(section->mr);

    return map->sections_nb++;

}

static void phys_section_destroy(MemoryRegion *mr)

{

    memory_region_unref(mr);

    if (mr->subpage) {

        subpage_t *subpage = container_of(mr, subpage_t, iomem);

        memory_region_destroy(&subpage->iomem);

        g_free(subpage);

    }

}

static void phys_sections_free(PhysPageMap *map)

{

    while (map->sections_nb > 0) {

        MemoryRegionSection *section = &map->sections[--map->sections_nb];

        phys_section_destroy(section->mr);

    }

    g_free(map->sections);

    g_free(map->nodes);

}

static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)

{

    subpage_t *subpage;

    hwaddr base = section->offset_within_address_space

        & TARGET_PAGE_MASK;

    MemoryRegionSection *existing = phys_page_find(d->phys_map, base,

                                                   d->map.nodes, d->map.sections);

    MemoryRegionSection subsection = {

        .offset_within_address_space = base,

        .size = int128_make64(TARGET_PAGE_SIZE),

    };

    hwaddr start, end;

    assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);

    if (!(existing->mr->subpage)) {

        subpage = subpage_init(d->as, base);

        subsection.address_space = d->as;

        subsection.mr = &subpage->iomem;

        phys_page_set(d, base >> TARGET_PAGE_BITS, 1,

                      phys_section_add(&d->map, &subsection));

    } else {

        subpage = container_of(existing->mr, subpage_t, iomem);

    }

    start = section->offset_within_address_space & ~TARGET_PAGE_MASK;

    end = start + int128_get64(section->size) - 1;

    subpage_register(subpage, start, end,

                     phys_section_add(&d->map, section));

}

static void register_multipage(AddressSpaceDispatch *d,

                               MemoryRegionSection *section)

{

    hwaddr start_addr = section->offset_within_address_space;

    uint16_t section_index = phys_section_add(&d->map, section);

    uint64_t num_pages = int128_get64(int128_rshift(section->size,

                                                    TARGET_PAGE_BITS));

    assert(num_pages);

    phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);

}

static void mem_add(MemoryListener *listener, MemoryRegionSection *section)

{

    AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);

    AddressSpaceDispatch *d = as->next_dispatch;

    MemoryRegionSection now = *section, remain = *section;

    Int128 page_size = int128_make64(TARGET_PAGE_SIZE);

    if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {

        uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)

                       - now.offset_within_address_space;

        now.size = int128_min(int128_make64(left), now.size);

        register_subpage(d, &now);

    } else {

        now.size = int128_zero();

    }

    while (int128_ne(remain.size, now.size)) {

        remain.size = int128_sub(remain.size, now.size);

        remain.offset_within_address_space += int128_get64(now.size);

        remain.offset_within_region += int128_get64(now.size);

        now = remain;

        if (int128_lt(remain.size, page_size)) {

            register_subpage(d, &now);

        } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {

            now.size = page_size;

            register_subpage(d, &now);

        } else {

            now.size = int128_and(now.size, int128_neg(page_size));

            register_multipage(d, &now);

        }

    }

}

void qemu_flush_coalesced_mmio_buffer(void)

{

    if (kvm_enabled())

        kvm_flush_coalesced_mmio_buffer();

}

void qemu_mutex_lock_ramlist(void)

{

    qemu_mutex_lock(&ram_list.mutex);

}

void qemu_mutex_unlock_ramlist(void)

{

    qemu_mutex_unlock(&ram_list.mutex);

}

#ifdef __linux__

#include <sys/vfs.h>

#define HUGETLBFS_MAGIC       0x958458f6

static long gethugepagesize(const char *path)

{

    struct statfs fs;

    int ret;

    do {

        ret = statfs(path, &fs);

    } while (ret != 0 && errno == EINTR);

    if (ret != 0) {

        perror(path);

        return 0;

    }

    if (fs.f_type != HUGETLBFS_MAGIC)

        fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);

    return fs.f_bsize;

}

static sigjmp_buf sigjump;

static void sigbus_handler(int signal)

{

    siglongjmp(sigjump, 1);

}

static void *file_ram_alloc(RAMBlock *block,

                            ram_addr_t memory,

                            const char *path)

{

    char *filename;

    char *sanitized_name;

    char *c;

    void *area;

    int fd;

    unsigned long hpagesize;

    hpagesize = gethugepagesize(path);

    if (!hpagesize) {

        return NULL;

    }

    if (memory < hpagesize) {

        return NULL;

    }

    if (kvm_enabled() && !kvm_has_sync_mmu()) {

        fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");

        return NULL;

    }

    /* Make name safe to use with mkstemp by replacing '/' with '_'. */

    sanitized_name = g_strdup(block->mr->name);

    for (c = sanitized_name; *c != '\0'; c++) {

        if (*c == '/')

            *c = '_';

    }

    filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,

                               sanitized_name);

    g_free(sanitized_name);

    fd = mkstemp(filename);

    if (fd < 0) {

        perror("unable to create backing store for hugepages");

        g_free(filename);

        return NULL;

    }

    unlink(filename);

    g_free(filename);

    memory = (memory+hpagesize-1) & ~(hpagesize-1);

    /*

     * ftruncate is not supported by hugetlbfs in older

     * hosts, so don't bother bailing out on errors.

     * If anything goes wrong with it under other filesystems,

     * mmap will fail.

     */

    if (ftruncate(fd, memory))

        perror("ftruncate");

    area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);

    if (area == MAP_FAILED) {

        perror("file_ram_alloc: can't mmap RAM pages");

        close(fd);

        return (NULL);

    }

    if (mem_prealloc) {

        int ret, i;

        struct sigaction act, oldact;

        sigset_t set, oldset;

        memset(&act, 0, sizeof(act));

        act.sa_handler = &sigbus_handler;

        act.sa_flags = 0;

        ret = sigaction(SIGBUS, &act, &oldact);

        if (ret) {

            perror("file_ram_alloc: failed to install signal handler");

            exit(1);

        }

        /* unblock SIGBUS */

        sigemptyset(&set);

        sigaddset(&set, SIGBUS);

        pthread_sigmask(SIG_UNBLOCK, &set, &oldset);

        if (sigsetjmp(sigjump, 1)) {

            fprintf(stderr, "file_ram_alloc: failed to preallocate pages\n");

            exit(1);

        }

        /* MAP_POPULATE silently ignores failures */

        for (i = 0; i < (memory/hpagesize); i++) {

            memset(area + (hpagesize*i), 0, 1);

        }

        ret = sigaction(SIGBUS, &oldact, NULL);

        if (ret) {

            perror("file_ram_alloc: failed to reinstall signal handler");

            exit(1);

        }

        pthread_sigmask(SIG_SETMASK, &oldset, NULL);

    }

    block->fd = fd;

    return area;

}

#else

static void *file_ram_alloc(RAMBlock *block,

                            ram_addr_t memory,

                            const char *path)

{

    fprintf(stderr, "-mem-path not supported on this host\n");

    exit(1);

}

#endif

static ram_addr_t find_ram_offset(ram_addr_t size)

{

    RAMBlock *block, *next_block;

    ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;

    assert(size != 0); /* it would hand out same offset multiple times */

    if (QTAILQ_EMPTY(&ram_list.blocks))

        return 0;

    QTAILQ_FOREACH(block, &ram_list.blocks, next) {

        ram_addr_t end, next = RAM_ADDR_MAX;

        end = block->offset + block->length;

        QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {

            if (next_block->offset >= end) {

                next = MIN(next, next_block->offset);

            }

        }

        if (next - end >= size && next - end < mingap) {

            offset = end;

            mingap = next - end;

        }

    }

    if (offset == RAM_ADDR_MAX) {

        fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",

                (uint64_t)size);

        abort();

    }

    return offset;

}

ram_addr_t last_ram_offset(void)

{

    RAMBlock *block;

    ram_addr_t last = 0;

    QTAILQ_FOREACH(block, &ram_list.blocks, next)

        last = MAX(last, block->offset + block->length);

    return last;

}

static void qemu_ram_setup_dump(void *addr, ram_addr_t size)

{

    int ret;

    /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */

    if (!qemu_opt_get_bool(qemu_get_machine_opts(),

                           "dump-guest-core", true)) {

        ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);

        if (ret) {

            perror("qemu_madvise");

            fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "

                            "but dump_guest_core=off specified\n");

        }

    }

}

void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)

{

    RAMBlock *new_block, *block;

    new_block = NULL;

    QTAILQ_FOREACH(block, &ram_list.blocks, next) {

        if (block->offset == addr) {

            new_block = block;

            break;

        }

    }

    assert(new_block);

    assert(!new_block->idstr[0]);

    if (dev) {

        char *id = qdev_get_dev_path(dev);

        if (id) {

            snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);

            g_free(id);

        }

    }

    pstrcat(new_block->idstr, sizeof(new_block->idstr), name);

    /* This assumes the iothread lock is taken here too.  */

    qemu_mutex_lock_ramlist();

    QTAILQ_FOREACH(block, &ram_list.blocks, next) {

        if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {

            fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",

                    new_block->idstr);

            abort();

        }

    }

    qemu_mutex_unlock_ramlist();

}

static int memory_try_enable_merging(void *addr, size_t len)

{

    if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {

        /* disabled by the user */

        return 0;

    }

    return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);

}

ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,

                                   MemoryRegion *mr)

{

    RAMBlock *block, *new_block;

    ram_addr_t old_ram_size, new_ram_size;

    old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;

    size = TARGET_PAGE_ALIGN(size);

    new_block = g_malloc0(sizeof(*new_block));

    new_block->fd = -1;

    /* This assumes the iothread lock is taken here too.  */

    qemu_mutex_lock_ramlist();

    new_block->mr = mr;

    new_block->offset = find_ram_offset(size);

    if (host) {

        new_block->host = host;

        new_block->flags |= RAM_PREALLOC_MASK;

    } else if (xen_enabled()) {

        if (mem_path) {

            fprintf(stderr, "-mem-path not supported with Xen\n");

            exit(1);

        }

        xen_ram_alloc(new_block->offset, size, mr);

    } else {

        if (mem_path) {

            if (phys_mem_alloc != qemu_anon_ram_alloc) {

                /*

                 * file_ram_alloc() needs to allocate just like

                 * phys_mem_alloc, but we haven't bothered to provide

                 * a hook there.

                 */

                fprintf(stderr,

                        "-mem-path not supported with this accelerator\n");

                exit(1);

            }

            new_block->host = file_ram_alloc(new_block, size, mem_path);

        }

        if (!new_block->host) {

            new_block->host = phys_mem_alloc(size);

            if (!new_block->host) {

                fprintf(stderr, "Cannot set up guest memory '%s': %s\n",

                        new_block->mr->name, strerror(errno));

                exit(1);

            }

            memory_try_enable_merging(new_block->host, size);

        }

    }

    new_block->length = size;

    /* Keep the list sorted from biggest to smallest block.  */

    QTAILQ_FOREACH(block, &ram_list.blocks, next) {

        if (block->length < new_block->length) {

            break;

        }

    }

    if (block) {

        QTAILQ_INSERT_BEFORE(block, new_block, next);

    } else {

        QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);

    }

    ram_list.mru_block = NULL;

    ram_list.version++;

    qemu_mutex_unlock_ramlist();

    new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;

    if (new_ram_size > old_ram_size) {

        int i;

        for (i = 0; i < DIRTY_MEMORY_NUM; i++) {

            ram_list.dirty_memory[i] =

                bitmap_zero_extend(ram_list.dirty_memory[i],

                                   old_ram_size, new_ram_size);

       }

    }

    cpu_physical_memory_set_dirty_range(new_block->offset, size);

    qemu_ram_setup_dump(new_block->host, size);

    qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);

    qemu_madvise(new_block->host, size, QEMU_MADV_DONTFORK);

    if (kvm_enabled())

        kvm_setup_guest_memory(new_block->host, size);

    return new_block->offset;

}

ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)

{

    return qemu_ram_alloc_from_ptr(size, NULL, mr);

}

void qemu_ram_free_from_ptr(ram_addr_t addr)

{

    RAMBlock *block;

    /* This assumes the iothread lock is taken here too.  */

    qemu_mutex_lock_ramlist();

    QTAILQ_FOREACH(block, &ram_list.blocks, next) {

        if (addr == block->offset) {

            QTAILQ_REMOVE(&ram_list.blocks, block, next);

            ram_list.mru_block = NULL;

            ram_list.version++;

            g_free(block);

            break;

        }

    }

    qemu_mutex_unlock_ramlist();

}

void qemu_ram_free(ram_addr_t addr)

{

    RAMBlock *block;

    /* This assumes the iothread lock is taken here too.  */

    qemu_mutex_lock_ramlist();

    QTAILQ_FOREACH(block, &ram_list.blocks, next) {

        if (addr == block->offset) {

            QTAILQ_REMOVE(&ram_list.blocks, block, next);

            ram_list.mru_block = NULL;

            ram_list.version++;

            if (block->flags & RAM_PREALLOC_MASK) {

                ;

            } else if (xen_enabled()) {

                xen_invalidate_map_cache_entry(block->host);

#ifndef _WIN32

            } else if (block->fd >= 0) {

                munmap(block->host, block->length);

                close(block->fd);

#endif

            } else {

                qemu_anon_ram_free(block->host, block->length);

            }

            g_free(block);

            break;

        }

    }

    qemu_mutex_unlock_ramlist();

}

#ifndef _WIN32

void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)

{

    RAMBlock *block;

    ram_addr_t offset;

    int flags;

    void *area, *vaddr;

    QTAILQ_FOREACH(block, &ram_list.blocks, next) {

        offset = addr - block->offset;

        if (offset < block->length) {

            vaddr = block->host + offset;

            if (block->flags & RAM_PREALLOC_MASK) {

                ;

            } else if (xen_enabled()) {

                abort();

            } else {

                flags = MAP_FIXED;

                munmap(vaddr, length);

                if (block->fd >= 0) {

#ifdef MAP_POPULATE

                    flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :

                        MAP_PRIVATE;

#else

                    flags |= MAP_PRIVATE;