Archipelago » qemu

/*

 *  x86 segmentation related helpers:

 *  TSS, interrupts, system calls, jumps and call/task gates, descriptors

 *

 *  Copyright (c) 2003 Fabrice Bellard

 *

 * This library is free software; you can redistribute it and/or

 * modify it under the terms of the GNU Lesser General Public

 * License as published by the Free Software Foundation; either

 * version 2 of the License, or (at your option) any later version.

 *

 * This library is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * Lesser General Public License for more details.

 *

 * You should have received a copy of the GNU Lesser General Public

 * License along with this library; if not, see <http://www.gnu.org/licenses/>.

 */

#include "cpu.h"

#include "qemu-log.h"

#include "helper.h"

//#define DEBUG_PCALL

#if !defined(CONFIG_USER_ONLY)

#include "softmmu_exec.h"

#endif /* !defined(CONFIG_USER_ONLY) */

#ifdef DEBUG_PCALL

# define LOG_PCALL(...) qemu_log_mask(CPU_LOG_PCALL, ## __VA_ARGS__)

# define LOG_PCALL_STATE(env)                                  \

    log_cpu_state_mask(CPU_LOG_PCALL, (env), CPU_DUMP_CCOP)

#else

# define LOG_PCALL(...) do { } while (0)

# define LOG_PCALL_STATE(env) do { } while (0)

#endif

/* return non zero if error */

static inline int load_segment(CPUX86State *env, uint32_t *e1_ptr,

                               uint32_t *e2_ptr, int selector)

{

    SegmentCache *dt;

    int index;

    target_ulong ptr;

    if (selector & 0x4) {

        dt = &env->ldt;

    } else {

        dt = &env->gdt;

    }

    index = selector & ~7;

    if ((index + 7) > dt->limit) {

        return -1;

    }

    ptr = dt->base + index;

    *e1_ptr = cpu_ldl_kernel(env, ptr);

    *e2_ptr = cpu_ldl_kernel(env, ptr + 4);

    return 0;

}

static inline unsigned int get_seg_limit(uint32_t e1, uint32_t e2)

{

    unsigned int limit;

    limit = (e1 & 0xffff) | (e2 & 0x000f0000);

    if (e2 & DESC_G_MASK) {

        limit = (limit << 12) | 0xfff;

    }

    return limit;

}

static inline uint32_t get_seg_base(uint32_t e1, uint32_t e2)

{

    return (e1 >> 16) | ((e2 & 0xff) << 16) | (e2 & 0xff000000);

}

static inline void load_seg_cache_raw_dt(SegmentCache *sc, uint32_t e1,

                                         uint32_t e2)

{

    sc->base = get_seg_base(e1, e2);

    sc->limit = get_seg_limit(e1, e2);

    sc->flags = e2;

}

/* init the segment cache in vm86 mode. */

static inline void load_seg_vm(CPUX86State *env, int seg, int selector)

{

    selector &= 0xffff;

    cpu_x86_load_seg_cache(env, seg, selector,

                           (selector << 4), 0xffff, 0);

}

static inline void get_ss_esp_from_tss(CPUX86State *env, uint32_t *ss_ptr,

                                       uint32_t *esp_ptr, int dpl)

{

    int type, index, shift;

#if 0

    {

        int i;

        printf("TR: base=%p limit=%x\n", env->tr.base, env->tr.limit);

        for (i = 0; i < env->tr.limit; i++) {

            printf("%02x ", env->tr.base[i]);

            if ((i & 7) == 7) {

                printf("\n");

            }

        }

        printf("\n");

    }

#endif

    if (!(env->tr.flags & DESC_P_MASK)) {

        cpu_abort(env, "invalid tss");

    }

    type = (env->tr.flags >> DESC_TYPE_SHIFT) & 0xf;

    if ((type & 7) != 1) {

        cpu_abort(env, "invalid tss type");

    }

    shift = type >> 3;

    index = (dpl * 4 + 2) << shift;

    if (index + (4 << shift) - 1 > env->tr.limit) {

        raise_exception_err(env, EXCP0A_TSS, env->tr.selector & 0xfffc);

    }

    if (shift == 0) {

        *esp_ptr = cpu_lduw_kernel(env, env->tr.base + index);

        *ss_ptr = cpu_lduw_kernel(env, env->tr.base + index + 2);

    } else {

        *esp_ptr = cpu_ldl_kernel(env, env->tr.base + index);

        *ss_ptr = cpu_lduw_kernel(env, env->tr.base + index + 4);

    }

}

/* XXX: merge with load_seg() */

static void tss_load_seg(CPUX86State *env, int seg_reg, int selector)

{

    uint32_t e1, e2;

    int rpl, dpl, cpl;

    if ((selector & 0xfffc) != 0) {

        if (load_segment(env, &e1, &e2, selector) != 0) {

            raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);

        }

        if (!(e2 & DESC_S_MASK)) {

            raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);

        }

        rpl = selector & 3;

        dpl = (e2 >> DESC_DPL_SHIFT) & 3;

        cpl = env->hflags & HF_CPL_MASK;

        if (seg_reg == R_CS) {

            if (!(e2 & DESC_CS_MASK)) {

                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);

            }

            /* XXX: is it correct? */

            if (dpl != rpl) {

                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);

            }

            if ((e2 & DESC_C_MASK) && dpl > rpl) {

                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);

            }

        } else if (seg_reg == R_SS) {

            /* SS must be writable data */

            if ((e2 & DESC_CS_MASK) || !(e2 & DESC_W_MASK)) {

                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);

            }

            if (dpl != cpl || dpl != rpl) {

                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);

            }

        } else {

            /* not readable code */

            if ((e2 & DESC_CS_MASK) && !(e2 & DESC_R_MASK)) {

                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);

            }

            /* if data or non conforming code, checks the rights */

            if (((e2 >> DESC_TYPE_SHIFT) & 0xf) < 12) {

                if (dpl < cpl || dpl < rpl) {

                    raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);

                }

            }

        }

        if (!(e2 & DESC_P_MASK)) {

            raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);

        }

        cpu_x86_load_seg_cache(env, seg_reg, selector,

                               get_seg_base(e1, e2),

                               get_seg_limit(e1, e2),

                               e2);

    } else {

        if (seg_reg == R_SS || seg_reg == R_CS) {

            raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);

        }

    }

}

#define SWITCH_TSS_JMP  0

#define SWITCH_TSS_IRET 1

#define SWITCH_TSS_CALL 2

/* XXX: restore CPU state in registers (PowerPC case) */

static void switch_tss(CPUX86State *env, int tss_selector,

                       uint32_t e1, uint32_t e2, int source,

                       uint32_t next_eip)

{

    int tss_limit, tss_limit_max, type, old_tss_limit_max, old_type, v1, v2, i;

    target_ulong tss_base;

    uint32_t new_regs[8], new_segs[6];

    uint32_t new_eflags, new_eip, new_cr3, new_ldt, new_trap;

    uint32_t old_eflags, eflags_mask;

    SegmentCache *dt;

    int index;

    target_ulong ptr;

    type = (e2 >> DESC_TYPE_SHIFT) & 0xf;

    LOG_PCALL("switch_tss: sel=0x%04x type=%d src=%d\n", tss_selector, type,

              source);

    /* if task gate, we read the TSS segment and we load it */

    if (type == 5) {

        if (!(e2 & DESC_P_MASK)) {

            raise_exception_err(env, EXCP0B_NOSEG, tss_selector & 0xfffc);

        }

        tss_selector = e1 >> 16;

        if (tss_selector & 4) {

            raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);

        }

        if (load_segment(env, &e1, &e2, tss_selector) != 0) {

            raise_exception_err(env, EXCP0D_GPF, tss_selector & 0xfffc);

        }

        if (e2 & DESC_S_MASK) {

            raise_exception_err(env, EXCP0D_GPF, tss_selector & 0xfffc);

        }

        type = (e2 >> DESC_TYPE_SHIFT) & 0xf;

        if ((type & 7) != 1) {

            raise_exception_err(env, EXCP0D_GPF, tss_selector & 0xfffc);

        }

    }

    if (!(e2 & DESC_P_MASK)) {

        raise_exception_err(env, EXCP0B_NOSEG, tss_selector & 0xfffc);

    }

    if (type & 8) {

        tss_limit_max = 103;

    } else {

        tss_limit_max = 43;

    }

    tss_limit = get_seg_limit(e1, e2);

    tss_base = get_seg_base(e1, e2);

    if ((tss_selector & 4) != 0 ||

        tss_limit < tss_limit_max) {

        raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);

    }

    old_type = (env->tr.flags >> DESC_TYPE_SHIFT) & 0xf;

    if (old_type & 8) {

        old_tss_limit_max = 103;

    } else {

        old_tss_limit_max = 43;

    }

    /* read all the registers from the new TSS */

    if (type & 8) {

        /* 32 bit */

        new_cr3 = cpu_ldl_kernel(env, tss_base + 0x1c);

        new_eip = cpu_ldl_kernel(env, tss_base + 0x20);

        new_eflags = cpu_ldl_kernel(env, tss_base + 0x24);

        for (i = 0; i < 8; i++) {

            new_regs[i] = cpu_ldl_kernel(env, tss_base + (0x28 + i * 4));

        }

        for (i = 0; i < 6; i++) {

            new_segs[i] = cpu_lduw_kernel(env, tss_base + (0x48 + i * 4));

        }

        new_ldt = cpu_lduw_kernel(env, tss_base + 0x60);

        new_trap = cpu_ldl_kernel(env, tss_base + 0x64);

    } else {

        /* 16 bit */

        new_cr3 = 0;

        new_eip = cpu_lduw_kernel(env, tss_base + 0x0e);

        new_eflags = cpu_lduw_kernel(env, tss_base + 0x10);

        for (i = 0; i < 8; i++) {

            new_regs[i] = cpu_lduw_kernel(env, tss_base + (0x12 + i * 2)) |

                0xffff0000;

        }

        for (i = 0; i < 4; i++) {

            new_segs[i] = cpu_lduw_kernel(env, tss_base + (0x22 + i * 4));

        }

        new_ldt = cpu_lduw_kernel(env, tss_base + 0x2a);

        new_segs[R_FS] = 0;

        new_segs[R_GS] = 0;

        new_trap = 0;

    }

    /* XXX: avoid a compiler warning, see

     http://support.amd.com/us/Processor_TechDocs/24593.pdf

     chapters 12.2.5 and 13.2.4 on how to implement TSS Trap bit */

    (void)new_trap;

    /* NOTE: we must avoid memory exceptions during the task switch,

       so we make dummy accesses before */

    /* XXX: it can still fail in some cases, so a bigger hack is

       necessary to valid the TLB after having done the accesses */

    v1 = cpu_ldub_kernel(env, env->tr.base);

    v2 = cpu_ldub_kernel(env, env->tr.base + old_tss_limit_max);

    cpu_stb_kernel(env, env->tr.base, v1);

    cpu_stb_kernel(env, env->tr.base + old_tss_limit_max, v2);

    /* clear busy bit (it is restartable) */

    if (source == SWITCH_TSS_JMP || source == SWITCH_TSS_IRET) {

        target_ulong ptr;

        uint32_t e2;

        ptr = env->gdt.base + (env->tr.selector & ~7);

        e2 = cpu_ldl_kernel(env, ptr + 4);

        e2 &= ~DESC_TSS_BUSY_MASK;

        cpu_stl_kernel(env, ptr + 4, e2);

    }

    old_eflags = cpu_compute_eflags(env);

    if (source == SWITCH_TSS_IRET) {

        old_eflags &= ~NT_MASK;

    }

    /* save the current state in the old TSS */

    if (type & 8) {

        /* 32 bit */

        cpu_stl_kernel(env, env->tr.base + 0x20, next_eip);

        cpu_stl_kernel(env, env->tr.base + 0x24, old_eflags);

        cpu_stl_kernel(env, env->tr.base + (0x28 + 0 * 4), EAX);

        cpu_stl_kernel(env, env->tr.base + (0x28 + 1 * 4), ECX);

        cpu_stl_kernel(env, env->tr.base + (0x28 + 2 * 4), EDX);

        cpu_stl_kernel(env, env->tr.base + (0x28 + 3 * 4), EBX);

        cpu_stl_kernel(env, env->tr.base + (0x28 + 4 * 4), ESP);

        cpu_stl_kernel(env, env->tr.base + (0x28 + 5 * 4), EBP);

        cpu_stl_kernel(env, env->tr.base + (0x28 + 6 * 4), ESI);

        cpu_stl_kernel(env, env->tr.base + (0x28 + 7 * 4), EDI);

        for (i = 0; i < 6; i++) {

            cpu_stw_kernel(env, env->tr.base + (0x48 + i * 4),

                           env->segs[i].selector);

        }

    } else {

        /* 16 bit */

        cpu_stw_kernel(env, env->tr.base + 0x0e, next_eip);

        cpu_stw_kernel(env, env->tr.base + 0x10, old_eflags);

        cpu_stw_kernel(env, env->tr.base + (0x12 + 0 * 2), EAX);

        cpu_stw_kernel(env, env->tr.base + (0x12 + 1 * 2), ECX);

        cpu_stw_kernel(env, env->tr.base + (0x12 + 2 * 2), EDX);

        cpu_stw_kernel(env, env->tr.base + (0x12 + 3 * 2), EBX);

        cpu_stw_kernel(env, env->tr.base + (0x12 + 4 * 2), ESP);

        cpu_stw_kernel(env, env->tr.base + (0x12 + 5 * 2), EBP);

        cpu_stw_kernel(env, env->tr.base + (0x12 + 6 * 2), ESI);

        cpu_stw_kernel(env, env->tr.base + (0x12 + 7 * 2), EDI);

        for (i = 0; i < 4; i++) {

            cpu_stw_kernel(env, env->tr.base + (0x22 + i * 4),

                           env->segs[i].selector);

        }

    }

    /* now if an exception occurs, it will occurs in the next task

       context */

    if (source == SWITCH_TSS_CALL) {

        cpu_stw_kernel(env, tss_base, env->tr.selector);

        new_eflags |= NT_MASK;

    }

    /* set busy bit */

    if (source == SWITCH_TSS_JMP || source == SWITCH_TSS_CALL) {

        target_ulong ptr;

        uint32_t e2;

        ptr = env->gdt.base + (tss_selector & ~7);

        e2 = cpu_ldl_kernel(env, ptr + 4);

        e2 |= DESC_TSS_BUSY_MASK;

        cpu_stl_kernel(env, ptr + 4, e2);

    }

    /* set the new CPU state */

    /* from this point, any exception which occurs can give problems */

    env->cr[0] |= CR0_TS_MASK;

    env->hflags |= HF_TS_MASK;

    env->tr.selector = tss_selector;

    env->tr.base = tss_base;

    env->tr.limit = tss_limit;

    env->tr.flags = e2 & ~DESC_TSS_BUSY_MASK;

    if ((type & 8) && (env->cr[0] & CR0_PG_MASK)) {

        cpu_x86_update_cr3(env, new_cr3);

    }

    /* load all registers without an exception, then reload them with

       possible exception */

    env->eip = new_eip;

    eflags_mask = TF_MASK | AC_MASK | ID_MASK |

        IF_MASK | IOPL_MASK | VM_MASK | RF_MASK | NT_MASK;

    if (!(type & 8)) {

        eflags_mask &= 0xffff;

    }

    cpu_load_eflags(env, new_eflags, eflags_mask);

    /* XXX: what to do in 16 bit case? */

    EAX = new_regs[0];

    ECX = new_regs[1];

    EDX = new_regs[2];

    EBX = new_regs[3];

    ESP = new_regs[4];

    EBP = new_regs[5];

    ESI = new_regs[6];

    EDI = new_regs[7];

    if (new_eflags & VM_MASK) {

        for (i = 0; i < 6; i++) {

            load_seg_vm(env, i, new_segs[i]);

        }

        /* in vm86, CPL is always 3 */

        cpu_x86_set_cpl(env, 3);

    } else {

        /* CPL is set the RPL of CS */

        cpu_x86_set_cpl(env, new_segs[R_CS] & 3);

        /* first just selectors as the rest may trigger exceptions */

        for (i = 0; i < 6; i++) {

            cpu_x86_load_seg_cache(env, i, new_segs[i], 0, 0, 0);

        }

    }

    env->ldt.selector = new_ldt & ~4;

    env->ldt.base = 0;

    env->ldt.limit = 0;

    env->ldt.flags = 0;

    /* load the LDT */

    if (new_ldt & 4) {

        raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);

    }

    if ((new_ldt & 0xfffc) != 0) {

        dt = &env->gdt;

        index = new_ldt & ~7;

        if ((index + 7) > dt->limit) {

            raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);

        }

        ptr = dt->base + index;

        e1 = cpu_ldl_kernel(env, ptr);

        e2 = cpu_ldl_kernel(env, ptr + 4);

        if ((e2 & DESC_S_MASK) || ((e2 >> DESC_TYPE_SHIFT) & 0xf) != 2) {

            raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);

        }

        if (!(e2 & DESC_P_MASK)) {

            raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);

        }

        load_seg_cache_raw_dt(&env->ldt, e1, e2);

    }

    /* load the segments */

    if (!(new_eflags & VM_MASK)) {

        tss_load_seg(env, R_CS, new_segs[R_CS]);

        tss_load_seg(env, R_SS, new_segs[R_SS]);

        tss_load_seg(env, R_ES, new_segs[R_ES]);

        tss_load_seg(env, R_DS, new_segs[R_DS]);

        tss_load_seg(env, R_FS, new_segs[R_FS]);

        tss_load_seg(env, R_GS, new_segs[R_GS]);

    }

    /* check that EIP is in the CS segment limits */

    if (new_eip > env->segs[R_CS].limit) {

        /* XXX: different exception if CALL? */

        raise_exception_err(env, EXCP0D_GPF, 0);

    }

#ifndef CONFIG_USER_ONLY

    /* reset local breakpoints */

    if (env->dr[7] & 0x55) {

        for (i = 0; i < 4; i++) {

            if (hw_breakpoint_enabled(env->dr[7], i) == 0x1) {

                hw_breakpoint_remove(env, i);

            }

        }

        env->dr[7] &= ~0x55;

    }

#endif

}

static inline unsigned int get_sp_mask(unsigned int e2)

{

    if (e2 & DESC_B_MASK) {

        return 0xffffffff;

    } else {

        return 0xffff;

    }

}

static int exception_has_error_code(int intno)

{

    switch (intno) {

    case 8:

    case 10:

    case 11:

    case 12:

    case 13:

    case 14:

    case 17:

        return 1;

    }

    return 0;

}

#ifdef TARGET_X86_64

#define SET_ESP(val, sp_mask)                           \

    do {                                                \

        if ((sp_mask) == 0xffff) {                      \

            ESP = (ESP & ~0xffff) | ((val) & 0xffff);   \

        } else if ((sp_mask) == 0xffffffffLL) {         \

            ESP = (uint32_t)(val);                      \

        } else {                                        \

            ESP = (val);                                \

        }                                               \

    } while (0)

#else

#define SET_ESP(val, sp_mask)                           \

    do {                                                \

        ESP = (ESP & ~(sp_mask)) | ((val) & (sp_mask)); \

    } while (0)

#endif

/* in 64-bit machines, this can overflow. So this segment addition macro

 * can be used to trim the value to 32-bit whenever needed */

#define SEG_ADDL(ssp, sp, sp_mask) ((uint32_t)((ssp) + (sp & (sp_mask))))

/* XXX: add a is_user flag to have proper security support */

#define PUSHW(ssp, sp, sp_mask, val)                             \

    {                                                            \

        sp -= 2;                                                 \

        cpu_stw_kernel(env, (ssp) + (sp & (sp_mask)), (val));    \

    }

#define PUSHL(ssp, sp, sp_mask, val)                                    \

    {                                                                   \

        sp -= 4;                                                        \

        cpu_stl_kernel(env, SEG_ADDL(ssp, sp, sp_mask), (uint32_t)(val)); \

    }

#define POPW(ssp, sp, sp_mask, val)                              \

    {                                                            \

        val = cpu_lduw_kernel(env, (ssp) + (sp & (sp_mask)));    \

        sp += 2;                                                 \

    }

#define POPL(ssp, sp, sp_mask, val)                                     \

    {                                                                   \

        val = (uint32_t)cpu_ldl_kernel(env, SEG_ADDL(ssp, sp, sp_mask)); \

        sp += 4;                                                        \

    }

/* protected mode interrupt */

static void do_interrupt_protected(CPUX86State *env, int intno, int is_int,

                                   int error_code, unsigned int next_eip,

                                   int is_hw)

{

    SegmentCache *dt;

    target_ulong ptr, ssp;

    int type, dpl, selector, ss_dpl, cpl;

    int has_error_code, new_stack, shift;

    uint32_t e1, e2, offset, ss = 0, esp, ss_e1 = 0, ss_e2 = 0;

    uint32_t old_eip, sp_mask;

    has_error_code = 0;

    if (!is_int && !is_hw) {

        has_error_code = exception_has_error_code(intno);

    }

    if (is_int) {

        old_eip = next_eip;

    } else {

        old_eip = env->eip;

    }

    dt = &env->idt;

    if (intno * 8 + 7 > dt->limit) {

        raise_exception_err(env, EXCP0D_GPF, intno * 8 + 2);

    }

    ptr = dt->base + intno * 8;

    e1 = cpu_ldl_kernel(env, ptr);

    e2 = cpu_ldl_kernel(env, ptr + 4);

    /* check gate type */

    type = (e2 >> DESC_TYPE_SHIFT) & 0x1f;

    switch (type) {

    case 5: /* task gate */

        /* must do that check here to return the correct error code */

        if (!(e2 & DESC_P_MASK)) {

            raise_exception_err(env, EXCP0B_NOSEG, intno * 8 + 2);

        }

        switch_tss(env, intno * 8, e1, e2, SWITCH_TSS_CALL, old_eip);

        if (has_error_code) {

            int type;

            uint32_t mask;

            /* push the error code */

            type = (env->tr.flags >> DESC_TYPE_SHIFT) & 0xf;

            shift = type >> 3;

            if (env->segs[R_SS].flags & DESC_B_MASK) {

                mask = 0xffffffff;

            } else {

                mask = 0xffff;

            }

            esp = (ESP - (2 << shift)) & mask;

            ssp = env->segs[R_SS].base + esp;

            if (shift) {

                cpu_stl_kernel(env, ssp, error_code);

            } else {

                cpu_stw_kernel(env, ssp, error_code);

            }

            SET_ESP(esp, mask);

        }

        return;

    case 6: /* 286 interrupt gate */

    case 7: /* 286 trap gate */

    case 14: /* 386 interrupt gate */

    case 15: /* 386 trap gate */

        break;

    default:

        raise_exception_err(env, EXCP0D_GPF, intno * 8 + 2);

        break;

    }

    dpl = (e2 >> DESC_DPL_SHIFT) & 3;

    cpl = env->hflags & HF_CPL_MASK;

    /* check privilege if software int */

    if (is_int && dpl < cpl) {

        raise_exception_err(env, EXCP0D_GPF, intno * 8 + 2);

    }

    /* check valid bit */

    if (!(e2 & DESC_P_MASK)) {

        raise_exception_err(env, EXCP0B_NOSEG, intno * 8 + 2);

    }

    selector = e1 >> 16;

    offset = (e2 & 0xffff0000) | (e1 & 0x0000ffff);

    if ((selector & 0xfffc) == 0) {

        raise_exception_err(env, EXCP0D_GPF, 0);

    }

    if (load_segment(env, &e1, &e2, selector) != 0) {

        raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);

    }

    if (!(e2 & DESC_S_MASK) || !(e2 & (DESC_CS_MASK))) {

        raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);

    }

    dpl = (e2 >> DESC_DPL_SHIFT) & 3;

    if (dpl > cpl) {

        raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);

    }

    if (!(e2 & DESC_P_MASK)) {

        raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);

    }

    if (!(e2 & DESC_C_MASK) && dpl < cpl) {

        /* to inner privilege */

        get_ss_esp_from_tss(env, &ss, &esp, dpl);

        if ((ss & 0xfffc) == 0) {

            raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);

        }

        if ((ss & 3) != dpl) {

            raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);

        }

        if (load_segment(env, &ss_e1, &ss_e2, ss) != 0) {

            raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);

        }

        ss_dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;

        if (ss_dpl != dpl) {

            raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);

        }

        if (!(ss_e2 & DESC_S_MASK) ||

            (ss_e2 & DESC_CS_MASK) ||

            !(ss_e2 & DESC_W_MASK)) {

            raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);

        }

        if (!(ss_e2 & DESC_P_MASK)) {

            raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);

        }

        new_stack = 1;

        sp_mask = get_sp_mask(ss_e2);

        ssp = get_seg_base(ss_e1, ss_e2);

    } else if ((e2 & DESC_C_MASK) || dpl == cpl) {

        /* to same privilege */

        if (env->eflags & VM_MASK) {

            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);

        }

        new_stack = 0;

        sp_mask = get_sp_mask(env->segs[R_SS].flags);

        ssp = env->segs[R_SS].base;

        esp = ESP;

        dpl = cpl;

    } else {

        raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);

        new_stack = 0; /* avoid warning */

        sp_mask = 0; /* avoid warning */

        ssp = 0; /* avoid warning */

        esp = 0; /* avoid warning */

    }

    shift = type >> 3;

#if 0

    /* XXX: check that enough room is available */

    push_size = 6 + (new_stack << 2) + (has_error_code << 1);

    if (env->eflags & VM_MASK) {

        push_size += 8;

    }

    push_size <<= shift;

#endif

    if (shift == 1) {

        if (new_stack) {

            if (env->eflags & VM_MASK) {

                PUSHL(ssp, esp, sp_mask, env->segs[R_GS].selector);

                PUSHL(ssp, esp, sp_mask, env->segs[R_FS].selector);

                PUSHL(ssp, esp, sp_mask, env->segs[R_DS].selector);

                PUSHL(ssp, esp, sp_mask, env->segs[R_ES].selector);

            }

            PUSHL(ssp, esp, sp_mask, env->segs[R_SS].selector);

            PUSHL(ssp, esp, sp_mask, ESP);

        }

        PUSHL(ssp, esp, sp_mask, cpu_compute_eflags(env));

        PUSHL(ssp, esp, sp_mask, env->segs[R_CS].selector);

        PUSHL(ssp, esp, sp_mask, old_eip);

        if (has_error_code) {

            PUSHL(ssp, esp, sp_mask, error_code);

        }

    } else {

        if (new_stack) {

            if (env->eflags & VM_MASK) {

                PUSHW(ssp, esp, sp_mask, env->segs[R_GS].selector);

                PUSHW(ssp, esp, sp_mask, env->segs[R_FS].selector);

                PUSHW(ssp, esp, sp_mask, env->segs[R_DS].selector);

                PUSHW(ssp, esp, sp_mask, env->segs[R_ES].selector);

            }

            PUSHW(ssp, esp, sp_mask, env->segs[R_SS].selector);

            PUSHW(ssp, esp, sp_mask, ESP);

        }

        PUSHW(ssp, esp, sp_mask, cpu_compute_eflags(env));

        PUSHW(ssp, esp, sp_mask, env->segs[R_CS].selector);

        PUSHW(ssp, esp, sp_mask, old_eip);

        if (has_error_code) {

            PUSHW(ssp, esp, sp_mask, error_code);

        }

    }

    if (new_stack) {

        if (env->eflags & VM_MASK) {

            cpu_x86_load_seg_cache(env, R_ES, 0, 0, 0, 0);

            cpu_x86_load_seg_cache(env, R_DS, 0, 0, 0, 0);

            cpu_x86_load_seg_cache(env, R_FS, 0, 0, 0, 0);

            cpu_x86_load_seg_cache(env, R_GS, 0, 0, 0, 0);

        }

        ss = (ss & ~3) | dpl;

        cpu_x86_load_seg_cache(env, R_SS, ss,

                               ssp, get_seg_limit(ss_e1, ss_e2), ss_e2);

    }

    SET_ESP(esp, sp_mask);

    selector = (selector & ~3) | dpl;

    cpu_x86_load_seg_cache(env, R_CS, selector,

                   get_seg_base(e1, e2),

                   get_seg_limit(e1, e2),

                   e2);

    cpu_x86_set_cpl(env, dpl);

    env->eip = offset;

    /* interrupt gate clear IF mask */

    if ((type & 1) == 0) {

        env->eflags &= ~IF_MASK;

    }

    env->eflags &= ~(TF_MASK | VM_MASK | RF_MASK | NT_MASK);

}

#ifdef TARGET_X86_64

#define PUSHQ(sp, val)                          \

    {                                           \

        sp -= 8;                                \

        cpu_stq_kernel(env, sp, (val));         \

    }

#define POPQ(sp, val)                           \

    {                                           \

        val = cpu_ldq_kernel(env, sp);          \

        sp += 8;                                \

    }

static inline target_ulong get_rsp_from_tss(CPUX86State *env, int level)

{

    int index;

#if 0

    printf("TR: base=" TARGET_FMT_lx " limit=%x\n",

           env->tr.base, env->tr.limit);

#endif

    if (!(env->tr.flags & DESC_P_MASK)) {

        cpu_abort(env, "invalid tss");

    }

    index = 8 * level + 4;

    if ((index + 7) > env->tr.limit) {

        raise_exception_err(env, EXCP0A_TSS, env->tr.selector & 0xfffc);

    }

    return cpu_ldq_kernel(env, env->tr.base + index);

}

/* 64 bit interrupt */

static void do_interrupt64(CPUX86State *env, int intno, int is_int,

                           int error_code, target_ulong next_eip, int is_hw)

{

    SegmentCache *dt;

    target_ulong ptr;

    int type, dpl, selector, cpl, ist;

    int has_error_code, new_stack;

    uint32_t e1, e2, e3, ss;

    target_ulong old_eip, esp, offset;

    has_error_code = 0;

    if (!is_int && !is_hw) {

        has_error_code = exception_has_error_code(intno);

    }

    if (is_int) {

        old_eip = next_eip;

    } else {

        old_eip = env->eip;

    }

    dt = &env->idt;

    if (intno * 16 + 15 > dt->limit) {

        raise_exception_err(env, EXCP0D_GPF, intno * 16 + 2);

    }

    ptr = dt->base + intno * 16;

    e1 = cpu_ldl_kernel(env, ptr);

    e2 = cpu_ldl_kernel(env, ptr + 4);

    e3 = cpu_ldl_kernel(env, ptr + 8);

    /* check gate type */

    type = (e2 >> DESC_TYPE_SHIFT) & 0x1f;

    switch (type) {

    case 14: /* 386 interrupt gate */

    case 15: /* 386 trap gate */

        break;

    default:

        raise_exception_err(env, EXCP0D_GPF, intno * 16 + 2);

        break;

    }

    dpl = (e2 >> DESC_DPL_SHIFT) & 3;

    cpl = env->hflags & HF_CPL_MASK;

    /* check privilege if software int */

    if (is_int && dpl < cpl) {

        raise_exception_err(env, EXCP0D_GPF, intno * 16 + 2);

    }

    /* check valid bit */

    if (!(e2 & DESC_P_MASK)) {

        raise_exception_err(env, EXCP0B_NOSEG, intno * 16 + 2);

    }

    selector = e1 >> 16;

    offset = ((target_ulong)e3 << 32) | (e2 & 0xffff0000) | (e1 & 0x0000ffff);

    ist = e2 & 7;

    if ((selector & 0xfffc) == 0) {

        raise_exception_err(env, EXCP0D_GPF, 0);

    }

    if (load_segment(env, &e1, &e2, selector) != 0) {

        raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);

    }

    if (!(e2 & DESC_S_MASK) || !(e2 & (DESC_CS_MASK))) {

        raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);

    }

    dpl = (e2 >> DESC_DPL_SHIFT) & 3;

    if (dpl > cpl) {

        raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);

    }

    if (!(e2 & DESC_P_MASK)) {

        raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);

    }

    if (!(e2 & DESC_L_MASK) || (e2 & DESC_B_MASK)) {

        raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);

    }

    if ((!(e2 & DESC_C_MASK) && dpl < cpl) || ist != 0) {

        /* to inner privilege */

        if (ist != 0) {

            esp = get_rsp_from_tss(env, ist + 3);

        } else {

            esp = get_rsp_from_tss(env, dpl);

        }

        esp &= ~0xfLL; /* align stack */

        ss = 0;

        new_stack = 1;

    } else if ((e2 & DESC_C_MASK) || dpl == cpl) {

        /* to same privilege */

        if (env->eflags & VM_MASK) {

            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);

        }

        new_stack = 0;

        if (ist != 0) {

            esp = get_rsp_from_tss(env, ist + 3);

        } else {

            esp = ESP;

        }

        esp &= ~0xfLL; /* align stack */