Archipelago » qemu

/*

 * Tiny Code Generator for QEMU

 *

 * Copyright (c) 2008 Fabrice Bellard

 *

 * Permission is hereby granted, free of charge, to any person obtaining a copy

 * of this software and associated documentation files (the "Software"), to deal

 * in the Software without restriction, including without limitation the rights

 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

 * copies of the Software, and to permit persons to whom the Software is

 * furnished to do so, subject to the following conditions:

 *

 * The above copyright notice and this permission notice shall be included in

 * all copies or substantial portions of the Software.

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL

 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN

 * THE SOFTWARE.

 */

#ifndef NDEBUG

static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {

#if TCG_TARGET_REG_BITS == 64

    "%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi",

    "%r8",  "%r9",  "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",

#else

    "%eax", "%ecx", "%edx", "%ebx", "%esp", "%ebp", "%esi", "%edi",

#endif

};

#endif

static const int tcg_target_reg_alloc_order[] = {

#if TCG_TARGET_REG_BITS == 64

    TCG_REG_RBP,

    TCG_REG_RBX,

    TCG_REG_R12,

    TCG_REG_R13,

    TCG_REG_R14,

    TCG_REG_R15,

    TCG_REG_R10,

    TCG_REG_R11,

    TCG_REG_R9,

    TCG_REG_R8,

    TCG_REG_RCX,

    TCG_REG_RDX,

    TCG_REG_RSI,

    TCG_REG_RDI,

    TCG_REG_RAX,

#else

    TCG_REG_EBX,

    TCG_REG_ESI,

    TCG_REG_EDI,

    TCG_REG_EBP,

    TCG_REG_ECX,

    TCG_REG_EDX,

    TCG_REG_EAX,

#endif

};

static const int tcg_target_call_iarg_regs[] = {

#if TCG_TARGET_REG_BITS == 64

#if defined(_WIN64)

    TCG_REG_RCX,

    TCG_REG_RDX,

#else

    TCG_REG_RDI,

    TCG_REG_RSI,

    TCG_REG_RDX,

    TCG_REG_RCX,

#endif

    TCG_REG_R8,

    TCG_REG_R9,

#else

    /* 32 bit mode uses stack based calling convention (GCC default). */

#endif

};

static const int tcg_target_call_oarg_regs[] = {

    TCG_REG_EAX,

#if TCG_TARGET_REG_BITS == 32

    TCG_REG_EDX

#endif

};

/* Registers used with L constraint, which are the first argument 

   registers on x86_64, and two random call clobbered registers on

   i386. */

#if TCG_TARGET_REG_BITS == 64

# define TCG_REG_L0 tcg_target_call_iarg_regs[0]

# define TCG_REG_L1 tcg_target_call_iarg_regs[1]

#else

# define TCG_REG_L0 TCG_REG_EAX

# define TCG_REG_L1 TCG_REG_EDX

#endif

/* For 32-bit, we are going to attempt to determine at runtime whether cmov

   is available.  However, the host compiler must supply <cpuid.h>, as we're

   not going to go so far as our own inline assembly.  */

#if TCG_TARGET_REG_BITS == 64

# define have_cmov 1

#elif defined(CONFIG_CPUID_H)

#include <cpuid.h>

static bool have_cmov;

#else

# define have_cmov 0

#endif

static uint8_t *tb_ret_addr;

static void patch_reloc(uint8_t *code_ptr, int type,

                        tcg_target_long value, tcg_target_long addend)

{

    value += addend;

    switch(type) {

    case R_386_PC32:

        value -= (uintptr_t)code_ptr;

        if (value != (int32_t)value) {

            tcg_abort();

        }

        *(uint32_t *)code_ptr = value;

        break;

    case R_386_PC8:

        value -= (uintptr_t)code_ptr;

        if (value != (int8_t)value) {

            tcg_abort();

        }

        *(uint8_t *)code_ptr = value;

        break;

    default:

        tcg_abort();

    }

}

/* parse target specific constraints */

static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)

{

    const char *ct_str;

    ct_str = *pct_str;

    switch(ct_str[0]) {

    case 'a':

        ct->ct |= TCG_CT_REG;

        tcg_regset_set_reg(ct->u.regs, TCG_REG_EAX);

        break;

    case 'b':

        ct->ct |= TCG_CT_REG;

        tcg_regset_set_reg(ct->u.regs, TCG_REG_EBX);

        break;

    case 'c':

        ct->ct |= TCG_CT_REG;

        tcg_regset_set_reg(ct->u.regs, TCG_REG_ECX);

        break;

    case 'd':

        ct->ct |= TCG_CT_REG;

        tcg_regset_set_reg(ct->u.regs, TCG_REG_EDX);

        break;

    case 'S':

        ct->ct |= TCG_CT_REG;

        tcg_regset_set_reg(ct->u.regs, TCG_REG_ESI);

        break;

    case 'D':

        ct->ct |= TCG_CT_REG;

        tcg_regset_set_reg(ct->u.regs, TCG_REG_EDI);

        break;

    case 'q':

        ct->ct |= TCG_CT_REG;

        if (TCG_TARGET_REG_BITS == 64) {

            tcg_regset_set32(ct->u.regs, 0, 0xffff);

        } else {

            tcg_regset_set32(ct->u.regs, 0, 0xf);

        }

        break;

    case 'Q':

        ct->ct |= TCG_CT_REG;

        tcg_regset_set32(ct->u.regs, 0, 0xf);

        break;

    case 'r':

        ct->ct |= TCG_CT_REG;

        if (TCG_TARGET_REG_BITS == 64) {

            tcg_regset_set32(ct->u.regs, 0, 0xffff);

        } else {

            tcg_regset_set32(ct->u.regs, 0, 0xff);

        }

        break;

        /* qemu_ld/st address constraint */

    case 'L':

        ct->ct |= TCG_CT_REG;

#if TCG_TARGET_REG_BITS == 64

            tcg_regset_set32(ct->u.regs, 0, 0xffff);

#else

            tcg_regset_set32(ct->u.regs, 0, 0xff);

#endif

        tcg_regset_reset_reg(ct->u.regs, TCG_REG_L0);

        tcg_regset_reset_reg(ct->u.regs, TCG_REG_L1);

        break;

    case 'e':

        ct->ct |= TCG_CT_CONST_S32;

        break;

    case 'Z':

        ct->ct |= TCG_CT_CONST_U32;

        break;

    default:

        return -1;

    }

    ct_str++;

    *pct_str = ct_str;

    return 0;

}

/* test if a constant matches the constraint */

static inline int tcg_target_const_match(tcg_target_long val,

                                         const TCGArgConstraint *arg_ct)

{

    int ct = arg_ct->ct;

    if (ct & TCG_CT_CONST) {

        return 1;

    }

    if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) {

        return 1;

    }

    if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) {

        return 1;

    }

    return 0;

}

#if TCG_TARGET_REG_BITS == 64

# define LOWREGMASK(x)        ((x) & 7)

#else

# define LOWREGMASK(x)        (x)

#endif

#define P_EXT                0x100                /* 0x0f opcode prefix */

#define P_DATA16        0x200                /* 0x66 opcode prefix */

#if TCG_TARGET_REG_BITS == 64

# define P_ADDR32        0x400                /* 0x67 opcode prefix */

# define P_REXW                0x800                /* Set REX.W = 1 */

# define P_REXB_R        0x1000                /* REG field as byte register */

# define P_REXB_RM        0x2000                /* R/M field as byte register */

# define P_GS           0x4000          /* gs segment override */

#else

# define P_ADDR32        0

# define P_REXW                0

# define P_REXB_R        0

# define P_REXB_RM        0

# define P_GS           0

#endif

#define OPC_ARITH_EvIz        (0x81)

#define OPC_ARITH_EvIb        (0x83)

#define OPC_ARITH_GvEv        (0x03)                /* ... plus (ARITH_FOO << 3) */

#define OPC_ADD_GvEv        (OPC_ARITH_GvEv | (ARITH_ADD << 3))

#define OPC_BSWAP        (0xc8 | P_EXT)

#define OPC_CALL_Jz        (0xe8)

#define OPC_CMOVCC      (0x40 | P_EXT)  /* ... plus condition code */

#define OPC_CMP_GvEv        (OPC_ARITH_GvEv | (ARITH_CMP << 3))

#define OPC_DEC_r32        (0x48)

#define OPC_IMUL_GvEv        (0xaf | P_EXT)

#define OPC_IMUL_GvEvIb        (0x6b)

#define OPC_IMUL_GvEvIz        (0x69)

#define OPC_INC_r32        (0x40)

#define OPC_JCC_long        (0x80 | P_EXT)        /* ... plus condition code */

#define OPC_JCC_short        (0x70)                /* ... plus condition code */

#define OPC_JMP_long        (0xe9)

#define OPC_JMP_short        (0xeb)

#define OPC_LEA         (0x8d)

#define OPC_MOVB_EvGv        (0x88)                /* stores, more or less */

#define OPC_MOVL_EvGv        (0x89)                /* stores, more or less */

#define OPC_MOVL_GvEv        (0x8b)                /* loads, more or less */

#define OPC_MOVB_EvIz   (0xc6)

#define OPC_MOVL_EvIz        (0xc7)

#define OPC_MOVL_Iv     (0xb8)

#define OPC_MOVSBL        (0xbe | P_EXT)

#define OPC_MOVSWL        (0xbf | P_EXT)

#define OPC_MOVSLQ        (0x63 | P_REXW)

#define OPC_MOVZBL        (0xb6 | P_EXT)

#define OPC_MOVZWL        (0xb7 | P_EXT)

#define OPC_POP_r32        (0x58)

#define OPC_PUSH_r32        (0x50)

#define OPC_PUSH_Iv        (0x68)

#define OPC_PUSH_Ib        (0x6a)

#define OPC_RET                (0xc3)

#define OPC_SETCC        (0x90 | P_EXT | P_REXB_RM) /* ... plus cc */

#define OPC_SHIFT_1        (0xd1)

#define OPC_SHIFT_Ib        (0xc1)

#define OPC_SHIFT_cl        (0xd3)

#define OPC_TESTL        (0x85)

#define OPC_XCHG_ax_r32        (0x90)

#define OPC_GRP3_Ev        (0xf7)

#define OPC_GRP5        (0xff)

/* Group 1 opcode extensions for 0x80-0x83.

   These are also used as modifiers for OPC_ARITH.  */

#define ARITH_ADD 0

#define ARITH_OR  1

#define ARITH_ADC 2

#define ARITH_SBB 3

#define ARITH_AND 4

#define ARITH_SUB 5

#define ARITH_XOR 6

#define ARITH_CMP 7

/* Group 2 opcode extensions for 0xc0, 0xc1, 0xd0-0xd3.  */

#define SHIFT_ROL 0

#define SHIFT_ROR 1

#define SHIFT_SHL 4

#define SHIFT_SHR 5

#define SHIFT_SAR 7

/* Group 3 opcode extensions for 0xf6, 0xf7.  To be used with OPC_GRP3.  */

#define EXT3_NOT   2

#define EXT3_NEG   3

#define EXT3_MUL   4

#define EXT3_IMUL  5

#define EXT3_DIV   6

#define EXT3_IDIV  7

/* Group 5 opcode extensions for 0xff.  To be used with OPC_GRP5.  */

#define EXT5_INC_Ev        0

#define EXT5_DEC_Ev        1

#define EXT5_CALLN_Ev        2

#define EXT5_JMPN_Ev        4

/* Condition codes to be added to OPC_JCC_{long,short}.  */

#define JCC_JMP (-1)

#define JCC_JO  0x0

#define JCC_JNO 0x1

#define JCC_JB  0x2

#define JCC_JAE 0x3

#define JCC_JE  0x4

#define JCC_JNE 0x5

#define JCC_JBE 0x6

#define JCC_JA  0x7

#define JCC_JS  0x8

#define JCC_JNS 0x9

#define JCC_JP  0xa

#define JCC_JNP 0xb

#define JCC_JL  0xc

#define JCC_JGE 0xd

#define JCC_JLE 0xe

#define JCC_JG  0xf

static const uint8_t tcg_cond_to_jcc[] = {

    [TCG_COND_EQ] = JCC_JE,

    [TCG_COND_NE] = JCC_JNE,

    [TCG_COND_LT] = JCC_JL,

    [TCG_COND_GE] = JCC_JGE,

    [TCG_COND_LE] = JCC_JLE,

    [TCG_COND_GT] = JCC_JG,

    [TCG_COND_LTU] = JCC_JB,

    [TCG_COND_GEU] = JCC_JAE,

    [TCG_COND_LEU] = JCC_JBE,

    [TCG_COND_GTU] = JCC_JA,

};

#if TCG_TARGET_REG_BITS == 64

static void tcg_out_opc(TCGContext *s, int opc, int r, int rm, int x)

{

    int rex;

    if (opc & P_GS) {

        tcg_out8(s, 0x65);

    }

    if (opc & P_DATA16) {

        /* We should never be asking for both 16 and 64-bit operation.  */

        assert((opc & P_REXW) == 0);

        tcg_out8(s, 0x66);

    }

    if (opc & P_ADDR32) {

        tcg_out8(s, 0x67);

    }

    rex = 0;

    rex |= (opc & P_REXW) >> 8;                /* REX.W */

    rex |= (r & 8) >> 1;                /* REX.R */

    rex |= (x & 8) >> 2;                /* REX.X */

    rex |= (rm & 8) >> 3;                /* REX.B */

    /* P_REXB_{R,RM} indicates that the given register is the low byte.

       For %[abcd]l we need no REX prefix, but for %{si,di,bp,sp}l we do,

       as otherwise the encoding indicates %[abcd]h.  Note that the values

       that are ORed in merely indicate that the REX byte must be present;

       those bits get discarded in output.  */

    rex |= opc & (r >= 4 ? P_REXB_R : 0);

    rex |= opc & (rm >= 4 ? P_REXB_RM : 0);

    if (rex) {

        tcg_out8(s, (uint8_t)(rex | 0x40));

    }

    if (opc & P_EXT) {

        tcg_out8(s, 0x0f);

    }

    tcg_out8(s, opc);

}

#else

static void tcg_out_opc(TCGContext *s, int opc)

{

    if (opc & P_DATA16) {

        tcg_out8(s, 0x66);

    }

    if (opc & P_EXT) {

        tcg_out8(s, 0x0f);

    }

    tcg_out8(s, opc);

}

/* Discard the register arguments to tcg_out_opc early, so as not to penalize

   the 32-bit compilation paths.  This method works with all versions of gcc,

   whereas relying on optimization may not be able to exclude them.  */

#define tcg_out_opc(s, opc, r, rm, x)  (tcg_out_opc)(s, opc)

#endif

static void tcg_out_modrm(TCGContext *s, int opc, int r, int rm)

{

    tcg_out_opc(s, opc, r, rm, 0);

    tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));

}

/* Output an opcode with a full "rm + (index<<shift) + offset" address mode.

   We handle either RM and INDEX missing with a negative value.  In 64-bit

   mode for absolute addresses, ~RM is the size of the immediate operand

   that will follow the instruction.  */

static void tcg_out_modrm_sib_offset(TCGContext *s, int opc, int r, int rm,

                                     int index, int shift,

                                     tcg_target_long offset)

{

    int mod, len;

    if (index < 0 && rm < 0) {

        if (TCG_TARGET_REG_BITS == 64) {

            /* Try for a rip-relative addressing mode.  This has replaced

               the 32-bit-mode absolute addressing encoding.  */

            tcg_target_long pc = (tcg_target_long)s->code_ptr + 5 + ~rm;

            tcg_target_long disp = offset - pc;

            if (disp == (int32_t)disp) {

                tcg_out_opc(s, opc, r, 0, 0);

                tcg_out8(s, (LOWREGMASK(r) << 3) | 5);

                tcg_out32(s, disp);

                return;

            }

            /* Try for an absolute address encoding.  This requires the

               use of the MODRM+SIB encoding and is therefore larger than

               rip-relative addressing.  */

            if (offset == (int32_t)offset) {

                tcg_out_opc(s, opc, r, 0, 0);

                tcg_out8(s, (LOWREGMASK(r) << 3) | 4);

                tcg_out8(s, (4 << 3) | 5);

                tcg_out32(s, offset);

                return;

            }

            /* ??? The memory isn't directly addressable.  */

            tcg_abort();

        } else {

            /* Absolute address.  */

            tcg_out_opc(s, opc, r, 0, 0);

            tcg_out8(s, (r << 3) | 5);

            tcg_out32(s, offset);

            return;

        }

    }

    /* Find the length of the immediate addend.  Note that the encoding

       that would be used for (%ebp) indicates absolute addressing.  */

    if (rm < 0) {

        mod = 0, len = 4, rm = 5;

    } else if (offset == 0 && LOWREGMASK(rm) != TCG_REG_EBP) {

        mod = 0, len = 0;

    } else if (offset == (int8_t)offset) {

        mod = 0x40, len = 1;

    } else {

        mod = 0x80, len = 4;

    }

    /* Use a single byte MODRM format if possible.  Note that the encoding

       that would be used for %esp is the escape to the two byte form.  */

    if (index < 0 && LOWREGMASK(rm) != TCG_REG_ESP) {

        /* Single byte MODRM format.  */

        tcg_out_opc(s, opc, r, rm, 0);

        tcg_out8(s, mod | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));

    } else {

        /* Two byte MODRM+SIB format.  */

        /* Note that the encoding that would place %esp into the index

           field indicates no index register.  In 64-bit mode, the REX.X

           bit counts, so %r12 can be used as the index.  */

        if (index < 0) {

            index = 4;

        } else {

            assert(index != TCG_REG_ESP);

        }

        tcg_out_opc(s, opc, r, rm, index);

        tcg_out8(s, mod | (LOWREGMASK(r) << 3) | 4);

        tcg_out8(s, (shift << 6) | (LOWREGMASK(index) << 3) | LOWREGMASK(rm));

    }

    if (len == 1) {

        tcg_out8(s, offset);

    } else if (len == 4) {

        tcg_out32(s, offset);

    }

}

/* A simplification of the above with no index or shift.  */

static inline void tcg_out_modrm_offset(TCGContext *s, int opc, int r,

                                        int rm, tcg_target_long offset)

{

    tcg_out_modrm_sib_offset(s, opc, r, rm, -1, 0, offset);

}

/* Generate dest op= src.  Uses the same ARITH_* codes as tgen_arithi.  */

static inline void tgen_arithr(TCGContext *s, int subop, int dest, int src)

{

    /* Propagate an opcode prefix, such as P_REXW.  */

    int ext = subop & ~0x7;

    subop &= 0x7;

    tcg_out_modrm(s, OPC_ARITH_GvEv + (subop << 3) + ext, dest, src);

}

static inline void tcg_out_mov(TCGContext *s, TCGType type,

                               TCGReg ret, TCGReg arg)

{

    if (arg != ret) {

        int opc = OPC_MOVL_GvEv + (type == TCG_TYPE_I64 ? P_REXW : 0);

        tcg_out_modrm(s, opc, ret, arg);

    }

}

static void tcg_out_movi(TCGContext *s, TCGType type,

                         TCGReg ret, tcg_target_long arg)

{

    if (arg == 0) {

        tgen_arithr(s, ARITH_XOR, ret, ret);

        return;

    } else if (arg == (uint32_t)arg || type == TCG_TYPE_I32) {

        tcg_out_opc(s, OPC_MOVL_Iv + LOWREGMASK(ret), 0, ret, 0);

        tcg_out32(s, arg);

    } else if (arg == (int32_t)arg) {

        tcg_out_modrm(s, OPC_MOVL_EvIz + P_REXW, 0, ret);

        tcg_out32(s, arg);

    } else {

        tcg_out_opc(s, OPC_MOVL_Iv + P_REXW + LOWREGMASK(ret), 0, ret, 0);

        tcg_out32(s, arg);

        tcg_out32(s, arg >> 31 >> 1);

    }

}

static inline void tcg_out_pushi(TCGContext *s, tcg_target_long val)

{

    if (val == (int8_t)val) {

        tcg_out_opc(s, OPC_PUSH_Ib, 0, 0, 0);

        tcg_out8(s, val);

    } else if (val == (int32_t)val) {

        tcg_out_opc(s, OPC_PUSH_Iv, 0, 0, 0);

        tcg_out32(s, val);

    } else {

        tcg_abort();

    }

}

static inline void tcg_out_push(TCGContext *s, int reg)

{

    tcg_out_opc(s, OPC_PUSH_r32 + LOWREGMASK(reg), 0, reg, 0);

}

static inline void tcg_out_pop(TCGContext *s, int reg)

{

    tcg_out_opc(s, OPC_POP_r32 + LOWREGMASK(reg), 0, reg, 0);

}

static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,

                              TCGReg arg1, tcg_target_long arg2)

{

    int opc = OPC_MOVL_GvEv + (type == TCG_TYPE_I64 ? P_REXW : 0);

    tcg_out_modrm_offset(s, opc, ret, arg1, arg2);

}

static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,

                              TCGReg arg1, tcg_target_long arg2)

{

    int opc = OPC_MOVL_EvGv + (type == TCG_TYPE_I64 ? P_REXW : 0);

    tcg_out_modrm_offset(s, opc, arg, arg1, arg2);

}

static void tcg_out_shifti(TCGContext *s, int subopc, int reg, int count)

{

    /* Propagate an opcode prefix, such as P_DATA16.  */

    int ext = subopc & ~0x7;

    subopc &= 0x7;

    if (count == 1) {

        tcg_out_modrm(s, OPC_SHIFT_1 + ext, subopc, reg);

    } else {

        tcg_out_modrm(s, OPC_SHIFT_Ib + ext, subopc, reg);

        tcg_out8(s, count);

    }

}

static inline void tcg_out_bswap32(TCGContext *s, int reg)

{

    tcg_out_opc(s, OPC_BSWAP + LOWREGMASK(reg), 0, reg, 0);

}

static inline void tcg_out_rolw_8(TCGContext *s, int reg)

{

    tcg_out_shifti(s, SHIFT_ROL + P_DATA16, reg, 8);

}

static inline void tcg_out_ext8u(TCGContext *s, int dest, int src)

{

    /* movzbl */

    assert(src < 4 || TCG_TARGET_REG_BITS == 64);

    tcg_out_modrm(s, OPC_MOVZBL + P_REXB_RM, dest, src);

}

static void tcg_out_ext8s(TCGContext *s, int dest, int src, int rexw)

{

    /* movsbl */

    assert(src < 4 || TCG_TARGET_REG_BITS == 64);

    tcg_out_modrm(s, OPC_MOVSBL + P_REXB_RM + rexw, dest, src);

}

static inline void tcg_out_ext16u(TCGContext *s, int dest, int src)

{

    /* movzwl */

    tcg_out_modrm(s, OPC_MOVZWL, dest, src);

}

static inline void tcg_out_ext16s(TCGContext *s, int dest, int src, int rexw)

{

    /* movsw[lq] */

    tcg_out_modrm(s, OPC_MOVSWL + rexw, dest, src);

}

static inline void tcg_out_ext32u(TCGContext *s, int dest, int src)

{

    /* 32-bit mov zero extends.  */

    tcg_out_modrm(s, OPC_MOVL_GvEv, dest, src);

}

static inline void tcg_out_ext32s(TCGContext *s, int dest, int src)

{

    tcg_out_modrm(s, OPC_MOVSLQ, dest, src);

}

static inline void tcg_out_bswap64(TCGContext *s, int reg)

{

    tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0);

}

static void tgen_arithi(TCGContext *s, int c, int r0,

                        tcg_target_long val, int cf)

{

    int rexw = 0;

    if (TCG_TARGET_REG_BITS == 64) {

        rexw = c & -8;

        c &= 7;

    }

    /* ??? While INC is 2 bytes shorter than ADDL $1, they also induce

       partial flags update stalls on Pentium4 and are not recommended

       by current Intel optimization manuals.  */

    if (!cf && (c == ARITH_ADD || c == ARITH_SUB) && (val == 1 || val == -1)) {

        int is_inc = (c == ARITH_ADD) ^ (val < 0);

        if (TCG_TARGET_REG_BITS == 64) {

            /* The single-byte increment encodings are re-tasked as the

               REX prefixes.  Use the MODRM encoding.  */

            tcg_out_modrm(s, OPC_GRP5 + rexw,

                          (is_inc ? EXT5_INC_Ev : EXT5_DEC_Ev), r0);

        } else {

            tcg_out8(s, (is_inc ? OPC_INC_r32 : OPC_DEC_r32) + r0);

        }

        return;

    }

    if (c == ARITH_AND) {

        if (TCG_TARGET_REG_BITS == 64) {

            if (val == 0xffffffffu) {

                tcg_out_ext32u(s, r0, r0);

                return;

            }

            if (val == (uint32_t)val) {

                /* AND with no high bits set can use a 32-bit operation.  */

                rexw = 0;

            }

        }

        if (val == 0xffu && (r0 < 4 || TCG_TARGET_REG_BITS == 64)) {

            tcg_out_ext8u(s, r0, r0);

            return;

        }

        if (val == 0xffffu) {

            tcg_out_ext16u(s, r0, r0);

            return;

        }

    }

    if (val == (int8_t)val) {

        tcg_out_modrm(s, OPC_ARITH_EvIb + rexw, c, r0);

        tcg_out8(s, val);

        return;

    }

    if (rexw == 0 || val == (int32_t)val) {

        tcg_out_modrm(s, OPC_ARITH_EvIz + rexw, c, r0);

        tcg_out32(s, val);

        return;

    }

    tcg_abort();

}

static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)

{

    if (val != 0) {

        tgen_arithi(s, ARITH_ADD + P_REXW, reg, val, 0);

    }

}

/* Use SMALL != 0 to force a short forward branch.  */

static void tcg_out_jxx(TCGContext *s, int opc, int label_index, int small)

{

    int32_t val, val1;

    TCGLabel *l = &s->labels[label_index];

    if (l->has_value) {

        val = l->u.value - (tcg_target_long)s->code_ptr;

        val1 = val - 2;

        if ((int8_t)val1 == val1) {

            if (opc == -1) {

                tcg_out8(s, OPC_JMP_short);

            } else {

                tcg_out8(s, OPC_JCC_short + opc);

            }

            tcg_out8(s, val1);

        } else {

            if (small) {

                tcg_abort();

            }

            if (opc == -1) {

                tcg_out8(s, OPC_JMP_long);

                tcg_out32(s, val - 5);

            } else {

                tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0);

                tcg_out32(s, val - 6);

            }

        }

    } else if (small) {

        if (opc == -1) {

            tcg_out8(s, OPC_JMP_short);

        } else {

            tcg_out8(s, OPC_JCC_short + opc);

        }

        tcg_out_reloc(s, s->code_ptr, R_386_PC8, label_index, -1);

        s->code_ptr += 1;

    } else {

        if (opc == -1) {

            tcg_out8(s, OPC_JMP_long);

        } else {

            tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0);

        }

        tcg_out_reloc(s, s->code_ptr, R_386_PC32, label_index, -4);

        s->code_ptr += 4;

    }

}

static void tcg_out_cmp(TCGContext *s, TCGArg arg1, TCGArg arg2,

                        int const_arg2, int rexw)

{

    if (const_arg2) {

        if (arg2 == 0) {

            /* test r, r */

            tcg_out_modrm(s, OPC_TESTL + rexw, arg1, arg1);

        } else {

            tgen_arithi(s, ARITH_CMP + rexw, arg1, arg2, 0);

        }

    } else {

        tgen_arithr(s, ARITH_CMP + rexw, arg1, arg2);

    }

}

static void tcg_out_brcond32(TCGContext *s, TCGCond cond,

                             TCGArg arg1, TCGArg arg2, int const_arg2,

                             int label_index, int small)

{

    tcg_out_cmp(s, arg1, arg2, const_arg2, 0);

    tcg_out_jxx(s, tcg_cond_to_jcc[cond], label_index, small);

}

#if TCG_TARGET_REG_BITS == 64

static void tcg_out_brcond64(TCGContext *s, TCGCond cond,

                             TCGArg arg1, TCGArg arg2, int const_arg2,

                             int label_index, int small)

{

    tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW);

    tcg_out_jxx(s, tcg_cond_to_jcc[cond], label_index, small);

}

#else

/* XXX: we implement it at the target level to avoid having to

   handle cross basic blocks temporaries */

static void tcg_out_brcond2(TCGContext *s, const TCGArg *args,

                            const int *const_args, int small)

{

    int label_next;

    label_next = gen_new_label();

    switch(args[4]) {

    case TCG_COND_EQ:

        tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2],

                         label_next, 1);

        tcg_out_brcond32(s, TCG_COND_EQ, args[1], args[3], const_args[3],

                         args[5], small);

        break;

    case TCG_COND_NE:

        tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2],

                         args[5], small);

        tcg_out_brcond32(s, TCG_COND_NE, args[1], args[3], const_args[3],

                         args[5], small);

        break;

    case TCG_COND_LT:

        tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3],

                         args[5], small);

        tcg_out_jxx(s, JCC_JNE, label_next, 1);

        tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2],

                         args[5], small);

        break;

    case TCG_COND_LE:

        tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3],

                         args[5], small);

        tcg_out_jxx(s, JCC_JNE, label_next, 1);

        tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2],

                         args[5], small);

        break;

    case TCG_COND_GT:

        tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3],

                         args[5], small);

        tcg_out_jxx(s, JCC_JNE, label_next, 1);

        tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2],

                         args[5], small);

        break;

    case TCG_COND_GE:

        tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3],

                         args[5], small);

        tcg_out_jxx(s, JCC_JNE, label_next, 1);

        tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2],

                         args[5], small);

        break;

    case TCG_COND_LTU:

        tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3],

                         args[5], small);

        tcg_out_jxx(s, JCC_JNE, label_next, 1);

        tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2],

                         args[5], small);

        break;