Archipelago » qemu

/*

 * Tiny Code Generator for QEMU

 *

 * Copyright (c) 2008 Fabrice Bellard

 *

 * Permission is hereby granted, free of charge, to any person obtaining a copy

 * of this software and associated documentation files (the "Software"), to deal

 * in the Software without restriction, including without limitation the rights

 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

 * copies of the Software, and to permit persons to whom the Software is

 * furnished to do so, subject to the following conditions:

 *

 * The above copyright notice and this permission notice shall be included in

 * all copies or substantial portions of the Software.

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL

 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN

 * THE SOFTWARE.

 */

/* define it to use liveness analysis (better code) */

#define USE_LIVENESS_ANALYSIS

#define USE_TCG_OPTIMIZATIONS

#include "config.h"

/* Define to jump the ELF file used to communicate with GDB.  */

#undef DEBUG_JIT

#if !defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG)

/* define it to suppress various consistency checks (faster) */

#define NDEBUG

#endif

#include "qemu-common.h"

#include "qemu/cache-utils.h"

#include "qemu/host-utils.h"

#include "qemu/timer.h"

/* Note: the long term plan is to reduce the dependancies on the QEMU

   CPU definitions. Currently they are used for qemu_ld/st

   instructions */

#define NO_CPU_IO_DEFS

#include "cpu.h"

#include "tcg-op.h"

#if UINTPTR_MAX == UINT32_MAX

# define ELF_CLASS  ELFCLASS32

#else

# define ELF_CLASS  ELFCLASS64

#endif

#ifdef HOST_WORDS_BIGENDIAN

# define ELF_DATA   ELFDATA2MSB

#else

# define ELF_DATA   ELFDATA2LSB

#endif

#include "elf.h"

/* Forward declarations for functions declared in tcg-target.c and used here. */

static void tcg_target_init(TCGContext *s);

static void tcg_target_qemu_prologue(TCGContext *s);

static void patch_reloc(uint8_t *code_ptr, int type, 

                        intptr_t value, intptr_t addend);

/* The CIE and FDE header definitions will be common to all hosts.  */

typedef struct {

    uint32_t len __attribute__((aligned((sizeof(void *)))));

    uint32_t id;

    uint8_t version;

    char augmentation[1];

    uint8_t code_align;

    uint8_t data_align;

    uint8_t return_column;

} DebugFrameCIE;

typedef struct QEMU_PACKED {

    uint32_t len __attribute__((aligned((sizeof(void *)))));

    uint32_t cie_offset;

    uintptr_t func_start;

    uintptr_t func_len;

} DebugFrameFDEHeader;

static void tcg_register_jit_int(void *buf, size_t size,

                                 void *debug_frame, size_t debug_frame_size)

    __attribute__((unused));

/* Forward declarations for functions declared and used in tcg-target.c. */

static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str);

static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,

                       intptr_t arg2);

static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);

static void tcg_out_movi(TCGContext *s, TCGType type,

                         TCGReg ret, tcg_target_long arg);

static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,

                       const int *const_args);

static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,

                       intptr_t arg2);

static int tcg_target_const_match(tcg_target_long val,

                                  const TCGArgConstraint *arg_ct);

static void tcg_out_tb_init(TCGContext *s);

static void tcg_out_tb_finalize(TCGContext *s);

TCGOpDef tcg_op_defs[] = {

#define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags },

#include "tcg-opc.h"

#undef DEF

};

const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs);

static TCGRegSet tcg_target_available_regs[2];

static TCGRegSet tcg_target_call_clobber_regs;

static inline void tcg_out8(TCGContext *s, uint8_t v)

{

    *s->code_ptr++ = v;

}

static inline void tcg_out16(TCGContext *s, uint16_t v)

{

    uint8_t *p = s->code_ptr;

    *(uint16_t *)p = v;

    s->code_ptr = p + 2;

}

static inline void tcg_out32(TCGContext *s, uint32_t v)

{

    uint8_t *p = s->code_ptr;

    *(uint32_t *)p = v;

    s->code_ptr = p + 4;

}

static inline void tcg_out64(TCGContext *s, uint64_t v)

{

    uint8_t *p = s->code_ptr;

    *(uint64_t *)p = v;

    s->code_ptr = p + 8;

}

/* label relocation processing */

static void tcg_out_reloc(TCGContext *s, uint8_t *code_ptr, int type,

                          int label_index, intptr_t addend)

{

    TCGLabel *l;

    TCGRelocation *r;

    l = &s->labels[label_index];

    if (l->has_value) {

        /* FIXME: This may break relocations on RISC targets that

           modify instruction fields in place.  The caller may not have 

           written the initial value.  */

        patch_reloc(code_ptr, type, l->u.value, addend);

    } else {

        /* add a new relocation entry */

        r = tcg_malloc(sizeof(TCGRelocation));

        r->type = type;

        r->ptr = code_ptr;

        r->addend = addend;

        r->next = l->u.first_reloc;

        l->u.first_reloc = r;

    }

}

static void tcg_out_label(TCGContext *s, int label_index, void *ptr)

{

    TCGLabel *l;

    TCGRelocation *r;

    intptr_t value = (intptr_t)ptr;

    l = &s->labels[label_index];

    if (l->has_value) {

        tcg_abort();

    }

    r = l->u.first_reloc;

    while (r != NULL) {

        patch_reloc(r->ptr, r->type, value, r->addend);

        r = r->next;

    }

    l->has_value = 1;

    l->u.value = value;

}

int gen_new_label(void)

{

    TCGContext *s = &tcg_ctx;

    int idx;

    TCGLabel *l;

    if (s->nb_labels >= TCG_MAX_LABELS)

        tcg_abort();

    idx = s->nb_labels++;

    l = &s->labels[idx];

    l->has_value = 0;

    l->u.first_reloc = NULL;

    return idx;

}

#include "tcg-target.c"

/* pool based memory allocation */

void *tcg_malloc_internal(TCGContext *s, int size)

{

    TCGPool *p;

    int pool_size;

    if (size > TCG_POOL_CHUNK_SIZE) {

        /* big malloc: insert a new pool (XXX: could optimize) */

        p = g_malloc(sizeof(TCGPool) + size);

        p->size = size;

        p->next = s->pool_first_large;

        s->pool_first_large = p;

        return p->data;

    } else {

        p = s->pool_current;

        if (!p) {

            p = s->pool_first;

            if (!p)

                goto new_pool;

        } else {

            if (!p->next) {

            new_pool:

                pool_size = TCG_POOL_CHUNK_SIZE;

                p = g_malloc(sizeof(TCGPool) + pool_size);

                p->size = pool_size;

                p->next = NULL;

                if (s->pool_current) 

                    s->pool_current->next = p;

                else

                    s->pool_first = p;

            } else {

                p = p->next;

            }

        }

    }

    s->pool_current = p;

    s->pool_cur = p->data + size;

    s->pool_end = p->data + p->size;

    return p->data;

}

void tcg_pool_reset(TCGContext *s)

{

    TCGPool *p, *t;

    for (p = s->pool_first_large; p; p = t) {

        t = p->next;

        g_free(p);

    }

    s->pool_first_large = NULL;

    s->pool_cur = s->pool_end = NULL;

    s->pool_current = NULL;

}

#include "helper.h"

typedef struct TCGHelperInfo {

    void *func;

    const char *name;

} TCGHelperInfo;

static const TCGHelperInfo all_helpers[] = {

#define GEN_HELPER 2

#include "helper.h"

    /* Include tcg-runtime.c functions.  */

    { tcg_helper_div_i32, "div_i32" },

    { tcg_helper_rem_i32, "rem_i32" },

    { tcg_helper_divu_i32, "divu_i32" },

    { tcg_helper_remu_i32, "remu_i32" },

    { tcg_helper_shl_i64, "shl_i64" },

    { tcg_helper_shr_i64, "shr_i64" },

    { tcg_helper_sar_i64, "sar_i64" },

    { tcg_helper_div_i64, "div_i64" },

    { tcg_helper_rem_i64, "rem_i64" },

    { tcg_helper_divu_i64, "divu_i64" },

    { tcg_helper_remu_i64, "remu_i64" },

    { tcg_helper_mulsh_i64, "mulsh_i64" },

    { tcg_helper_muluh_i64, "muluh_i64" },

};

void tcg_context_init(TCGContext *s)

{

    int op, total_args, n, i;

    TCGOpDef *def;

    TCGArgConstraint *args_ct;

    int *sorted_args;

    GHashTable *helper_table;

    memset(s, 0, sizeof(*s));

    s->nb_globals = 0;

    /* Count total number of arguments and allocate the corresponding

       space */

    total_args = 0;

    for(op = 0; op < NB_OPS; op++) {

        def = &tcg_op_defs[op];

        n = def->nb_iargs + def->nb_oargs;

        total_args += n;

    }

    args_ct = g_malloc(sizeof(TCGArgConstraint) * total_args);

    sorted_args = g_malloc(sizeof(int) * total_args);

    for(op = 0; op < NB_OPS; op++) {

        def = &tcg_op_defs[op];

        def->args_ct = args_ct;

        def->sorted_args = sorted_args;

        n = def->nb_iargs + def->nb_oargs;

        sorted_args += n;

        args_ct += n;

    }

    /* Register helpers.  */

    /* Use g_direct_hash/equal for direct pointer comparisons on func.  */

    s->helpers = helper_table = g_hash_table_new(NULL, NULL);

    for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {

        g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,

                            (gpointer)all_helpers[i].name);

    }

    tcg_target_init(s);

}

void tcg_prologue_init(TCGContext *s)

{

    /* init global prologue and epilogue */

    s->code_buf = s->code_gen_prologue;

    s->code_ptr = s->code_buf;

    tcg_target_qemu_prologue(s);

    flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);

#ifdef DEBUG_DISAS

    if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {

        size_t size = s->code_ptr - s->code_buf;

        qemu_log("PROLOGUE: [size=%zu]\n", size);

        log_disas(s->code_buf, size);

        qemu_log("\n");

        qemu_log_flush();

    }

#endif

}

void tcg_set_frame(TCGContext *s, int reg, intptr_t start, intptr_t size)

{

    s->frame_start = start;

    s->frame_end = start + size;

    s->frame_reg = reg;

}

void tcg_func_start(TCGContext *s)

{

    int i;

    tcg_pool_reset(s);

    s->nb_temps = s->nb_globals;

    for(i = 0; i < (TCG_TYPE_COUNT * 2); i++)

        s->first_free_temp[i] = -1;

    s->labels = tcg_malloc(sizeof(TCGLabel) * TCG_MAX_LABELS);

    s->nb_labels = 0;

    s->current_frame_offset = s->frame_start;

#ifdef CONFIG_DEBUG_TCG

    s->goto_tb_issue_mask = 0;

#endif

    s->gen_opc_ptr = s->gen_opc_buf;

    s->gen_opparam_ptr = s->gen_opparam_buf;

    s->be = tcg_malloc(sizeof(TCGBackendData));

}

static inline void tcg_temp_alloc(TCGContext *s, int n)

{

    if (n > TCG_MAX_TEMPS)

        tcg_abort();

}

static inline int tcg_global_reg_new_internal(TCGType type, int reg,

                                              const char *name)

{

    TCGContext *s = &tcg_ctx;

    TCGTemp *ts;

    int idx;

#if TCG_TARGET_REG_BITS == 32

    if (type != TCG_TYPE_I32)

        tcg_abort();

#endif

    if (tcg_regset_test_reg(s->reserved_regs, reg))

        tcg_abort();

    idx = s->nb_globals;

    tcg_temp_alloc(s, s->nb_globals + 1);

    ts = &s->temps[s->nb_globals];

    ts->base_type = type;

    ts->type = type;

    ts->fixed_reg = 1;

    ts->reg = reg;

    ts->name = name;

    s->nb_globals++;

    tcg_regset_set_reg(s->reserved_regs, reg);

    return idx;

}

TCGv_i32 tcg_global_reg_new_i32(int reg, const char *name)

{

    int idx;

    idx = tcg_global_reg_new_internal(TCG_TYPE_I32, reg, name);

    return MAKE_TCGV_I32(idx);

}

TCGv_i64 tcg_global_reg_new_i64(int reg, const char *name)

{

    int idx;

    idx = tcg_global_reg_new_internal(TCG_TYPE_I64, reg, name);

    return MAKE_TCGV_I64(idx);

}

static inline int tcg_global_mem_new_internal(TCGType type, int reg,

                                              intptr_t offset,

                                              const char *name)

{

    TCGContext *s = &tcg_ctx;

    TCGTemp *ts;

    int idx;

    idx = s->nb_globals;

#if TCG_TARGET_REG_BITS == 32

    if (type == TCG_TYPE_I64) {

        char buf[64];

        tcg_temp_alloc(s, s->nb_globals + 2);

        ts = &s->temps[s->nb_globals];

        ts->base_type = type;

        ts->type = TCG_TYPE_I32;

        ts->fixed_reg = 0;

        ts->mem_allocated = 1;

        ts->mem_reg = reg;

#ifdef TCG_TARGET_WORDS_BIGENDIAN

        ts->mem_offset = offset + 4;

#else

        ts->mem_offset = offset;

#endif

        pstrcpy(buf, sizeof(buf), name);

        pstrcat(buf, sizeof(buf), "_0");

        ts->name = strdup(buf);

        ts++;

        ts->base_type = type;

        ts->type = TCG_TYPE_I32;

        ts->fixed_reg = 0;

        ts->mem_allocated = 1;

        ts->mem_reg = reg;

#ifdef TCG_TARGET_WORDS_BIGENDIAN

        ts->mem_offset = offset;

#else

        ts->mem_offset = offset + 4;

#endif

        pstrcpy(buf, sizeof(buf), name);

        pstrcat(buf, sizeof(buf), "_1");

        ts->name = strdup(buf);

        s->nb_globals += 2;

    } else

#endif

    {

        tcg_temp_alloc(s, s->nb_globals + 1);

        ts = &s->temps[s->nb_globals];

        ts->base_type = type;

        ts->type = type;

        ts->fixed_reg = 0;

        ts->mem_allocated = 1;

        ts->mem_reg = reg;

        ts->mem_offset = offset;

        ts->name = name;

        s->nb_globals++;

    }

    return idx;

}

TCGv_i32 tcg_global_mem_new_i32(int reg, intptr_t offset, const char *name)

{

    int idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);

    return MAKE_TCGV_I32(idx);

}

TCGv_i64 tcg_global_mem_new_i64(int reg, intptr_t offset, const char *name)

{

    int idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);

    return MAKE_TCGV_I64(idx);

}

static inline int tcg_temp_new_internal(TCGType type, int temp_local)

{

    TCGContext *s = &tcg_ctx;

    TCGTemp *ts;

    int idx, k;

    k = type;

    if (temp_local)

        k += TCG_TYPE_COUNT;

    idx = s->first_free_temp[k];

    if (idx != -1) {

        /* There is already an available temp with the

           right type */

        ts = &s->temps[idx];

        s->first_free_temp[k] = ts->next_free_temp;

        ts->temp_allocated = 1;

        assert(ts->temp_local == temp_local);

    } else {

        idx = s->nb_temps;

#if TCG_TARGET_REG_BITS == 32

        if (type == TCG_TYPE_I64) {

            tcg_temp_alloc(s, s->nb_temps + 2);

            ts = &s->temps[s->nb_temps];

            ts->base_type = type;

            ts->type = TCG_TYPE_I32;

            ts->temp_allocated = 1;

            ts->temp_local = temp_local;

            ts->name = NULL;

            ts++;

            ts->base_type = TCG_TYPE_I32;

            ts->type = TCG_TYPE_I32;

            ts->temp_allocated = 1;

            ts->temp_local = temp_local;

            ts->name = NULL;

            s->nb_temps += 2;

        } else

#endif

        {

            tcg_temp_alloc(s, s->nb_temps + 1);

            ts = &s->temps[s->nb_temps];

            ts->base_type = type;

            ts->type = type;

            ts->temp_allocated = 1;

            ts->temp_local = temp_local;

            ts->name = NULL;

            s->nb_temps++;

        }

    }

#if defined(CONFIG_DEBUG_TCG)

    s->temps_in_use++;

#endif

    return idx;

}

TCGv_i32 tcg_temp_new_internal_i32(int temp_local)

{

    int idx;

    idx = tcg_temp_new_internal(TCG_TYPE_I32, temp_local);

    return MAKE_TCGV_I32(idx);

}

TCGv_i64 tcg_temp_new_internal_i64(int temp_local)

{

    int idx;

    idx = tcg_temp_new_internal(TCG_TYPE_I64, temp_local);

    return MAKE_TCGV_I64(idx);

}

static inline void tcg_temp_free_internal(int idx)

{

    TCGContext *s = &tcg_ctx;

    TCGTemp *ts;

    int k;

#if defined(CONFIG_DEBUG_TCG)

    s->temps_in_use--;

    if (s->temps_in_use < 0) {

        fprintf(stderr, "More temporaries freed than allocated!\n");

    }

#endif

    assert(idx >= s->nb_globals && idx < s->nb_temps);

    ts = &s->temps[idx];

    assert(ts->temp_allocated != 0);

    ts->temp_allocated = 0;

    k = ts->base_type;

    if (ts->temp_local)

        k += TCG_TYPE_COUNT;

    ts->next_free_temp = s->first_free_temp[k];

    s->first_free_temp[k] = idx;

}

void tcg_temp_free_i32(TCGv_i32 arg)

{

    tcg_temp_free_internal(GET_TCGV_I32(arg));

}

void tcg_temp_free_i64(TCGv_i64 arg)

{

    tcg_temp_free_internal(GET_TCGV_I64(arg));

}

TCGv_i32 tcg_const_i32(int32_t val)

{

    TCGv_i32 t0;

    t0 = tcg_temp_new_i32();

    tcg_gen_movi_i32(t0, val);

    return t0;

}

TCGv_i64 tcg_const_i64(int64_t val)

{

    TCGv_i64 t0;

    t0 = tcg_temp_new_i64();

    tcg_gen_movi_i64(t0, val);

    return t0;

}

TCGv_i32 tcg_const_local_i32(int32_t val)

{

    TCGv_i32 t0;

    t0 = tcg_temp_local_new_i32();

    tcg_gen_movi_i32(t0, val);

    return t0;

}

TCGv_i64 tcg_const_local_i64(int64_t val)

{

    TCGv_i64 t0;

    t0 = tcg_temp_local_new_i64();

    tcg_gen_movi_i64(t0, val);

    return t0;

}

#if defined(CONFIG_DEBUG_TCG)

void tcg_clear_temp_count(void)

{

    TCGContext *s = &tcg_ctx;

    s->temps_in_use = 0;

}

int tcg_check_temp_count(void)

{

    TCGContext *s = &tcg_ctx;

    if (s->temps_in_use) {

        /* Clear the count so that we don't give another

         * warning immediately next time around.

         */

        s->temps_in_use = 0;

        return 1;

    }

    return 0;

}

#endif

/* Note: we convert the 64 bit args to 32 bit and do some alignment

   and endian swap. Maybe it would be better to do the alignment

   and endian swap in tcg_reg_alloc_call(). */

void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags,

                   int sizemask, TCGArg ret, int nargs, TCGArg *args)

{

    int i;

    int real_args;

    int nb_rets;

    TCGArg *nparam;

#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64

    for (i = 0; i < nargs; ++i) {

        int is_64bit = sizemask & (1 << (i+1)*2);

        int is_signed = sizemask & (2 << (i+1)*2);

        if (!is_64bit) {

            TCGv_i64 temp = tcg_temp_new_i64();

            TCGv_i64 orig = MAKE_TCGV_I64(args[i]);

            if (is_signed) {

                tcg_gen_ext32s_i64(temp, orig);

            } else {

                tcg_gen_ext32u_i64(temp, orig);

            }

            args[i] = GET_TCGV_I64(temp);

        }

    }

#endif /* TCG_TARGET_EXTEND_ARGS */

    *s->gen_opc_ptr++ = INDEX_op_call;

    nparam = s->gen_opparam_ptr++;

    if (ret != TCG_CALL_DUMMY_ARG) {

#if TCG_TARGET_REG_BITS < 64

        if (sizemask & 1) {

#ifdef TCG_TARGET_WORDS_BIGENDIAN

            *s->gen_opparam_ptr++ = ret + 1;

            *s->gen_opparam_ptr++ = ret;

#else

            *s->gen_opparam_ptr++ = ret;

            *s->gen_opparam_ptr++ = ret + 1;

#endif

            nb_rets = 2;

        } else

#endif

        {

            *s->gen_opparam_ptr++ = ret;

            nb_rets = 1;

        }

    } else {

        nb_rets = 0;

    }

    real_args = 0;

    for (i = 0; i < nargs; i++) {

#if TCG_TARGET_REG_BITS < 64

        int is_64bit = sizemask & (1 << (i+1)*2);

        if (is_64bit) {

#ifdef TCG_TARGET_CALL_ALIGN_ARGS

            /* some targets want aligned 64 bit args */

            if (real_args & 1) {

                *s->gen_opparam_ptr++ = TCG_CALL_DUMMY_ARG;

                real_args++;

            }

#endif

            /* If stack grows up, then we will be placing successive

               arguments at lower addresses, which means we need to

               reverse the order compared to how we would normally

               treat either big or little-endian.  For those arguments

               that will wind up in registers, this still works for

               HPPA (the only current STACK_GROWSUP target) since the

               argument registers are *also* allocated in decreasing

               order.  If another such target is added, this logic may

               have to get more complicated to differentiate between

               stack arguments and register arguments.  */

#if defined(TCG_TARGET_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)

            *s->gen_opparam_ptr++ = args[i] + 1;

            *s->gen_opparam_ptr++ = args[i];

#else

            *s->gen_opparam_ptr++ = args[i];

            *s->gen_opparam_ptr++ = args[i] + 1;

#endif

            real_args += 2;

            continue;

        }

#endif /* TCG_TARGET_REG_BITS < 64 */

        *s->gen_opparam_ptr++ = args[i];

        real_args++;

    }

    *s->gen_opparam_ptr++ = GET_TCGV_PTR(func);

    *s->gen_opparam_ptr++ = flags;

    *nparam = (nb_rets << 16) | (real_args + 1);

    /* total parameters, needed to go backward in the instruction stream */

    *s->gen_opparam_ptr++ = 1 + nb_rets + real_args + 3;

#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64

    for (i = 0; i < nargs; ++i) {

        int is_64bit = sizemask & (1 << (i+1)*2);

        if (!is_64bit) {

            TCGv_i64 temp = MAKE_TCGV_I64(args[i]);

            tcg_temp_free_i64(temp);

        }

    }

#endif /* TCG_TARGET_EXTEND_ARGS */

}

#if TCG_TARGET_REG_BITS == 32

void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,

                        int c, int right, int arith)

{

    if (c == 0) {

        tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));

        tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));

    } else if (c >= 32) {

        c -= 32;

        if (right) {

            if (arith) {

                tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);

                tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);

            } else {

                tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);

                tcg_gen_movi_i32(TCGV_HIGH(ret), 0);

            }

        } else {

            tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);

            tcg_gen_movi_i32(TCGV_LOW(ret), 0);

        }

    } else {

        TCGv_i32 t0, t1;

        t0 = tcg_temp_new_i32();

        t1 = tcg_temp_new_i32();

        if (right) {

            tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c);

            if (arith)

                tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c);

            else

                tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c);

            tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);

            tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0);

            tcg_gen_mov_i32(TCGV_HIGH(ret), t1);

        } else {

            tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);

            /* Note: ret can be the same as arg1, so we use t1 */

            tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c);

            tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);

            tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0);

            tcg_gen_mov_i32(TCGV_LOW(ret), t1);

        }

        tcg_temp_free_i32(t0);

        tcg_temp_free_i32(t1);

    }

}

#endif

static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st)

{

    switch (op & MO_SIZE) {

    case MO_8:

        op &= ~MO_BSWAP;

        break;

    case MO_16:

        break;

    case MO_32:

        if (!is64) {

            op &= ~MO_SIGN;

        }

        break;

    case MO_64:

        if (!is64) {

            tcg_abort();

        }

        break;

    }

    if (st) {

        op &= ~MO_SIGN;

    }

    return op;

}

static const TCGOpcode old_ld_opc[8] = {

    [MO_UB] = INDEX_op_qemu_ld8u,

    [MO_SB] = INDEX_op_qemu_ld8s,

    [MO_UW] = INDEX_op_qemu_ld16u,

    [MO_SW] = INDEX_op_qemu_ld16s,

#if TCG_TARGET_REG_BITS == 32

    [MO_UL] = INDEX_op_qemu_ld32,

    [MO_SL] = INDEX_op_qemu_ld32,

#else

    [MO_UL] = INDEX_op_qemu_ld32u,

    [MO_SL] = INDEX_op_qemu_ld32s,

#endif

    [MO_Q]  = INDEX_op_qemu_ld64,

};

static const TCGOpcode old_st_opc[4] = {

    [MO_UB] = INDEX_op_qemu_st8,

    [MO_UW] = INDEX_op_qemu_st16,

    [MO_UL] = INDEX_op_qemu_st32,

    [MO_Q]  = INDEX_op_qemu_st64,

};

void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)

{

    memop = tcg_canonicalize_memop(memop, 0, 0);

    if (TCG_TARGET_HAS_new_ldst) {

        *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i32;

        tcg_add_param_i32(val);

        tcg_add_param_tl(addr);

        *tcg_ctx.gen_opparam_ptr++ = memop;

        *tcg_ctx.gen_opparam_ptr++ = idx;

        return;

    }

    /* The old opcodes only support target-endian memory operations.  */

    assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);

    assert(old_ld_opc[memop & MO_SSIZE] != 0);

    if (TCG_TARGET_REG_BITS == 32) {

        *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE];

        tcg_add_param_i32(val);

        tcg_add_param_tl(addr);

        *tcg_ctx.gen_opparam_ptr++ = idx;

    } else {

        TCGv_i64 val64 = tcg_temp_new_i64();

        *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE];

        tcg_add_param_i64(val64);

        tcg_add_param_tl(addr);

        *tcg_ctx.gen_opparam_ptr++ = idx;

        tcg_gen_trunc_i64_i32(val, val64);

        tcg_temp_free_i64(val64);

    }

}

void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)

{

    memop = tcg_canonicalize_memop(memop, 0, 1);

    if (TCG_TARGET_HAS_new_ldst) {

        *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i32;

        tcg_add_param_i32(val);

        tcg_add_param_tl(addr);

        *tcg_ctx.gen_opparam_ptr++ = memop;

        *tcg_ctx.gen_opparam_ptr++ = idx;

        return;

    }

    /* The old opcodes only support target-endian memory operations.  */

    assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);

    assert(old_st_opc[memop & MO_SIZE] != 0);

    if (TCG_TARGET_REG_BITS == 32) {

        *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE];

        tcg_add_param_i32(val);

        tcg_add_param_tl(addr);

        *tcg_ctx.gen_opparam_ptr++ = idx;

    } else {

        TCGv_i64 val64 = tcg_temp_new_i64();

        tcg_gen_extu_i32_i64(val64, val);

        *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE];

        tcg_add_param_i64(val64);

        tcg_add_param_tl(addr);

        *tcg_ctx.gen_opparam_ptr++ = idx;

        tcg_temp_free_i64(val64);

    }

}

void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)

{

    memop = tcg_canonicalize_memop(memop, 1, 0);

#if TCG_TARGET_REG_BITS == 32

    if ((memop & MO_SIZE) < MO_64) {

        tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);

        if (memop & MO_SIGN) {

            tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);

        } else {

            tcg_gen_movi_i32(TCGV_HIGH(val), 0);

        }

        return;

    }

#endif

    if (TCG_TARGET_HAS_new_ldst) {

        *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i64;

        tcg_add_param_i64(val);

        tcg_add_param_tl(addr);

        *tcg_ctx.gen_opparam_ptr++ = memop;

        *tcg_ctx.gen_opparam_ptr++ = idx;

        return;

    }

    /* The old opcodes only support target-endian memory operations.  */

    assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);

    assert(old_ld_opc[memop & MO_SSIZE] != 0);

    *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE];

    tcg_add_param_i64(val);

    tcg_add_param_tl(addr);

    *tcg_ctx.gen_opparam_ptr++ = idx;

}

void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)

{

    memop = tcg_canonicalize_memop(memop, 1, 1);

#if TCG_TARGET_REG_BITS == 32

    if ((memop & MO_SIZE) < MO_64) {

        tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);

        return;

    }

#endif

    if (TCG_TARGET_HAS_new_ldst) {

        *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i64;

        tcg_add_param_i64(val);

        tcg_add_param_tl(addr);

        *tcg_ctx.gen_opparam_ptr++ = memop;

        *tcg_ctx.gen_opparam_ptr++ = idx;

        return;

    }

    /* The old opcodes only support target-endian memory operations.  */

    assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);

    assert(old_st_opc[memop & MO_SIZE] != 0);

    *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE];

    tcg_add_param_i64(val);

    tcg_add_param_tl(addr);

    *tcg_ctx.gen_opparam_ptr++ = idx;

}

static void tcg_reg_alloc_start(TCGContext *s)

{

    int i;

    TCGTemp *ts;

    for(i = 0; i < s->nb_globals; i++) {

        ts = &s->temps[i];

        if (ts->fixed_reg) {

            ts->val_type = TEMP_VAL_REG;

        } else {

            ts->val_type = TEMP_VAL_MEM;

        }

    }

    for(i = s->nb_globals; i < s->nb_temps; i++) {

        ts = &s->temps[i];

        if (ts->temp_local) {

            ts->val_type = TEMP_VAL_MEM;

        } else {

            ts->val_type = TEMP_VAL_DEAD;

        }

        ts->mem_allocated = 0;

        ts->fixed_reg = 0;

    }

    for(i = 0; i < TCG_TARGET_NB_REGS; i++) {

        s->reg_to_temp[i] = -1;

    }

}

static char *tcg_get_arg_str_idx(TCGContext *s, char *buf, int buf_size,

                                 int idx)

{

    TCGTemp *ts;

    assert(idx >= 0 && idx < s->nb_temps);

    ts = &s->temps[idx];

    if (idx < s->nb_globals) {

        pstrcpy(buf, buf_size, ts->name);

    } else {

        if (ts->temp_local) 

            snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);

        else

            snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);

    }

    return buf;

}

char *tcg_get_arg_str_i32(TCGContext *s, char *buf, int buf_size, TCGv_i32 arg)

{

    return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I32(arg));

}

char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg)

{

    return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I64(arg));

}

/* Find helper name.  */

static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val)

{

    const char *ret = NULL;

    if (s->helpers) {

        ret = g_hash_table_lookup(s->helpers, (gpointer)val);

    }

    return ret;

}

static const char * const cond_name[] =

{

    [TCG_COND_NEVER] = "never",

    [TCG_COND_ALWAYS] = "always",

    [TCG_COND_EQ] = "eq",

    [TCG_COND_NE] = "ne",

    [TCG_COND_LT] = "lt",

    [TCG_COND_GE] = "ge",

    [TCG_COND_LE] = "le",

    [TCG_COND_GT] = "gt",

    [TCG_COND_LTU] = "ltu",

    [TCG_COND_GEU] = "geu",

    [TCG_COND_LEU] = "leu",

    [TCG_COND_GTU] = "gtu"

};

static const char * const ldst_name[] =

{

    [MO_UB]   = "ub",

    [MO_SB]   = "sb",

    [MO_LEUW] = "leuw",

    [MO_LESW] = "lesw",

    [MO_LEUL] = "leul",

    [MO_LESL] = "lesl",

    [MO_LEQ]  = "leq",

    [MO_BEUW] = "beuw",

    [MO_BESW] = "besw",

    [MO_BEUL] = "beul",

    [MO_BESL] = "besl",

    [MO_BEQ]  = "beq",

};

void tcg_dump_ops(TCGContext *s)

{

    const uint16_t *opc_ptr;

    const TCGArg *args;

    TCGArg arg;

    TCGOpcode c;

    int i, k, nb_oargs, nb_iargs, nb_cargs, first_insn;

    const TCGOpDef *def;

    char buf[128];

    first_insn = 1;

    opc_ptr = s->gen_opc_buf;

    args = s->gen_opparam_buf;

    while (opc_ptr < s->gen_opc_ptr) {

        c = *opc_ptr++;

        def = &tcg_op_defs[c];

        if (c == INDEX_op_debug_insn_start) {

            uint64_t pc;

#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS

            pc = ((uint64_t)args[1] << 32) | args[0];

#else

            pc = args[0];

#endif

            if (!first_insn) {

                qemu_log("\n");

            }

            qemu_log(" ---- 0x%" PRIx64, pc);

            first_insn = 0;

            nb_oargs = def->nb_oargs;

            nb_iargs = def->nb_iargs;

            nb_cargs = def->nb_cargs;

        } else if (c == INDEX_op_call) {

            TCGArg arg;

            /* variable number of arguments */

            arg = *args++;

            nb_oargs = arg >> 16;

            nb_iargs = arg & 0xffff;

            nb_cargs = def->nb_cargs;

            qemu_log(" %s ", def->name);

            /* function name */

            qemu_log("%s",

                     tcg_get_arg_str_idx(s, buf, sizeof(buf),

                                         args[nb_oargs + nb_iargs - 1]));

            /* flags */

            qemu_log(",$0x%" TCG_PRIlx, args[nb_oargs + nb_iargs]);

            /* nb out args */

            qemu_log(",$%d", nb_oargs);

            for(i = 0; i < nb_oargs; i++) {

                qemu_log(",");

                qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),

                                                   args[i]));

            }

            for(i = 0; i < (nb_iargs - 1); i++) {

                qemu_log(",");

                if (args[nb_oargs + i] == TCG_CALL_DUMMY_ARG) {

                    qemu_log("<dummy>");

                } else {

                    qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),

                                                       args[nb_oargs + i]));

                }

            }

        } else if (c == INDEX_op_movi_i32 || c == INDEX_op_movi_i64) {

            tcg_target_ulong val;

            const char *name;

            nb_oargs = def->nb_oargs;

            nb_iargs = def->nb_iargs;

            nb_cargs = def->nb_cargs;

            qemu_log(" %s %s,$", def->name,

                     tcg_get_arg_str_idx(s, buf, sizeof(buf), args[0]));

            val = args[1];

            name = tcg_find_helper(s, val);

            if (name) {

                qemu_log("%s", name);

            } else {

                if (c == INDEX_op_movi_i32) {

                    qemu_log("0x%x", (uint32_t)val);

                } else {

                    qemu_log("0x%" PRIx64 , (uint64_t)val);

                }

            }

        } else {

            qemu_log(" %s ", def->name);

            if (c == INDEX_op_nopn) {

                /* variable number of arguments */

                nb_cargs = *args;

                nb_oargs = 0;

                nb_iargs = 0;

            } else {

                nb_oargs = def->nb_oargs;

                nb_iargs = def->nb_iargs;

                nb_cargs = def->nb_cargs;

            }

            k = 0;

            for(i = 0; i < nb_oargs; i++) {

                if (k != 0) {

                    qemu_log(",");

                }

                qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),

                                                   args[k++]));

            }

            for(i = 0; i < nb_iargs; i++) {

                if (k != 0) {

                    qemu_log(",");

                }

                qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),

                                                   args[k++]));

            }

            switch (c) {

            case INDEX_op_brcond_i32:

            case INDEX_op_setcond_i32:

            case INDEX_op_movcond_i32:

            case INDEX_op_brcond2_i32:

            case INDEX_op_setcond2_i32:

            case INDEX_op_brcond_i64:

            case INDEX_op_setcond_i64:

            case INDEX_op_movcond_i64:

                if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]]) {

                    qemu_log(",%s", cond_name[args[k++]]);

                } else {

                    qemu_log(",$0x%" TCG_PRIlx, args[k++]);

                }

                i = 1;

                break;

            case INDEX_op_qemu_ld_i32:

            case INDEX_op_qemu_st_i32:

            case INDEX_op_qemu_ld_i64:

            case INDEX_op_qemu_st_i64:

                if (args[k] < ARRAY_SIZE(ldst_name) && ldst_name[args[k]]) {

                    qemu_log(",%s", ldst_name[args[k++]]);

                } else {

                    qemu_log(",$0x%" TCG_PRIlx, args[k++]);

                }

                i = 1;

                break;

            default:

                i = 0;

                break;

            }

            for(; i < nb_cargs; i++) {

                if (k != 0) {

                    qemu_log(",");

                }

                arg = args[k++];

                qemu_log("$0x%" TCG_PRIlx, arg);

            }

        }

        qemu_log("\n");

        args += nb_iargs + nb_oargs + nb_cargs;

    }

}

/* we give more priority to constraints with less registers */

static int get_constraint_priority(const TCGOpDef *def, int k)

{

    const TCGArgConstraint *arg_ct;

    int i, n;

    arg_ct = &def->args_ct[k];

    if (arg_ct->ct & TCG_CT_ALIAS) {

        /* an alias is equivalent to a single register */

        n = 1;

    } else {

        if (!(arg_ct->ct & TCG_CT_REG))

            return 0;

        n = 0;

        for(i = 0; i < TCG_TARGET_NB_REGS; i++) {

            if (tcg_regset_test_reg(arg_ct->u.regs, i))

                n++;

        }

    }

    return TCG_TARGET_NB_REGS - n + 1;

}

/* sort from highest priority to lowest */

static void sort_constraints(TCGOpDef *def, int start, int n)

{

    int i, j, p1, p2, tmp;

    for(i = 0; i < n; i++)

        def->sorted_args[start + i] = start + i;

    if (n <= 1)

        return;

    for(i = 0; i < n - 1; i++) {

        for(j = i + 1; j < n; j++) {

            p1 = get_constraint_priority(def, def->sorted_args[start + i]);

            p2 = get_constraint_priority(def, def->sorted_args[start + j]);

            if (p1 < p2) {

                tmp = def->sorted_args[start + i];

                def->sorted_args[start + i] = def->sorted_args[start + j];

                def->sorted_args[start + j] = tmp;

            }

        }

    }

}

void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs)

{

    TCGOpcode op;

    TCGOpDef *def;

    const char *ct_str;

    int i, nb_args;

    for(;;) {

        if (tdefs->op == (TCGOpcode)-1)

            break;

        op = tdefs->op;

        assert((unsigned)op < NB_OPS);

        def = &tcg_op_defs[op];

#if defined(CONFIG_DEBUG_TCG)

        /* Duplicate entry in op definitions? */

        assert(!def->used);

        def->used = 1;

#endif

        nb_args = def->nb_iargs + def->nb_oargs;

        for(i = 0; i < nb_args; i++) {

            ct_str = tdefs->args_ct_str[i];

            /* Incomplete TCGTargetOpDef entry? */

            assert(ct_str != NULL);

            tcg_regset_clear(def->args_ct[i].u.regs);

            def->args_ct[i].ct = 0;

            if (ct_str[0] >= '0' && ct_str[0] <= '9') {

                int oarg;

                oarg = ct_str[0] - '0';

                assert(oarg < def->nb_oargs);

                assert(def->args_ct[oarg].ct & TCG_CT_REG);

                /* TCG_CT_ALIAS is for the output arguments. The input

                   argument is tagged with TCG_CT_IALIAS. */

                def->args_ct[i] = def->args_ct[oarg];

                def->args_ct[oarg].ct = TCG_CT_ALIAS;

                def->args_ct[oarg].alias_index = i;

                def->args_ct[i].ct |= TCG_CT_IALIAS;

                def->args_ct[i].alias_index = oarg;

            } else {

                for(;;) {

                    if (*ct_str == '\0')

                        break;

                    switch(*ct_str) {

                    case 'i':

                        def->args_ct[i].ct |= TCG_CT_CONST;

                        ct_str++;

                        break;

                    default:

                        if (target_parse_constraint(&def->args_ct[i], &ct_str) < 0) {

                            fprintf(stderr, "Invalid constraint '%s' for arg %d of operation '%s'\n",

                                    ct_str, i, def->name);

                            exit(1);

                        }

                    }

                }

            }

        }

        /* TCGTargetOpDef entry with too much information? */

        assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);

        /* sort the constraints (XXX: this is just an heuristic) */

        sort_constraints(def, 0, def->nb_oargs);

        sort_constraints(def, def->nb_oargs, def->nb_iargs);

#if 0

        {

            int i;

            printf("%s: sorted=", def->name);

            for(i = 0; i < def->nb_oargs + def->nb_iargs; i++)