Archipelago » qemu

/*

 *  AArch64 translation

 *

 *  Copyright (c) 2013 Alexander Graf <agraf@suse.de>

 *

 * This library is free software; you can redistribute it and/or

 * modify it under the terms of the GNU Lesser General Public

 * License as published by the Free Software Foundation; either

 * version 2 of the License, or (at your option) any later version.

 *

 * This library is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * Lesser General Public License for more details.

 *

 * You should have received a copy of the GNU Lesser General Public

 * License along with this library; if not, see <http://www.gnu.org/licenses/>.

 */

#include <stdarg.h>

#include <stdlib.h>

#include <stdio.h>

#include <string.h>

#include <inttypes.h>

#include "cpu.h"

#include "tcg-op.h"

#include "qemu/log.h"

#include "translate.h"

#include "qemu/host-utils.h"

#include "exec/gen-icount.h"

#include "helper.h"

#define GEN_HELPER 1

#include "helper.h"

static TCGv_i64 cpu_X[32];

static TCGv_i64 cpu_pc;

static TCGv_i32 cpu_NF, cpu_ZF, cpu_CF, cpu_VF;

/* Load/store exclusive handling */

static TCGv_i64 cpu_exclusive_addr;

static TCGv_i64 cpu_exclusive_val;

static TCGv_i64 cpu_exclusive_high;

#ifdef CONFIG_USER_ONLY

static TCGv_i64 cpu_exclusive_test;

static TCGv_i32 cpu_exclusive_info;

#endif

static const char *regnames[] = {

    "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",

    "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",

    "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",

    "x24", "x25", "x26", "x27", "x28", "x29", "lr", "sp"

};

enum a64_shift_type {

    A64_SHIFT_TYPE_LSL = 0,

    A64_SHIFT_TYPE_LSR = 1,

    A64_SHIFT_TYPE_ASR = 2,

    A64_SHIFT_TYPE_ROR = 3

};

/* Table based decoder typedefs - used when the relevant bits for decode

 * are too awkwardly scattered across the instruction (eg SIMD).

 */

typedef void AArch64DecodeFn(DisasContext *s, uint32_t insn);

typedef struct AArch64DecodeTable {

    uint32_t pattern;

    uint32_t mask;

    AArch64DecodeFn *disas_fn;

} AArch64DecodeTable;

/* Function prototype for gen_ functions for calling Neon helpers */

typedef void NeonGenTwoOpFn(TCGv_i32, TCGv_i32, TCGv_i32);

typedef void NeonGenTwoOpEnvFn(TCGv_i32, TCGv_ptr, TCGv_i32, TCGv_i32);

typedef void NeonGenNarrowFn(TCGv_i32, TCGv_i64);

typedef void NeonGenNarrowEnvFn(TCGv_i32, TCGv_ptr, TCGv_i64);

typedef void NeonGenTwoSingleOPFn(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_ptr);

typedef void NeonGenTwoDoubleOPFn(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_ptr);

/* initialize TCG globals.  */

void a64_translate_init(void)

{

    int i;

    cpu_pc = tcg_global_mem_new_i64(TCG_AREG0,

                                    offsetof(CPUARMState, pc),

                                    "pc");

    for (i = 0; i < 32; i++) {

        cpu_X[i] = tcg_global_mem_new_i64(TCG_AREG0,

                                          offsetof(CPUARMState, xregs[i]),

                                          regnames[i]);

    }

    cpu_NF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, NF), "NF");

    cpu_ZF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, ZF), "ZF");

    cpu_CF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, CF), "CF");

    cpu_VF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, VF), "VF");

    cpu_exclusive_addr = tcg_global_mem_new_i64(TCG_AREG0,

        offsetof(CPUARMState, exclusive_addr), "exclusive_addr");

    cpu_exclusive_val = tcg_global_mem_new_i64(TCG_AREG0,

        offsetof(CPUARMState, exclusive_val), "exclusive_val");

    cpu_exclusive_high = tcg_global_mem_new_i64(TCG_AREG0,

        offsetof(CPUARMState, exclusive_high), "exclusive_high");

#ifdef CONFIG_USER_ONLY

    cpu_exclusive_test = tcg_global_mem_new_i64(TCG_AREG0,

        offsetof(CPUARMState, exclusive_test), "exclusive_test");

    cpu_exclusive_info = tcg_global_mem_new_i32(TCG_AREG0,

        offsetof(CPUARMState, exclusive_info), "exclusive_info");

#endif

}

void aarch64_cpu_dump_state(CPUState *cs, FILE *f,

                            fprintf_function cpu_fprintf, int flags)

{

    ARMCPU *cpu = ARM_CPU(cs);

    CPUARMState *env = &cpu->env;

    uint32_t psr = pstate_read(env);

    int i;

    cpu_fprintf(f, "PC=%016"PRIx64"  SP=%016"PRIx64"\n",

            env->pc, env->xregs[31]);

    for (i = 0; i < 31; i++) {

        cpu_fprintf(f, "X%02d=%016"PRIx64, i, env->xregs[i]);

        if ((i % 4) == 3) {

            cpu_fprintf(f, "\n");

        } else {

            cpu_fprintf(f, " ");

        }

    }

    cpu_fprintf(f, "PSTATE=%08x (flags %c%c%c%c)\n",

                psr,

                psr & PSTATE_N ? 'N' : '-',

                psr & PSTATE_Z ? 'Z' : '-',

                psr & PSTATE_C ? 'C' : '-',

                psr & PSTATE_V ? 'V' : '-');

    cpu_fprintf(f, "\n");

    if (flags & CPU_DUMP_FPU) {

        int numvfpregs = 32;

        for (i = 0; i < numvfpregs; i += 2) {

            uint64_t vlo = float64_val(env->vfp.regs[i * 2]);

            uint64_t vhi = float64_val(env->vfp.regs[(i * 2) + 1]);

            cpu_fprintf(f, "q%02d=%016" PRIx64 ":%016" PRIx64 " ",

                        i, vhi, vlo);

            vlo = float64_val(env->vfp.regs[(i + 1) * 2]);

            vhi = float64_val(env->vfp.regs[((i + 1) * 2) + 1]);

            cpu_fprintf(f, "q%02d=%016" PRIx64 ":%016" PRIx64 "\n",

                        i + 1, vhi, vlo);

        }

        cpu_fprintf(f, "FPCR: %08x  FPSR: %08x\n",

                    vfp_get_fpcr(env), vfp_get_fpsr(env));

    }

}

static int get_mem_index(DisasContext *s)

{

#ifdef CONFIG_USER_ONLY

    return 1;

#else

    return s->user;

#endif

}

void gen_a64_set_pc_im(uint64_t val)

{

    tcg_gen_movi_i64(cpu_pc, val);

}

static void gen_exception(int excp)

{

    TCGv_i32 tmp = tcg_temp_new_i32();

    tcg_gen_movi_i32(tmp, excp);

    gen_helper_exception(cpu_env, tmp);

    tcg_temp_free_i32(tmp);

}

static void gen_exception_insn(DisasContext *s, int offset, int excp)

{

    gen_a64_set_pc_im(s->pc - offset);

    gen_exception(excp);

    s->is_jmp = DISAS_EXC;

}

static inline bool use_goto_tb(DisasContext *s, int n, uint64_t dest)

{

    /* No direct tb linking with singlestep or deterministic io */

    if (s->singlestep_enabled || (s->tb->cflags & CF_LAST_IO)) {

        return false;

    }

    /* Only link tbs from inside the same guest page */

    if ((s->tb->pc & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) {

        return false;

    }

    return true;

}

static inline void gen_goto_tb(DisasContext *s, int n, uint64_t dest)

{

    TranslationBlock *tb;

    tb = s->tb;

    if (use_goto_tb(s, n, dest)) {

        tcg_gen_goto_tb(n);

        gen_a64_set_pc_im(dest);

        tcg_gen_exit_tb((tcg_target_long)tb + n);

        s->is_jmp = DISAS_TB_JUMP;

    } else {

        gen_a64_set_pc_im(dest);

        if (s->singlestep_enabled) {

            gen_exception(EXCP_DEBUG);

        }

        tcg_gen_exit_tb(0);

        s->is_jmp = DISAS_JUMP;

    }

}

static void unallocated_encoding(DisasContext *s)

{

    gen_exception_insn(s, 4, EXCP_UDEF);

}

#define unsupported_encoding(s, insn)                                    \

    do {                                                                 \

        qemu_log_mask(LOG_UNIMP,                                         \

                      "%s:%d: unsupported instruction encoding 0x%08x "  \

                      "at pc=%016" PRIx64 "\n",                          \

                      __FILE__, __LINE__, insn, s->pc - 4);              \

        unallocated_encoding(s);                                         \

    } while (0);

static void init_tmp_a64_array(DisasContext *s)

{

#ifdef CONFIG_DEBUG_TCG

    int i;

    for (i = 0; i < ARRAY_SIZE(s->tmp_a64); i++) {

        TCGV_UNUSED_I64(s->tmp_a64[i]);

    }

#endif

    s->tmp_a64_count = 0;

}

static void free_tmp_a64(DisasContext *s)

{

    int i;

    for (i = 0; i < s->tmp_a64_count; i++) {

        tcg_temp_free_i64(s->tmp_a64[i]);

    }

    init_tmp_a64_array(s);

}

static TCGv_i64 new_tmp_a64(DisasContext *s)

{

    assert(s->tmp_a64_count < TMP_A64_MAX);

    return s->tmp_a64[s->tmp_a64_count++] = tcg_temp_new_i64();

}

static TCGv_i64 new_tmp_a64_zero(DisasContext *s)

{

    TCGv_i64 t = new_tmp_a64(s);

    tcg_gen_movi_i64(t, 0);

    return t;

}

/*

 * Register access functions

 *

 * These functions are used for directly accessing a register in where

 * changes to the final register value are likely to be made. If you

 * need to use a register for temporary calculation (e.g. index type

 * operations) use the read_* form.

 *

 * B1.2.1 Register mappings

 *

 * In instruction register encoding 31 can refer to ZR (zero register) or

 * the SP (stack pointer) depending on context. In QEMU's case we map SP

 * to cpu_X[31] and ZR accesses to a temporary which can be discarded.

 * This is the point of the _sp forms.

 */

static TCGv_i64 cpu_reg(DisasContext *s, int reg)

{

    if (reg == 31) {

        return new_tmp_a64_zero(s);

    } else {

        return cpu_X[reg];

    }

}

/* register access for when 31 == SP */

static TCGv_i64 cpu_reg_sp(DisasContext *s, int reg)

{

    return cpu_X[reg];

}

/* read a cpu register in 32bit/64bit mode. Returns a TCGv_i64

 * representing the register contents. This TCGv is an auto-freed

 * temporary so it need not be explicitly freed, and may be modified.

 */

static TCGv_i64 read_cpu_reg(DisasContext *s, int reg, int sf)

{

    TCGv_i64 v = new_tmp_a64(s);

    if (reg != 31) {

        if (sf) {

            tcg_gen_mov_i64(v, cpu_X[reg]);

        } else {

            tcg_gen_ext32u_i64(v, cpu_X[reg]);

        }

    } else {

        tcg_gen_movi_i64(v, 0);

    }

    return v;

}

static TCGv_i64 read_cpu_reg_sp(DisasContext *s, int reg, int sf)

{

    TCGv_i64 v = new_tmp_a64(s);

    if (sf) {

        tcg_gen_mov_i64(v, cpu_X[reg]);

    } else {

        tcg_gen_ext32u_i64(v, cpu_X[reg]);

    }

    return v;

}

/* Return the offset into CPUARMState of an element of specified

 * size, 'element' places in from the least significant end of

 * the FP/vector register Qn.

 */

static inline int vec_reg_offset(int regno, int element, TCGMemOp size)

{

    int offs = offsetof(CPUARMState, vfp.regs[regno * 2]);

#ifdef HOST_WORDS_BIGENDIAN

    /* This is complicated slightly because vfp.regs[2n] is

     * still the low half and  vfp.regs[2n+1] the high half

     * of the 128 bit vector, even on big endian systems.

     * Calculate the offset assuming a fully bigendian 128 bits,

     * then XOR to account for the order of the two 64 bit halves.

     */

    offs += (16 - ((element + 1) * (1 << size)));

    offs ^= 8;

#else

    offs += element * (1 << size);

#endif

    return offs;

}

/* Return the offset into CPUARMState of a slice (from

 * the least significant end) of FP register Qn (ie

 * Dn, Sn, Hn or Bn).

 * (Note that this is not the same mapping as for A32; see cpu.h)

 */

static inline int fp_reg_offset(int regno, TCGMemOp size)

{

    int offs = offsetof(CPUARMState, vfp.regs[regno * 2]);

#ifdef HOST_WORDS_BIGENDIAN

    offs += (8 - (1 << size));

#endif

    return offs;

}

/* Offset of the high half of the 128 bit vector Qn */

static inline int fp_reg_hi_offset(int regno)

{

    return offsetof(CPUARMState, vfp.regs[regno * 2 + 1]);

}

/* Convenience accessors for reading and writing single and double

 * FP registers. Writing clears the upper parts of the associated

 * 128 bit vector register, as required by the architecture.

 * Note that unlike the GP register accessors, the values returned

 * by the read functions must be manually freed.

 */

static TCGv_i64 read_fp_dreg(DisasContext *s, int reg)

{

    TCGv_i64 v = tcg_temp_new_i64();

    tcg_gen_ld_i64(v, cpu_env, fp_reg_offset(reg, MO_64));

    return v;

}

static TCGv_i32 read_fp_sreg(DisasContext *s, int reg)

{

    TCGv_i32 v = tcg_temp_new_i32();

    tcg_gen_ld_i32(v, cpu_env, fp_reg_offset(reg, MO_32));

    return v;

}

static void write_fp_dreg(DisasContext *s, int reg, TCGv_i64 v)

{

    TCGv_i64 tcg_zero = tcg_const_i64(0);

    tcg_gen_st_i64(v, cpu_env, fp_reg_offset(reg, MO_64));

    tcg_gen_st_i64(tcg_zero, cpu_env, fp_reg_hi_offset(reg));

    tcg_temp_free_i64(tcg_zero);

}

static void write_fp_sreg(DisasContext *s, int reg, TCGv_i32 v)

{

    TCGv_i64 tmp = tcg_temp_new_i64();

    tcg_gen_extu_i32_i64(tmp, v);

    write_fp_dreg(s, reg, tmp);

    tcg_temp_free_i64(tmp);

}

static TCGv_ptr get_fpstatus_ptr(void)

{

    TCGv_ptr statusptr = tcg_temp_new_ptr();

    int offset;

    /* In A64 all instructions (both FP and Neon) use the FPCR;

     * there is no equivalent of the A32 Neon "standard FPSCR value"

     * and all operations use vfp.fp_status.

     */

    offset = offsetof(CPUARMState, vfp.fp_status);

    tcg_gen_addi_ptr(statusptr, cpu_env, offset);

    return statusptr;

}

/* Set ZF and NF based on a 64 bit result. This is alas fiddlier

 * than the 32 bit equivalent.

 */

static inline void gen_set_NZ64(TCGv_i64 result)

{

    TCGv_i64 flag = tcg_temp_new_i64();

    tcg_gen_setcondi_i64(TCG_COND_NE, flag, result, 0);

    tcg_gen_trunc_i64_i32(cpu_ZF, flag);

    tcg_gen_shri_i64(flag, result, 32);

    tcg_gen_trunc_i64_i32(cpu_NF, flag);

    tcg_temp_free_i64(flag);

}

/* Set NZCV as for a logical operation: NZ as per result, CV cleared. */

static inline void gen_logic_CC(int sf, TCGv_i64 result)

{

    if (sf) {

        gen_set_NZ64(result);

    } else {

        tcg_gen_trunc_i64_i32(cpu_ZF, result);

        tcg_gen_trunc_i64_i32(cpu_NF, result);

    }

    tcg_gen_movi_i32(cpu_CF, 0);

    tcg_gen_movi_i32(cpu_VF, 0);

}

/* dest = T0 + T1; compute C, N, V and Z flags */

static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)

{

    if (sf) {

        TCGv_i64 result, flag, tmp;

        result = tcg_temp_new_i64();

        flag = tcg_temp_new_i64();

        tmp = tcg_temp_new_i64();

        tcg_gen_movi_i64(tmp, 0);

        tcg_gen_add2_i64(result, flag, t0, tmp, t1, tmp);

        tcg_gen_trunc_i64_i32(cpu_CF, flag);

        gen_set_NZ64(result);

        tcg_gen_xor_i64(flag, result, t0);

        tcg_gen_xor_i64(tmp, t0, t1);

        tcg_gen_andc_i64(flag, flag, tmp);

        tcg_temp_free_i64(tmp);

        tcg_gen_shri_i64(flag, flag, 32);

        tcg_gen_trunc_i64_i32(cpu_VF, flag);

        tcg_gen_mov_i64(dest, result);

        tcg_temp_free_i64(result);

        tcg_temp_free_i64(flag);

    } else {

        /* 32 bit arithmetic */

        TCGv_i32 t0_32 = tcg_temp_new_i32();

        TCGv_i32 t1_32 = tcg_temp_new_i32();

        TCGv_i32 tmp = tcg_temp_new_i32();

        tcg_gen_movi_i32(tmp, 0);

        tcg_gen_trunc_i64_i32(t0_32, t0);

        tcg_gen_trunc_i64_i32(t1_32, t1);

        tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, t1_32, tmp);

        tcg_gen_mov_i32(cpu_ZF, cpu_NF);

        tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);

        tcg_gen_xor_i32(tmp, t0_32, t1_32);

        tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);

        tcg_gen_extu_i32_i64(dest, cpu_NF);

        tcg_temp_free_i32(tmp);

        tcg_temp_free_i32(t0_32);

        tcg_temp_free_i32(t1_32);

    }

}

/* dest = T0 - T1; compute C, N, V and Z flags */

static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)

{

    if (sf) {

        /* 64 bit arithmetic */

        TCGv_i64 result, flag, tmp;

        result = tcg_temp_new_i64();

        flag = tcg_temp_new_i64();

        tcg_gen_sub_i64(result, t0, t1);

        gen_set_NZ64(result);

        tcg_gen_setcond_i64(TCG_COND_GEU, flag, t0, t1);

        tcg_gen_trunc_i64_i32(cpu_CF, flag);

        tcg_gen_xor_i64(flag, result, t0);

        tmp = tcg_temp_new_i64();

        tcg_gen_xor_i64(tmp, t0, t1);

        tcg_gen_and_i64(flag, flag, tmp);

        tcg_temp_free_i64(tmp);

        tcg_gen_shri_i64(flag, flag, 32);

        tcg_gen_trunc_i64_i32(cpu_VF, flag);

        tcg_gen_mov_i64(dest, result);

        tcg_temp_free_i64(flag);

        tcg_temp_free_i64(result);

    } else {

        /* 32 bit arithmetic */

        TCGv_i32 t0_32 = tcg_temp_new_i32();

        TCGv_i32 t1_32 = tcg_temp_new_i32();

        TCGv_i32 tmp;

        tcg_gen_trunc_i64_i32(t0_32, t0);

        tcg_gen_trunc_i64_i32(t1_32, t1);

        tcg_gen_sub_i32(cpu_NF, t0_32, t1_32);

        tcg_gen_mov_i32(cpu_ZF, cpu_NF);

        tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0_32, t1_32);

        tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);

        tmp = tcg_temp_new_i32();

        tcg_gen_xor_i32(tmp, t0_32, t1_32);

        tcg_temp_free_i32(t0_32);

        tcg_temp_free_i32(t1_32);

        tcg_gen_and_i32(cpu_VF, cpu_VF, tmp);

        tcg_temp_free_i32(tmp);

        tcg_gen_extu_i32_i64(dest, cpu_NF);

    }

}

/* dest = T0 + T1 + CF; do not compute flags. */

static void gen_adc(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)

{

    TCGv_i64 flag = tcg_temp_new_i64();

    tcg_gen_extu_i32_i64(flag, cpu_CF);

    tcg_gen_add_i64(dest, t0, t1);

    tcg_gen_add_i64(dest, dest, flag);

    tcg_temp_free_i64(flag);

    if (!sf) {

        tcg_gen_ext32u_i64(dest, dest);

    }

}

/* dest = T0 + T1 + CF; compute C, N, V and Z flags. */

static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)

{

    if (sf) {

        TCGv_i64 result, cf_64, vf_64, tmp;

        result = tcg_temp_new_i64();

        cf_64 = tcg_temp_new_i64();

        vf_64 = tcg_temp_new_i64();

        tmp = tcg_const_i64(0);

        tcg_gen_extu_i32_i64(cf_64, cpu_CF);

        tcg_gen_add2_i64(result, cf_64, t0, tmp, cf_64, tmp);

        tcg_gen_add2_i64(result, cf_64, result, cf_64, t1, tmp);

        tcg_gen_trunc_i64_i32(cpu_CF, cf_64);

        gen_set_NZ64(result);

        tcg_gen_xor_i64(vf_64, result, t0);

        tcg_gen_xor_i64(tmp, t0, t1);

        tcg_gen_andc_i64(vf_64, vf_64, tmp);

        tcg_gen_shri_i64(vf_64, vf_64, 32);

        tcg_gen_trunc_i64_i32(cpu_VF, vf_64);

        tcg_gen_mov_i64(dest, result);

        tcg_temp_free_i64(tmp);

        tcg_temp_free_i64(vf_64);

        tcg_temp_free_i64(cf_64);

        tcg_temp_free_i64(result);

    } else {

        TCGv_i32 t0_32, t1_32, tmp;

        t0_32 = tcg_temp_new_i32();

        t1_32 = tcg_temp_new_i32();

        tmp = tcg_const_i32(0);

        tcg_gen_trunc_i64_i32(t0_32, t0);

        tcg_gen_trunc_i64_i32(t1_32, t1);

        tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, cpu_CF, tmp);

        tcg_gen_add2_i32(cpu_NF, cpu_CF, cpu_NF, cpu_CF, t1_32, tmp);

        tcg_gen_mov_i32(cpu_ZF, cpu_NF);

        tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);

        tcg_gen_xor_i32(tmp, t0_32, t1_32);

        tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);

        tcg_gen_extu_i32_i64(dest, cpu_NF);

        tcg_temp_free_i32(tmp);

        tcg_temp_free_i32(t1_32);

        tcg_temp_free_i32(t0_32);

    }

}

/*

 * Load/Store generators

 */

/*

 * Store from GPR register to memory

 */

static void do_gpr_st(DisasContext *s, TCGv_i64 source,

                      TCGv_i64 tcg_addr, int size)

{

    g_assert(size <= 3);

    tcg_gen_qemu_st_i64(source, tcg_addr, get_mem_index(s), MO_TE + size);

}

/*

 * Load from memory to GPR register

 */

static void do_gpr_ld(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr,

                      int size, bool is_signed, bool extend)

{

    TCGMemOp memop = MO_TE + size;

    g_assert(size <= 3);

    if (is_signed) {

        memop += MO_SIGN;

    }

    tcg_gen_qemu_ld_i64(dest, tcg_addr, get_mem_index(s), memop);

    if (extend && is_signed) {

        g_assert(size < 3);

        tcg_gen_ext32u_i64(dest, dest);

    }

}

/*

 * Store from FP register to memory

 */

static void do_fp_st(DisasContext *s, int srcidx, TCGv_i64 tcg_addr, int size)

{

    /* This writes the bottom N bits of a 128 bit wide vector to memory */

    TCGv_i64 tmp = tcg_temp_new_i64();

    tcg_gen_ld_i64(tmp, cpu_env, fp_reg_offset(srcidx, MO_64));

    if (size < 4) {

        tcg_gen_qemu_st_i64(tmp, tcg_addr, get_mem_index(s), MO_TE + size);

    } else {

        TCGv_i64 tcg_hiaddr = tcg_temp_new_i64();

        tcg_gen_qemu_st_i64(tmp, tcg_addr, get_mem_index(s), MO_TEQ);

        tcg_gen_qemu_st64(tmp, tcg_addr, get_mem_index(s));

        tcg_gen_ld_i64(tmp, cpu_env, fp_reg_hi_offset(srcidx));

        tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8);

        tcg_gen_qemu_st_i64(tmp, tcg_hiaddr, get_mem_index(s), MO_TEQ);

        tcg_temp_free_i64(tcg_hiaddr);

    }

    tcg_temp_free_i64(tmp);

}

/*

 * Load from memory to FP register

 */

static void do_fp_ld(DisasContext *s, int destidx, TCGv_i64 tcg_addr, int size)

{

    /* This always zero-extends and writes to a full 128 bit wide vector */

    TCGv_i64 tmplo = tcg_temp_new_i64();

    TCGv_i64 tmphi;

    if (size < 4) {

        TCGMemOp memop = MO_TE + size;

        tmphi = tcg_const_i64(0);

        tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), memop);

    } else {

        TCGv_i64 tcg_hiaddr;

        tmphi = tcg_temp_new_i64();

        tcg_hiaddr = tcg_temp_new_i64();

        tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), MO_TEQ);

        tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8);

        tcg_gen_qemu_ld_i64(tmphi, tcg_hiaddr, get_mem_index(s), MO_TEQ);

        tcg_temp_free_i64(tcg_hiaddr);

    }

    tcg_gen_st_i64(tmplo, cpu_env, fp_reg_offset(destidx, MO_64));

    tcg_gen_st_i64(tmphi, cpu_env, fp_reg_hi_offset(destidx));

    tcg_temp_free_i64(tmplo);

    tcg_temp_free_i64(tmphi);

}

/*

 * Vector load/store helpers.

 *

 * The principal difference between this and a FP load is that we don't

 * zero extend as we are filling a partial chunk of the vector register.

 * These functions don't support 128 bit loads/stores, which would be

 * normal load/store operations.

 *

 * The _i32 versions are useful when operating on 32 bit quantities

 * (eg for floating point single or using Neon helper functions).

 */

/* Get value of an element within a vector register */

static void read_vec_element(DisasContext *s, TCGv_i64 tcg_dest, int srcidx,

                             int element, TCGMemOp memop)

{

    int vect_off = vec_reg_offset(srcidx, element, memop & MO_SIZE);

    switch (memop) {

    case MO_8:

        tcg_gen_ld8u_i64(tcg_dest, cpu_env, vect_off);

        break;

    case MO_16:

        tcg_gen_ld16u_i64(tcg_dest, cpu_env, vect_off);

        break;

    case MO_32:

        tcg_gen_ld32u_i64(tcg_dest, cpu_env, vect_off);

        break;

    case MO_8|MO_SIGN:

        tcg_gen_ld8s_i64(tcg_dest, cpu_env, vect_off);

        break;

    case MO_16|MO_SIGN:

        tcg_gen_ld16s_i64(tcg_dest, cpu_env, vect_off);

        break;

    case MO_32|MO_SIGN:

        tcg_gen_ld32s_i64(tcg_dest, cpu_env, vect_off);

        break;

    case MO_64:

    case MO_64|MO_SIGN:

        tcg_gen_ld_i64(tcg_dest, cpu_env, vect_off);

        break;

    default:

        g_assert_not_reached();

    }

}

static void read_vec_element_i32(DisasContext *s, TCGv_i32 tcg_dest, int srcidx,

                                 int element, TCGMemOp memop)

{

    int vect_off = vec_reg_offset(srcidx, element, memop & MO_SIZE);

    switch (memop) {

    case MO_8:

        tcg_gen_ld8u_i32(tcg_dest, cpu_env, vect_off);

        break;

    case MO_16:

        tcg_gen_ld16u_i32(tcg_dest, cpu_env, vect_off);

        break;

    case MO_8|MO_SIGN:

        tcg_gen_ld8s_i32(tcg_dest, cpu_env, vect_off);

        break;

    case MO_16|MO_SIGN:

        tcg_gen_ld16s_i32(tcg_dest, cpu_env, vect_off);

        break;

    case MO_32:

    case MO_32|MO_SIGN:

        tcg_gen_ld_i32(tcg_dest, cpu_env, vect_off);

        break;

    default:

        g_assert_not_reached();

    }

}

/* Set value of an element within a vector register */

static void write_vec_element(DisasContext *s, TCGv_i64 tcg_src, int destidx,

                              int element, TCGMemOp memop)

{

    int vect_off = vec_reg_offset(destidx, element, memop & MO_SIZE);

    switch (memop) {

    case MO_8:

        tcg_gen_st8_i64(tcg_src, cpu_env, vect_off);

        break;

    case MO_16:

        tcg_gen_st16_i64(tcg_src, cpu_env, vect_off);

        break;

    case MO_32:

        tcg_gen_st32_i64(tcg_src, cpu_env, vect_off);

        break;

    case MO_64:

        tcg_gen_st_i64(tcg_src, cpu_env, vect_off);

        break;

    default:

        g_assert_not_reached();

    }

}

static void write_vec_element_i32(DisasContext *s, TCGv_i32 tcg_src,

                                  int destidx, int element, TCGMemOp memop)

{

    int vect_off = vec_reg_offset(destidx, element, memop & MO_SIZE);

    switch (memop) {

    case MO_8:

        tcg_gen_st8_i32(tcg_src, cpu_env, vect_off);

        break;

    case MO_16:

        tcg_gen_st16_i32(tcg_src, cpu_env, vect_off);

        break;

    case MO_32:

        tcg_gen_st_i32(tcg_src, cpu_env, vect_off);

        break;

    default:

        g_assert_not_reached();

    }

}

/* Clear the high 64 bits of a 128 bit vector (in general non-quad

 * vector ops all need to do this).

 */

static void clear_vec_high(DisasContext *s, int rd)

{

    TCGv_i64 tcg_zero = tcg_const_i64(0);

    write_vec_element(s, tcg_zero, rd, 1, MO_64);

    tcg_temp_free_i64(tcg_zero);

}

/* Store from vector register to memory */

static void do_vec_st(DisasContext *s, int srcidx, int element,

                      TCGv_i64 tcg_addr, int size)

{

    TCGMemOp memop = MO_TE + size;

    TCGv_i64 tcg_tmp = tcg_temp_new_i64();

    read_vec_element(s, tcg_tmp, srcidx, element, size);

    tcg_gen_qemu_st_i64(tcg_tmp, tcg_addr, get_mem_index(s), memop);

    tcg_temp_free_i64(tcg_tmp);

}

/* Load from memory to vector register */

static void do_vec_ld(DisasContext *s, int destidx, int element,

                      TCGv_i64 tcg_addr, int size)

{

    TCGMemOp memop = MO_TE + size;

    TCGv_i64 tcg_tmp = tcg_temp_new_i64();

    tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr, get_mem_index(s), memop);

    write_vec_element(s, tcg_tmp, destidx, element, size);

    tcg_temp_free_i64(tcg_tmp);

}

/*

 * This utility function is for doing register extension with an

 * optional shift. You will likely want to pass a temporary for the

 * destination register. See DecodeRegExtend() in the ARM ARM.

 */

static void ext_and_shift_reg(TCGv_i64 tcg_out, TCGv_i64 tcg_in,

                              int option, unsigned int shift)

{

    int extsize = extract32(option, 0, 2);

    bool is_signed = extract32(option, 2, 1);

    if (is_signed) {

        switch (extsize) {

        case 0:

            tcg_gen_ext8s_i64(tcg_out, tcg_in);

            break;

        case 1:

            tcg_gen_ext16s_i64(tcg_out, tcg_in);

            break;

        case 2:

            tcg_gen_ext32s_i64(tcg_out, tcg_in);

            break;

        case 3:

            tcg_gen_mov_i64(tcg_out, tcg_in);

            break;

        }

    } else {

        switch (extsize) {

        case 0:

            tcg_gen_ext8u_i64(tcg_out, tcg_in);

            break;

        case 1:

            tcg_gen_ext16u_i64(tcg_out, tcg_in);

            break;

        case 2:

            tcg_gen_ext32u_i64(tcg_out, tcg_in);

            break;

        case 3: