Archipelago » qemu

/*

 * ARM NEON vector operations.

 *

 * Copyright (c) 2007 CodeSourcery.

 * Written by Paul Brook

 *

 * This code is licenced under the GPL.

 */

/* Note that for NEON an "l" prefix means it is a wide operation, unlike

   scalar arm ops where it means a word size operation.  */

#define SIGNBIT (uint32_t)0x80000000

/* ??? NEON ops should probably have their own float status.  */

#define NFS &env->vfp.fp_status

#define NEON_OP(name) void OPPROTO op_neon_##name (void)

/* Helper routines to perform bitwise copies between float and int.  */

static inline float32 vfp_itos(uint32_t i)

{

    union {

        uint32_t i;

        float32 s;

    } v;

    v.i = i;

    return v.s;

}

static inline uint32_t vfp_stoi(float32 s)

{

    union {

        uint32_t i;

        float32 s;

    } v;

    v.s = s;

    return v.i;

}

NEON_OP(getreg_T0)

{

    T0 = *(uint32_t *)((char *) env + PARAM1);

}

NEON_OP(getreg_T1)

{

    T1 = *(uint32_t *)((char *) env + PARAM1);

}

NEON_OP(setreg_T0)

{

    *(uint32_t *)((char *) env + PARAM1) = T0;

}

NEON_OP(setreg_T1)

{

    *(uint32_t *)((char *) env + PARAM1) = T1;

}

#define NEON_TYPE1(name, type) \

typedef struct \

{ \

    type v1; \

} neon_##name;

#ifdef WORDS_BIGENDIAN

#define NEON_TYPE2(name, type) \

typedef struct \

{ \

    type v2; \

    type v1; \

} neon_##name;

#define NEON_TYPE4(name, type) \

typedef struct \

{ \

    type v4; \

    type v3; \

    type v2; \

    type v1; \

} neon_##name;

#else

#define NEON_TYPE2(name, type) \

typedef struct \

{ \

    type v1; \

    type v2; \

} neon_##name;

#define NEON_TYPE4(name, type) \

typedef struct \

{ \

    type v1; \

    type v2; \

    type v3; \

    type v4; \

} neon_##name;

#endif

NEON_TYPE4(s8, int8_t)

NEON_TYPE4(u8, uint8_t)

NEON_TYPE2(s16, int16_t)

NEON_TYPE2(u16, uint16_t)

NEON_TYPE1(s32, int32_t)

NEON_TYPE1(u32, uint32_t)

#undef NEON_TYPE4

#undef NEON_TYPE2

#undef NEON_TYPE1

/* Copy from a uint32_t to a vector structure type.  */

#define NEON_UNPACK(vtype, dest, val) do { \

    union { \

        vtype v; \

        uint32_t i; \

    } conv_u; \

    conv_u.i = (val); \

    dest = conv_u.v; \

    } while(0)

/* Copy from a vector structure type to a uint32_t.  */

#define NEON_PACK(vtype, dest, val) do { \

    union { \

        vtype v; \

        uint32_t i; \

    } conv_u; \

    conv_u.v = (val); \

    dest = conv_u.i; \

    } while(0)

#define NEON_DO1 \

    NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);

#define NEON_DO2 \

    NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \

    NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);

#define NEON_DO4 \

    NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \

    NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \

    NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \

    NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);

#define NEON_VOP(name, vtype, n) \

NEON_OP(name) \

{ \

    vtype vsrc1; \

    vtype vsrc2; \

    vtype vdest; \

    NEON_UNPACK(vtype, vsrc1, T0); \

    NEON_UNPACK(vtype, vsrc2, T1); \

    NEON_DO##n; \

    NEON_PACK(vtype, T0, vdest); \

    FORCE_RET(); \

}

#define NEON_VOP1(name, vtype, n) \

NEON_OP(name) \

{ \

    vtype vsrc1; \

    vtype vdest; \

    NEON_UNPACK(vtype, vsrc1, T0); \

    NEON_DO##n; \

    NEON_PACK(vtype, T0, vdest); \

    FORCE_RET(); \

}

/* Pairwise operations.  */

/* For 32-bit elements each segment only contains a single element, so

   the elementwise and pairwise operations are the same.  */

#define NEON_PDO2 \

    NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \

    NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);

#define NEON_PDO4 \

    NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \

    NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \

    NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \

    NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \

#define NEON_POP(name, vtype, n) \

NEON_OP(name) \

{ \

    vtype vsrc1; \

    vtype vsrc2; \

    vtype vdest; \

    NEON_UNPACK(vtype, vsrc1, T0); \

    NEON_UNPACK(vtype, vsrc2, T1); \

    NEON_PDO##n; \

    NEON_PACK(vtype, T0, vdest); \

    FORCE_RET(); \

}

#define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1

NEON_VOP(hadd_s8, neon_s8, 4)

NEON_VOP(hadd_u8, neon_u8, 4)

NEON_VOP(hadd_s16, neon_s16, 2)

NEON_VOP(hadd_u16, neon_u16, 2)

#undef NEON_FN

NEON_OP(hadd_s32)

{

    int32_t src1 = T0;

    int32_t src2 = T1;

    int32_t dest;

    dest = (src1 >> 1) + (src2 >> 1);

    if (src1 & src2 & 1)

        dest++;

    T0 = dest;

    FORCE_RET();

}

NEON_OP(hadd_u32)

{

    uint32_t src1 = T0;

    uint32_t src2 = T1;

    uint32_t dest;

    dest = (src1 >> 1) + (src2 >> 1);

    if (src1 & src2 & 1)

        dest++;

    T0 = dest;

    FORCE_RET();

}

#define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1

NEON_VOP(rhadd_s8, neon_s8, 4)

NEON_VOP(rhadd_u8, neon_u8, 4)

NEON_VOP(rhadd_s16, neon_s16, 2)

NEON_VOP(rhadd_u16, neon_u16, 2)

#undef NEON_FN

NEON_OP(rhadd_s32)

{

    int32_t src1 = T0;

    int32_t src2 = T1;

    int32_t dest;

    dest = (src1 >> 1) + (src2 >> 1);

    if ((src1 | src2) & 1)

        dest++;

    T0 = dest;

    FORCE_RET();

}

NEON_OP(rhadd_u32)

{

    uint32_t src1 = T0;

    uint32_t src2 = T1;

    uint32_t dest;

    dest = (src1 >> 1) + (src2 >> 1);

    if ((src1 | src2) & 1)

        dest++;

    T0 = dest;

    FORCE_RET();

}

#define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1

NEON_VOP(hsub_s8, neon_s8, 4)

NEON_VOP(hsub_u8, neon_u8, 4)

NEON_VOP(hsub_s16, neon_s16, 2)

NEON_VOP(hsub_u16, neon_u16, 2)

#undef NEON_FN

NEON_OP(hsub_s32)

{

    int32_t src1 = T0;

    int32_t src2 = T1;

    int32_t dest;

    dest = (src1 >> 1) - (src2 >> 1);

    if ((~src1) & src2 & 1)

        dest--;

    T0 = dest;

    FORCE_RET();

}

NEON_OP(hsub_u32)

{

    uint32_t src1 = T0;

    uint32_t src2 = T1;

    uint32_t dest;

    dest = (src1 >> 1) - (src2 >> 1);

    if ((~src1) & src2 & 1)

        dest--;

    T0 = dest;

    FORCE_RET();

}

#define NEON_USAT(dest, src1, src2, type) do { \

    uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \

    if (tmp != (type)tmp) { \

        env->QF = 1; \

        dest = ~0; \

    } else { \

        dest = tmp; \

    }} while(0)

#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)

NEON_VOP(qadd_u8, neon_u8, 4)

#undef NEON_FN

#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)

NEON_VOP(qadd_u16, neon_u16, 2)

#undef NEON_FN

#undef NEON_USAT

#define NEON_SSAT(dest, src1, src2, type) do { \

    int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \

    if (tmp != (type)tmp) { \

        env->QF = 1; \

        if (src2 > 0) { \

            tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \

        } else { \

            tmp = 1 << (sizeof(type) * 8 - 1); \

        } \

    } \

    dest = tmp; \

    } while(0)

#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)

NEON_VOP(qadd_s8, neon_s8, 4)

#undef NEON_FN

#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)

NEON_VOP(qadd_s16, neon_s16, 2)

#undef NEON_FN

#undef NEON_SSAT

#define NEON_USAT(dest, src1, src2, type) do { \

    uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \

    if (tmp != (type)tmp) { \

        env->QF = 1; \

        dest = 0; \

    } else { \

        dest = tmp; \

    }} while(0)

#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)

NEON_VOP(qsub_u8, neon_u8, 4)

#undef NEON_FN

#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)

NEON_VOP(qsub_u16, neon_u16, 2)

#undef NEON_FN

#undef NEON_USAT

#define NEON_SSAT(dest, src1, src2, type) do { \

    int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \

    if (tmp != (type)tmp) { \

        env->QF = 1; \

        if (src2 < 0) { \

            tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \

        } else { \

            tmp = 1 << (sizeof(type) * 8 - 1); \

        } \

    } \

    dest = tmp; \

    } while(0)

#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)

NEON_VOP(qsub_s8, neon_s8, 4)

#undef NEON_FN

#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)

NEON_VOP(qsub_s16, neon_s16, 2)

#undef NEON_FN

#undef NEON_SSAT

#define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0

NEON_VOP(cgt_s8, neon_s8, 4)

NEON_VOP(cgt_u8, neon_u8, 4)

NEON_VOP(cgt_s16, neon_s16, 2)

NEON_VOP(cgt_u16, neon_u16, 2)

NEON_VOP(cgt_s32, neon_s32, 1)

NEON_VOP(cgt_u32, neon_u32, 1)

#undef NEON_FN

#define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0

NEON_VOP(cge_s8, neon_s8, 4)

NEON_VOP(cge_u8, neon_u8, 4)

NEON_VOP(cge_s16, neon_s16, 2)

NEON_VOP(cge_u16, neon_u16, 2)

NEON_VOP(cge_s32, neon_s32, 1)

NEON_VOP(cge_u32, neon_u32, 1)

#undef NEON_FN

#define NEON_FN(dest, src1, src2) do { \

    int8_t tmp; \

    tmp = (int8_t)src2; \

    if (tmp < 0) { \

        dest = src1 >> -tmp; \

    } else { \

        dest = src1 << tmp; \

    }} while (0)

NEON_VOP(shl_s8, neon_s8, 4)

NEON_VOP(shl_u8, neon_u8, 4)

NEON_VOP(shl_s16, neon_s16, 2)

NEON_VOP(shl_u16, neon_u16, 2)

NEON_VOP(shl_s32, neon_s32, 1)

NEON_VOP(shl_u32, neon_u32, 1)

#undef NEON_FN

NEON_OP(shl_u64)

{

    int8_t shift = env->vfp.scratch[0];

    uint64_t val = T0 | ((uint64_t)T1 << 32);

    if (shift < 0) {

        val >>= -shift;

    } else {

        val <<= shift;

    }

    T0 = val;

    T1 = val >> 32;

    FORCE_RET();

}

NEON_OP(shl_s64)

{

    int8_t shift = env->vfp.scratch[0];

    int64_t val = T0 | ((uint64_t)T1 << 32);

    if (shift < 0) {

        val >>= -shift;

    } else {

        val <<= shift;

    }

    T0 = val;

    T1 = val >> 32;

    FORCE_RET();

}

#define NEON_FN(dest, src1, src2) do { \

    int8_t tmp; \

    tmp = (int8_t)src1; \

    if (tmp < 0) { \

        dest = (src2 + (1 << (-1 - tmp))) >> -tmp; \

    } else { \

        dest = src2 << tmp; \

    }} while (0)

NEON_VOP(rshl_s8, neon_s8, 4)

NEON_VOP(rshl_u8, neon_u8, 4)

NEON_VOP(rshl_s16, neon_s16, 2)

NEON_VOP(rshl_u16, neon_u16, 2)

NEON_VOP(rshl_s32, neon_s32, 1)

NEON_VOP(rshl_u32, neon_u32, 1)

#undef NEON_FN

NEON_OP(rshl_u64)

{

    int8_t shift = env->vfp.scratch[0];

    uint64_t val = T0 | ((uint64_t)T1 << 32);

    if (shift < 0) {

        val = (val + ((uint64_t)1 << (-1 - shift))) >> -shift;

        val >>= -shift;

    } else {

        val <<= shift;

    }

    T0 = val;

    T1 = val >> 32;

    FORCE_RET();

}

NEON_OP(rshl_s64)

{

    int8_t shift = env->vfp.scratch[0];

    int64_t val = T0 | ((uint64_t)T1 << 32);

    if (shift < 0) {

        val = (val + ((int64_t)1 << (-1 - shift))) >> -shift;

    } else {

        val <<= shift;

    }

    T0 = val;

    T1 = val >> 32;

    FORCE_RET();

}

#define NEON_FN(dest, src1, src2) do { \

    int8_t tmp; \

    tmp = (int8_t)src1; \

    if (tmp < 0) { \

        dest = src2 >> -tmp; \

    } else { \

        dest = src2 << tmp; \

        if ((dest >> tmp) != src2) { \

            env->QF = 1; \

            dest = ~0; \

        } \

    }} while (0)

NEON_VOP(qshl_s8, neon_s8, 4)

NEON_VOP(qshl_s16, neon_s16, 2)

NEON_VOP(qshl_s32, neon_s32, 1)

#undef NEON_FN

NEON_OP(qshl_s64)

{

    int8_t shift = env->vfp.scratch[0];

    int64_t val = T0 | ((uint64_t)T1 << 32);

    if (shift < 0) {

        val >>= -shift;

    } else {

        int64_t tmp = val;

        val <<= shift;

        if ((val >> shift) != tmp) {

            env->QF = 1;

            val = (tmp >> 63) ^ 0x7fffffffffffffffULL;

        }

    }

    T0 = val;

    T1 = val >> 32;

    FORCE_RET();

}

#define NEON_FN(dest, src1, src2) do { \

    int8_t tmp; \

    tmp = (int8_t)src1; \

    if (tmp < 0) { \

        dest = src2 >> -tmp; \

    } else { \

        dest = src2 << tmp; \

        if ((dest >> tmp) != src2) { \

            env->QF = 1; \

            dest = src2 >> 31; \

        } \

    }} while (0)

NEON_VOP(qshl_u8, neon_u8, 4)

NEON_VOP(qshl_u16, neon_u16, 2)

NEON_VOP(qshl_u32, neon_u32, 1)

#undef NEON_FN

NEON_OP(qshl_u64)

{

    int8_t shift = env->vfp.scratch[0];

    uint64_t val = T0 | ((uint64_t)T1 << 32);

    if (shift < 0) {

        val >>= -shift;

    } else {

        uint64_t tmp = val;

        val <<= shift;

        if ((val >> shift) != tmp) {

            env->QF = 1;

            val = ~(uint64_t)0;

        }

    }

    T0 = val;

    T1 = val >> 32;

    FORCE_RET();

}

#define NEON_FN(dest, src1, src2) do { \

    int8_t tmp; \

    tmp = (int8_t)src1; \

    if (tmp < 0) { \

        dest = (src2 + (1 << (-1 - tmp))) >> -tmp; \

    } else { \

        dest = src2 << tmp; \

        if ((dest >> tmp) != src2) { \

            dest = ~0; \

        } \

    }} while (0)

NEON_VOP(qrshl_s8, neon_s8, 4)

NEON_VOP(qrshl_s16, neon_s16, 2)

NEON_VOP(qrshl_s32, neon_s32, 1)

#undef NEON_FN

#define NEON_FN(dest, src1, src2) do { \

    int8_t tmp; \

    tmp = (int8_t)src1; \

    if (tmp < 0) { \

        dest = (src2 + (1 << (-1 - tmp))) >> -tmp; \

    } else { \

        dest = src2 << tmp; \

        if ((dest >> tmp) != src2) { \

            env->QF = 1; \

            dest = src2 >> 31; \

        } \

    }} while (0)

NEON_VOP(qrshl_u8, neon_u8, 4)

NEON_VOP(qrshl_u16, neon_u16, 2)

NEON_VOP(qrshl_u32, neon_u32, 1)

#undef NEON_FN

#define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2

NEON_VOP(max_s8, neon_s8, 4)

NEON_VOP(max_u8, neon_u8, 4)

NEON_VOP(max_s16, neon_s16, 2)

NEON_VOP(max_u16, neon_u16, 2)

NEON_VOP(max_s32, neon_s32, 1)

NEON_VOP(max_u32, neon_u32, 1)

NEON_POP(pmax_s8, neon_s8, 4)

NEON_POP(pmax_u8, neon_u8, 4)

NEON_POP(pmax_s16, neon_s16, 2)

NEON_POP(pmax_u16, neon_u16, 2)

#undef NEON_FN

NEON_OP(max_f32)

{

    float32 f0 = vfp_itos(T0);

    float32 f1 = vfp_itos(T1);

    T0 = (float32_compare_quiet(f0, f1, NFS) == 1) ? T0 : T1;

    FORCE_RET();

}

#define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2

NEON_VOP(min_s8, neon_s8, 4)

NEON_VOP(min_u8, neon_u8, 4)

NEON_VOP(min_s16, neon_s16, 2)

NEON_VOP(min_u16, neon_u16, 2)

NEON_VOP(min_s32, neon_s32, 1)

NEON_VOP(min_u32, neon_u32, 1)

NEON_POP(pmin_s8, neon_s8, 4)

NEON_POP(pmin_u8, neon_u8, 4)

NEON_POP(pmin_s16, neon_s16, 2)

NEON_POP(pmin_u16, neon_u16, 2)

#undef NEON_FN

NEON_OP(min_f32)

{

    float32 f0 = vfp_itos(T0);

    float32 f1 = vfp_itos(T1);

    T0 = (float32_compare_quiet(f0, f1, NFS) == -1) ? T0 : T1;

    FORCE_RET();

}

#define NEON_FN(dest, src1, src2) \

    dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)

NEON_VOP(abd_s8, neon_s8, 4)

NEON_VOP(abd_u8, neon_u8, 4)

NEON_VOP(abd_s16, neon_s16, 2)

NEON_VOP(abd_u16, neon_u16, 2)

NEON_VOP(abd_s32, neon_s32, 1)

NEON_VOP(abd_u32, neon_u32, 1)

#undef NEON_FN

NEON_OP(abd_f32)

{

    float32 f0 = vfp_itos(T0);

    float32 f1 = vfp_itos(T1);

    T0 = vfp_stoi((float32_compare_quiet(f0, f1, NFS) == 1)

                  ? float32_sub(f0, f1, NFS)

                  : float32_sub(f1, f0, NFS));

    FORCE_RET();

}

#define NEON_FN(dest, src1, src2) dest = src1 + src2

NEON_VOP(add_u8, neon_u8, 4)

NEON_VOP(add_u16, neon_u16, 2)

NEON_POP(padd_u8, neon_u8, 4)

NEON_POP(padd_u16, neon_u16, 2)

#undef NEON_FN

NEON_OP(add_f32)

{

    T0 = vfp_stoi(float32_add(vfp_itos(T0), vfp_itos(T1), NFS));

    FORCE_RET();

}

#define NEON_FN(dest, src1, src2) dest = src1 - src2

NEON_VOP(sub_u8, neon_u8, 4)

NEON_VOP(sub_u16, neon_u16, 2)

#undef NEON_FN

NEON_OP(sub_f32)

{

    T0 = vfp_stoi(float32_sub(vfp_itos(T0), vfp_itos(T1), NFS));

    FORCE_RET();

}

#define NEON_FN(dest, src1, src2) dest = src2 - src1

NEON_VOP(rsb_u8, neon_u8, 4)

NEON_VOP(rsb_u16, neon_u16, 2)

#undef NEON_FN

NEON_OP(rsb_f32)

{

    T0 = vfp_stoi(float32_sub(vfp_itos(T1), vfp_itos(T0), NFS));

    FORCE_RET();

}

#define NEON_FN(dest, src1, src2) dest = src1 * src2

NEON_VOP(mul_u8, neon_u8, 4)

NEON_VOP(mul_u16, neon_u16, 2)

#undef NEON_FN

NEON_OP(mul_f32)

{

    T0 = vfp_stoi(float32_mul(vfp_itos(T0), vfp_itos(T1), NFS));

    FORCE_RET();

}

NEON_OP(mul_p8)

{

    T0 = helper_neon_mul_p8(T0, T1);

}

#define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0

NEON_VOP(tst_u8, neon_u8, 4)

NEON_VOP(tst_u16, neon_u16, 2)

NEON_VOP(tst_u32, neon_u32, 1)

#undef NEON_FN

#define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0

NEON_VOP(ceq_u8, neon_u8, 4)

NEON_VOP(ceq_u16, neon_u16, 2)

NEON_VOP(ceq_u32, neon_u32, 1)

#undef NEON_FN

#define NEON_QDMULH16(dest, src1, src2, round) do { \

    uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \

    if ((tmp ^ (tmp << 1)) & SIGNBIT) { \

        env->QF = 1; \

        tmp = (tmp >> 31) ^ ~SIGNBIT; \

    } \

    tmp <<= 1; \

    if (round) { \

        int32_t old = tmp; \

        tmp += 1 << 15; \

        if ((int32_t)tmp < old) { \

            env->QF = 1; \

            tmp = SIGNBIT - 1; \

        } \

    } \

    dest = tmp >> 16; \

    } while(0)

#define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)

NEON_VOP(qdmulh_s16, neon_s16, 2)

#undef NEON_FN

#define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)

NEON_VOP(qrdmulh_s16, neon_s16, 2)

#undef NEON_FN

#undef NEON_QDMULH16

#define SIGNBIT64 ((uint64_t)1 << 63)

#define NEON_QDMULH32(dest, src1, src2, round) do { \

    uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \

    if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \

        env->QF = 1; \

        tmp = (tmp >> 63) ^ ~SIGNBIT64; \

    } else { \

        tmp <<= 1; \

    } \

    if (round) { \

        int64_t old = tmp; \

        tmp += (int64_t)1 << 31; \

        if ((int64_t)tmp < old) { \

            env->QF = 1; \

            tmp = SIGNBIT64 - 1; \

        } \

    } \

    dest = tmp >> 32; \

    } while(0)

#define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)

NEON_VOP(qdmulh_s32, neon_s32, 1)

#undef NEON_FN

#define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)

NEON_VOP(qrdmulh_s32, neon_s32, 1)

#undef NEON_FN

#undef NEON_QDMULH32

/* Floating point comparisons produce an integer result.  */

#define NEON_VOP_FCMP(name, cmp) \

NEON_OP(name) \

{ \

    if (float32_compare_quiet(vfp_itos(T0), vfp_itos(T1), NFS) cmp 0) \

        T0 = -1; \

    else \

        T0 = 0; \

    FORCE_RET(); \

}

NEON_VOP_FCMP(ceq_f32, ==)

NEON_VOP_FCMP(cge_f32, >=)

NEON_VOP_FCMP(cgt_f32, >)

NEON_OP(acge_f32)

{

    float32 f0 = float32_abs(vfp_itos(T0));

    float32 f1 = float32_abs(vfp_itos(T1));

    T0 = (float32_compare_quiet(f0, f1,NFS) >= 0) ? -1 : 0;

    FORCE_RET();

}

NEON_OP(acgt_f32)

{

    float32 f0 = float32_abs(vfp_itos(T0));

    float32 f1 = float32_abs(vfp_itos(T1));

    T0 = (float32_compare_quiet(f0, f1, NFS) > 0) ? -1 : 0;

    FORCE_RET();

}

/* Narrowing instructions.  The named type is the destination type.  */

NEON_OP(narrow_u8)

{

    T0 = (T0 & 0xff) | ((T0 >> 8) & 0xff00)

         | ((T1 << 16) & 0xff0000) | (T1 << 24);

    FORCE_RET();

}

NEON_OP(narrow_sat_u8)

{

    neon_u16 src;

    neon_u8 dest;

#define SAT8(d, s) \

    if (s > 0xff) { \

        d = 0xff; \

        env->QF = 1; \

    } else  { \

        d = s; \

    }

    NEON_UNPACK(neon_u16, src, T0);

    SAT8(dest.v1, src.v1);

    SAT8(dest.v2, src.v2);

    NEON_UNPACK(neon_u16, src, T1);

    SAT8(dest.v3, src.v1);

    SAT8(dest.v4, src.v2);

    NEON_PACK(neon_u8, T0, dest);

    FORCE_RET();

#undef SAT8

}

NEON_OP(narrow_sat_s8)

{

    neon_s16 src;

    neon_s8 dest;

#define SAT8(d, s) \

    if (s != (uint8_t)s) { \

        d = (s >> 15) ^ 0x7f; \

        env->QF = 1; \

    } else  { \

        d = s; \

    }

    NEON_UNPACK(neon_s16, src, T0);

    SAT8(dest.v1, src.v1);

    SAT8(dest.v2, src.v2);

    NEON_UNPACK(neon_s16, src, T1);

    SAT8(dest.v3, src.v1);

    SAT8(dest.v4, src.v2);

    NEON_PACK(neon_s8, T0, dest);

    FORCE_RET();

#undef SAT8

}

NEON_OP(narrow_u16)

{

    T0 = (T0 & 0xffff) | (T1 << 16);

}

NEON_OP(narrow_sat_u16)

{

    if (T0 > 0xffff) {

        T0 = 0xffff;

        env->QF = 1;

    }

    if (T1 > 0xffff) {

        T1 = 0xffff;

        env->QF = 1;

    }

    T0 |= T1 << 16;

    FORCE_RET();

}

NEON_OP(narrow_sat_s16)

{

    if ((int32_t)T0 != (int16_t)T0) {

        T0 = ((int32_t)T0 >> 31) ^ 0x7fff;

        env->QF = 1;

    }

    if ((int32_t)T1 != (int16_t) T1) {

        T1 = ((int32_t)T1 >> 31) ^ 0x7fff;

        env->QF = 1;

    }

    T0 = (uint16_t)T0 | (T1 << 16);

    FORCE_RET();

}

NEON_OP(narrow_sat_u32)

{

    if (T1) {

        T0 = 0xffffffffu;

        env->QF = 1;

    }

    FORCE_RET();

}

NEON_OP(narrow_sat_s32)

{

    int32_t sign = (int32_t)T1 >> 31;

    if ((int32_t)T1 != sign) {

        T0 = sign ^ 0x7fffffff;

        env->QF = 1;

    }

    FORCE_RET();

}

/* Narrowing instructions.  Named type is the narrow type.  */

NEON_OP(narrow_high_u8)

{

    T0 = ((T0 >> 8) & 0xff) | ((T0 >> 16) & 0xff00)

        | ((T1 << 8) & 0xff0000) | (T1 & 0xff000000);

    FORCE_RET();

}

NEON_OP(narrow_high_u16)

{

    T0 = (T0 >> 16) | (T1 & 0xffff0000);

    FORCE_RET();

}

NEON_OP(narrow_high_round_u8)

{

    T0 = (((T0 + 0x80) >> 8) & 0xff) | (((T0 + 0x800000) >> 16) & 0xff00)

        | (((T1 + 0x80) << 8) & 0xff0000) | ((T1 + 0x800000) & 0xff000000);

    FORCE_RET();

}

NEON_OP(narrow_high_round_u16)

{

    T0 = ((T0 + 0x8000) >> 16) | ((T1 + 0x8000) & 0xffff0000);

    FORCE_RET();

}

NEON_OP(narrow_high_round_u32)

{

    if (T0 >= 0x80000000u)

        T0 = T1 + 1;

    else

        T0 = T1;