Statistics
| Branch: | Revision:

root / tcg / x86_64 / tcg-target.c @ 09aac126

History | View | Annotate | Download (40.6 kB)

1
/*
2
 * Tiny Code Generator for QEMU
3
 *
4
 * Copyright (c) 2008 Fabrice Bellard
5
 *
6
 * Permission is hereby granted, free of charge, to any person obtaining a copy
7
 * of this software and associated documentation files (the "Software"), to deal
8
 * in the Software without restriction, including without limitation the rights
9
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
 * copies of the Software, and to permit persons to whom the Software is
11
 * furnished to do so, subject to the following conditions:
12
 *
13
 * The above copyright notice and this permission notice shall be included in
14
 * all copies or substantial portions of the Software.
15
 *
16
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22
 * THE SOFTWARE.
23
 */
24

    
25
#ifndef NDEBUG
26
static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
27
    "%rax",
28
    "%rcx",
29
    "%rdx",
30
    "%rbx",
31
    "%rsp",
32
    "%rbp",
33
    "%rsi",
34
    "%rdi",
35
    "%r8",
36
    "%r9",
37
    "%r10",
38
    "%r11",
39
    "%r12",
40
    "%r13",
41
    "%r14",
42
    "%r15",
43
};
44
#endif
45

    
46
static const int tcg_target_reg_alloc_order[] = {
47
    TCG_REG_RBP,
48
    TCG_REG_RBX,
49
    TCG_REG_R12,
50
    TCG_REG_R13,
51
    TCG_REG_R14,
52
    TCG_REG_R15,
53
    TCG_REG_R10,
54
    TCG_REG_R11,
55
    TCG_REG_R9,
56
    TCG_REG_R8,
57
    TCG_REG_RCX,
58
    TCG_REG_RDX,
59
    TCG_REG_RSI,
60
    TCG_REG_RDI,
61
    TCG_REG_RAX,
62
};
63

    
64
static const int tcg_target_call_iarg_regs[6] = {
65
    TCG_REG_RDI,
66
    TCG_REG_RSI,
67
    TCG_REG_RDX,
68
    TCG_REG_RCX,
69
    TCG_REG_R8,
70
    TCG_REG_R9,
71
};
72

    
73
static const int tcg_target_call_oarg_regs[2] = {
74
    TCG_REG_RAX, 
75
    TCG_REG_RDX 
76
};
77

    
78
static uint8_t *tb_ret_addr;
79

    
80
static void patch_reloc(uint8_t *code_ptr, int type, 
81
                        tcg_target_long value, tcg_target_long addend)
82
{
83
    value += addend;
84
    switch(type) {
85
    case R_X86_64_32:
86
        if (value != (uint32_t)value)
87
            tcg_abort();
88
        *(uint32_t *)code_ptr = value;
89
        break;
90
    case R_X86_64_32S:
91
        if (value != (int32_t)value)
92
            tcg_abort();
93
        *(uint32_t *)code_ptr = value;
94
        break;
95
    case R_386_PC32:
96
        value -= (long)code_ptr;
97
        if (value != (int32_t)value)
98
            tcg_abort();
99
        *(uint32_t *)code_ptr = value;
100
        break;
101
    default:
102
        tcg_abort();
103
    }
104
}
105

    
106
/* maximum number of register used for input function arguments */
107
static inline int tcg_target_get_call_iarg_regs_count(int flags)
108
{
109
    return 6;
110
}
111

    
112
/* parse target specific constraints */
113
static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
114
{
115
    const char *ct_str;
116

    
117
    ct_str = *pct_str;
118
    switch(ct_str[0]) {
119
    case 'a':
120
        ct->ct |= TCG_CT_REG;
121
        tcg_regset_set_reg(ct->u.regs, TCG_REG_RAX);
122
        break;
123
    case 'b':
124
        ct->ct |= TCG_CT_REG;
125
        tcg_regset_set_reg(ct->u.regs, TCG_REG_RBX);
126
        break;
127
    case 'c':
128
        ct->ct |= TCG_CT_REG;
129
        tcg_regset_set_reg(ct->u.regs, TCG_REG_RCX);
130
        break;
131
    case 'd':
132
        ct->ct |= TCG_CT_REG;
133
        tcg_regset_set_reg(ct->u.regs, TCG_REG_RDX);
134
        break;
135
    case 'S':
136
        ct->ct |= TCG_CT_REG;
137
        tcg_regset_set_reg(ct->u.regs, TCG_REG_RSI);
138
        break;
139
    case 'D':
140
        ct->ct |= TCG_CT_REG;
141
        tcg_regset_set_reg(ct->u.regs, TCG_REG_RDI);
142
        break;
143
    case 'q':
144
        ct->ct |= TCG_CT_REG;
145
        tcg_regset_set32(ct->u.regs, 0, 0xf);
146
        break;
147
    case 'r':
148
        ct->ct |= TCG_CT_REG;
149
        tcg_regset_set32(ct->u.regs, 0, 0xffff);
150
        break;
151
    case 'L': /* qemu_ld/st constraint */
152
        ct->ct |= TCG_CT_REG;
153
        tcg_regset_set32(ct->u.regs, 0, 0xffff);
154
        tcg_regset_reset_reg(ct->u.regs, TCG_REG_RSI);
155
        tcg_regset_reset_reg(ct->u.regs, TCG_REG_RDI);
156
        break;
157
    case 'e':
158
        ct->ct |= TCG_CT_CONST_S32;
159
        break;
160
    case 'Z':
161
        ct->ct |= TCG_CT_CONST_U32;
162
        break;
163
    default:
164
        return -1;
165
    }
166
    ct_str++;
167
    *pct_str = ct_str;
168
    return 0;
169
}
170

    
171
/* test if a constant matches the constraint */
172
static inline int tcg_target_const_match(tcg_target_long val,
173
                                         const TCGArgConstraint *arg_ct)
174
{
175
    int ct;
176
    ct = arg_ct->ct;
177
    if (ct & TCG_CT_CONST)
178
        return 1;
179
    else if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val)
180
        return 1;
181
    else if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val)
182
        return 1;
183
    else
184
        return 0;
185
}
186

    
187
#define ARITH_ADD 0
188
#define ARITH_OR  1
189
#define ARITH_ADC 2
190
#define ARITH_SBB 3
191
#define ARITH_AND 4
192
#define ARITH_SUB 5
193
#define ARITH_XOR 6
194
#define ARITH_CMP 7
195

    
196
#define SHIFT_ROL 0
197
#define SHIFT_ROR 1
198
#define SHIFT_SHL 4
199
#define SHIFT_SHR 5
200
#define SHIFT_SAR 7
201

    
202
#define JCC_JMP (-1)
203
#define JCC_JO  0x0
204
#define JCC_JNO 0x1
205
#define JCC_JB  0x2
206
#define JCC_JAE 0x3
207
#define JCC_JE  0x4
208
#define JCC_JNE 0x5
209
#define JCC_JBE 0x6
210
#define JCC_JA  0x7
211
#define JCC_JS  0x8
212
#define JCC_JNS 0x9
213
#define JCC_JP  0xa
214
#define JCC_JNP 0xb
215
#define JCC_JL  0xc
216
#define JCC_JGE 0xd
217
#define JCC_JLE 0xe
218
#define JCC_JG  0xf
219

    
220
#define P_EXT                0x100                /* 0x0f opcode prefix */
221
#define P_REXW                0x200                /* set rex.w = 1 */
222
#define P_REXB_R        0x400                /* REG field as byte register */
223
#define P_REXB_RM        0x800                /* R/M field as byte register */
224
                                  
225
static const uint8_t tcg_cond_to_jcc[10] = {
226
    [TCG_COND_EQ] = JCC_JE,
227
    [TCG_COND_NE] = JCC_JNE,
228
    [TCG_COND_LT] = JCC_JL,
229
    [TCG_COND_GE] = JCC_JGE,
230
    [TCG_COND_LE] = JCC_JLE,
231
    [TCG_COND_GT] = JCC_JG,
232
    [TCG_COND_LTU] = JCC_JB,
233
    [TCG_COND_GEU] = JCC_JAE,
234
    [TCG_COND_LEU] = JCC_JBE,
235
    [TCG_COND_GTU] = JCC_JA,
236
};
237

    
238
static void tcg_out_opc(TCGContext *s, int opc, int r, int rm, int x)
239
{
240
    int rex = 0;
241

    
242
    rex |= (opc & P_REXW) >> 6;                /* REX.W */
243
    rex |= (r & 8) >> 1;                /* REX.R */
244
    rex |= (x & 8) >> 2;                /* REX.X */
245
    rex |= (rm & 8) >> 3;                /* REX.B */
246

    
247
    /* P_REXB_{R,RM} indicates that the given register is the low byte.
248
       For %[abcd]l we need no REX prefix, but for %{si,di,bp,sp}l we do,
249
       as otherwise the encoding indicates %[abcd]h.  Note that the values
250
       that are ORed in merely indicate that the REX byte must be present;
251
       those bits get discarded in output.  */
252
    rex |= opc & (r >= 4 ? P_REXB_R : 0);
253
    rex |= opc & (rm >= 4 ? P_REXB_RM : 0);
254

    
255
    if (rex) {
256
        tcg_out8(s, (uint8_t)(rex | 0x40));
257
    }
258
    if (opc & P_EXT) {
259
        tcg_out8(s, 0x0f);
260
    }
261
    tcg_out8(s, opc & 0xff);
262
}
263

    
264
static inline void tcg_out_modrm(TCGContext *s, int opc, int r, int rm)
265
{
266
    tcg_out_opc(s, opc, r, rm, 0);
267
    tcg_out8(s, 0xc0 | ((r & 7) << 3) | (rm & 7));
268
}
269

    
270
/* rm < 0 means no register index plus (-rm - 1 immediate bytes) */
271
static inline void tcg_out_modrm_offset(TCGContext *s, int opc, int r, int rm, 
272
                                        tcg_target_long offset)
273
{
274
    if (rm < 0) {
275
        tcg_target_long val;
276
        tcg_out_opc(s, opc, r, 0, 0);
277
        val = offset - ((tcg_target_long)s->code_ptr + 5 + (-rm - 1));
278
        if (val == (int32_t)val) {
279
            /* eip relative */
280
            tcg_out8(s, 0x05 | ((r & 7) << 3));
281
            tcg_out32(s, val);
282
        } else if (offset == (int32_t)offset) {
283
            tcg_out8(s, 0x04 | ((r & 7) << 3));
284
            tcg_out8(s, 0x25); /* sib */
285
            tcg_out32(s, offset);
286
        } else {
287
            tcg_abort();
288
        }
289
    } else if (offset == 0 && (rm & 7) != TCG_REG_RBP) {
290
        tcg_out_opc(s, opc, r, rm, 0);
291
        if ((rm & 7) == TCG_REG_RSP) {
292
            tcg_out8(s, 0x04 | ((r & 7) << 3));
293
            tcg_out8(s, 0x24);
294
        } else {
295
            tcg_out8(s, 0x00 | ((r & 7) << 3) | (rm & 7));
296
        }
297
    } else if ((int8_t)offset == offset) {
298
        tcg_out_opc(s, opc, r, rm, 0);
299
        if ((rm & 7) == TCG_REG_RSP) {
300
            tcg_out8(s, 0x44 | ((r & 7) << 3));
301
            tcg_out8(s, 0x24);
302
        } else {
303
            tcg_out8(s, 0x40 | ((r & 7) << 3) | (rm & 7));
304
        }
305
        tcg_out8(s, offset);
306
    } else {
307
        tcg_out_opc(s, opc, r, rm, 0);
308
        if ((rm & 7) == TCG_REG_RSP) {
309
            tcg_out8(s, 0x84 | ((r & 7) << 3));
310
            tcg_out8(s, 0x24);
311
        } else {
312
            tcg_out8(s, 0x80 | ((r & 7) << 3) | (rm & 7));
313
        }
314
        tcg_out32(s, offset);
315
    }
316
}
317

    
318
#if defined(CONFIG_SOFTMMU)
319
/* XXX: incomplete. index must be different from ESP */
320
static void tcg_out_modrm_offset2(TCGContext *s, int opc, int r, int rm, 
321
                                  int index, int shift,
322
                                  tcg_target_long offset)
323
{
324
    int mod;
325
    if (rm == -1)
326
        tcg_abort();
327
    if (offset == 0 && (rm & 7) != TCG_REG_RBP) {
328
        mod = 0;
329
    } else if (offset == (int8_t)offset) {
330
        mod = 0x40;
331
    } else if (offset == (int32_t)offset) {
332
        mod = 0x80;
333
    } else {
334
        tcg_abort();
335
    }
336
    if (index == -1) {
337
        tcg_out_opc(s, opc, r, rm, 0);
338
        if ((rm & 7) == TCG_REG_RSP) {
339
            tcg_out8(s, mod | ((r & 7) << 3) | 0x04);
340
            tcg_out8(s, 0x04 | (rm & 7));
341
        } else {
342
            tcg_out8(s, mod | ((r & 7) << 3) | (rm & 7));
343
        }
344
    } else {
345
        tcg_out_opc(s, opc, r, rm, index);
346
        tcg_out8(s, mod | ((r & 7) << 3) | 0x04);
347
        tcg_out8(s, (shift << 6) | ((index & 7) << 3) | (rm & 7));
348
    }
349
    if (mod == 0x40) {
350
        tcg_out8(s, offset);
351
    } else if (mod == 0x80) {
352
        tcg_out32(s, offset);
353
    }
354
}
355
#endif
356

    
357
static inline void tcg_out_mov(TCGContext *s, int ret, int arg)
358
{
359
    tcg_out_modrm(s, 0x8b | P_REXW, ret, arg);
360
}
361

    
362
static inline void tcg_out_movi(TCGContext *s, TCGType type, 
363
                                int ret, tcg_target_long arg)
364
{
365
    if (arg == 0) {
366
        tcg_out_modrm(s, 0x01 | (ARITH_XOR << 3), ret, ret); /* xor r0,r0 */
367
    } else if (arg == (uint32_t)arg || type == TCG_TYPE_I32) {
368
        tcg_out_opc(s, 0xb8 + (ret & 7), 0, ret, 0);
369
        tcg_out32(s, arg);
370
    } else if (arg == (int32_t)arg) {
371
        tcg_out_modrm(s, 0xc7 | P_REXW, 0, ret);
372
        tcg_out32(s, arg);
373
    } else {
374
        tcg_out_opc(s, (0xb8 + (ret & 7)) | P_REXW, 0, ret, 0);
375
        tcg_out32(s, arg);
376
        tcg_out32(s, arg >> 32);
377
    }
378
}
379

    
380
static void tcg_out_goto(TCGContext *s, int call, uint8_t *target)
381
{
382
    int32_t disp;
383

    
384
    disp = target - s->code_ptr - 5;
385
    if (disp == (target - s->code_ptr - 5)) {
386
        tcg_out8(s, call ? 0xe8 : 0xe9);
387
        tcg_out32(s, disp);
388
    } else {
389
        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R10, (tcg_target_long) target);
390
        tcg_out_modrm(s, 0xff, call ? 2 : 4, TCG_REG_R10);
391
    }
392
}
393

    
394
static inline void tcg_out_ld(TCGContext *s, TCGType type, int ret,
395
                              int arg1, tcg_target_long arg2)
396
{
397
    if (type == TCG_TYPE_I32)
398
        tcg_out_modrm_offset(s, 0x8b, ret, arg1, arg2); /* movl */
399
    else
400
        tcg_out_modrm_offset(s, 0x8b | P_REXW, ret, arg1, arg2); /* movq */
401
}
402

    
403
static inline void tcg_out_st(TCGContext *s, TCGType type, int arg,
404
                              int arg1, tcg_target_long arg2)
405
{
406
    if (type == TCG_TYPE_I32)
407
        tcg_out_modrm_offset(s, 0x89, arg, arg1, arg2); /* movl */
408
    else
409
        tcg_out_modrm_offset(s, 0x89 | P_REXW, arg, arg1, arg2); /* movq */
410
}
411

    
412
static inline void tgen_arithi32(TCGContext *s, int c, int r0, int32_t val)
413
{
414
    if ((c == ARITH_ADD && val == 1) || (c == ARITH_SUB && val == -1)) {
415
        /* inc */
416
        tcg_out_modrm(s, 0xff, 0, r0);
417
    } else if ((c == ARITH_ADD && val == -1) || (c == ARITH_SUB && val == 1)) {
418
        /* dec */
419
        tcg_out_modrm(s, 0xff, 1, r0);
420
    } else if (val == (int8_t)val) {
421
        tcg_out_modrm(s, 0x83, c, r0);
422
        tcg_out8(s, val);
423
    } else if (c == ARITH_AND && val == 0xffu) {
424
        /* movzbl */
425
        tcg_out_modrm(s, 0xb6 | P_EXT | P_REXB_RM, r0, r0);
426
    } else if (c == ARITH_AND && val == 0xffffu) {
427
        /* movzwl */
428
        tcg_out_modrm(s, 0xb7 | P_EXT, r0, r0);
429
    } else {
430
        tcg_out_modrm(s, 0x81, c, r0);
431
        tcg_out32(s, val);
432
    }
433
}
434

    
435
static inline void tgen_arithi64(TCGContext *s, int c, int r0, int64_t val)
436
{
437
    if ((c == ARITH_ADD && val == 1) || (c == ARITH_SUB && val == -1)) {
438
        /* inc */
439
        tcg_out_modrm(s, 0xff | P_REXW, 0, r0);
440
    } else if ((c == ARITH_ADD && val == -1) || (c == ARITH_SUB && val == 1)) {
441
        /* dec */
442
        tcg_out_modrm(s, 0xff | P_REXW, 1, r0);
443
    } else if (c == ARITH_AND && val == 0xffffffffu) {
444
        /* 32-bit mov zero extends */
445
        tcg_out_modrm(s, 0x8b, r0, r0);
446
    } else if (c == ARITH_AND && val == (uint32_t)val) {
447
        /* AND with no high bits set can use a 32-bit operation.  */
448
        tgen_arithi32(s, c, r0, (uint32_t)val);
449
    } else if (val == (int8_t)val) {
450
        tcg_out_modrm(s, 0x83 | P_REXW, c, r0);
451
        tcg_out8(s, val);
452
    } else if (val == (int32_t)val) {
453
        tcg_out_modrm(s, 0x81 | P_REXW, c, r0);
454
        tcg_out32(s, val);
455
    } else {
456
        tcg_abort();
457
    }
458
}
459

    
460
static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
461
{
462
    if (val != 0)
463
        tgen_arithi64(s, ARITH_ADD, reg, val);
464
}
465

    
466
static void tcg_out_jxx(TCGContext *s, int opc, int label_index)
467
{
468
    int32_t val, val1;
469
    TCGLabel *l = &s->labels[label_index];
470
    
471
    if (l->has_value) {
472
        val = l->u.value - (tcg_target_long)s->code_ptr;
473
        val1 = val - 2;
474
        if ((int8_t)val1 == val1) {
475
            if (opc == -1)
476
                tcg_out8(s, 0xeb);
477
            else
478
                tcg_out8(s, 0x70 + opc);
479
            tcg_out8(s, val1);
480
        } else {
481
            if (opc == -1) {
482
                tcg_out8(s, 0xe9);
483
                tcg_out32(s, val - 5);
484
            } else {
485
                tcg_out8(s, 0x0f);
486
                tcg_out8(s, 0x80 + opc);
487
                tcg_out32(s, val - 6);
488
            }
489
        }
490
    } else {
491
        if (opc == -1) {
492
            tcg_out8(s, 0xe9);
493
        } else {
494
            tcg_out8(s, 0x0f);
495
            tcg_out8(s, 0x80 + opc);
496
        }
497
        tcg_out_reloc(s, s->code_ptr, R_386_PC32, label_index, -4);
498
        s->code_ptr += 4;
499
    }
500
}
501

    
502
static void tcg_out_brcond(TCGContext *s, int cond, 
503
                           TCGArg arg1, TCGArg arg2, int const_arg2,
504
                           int label_index, int rexw)
505
{
506
    if (const_arg2) {
507
        if (arg2 == 0) {
508
            /* test r, r */
509
            tcg_out_modrm(s, 0x85 | rexw, arg1, arg1);
510
        } else {
511
            if (rexw)
512
                tgen_arithi64(s, ARITH_CMP, arg1, arg2);
513
            else
514
                tgen_arithi32(s, ARITH_CMP, arg1, arg2);
515
        }
516
    } else {
517
        tcg_out_modrm(s, 0x01 | (ARITH_CMP << 3) | rexw, arg2, arg1);
518
    }
519
    tcg_out_jxx(s, tcg_cond_to_jcc[cond], label_index);
520
}
521

    
522
#if defined(CONFIG_SOFTMMU)
523

    
524
#include "../../softmmu_defs.h"
525

    
526
static void *qemu_ld_helpers[4] = {
527
    __ldb_mmu,
528
    __ldw_mmu,
529
    __ldl_mmu,
530
    __ldq_mmu,
531
};
532

    
533
static void *qemu_st_helpers[4] = {
534
    __stb_mmu,
535
    __stw_mmu,
536
    __stl_mmu,
537
    __stq_mmu,
538
};
539
#endif
540

    
541
static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
542
                            int opc)
543
{
544
    int addr_reg, data_reg, r0, r1, mem_index, s_bits, bswap, rexw;
545
    int32_t offset;
546
#if defined(CONFIG_SOFTMMU)
547
    uint8_t *label1_ptr, *label2_ptr;
548
#endif
549

    
550
    data_reg = *args++;
551
    addr_reg = *args++;
552
    mem_index = *args;
553
    s_bits = opc & 3;
554

    
555
    r0 = TCG_REG_RDI;
556
    r1 = TCG_REG_RSI;
557

    
558
#if TARGET_LONG_BITS == 32
559
    rexw = 0;
560
#else
561
    rexw = P_REXW;
562
#endif
563
#if defined(CONFIG_SOFTMMU)
564
    /* mov */
565
    tcg_out_modrm(s, 0x8b | rexw, r1, addr_reg);
566

    
567
    /* mov */
568
    tcg_out_modrm(s, 0x8b | rexw, r0, addr_reg);
569
 
570
    tcg_out_modrm(s, 0xc1 | rexw, 5, r1); /* shr $x, r1 */
571
    tcg_out8(s, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); 
572
    
573
    tcg_out_modrm(s, 0x81 | rexw, 4, r0); /* andl $x, r0 */
574
    tcg_out32(s, TARGET_PAGE_MASK | ((1 << s_bits) - 1));
575
    
576
    tcg_out_modrm(s, 0x81, 4, r1); /* andl $x, r1 */
577
    tcg_out32(s, (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
578

    
579
    /* lea offset(r1, env), r1 */
580
    tcg_out_modrm_offset2(s, 0x8d | P_REXW, r1, r1, TCG_AREG0, 0,
581
                          offsetof(CPUState, tlb_table[mem_index][0].addr_read));
582

    
583
    /* cmp 0(r1), r0 */
584
    tcg_out_modrm_offset(s, 0x3b | rexw, r0, r1, 0);
585
    
586
    /* mov */
587
    tcg_out_modrm(s, 0x8b | rexw, r0, addr_reg);
588
    
589
    /* je label1 */
590
    tcg_out8(s, 0x70 + JCC_JE);
591
    label1_ptr = s->code_ptr;
592
    s->code_ptr++;
593

    
594
    /* XXX: move that code at the end of the TB */
595
    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_RSI, mem_index);
596
    tcg_out_goto(s, 1, qemu_ld_helpers[s_bits]);
597

    
598
    switch(opc) {
599
    case 0 | 4:
600
        /* movsbq */
601
        tcg_out_modrm(s, 0xbe | P_EXT | P_REXW, data_reg, TCG_REG_RAX);
602
        break;
603
    case 1 | 4:
604
        /* movswq */
605
        tcg_out_modrm(s, 0xbf | P_EXT | P_REXW, data_reg, TCG_REG_RAX);
606
        break;
607
    case 2 | 4:
608
        /* movslq */
609
        tcg_out_modrm(s, 0x63 | P_REXW, data_reg, TCG_REG_RAX);
610
        break;
611
    case 0:
612
        /* movzbq */
613
        tcg_out_modrm(s, 0xb6 | P_EXT | P_REXW, data_reg, TCG_REG_RAX);
614
        break;
615
    case 1:
616
        /* movzwq */
617
        tcg_out_modrm(s, 0xb7 | P_EXT | P_REXW, data_reg, TCG_REG_RAX);
618
        break;
619
    case 2:
620
    default:
621
        /* movl */
622
        tcg_out_modrm(s, 0x8b, data_reg, TCG_REG_RAX);
623
        break;
624
    case 3:
625
        tcg_out_mov(s, data_reg, TCG_REG_RAX);
626
        break;
627
    }
628

    
629
    /* jmp label2 */
630
    tcg_out8(s, 0xeb);
631
    label2_ptr = s->code_ptr;
632
    s->code_ptr++;
633
    
634
    /* label1: */
635
    *label1_ptr = s->code_ptr - label1_ptr - 1;
636

    
637
    /* add x(r1), r0 */
638
    tcg_out_modrm_offset(s, 0x03 | P_REXW, r0, r1, offsetof(CPUTLBEntry, addend) - 
639
                         offsetof(CPUTLBEntry, addr_read));
640
    offset = 0;
641
#else
642
    if (GUEST_BASE == (int32_t)GUEST_BASE) {
643
        r0 = addr_reg;
644
        offset = GUEST_BASE;
645
    } else {
646
        offset = 0;
647
        /* movq $GUEST_BASE, r0 */
648
        tcg_out_opc(s, (0xb8 + (r0 & 7)) | P_REXW, 0, r0, 0);
649
        tcg_out32(s, GUEST_BASE);
650
        tcg_out32(s, GUEST_BASE >> 32);
651
        /* addq addr_reg, r0 */
652
        tcg_out_modrm(s, 0x01 | P_REXW, addr_reg, r0);
653
    }
654
#endif    
655

    
656
#ifdef TARGET_WORDS_BIGENDIAN
657
    bswap = 1;
658
#else
659
    bswap = 0;
660
#endif
661
    switch(opc) {
662
    case 0:
663
        /* movzbl */
664
        tcg_out_modrm_offset(s, 0xb6 | P_EXT, data_reg, r0, offset);
665
        break;
666
    case 0 | 4:
667
        /* movsbX */
668
        tcg_out_modrm_offset(s, 0xbe | P_EXT | rexw, data_reg, r0, offset);
669
        break;
670
    case 1:
671
        /* movzwl */
672
        tcg_out_modrm_offset(s, 0xb7 | P_EXT, data_reg, r0, offset);
673
        if (bswap) {
674
            /* rolw $8, data_reg */
675
            tcg_out8(s, 0x66); 
676
            tcg_out_modrm(s, 0xc1, 0, data_reg);
677
            tcg_out8(s, 8);
678
        }
679
        break;
680
    case 1 | 4:
681
        if (bswap) {
682
            /* movzwl */
683
            tcg_out_modrm_offset(s, 0xb7 | P_EXT, data_reg, r0, offset);
684
            /* rolw $8, data_reg */
685
            tcg_out8(s, 0x66); 
686
            tcg_out_modrm(s, 0xc1, 0, data_reg);
687
            tcg_out8(s, 8);
688

    
689
            /* movswX data_reg, data_reg */
690
            tcg_out_modrm(s, 0xbf | P_EXT | rexw, data_reg, data_reg);
691
        } else {
692
            /* movswX */
693
            tcg_out_modrm_offset(s, 0xbf | P_EXT | rexw, data_reg, r0, offset);
694
        }
695
        break;
696
    case 2:
697
        /* movl (r0), data_reg */
698
        tcg_out_modrm_offset(s, 0x8b, data_reg, r0, offset);
699
        if (bswap) {
700
            /* bswap */
701
            tcg_out_opc(s, (0xc8 + (data_reg & 7)) | P_EXT, 0, data_reg, 0);
702
        }
703
        break;
704
    case 2 | 4:
705
        if (bswap) {
706
            /* movl (r0), data_reg */
707
            tcg_out_modrm_offset(s, 0x8b, data_reg, r0, offset);
708
            /* bswap */
709
            tcg_out_opc(s, (0xc8 + (data_reg & 7)) | P_EXT, 0, data_reg, 0);
710
            /* movslq */
711
            tcg_out_modrm(s, 0x63 | P_REXW, data_reg, data_reg);
712
        } else {
713
            /* movslq */
714
            tcg_out_modrm_offset(s, 0x63 | P_REXW, data_reg, r0, offset);
715
        }
716
        break;
717
    case 3:
718
        /* movq (r0), data_reg */
719
        tcg_out_modrm_offset(s, 0x8b | P_REXW, data_reg, r0, offset);
720
        if (bswap) {
721
            /* bswap */
722
            tcg_out_opc(s, (0xc8 + (data_reg & 7)) | P_EXT | P_REXW, 0, data_reg, 0);
723
        }
724
        break;
725
    default:
726
        tcg_abort();
727
    }
728

    
729
#if defined(CONFIG_SOFTMMU)
730
    /* label2: */
731
    *label2_ptr = s->code_ptr - label2_ptr - 1;
732
#endif
733
}
734

    
735
static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
736
                            int opc)
737
{
738
    int addr_reg, data_reg, r0, r1, mem_index, s_bits, bswap, rexw;
739
    int32_t offset;
740
#if defined(CONFIG_SOFTMMU)
741
    uint8_t *label1_ptr, *label2_ptr;
742
#endif
743

    
744
    data_reg = *args++;
745
    addr_reg = *args++;
746
    mem_index = *args;
747

    
748
    s_bits = opc;
749

    
750
    r0 = TCG_REG_RDI;
751
    r1 = TCG_REG_RSI;
752

    
753
#if TARGET_LONG_BITS == 32
754
    rexw = 0;
755
#else
756
    rexw = P_REXW;
757
#endif
758
#if defined(CONFIG_SOFTMMU)
759
    /* mov */
760
    tcg_out_modrm(s, 0x8b | rexw, r1, addr_reg);
761

    
762
    /* mov */
763
    tcg_out_modrm(s, 0x8b | rexw, r0, addr_reg);
764
 
765
    tcg_out_modrm(s, 0xc1 | rexw, 5, r1); /* shr $x, r1 */
766
    tcg_out8(s, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); 
767
    
768
    tcg_out_modrm(s, 0x81 | rexw, 4, r0); /* andl $x, r0 */
769
    tcg_out32(s, TARGET_PAGE_MASK | ((1 << s_bits) - 1));
770
    
771
    tcg_out_modrm(s, 0x81, 4, r1); /* andl $x, r1 */
772
    tcg_out32(s, (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
773

    
774
    /* lea offset(r1, env), r1 */
775
    tcg_out_modrm_offset2(s, 0x8d | P_REXW, r1, r1, TCG_AREG0, 0,
776
                          offsetof(CPUState, tlb_table[mem_index][0].addr_write));
777

    
778
    /* cmp 0(r1), r0 */
779
    tcg_out_modrm_offset(s, 0x3b | rexw, r0, r1, 0);
780
    
781
    /* mov */
782
    tcg_out_modrm(s, 0x8b | rexw, r0, addr_reg);
783
    
784
    /* je label1 */
785
    tcg_out8(s, 0x70 + JCC_JE);
786
    label1_ptr = s->code_ptr;
787
    s->code_ptr++;
788

    
789
    /* XXX: move that code at the end of the TB */
790
    switch(opc) {
791
    case 0:
792
        /* movzbl */
793
        tcg_out_modrm(s, 0xb6 | P_EXT | P_REXB_RM, TCG_REG_RSI, data_reg);
794
        break;
795
    case 1:
796
        /* movzwl */
797
        tcg_out_modrm(s, 0xb7 | P_EXT, TCG_REG_RSI, data_reg);
798
        break;
799
    case 2:
800
        /* movl */
801
        tcg_out_modrm(s, 0x8b, TCG_REG_RSI, data_reg);
802
        break;
803
    default:
804
    case 3:
805
        tcg_out_mov(s, TCG_REG_RSI, data_reg);
806
        break;
807
    }
808
    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_RDX, mem_index);
809
    tcg_out_goto(s, 1, qemu_st_helpers[s_bits]);
810

    
811
    /* jmp label2 */
812
    tcg_out8(s, 0xeb);
813
    label2_ptr = s->code_ptr;
814
    s->code_ptr++;
815
    
816
    /* label1: */
817
    *label1_ptr = s->code_ptr - label1_ptr - 1;
818

    
819
    /* add x(r1), r0 */
820
    tcg_out_modrm_offset(s, 0x03 | P_REXW, r0, r1, offsetof(CPUTLBEntry, addend) - 
821
                         offsetof(CPUTLBEntry, addr_write));
822
    offset = 0;
823
#else
824
    if (GUEST_BASE == (int32_t)GUEST_BASE) {
825
        r0 = addr_reg;
826
        offset = GUEST_BASE;
827
    } else {
828
        offset = 0;
829
        /* movq $GUEST_BASE, r0 */
830
        tcg_out_opc(s, (0xb8 + (r0 & 7)) | P_REXW, 0, r0, 0);
831
        tcg_out32(s, GUEST_BASE);
832
        tcg_out32(s, GUEST_BASE >> 32);
833
        /* addq addr_reg, r0 */
834
        tcg_out_modrm(s, 0x01 | P_REXW, addr_reg, r0);
835
    }
836
#endif
837

    
838
#ifdef TARGET_WORDS_BIGENDIAN
839
    bswap = 1;
840
#else
841
    bswap = 0;
842
#endif
843
    switch(opc) {
844
    case 0:
845
        /* movb */
846
        tcg_out_modrm_offset(s, 0x88 | P_REXB_R, data_reg, r0, offset);
847
        break;
848
    case 1:
849
        if (bswap) {
850
            tcg_out_modrm(s, 0x8b, r1, data_reg); /* movl */
851
            tcg_out8(s, 0x66); /* rolw $8, %ecx */
852
            tcg_out_modrm(s, 0xc1, 0, r1);
853
            tcg_out8(s, 8);
854
            data_reg = r1;
855
        }
856
        /* movw */
857
        tcg_out8(s, 0x66);
858
        tcg_out_modrm_offset(s, 0x89, data_reg, r0, offset);
859
        break;
860
    case 2:
861
        if (bswap) {
862
            tcg_out_modrm(s, 0x8b, r1, data_reg); /* movl */
863
            /* bswap data_reg */
864
            tcg_out_opc(s, (0xc8 + r1) | P_EXT, 0, r1, 0);
865
            data_reg = r1;
866
        }
867
        /* movl */
868
        tcg_out_modrm_offset(s, 0x89, data_reg, r0, offset);
869
        break;
870
    case 3:
871
        if (bswap) {
872
            tcg_out_mov(s, r1, data_reg);
873
            /* bswap data_reg */
874
            tcg_out_opc(s, (0xc8 + r1) | P_EXT | P_REXW, 0, r1, 0);
875
            data_reg = r1;
876
        }
877
        /* movq */
878
        tcg_out_modrm_offset(s, 0x89 | P_REXW, data_reg, r0, offset);
879
        break;
880
    default:
881
        tcg_abort();
882
    }
883

    
884
#if defined(CONFIG_SOFTMMU)
885
    /* label2: */
886
    *label2_ptr = s->code_ptr - label2_ptr - 1;
887
#endif
888
}
889

    
890
static inline void tcg_out_op(TCGContext *s, int opc, const TCGArg *args,
891
                              const int *const_args)
892
{
893
    int c;
894
    
895
    switch(opc) {
896
    case INDEX_op_exit_tb:
897
        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RAX, args[0]);
898
        tcg_out_goto(s, 0, tb_ret_addr);
899
        break;
900
    case INDEX_op_goto_tb:
901
        if (s->tb_jmp_offset) {
902
            /* direct jump method */
903
            tcg_out8(s, 0xe9); /* jmp im */
904
            s->tb_jmp_offset[args[0]] = s->code_ptr - s->code_buf;
905
            tcg_out32(s, 0);
906
        } else {
907
            /* indirect jump method */
908
            /* jmp Ev */
909
            tcg_out_modrm_offset(s, 0xff, 4, -1, 
910
                                 (tcg_target_long)(s->tb_next + 
911
                                                   args[0]));
912
        }
913
        s->tb_next_offset[args[0]] = s->code_ptr - s->code_buf;
914
        break;
915
    case INDEX_op_call:
916
        if (const_args[0]) {
917
            tcg_out_goto(s, 1, (void *) args[0]);
918
        } else {
919
            tcg_out_modrm(s, 0xff, 2, args[0]);
920
        }
921
        break;
922
    case INDEX_op_jmp:
923
        if (const_args[0]) {
924
            tcg_out_goto(s, 0, (void *) args[0]);
925
        } else {
926
            tcg_out_modrm(s, 0xff, 4, args[0]);
927
        }
928
        break;
929
    case INDEX_op_br:
930
        tcg_out_jxx(s, JCC_JMP, args[0]);
931
        break;
932
    case INDEX_op_movi_i32:
933
        tcg_out_movi(s, TCG_TYPE_I32, args[0], (uint32_t)args[1]);
934
        break;
935
    case INDEX_op_movi_i64:
936
        tcg_out_movi(s, TCG_TYPE_I64, args[0], args[1]);
937
        break;
938
    case INDEX_op_ld8u_i32:
939
    case INDEX_op_ld8u_i64:
940
        /* movzbl */
941
        tcg_out_modrm_offset(s, 0xb6 | P_EXT, args[0], args[1], args[2]);
942
        break;
943
    case INDEX_op_ld8s_i32:
944
        /* movsbl */
945
        tcg_out_modrm_offset(s, 0xbe | P_EXT, args[0], args[1], args[2]);
946
        break;
947
    case INDEX_op_ld8s_i64:
948
        /* movsbq */
949
        tcg_out_modrm_offset(s, 0xbe | P_EXT | P_REXW, args[0], args[1], args[2]);
950
        break;
951
    case INDEX_op_ld16u_i32:
952
    case INDEX_op_ld16u_i64:
953
        /* movzwl */
954
        tcg_out_modrm_offset(s, 0xb7 | P_EXT, args[0], args[1], args[2]);
955
        break;
956
    case INDEX_op_ld16s_i32:
957
        /* movswl */
958
        tcg_out_modrm_offset(s, 0xbf | P_EXT, args[0], args[1], args[2]);
959
        break;
960
    case INDEX_op_ld16s_i64:
961
        /* movswq */
962
        tcg_out_modrm_offset(s, 0xbf | P_EXT | P_REXW, args[0], args[1], args[2]);
963
        break;
964
    case INDEX_op_ld_i32:
965
    case INDEX_op_ld32u_i64:
966
        /* movl */
967
        tcg_out_modrm_offset(s, 0x8b, args[0], args[1], args[2]);
968
        break;
969
    case INDEX_op_ld32s_i64:
970
        /* movslq */
971
        tcg_out_modrm_offset(s, 0x63 | P_REXW, args[0], args[1], args[2]);
972
        break;
973
    case INDEX_op_ld_i64:
974
        /* movq */
975
        tcg_out_modrm_offset(s, 0x8b | P_REXW, args[0], args[1], args[2]);
976
        break;
977
        
978
    case INDEX_op_st8_i32:
979
    case INDEX_op_st8_i64:
980
        /* movb */
981
        tcg_out_modrm_offset(s, 0x88 | P_REXB_R, args[0], args[1], args[2]);
982
        break;
983
    case INDEX_op_st16_i32:
984
    case INDEX_op_st16_i64:
985
        /* movw */
986
        tcg_out8(s, 0x66);
987
        tcg_out_modrm_offset(s, 0x89, args[0], args[1], args[2]);
988
        break;
989
    case INDEX_op_st_i32:
990
    case INDEX_op_st32_i64:
991
        /* movl */
992
        tcg_out_modrm_offset(s, 0x89, args[0], args[1], args[2]);
993
        break;
994
    case INDEX_op_st_i64:
995
        /* movq */
996
        tcg_out_modrm_offset(s, 0x89 | P_REXW, args[0], args[1], args[2]);
997
        break;
998

    
999
    case INDEX_op_sub_i32:
1000
        c = ARITH_SUB;
1001
        goto gen_arith32;
1002
    case INDEX_op_and_i32:
1003
        c = ARITH_AND;
1004
        goto gen_arith32;
1005
    case INDEX_op_or_i32:
1006
        c = ARITH_OR;
1007
        goto gen_arith32;
1008
    case INDEX_op_xor_i32:
1009
        c = ARITH_XOR;
1010
        goto gen_arith32;
1011
    case INDEX_op_add_i32:
1012
        c = ARITH_ADD;
1013
    gen_arith32:
1014
        if (const_args[2]) {
1015
            tgen_arithi32(s, c, args[0], args[2]);
1016
        } else {
1017
            tcg_out_modrm(s, 0x01 | (c << 3), args[2], args[0]);
1018
        }
1019
        break;
1020

    
1021
    case INDEX_op_sub_i64:
1022
        c = ARITH_SUB;
1023
        goto gen_arith64;
1024
    case INDEX_op_and_i64:
1025
        c = ARITH_AND;
1026
        goto gen_arith64;
1027
    case INDEX_op_or_i64:
1028
        c = ARITH_OR;
1029
        goto gen_arith64;
1030
    case INDEX_op_xor_i64:
1031
        c = ARITH_XOR;
1032
        goto gen_arith64;
1033
    case INDEX_op_add_i64:
1034
        c = ARITH_ADD;
1035
    gen_arith64:
1036
        if (const_args[2]) {
1037
            tgen_arithi64(s, c, args[0], args[2]);
1038
        } else {
1039
            tcg_out_modrm(s, 0x01 | (c << 3) | P_REXW, args[2], args[0]);
1040
        }
1041
        break;
1042

    
1043
    case INDEX_op_mul_i32:
1044
        if (const_args[2]) {
1045
            int32_t val;
1046
            val = args[2];
1047
            if (val == (int8_t)val) {
1048
                tcg_out_modrm(s, 0x6b, args[0], args[0]);
1049
                tcg_out8(s, val);
1050
            } else {
1051
                tcg_out_modrm(s, 0x69, args[0], args[0]);
1052
                tcg_out32(s, val);
1053
            }
1054
        } else {
1055
            tcg_out_modrm(s, 0xaf | P_EXT, args[0], args[2]);
1056
        }
1057
        break;
1058
    case INDEX_op_mul_i64:
1059
        if (const_args[2]) {
1060
            int32_t val;
1061
            val = args[2];
1062
            if (val == (int8_t)val) {
1063
                tcg_out_modrm(s, 0x6b | P_REXW, args[0], args[0]);
1064
                tcg_out8(s, val);
1065
            } else {
1066
                tcg_out_modrm(s, 0x69 | P_REXW, args[0], args[0]);
1067
                tcg_out32(s, val);
1068
            }
1069
        } else {
1070
            tcg_out_modrm(s, 0xaf | P_EXT | P_REXW, args[0], args[2]);
1071
        }
1072
        break;
1073
    case INDEX_op_div2_i32:
1074
        tcg_out_modrm(s, 0xf7, 7, args[4]);
1075
        break;
1076
    case INDEX_op_divu2_i32:
1077
        tcg_out_modrm(s, 0xf7, 6, args[4]);
1078
        break;
1079
    case INDEX_op_div2_i64:
1080
        tcg_out_modrm(s, 0xf7 | P_REXW, 7, args[4]);
1081
        break;
1082
    case INDEX_op_divu2_i64:
1083
        tcg_out_modrm(s, 0xf7 | P_REXW, 6, args[4]);
1084
        break;
1085

    
1086
    case INDEX_op_shl_i32:
1087
        c = SHIFT_SHL;
1088
    gen_shift32:
1089
        if (const_args[2]) {
1090
            if (args[2] == 1) {
1091
                tcg_out_modrm(s, 0xd1, c, args[0]);
1092
            } else {
1093
                tcg_out_modrm(s, 0xc1, c, args[0]);
1094
                tcg_out8(s, args[2]);
1095
            }
1096
        } else {
1097
            tcg_out_modrm(s, 0xd3, c, args[0]);
1098
        }
1099
        break;
1100
    case INDEX_op_shr_i32:
1101
        c = SHIFT_SHR;
1102
        goto gen_shift32;
1103
    case INDEX_op_sar_i32:
1104
        c = SHIFT_SAR;
1105
        goto gen_shift32;
1106
    case INDEX_op_rotl_i32:
1107
        c = SHIFT_ROL;
1108
        goto gen_shift32;
1109
    case INDEX_op_rotr_i32:
1110
        c = SHIFT_ROR;
1111
        goto gen_shift32;
1112

    
1113
    case INDEX_op_shl_i64:
1114
        c = SHIFT_SHL;
1115
    gen_shift64:
1116
        if (const_args[2]) {
1117
            if (args[2] == 1) {
1118
                tcg_out_modrm(s, 0xd1 | P_REXW, c, args[0]);
1119
            } else {
1120
                tcg_out_modrm(s, 0xc1 | P_REXW, c, args[0]);
1121
                tcg_out8(s, args[2]);
1122
            }
1123
        } else {
1124
            tcg_out_modrm(s, 0xd3 | P_REXW, c, args[0]);
1125
        }
1126
        break;
1127
    case INDEX_op_shr_i64:
1128
        c = SHIFT_SHR;
1129
        goto gen_shift64;
1130
    case INDEX_op_sar_i64:
1131
        c = SHIFT_SAR;
1132
        goto gen_shift64;
1133
    case INDEX_op_rotl_i64:
1134
        c = SHIFT_ROL;
1135
        goto gen_shift64;
1136
    case INDEX_op_rotr_i64:
1137
        c = SHIFT_ROR;
1138
        goto gen_shift64;
1139

    
1140
    case INDEX_op_brcond_i32:
1141
        tcg_out_brcond(s, args[2], args[0], args[1], const_args[1], 
1142
                       args[3], 0);
1143
        break;
1144
    case INDEX_op_brcond_i64:
1145
        tcg_out_brcond(s, args[2], args[0], args[1], const_args[1], 
1146
                       args[3], P_REXW);
1147
        break;
1148

    
1149
    case INDEX_op_bswap16_i32:
1150
    case INDEX_op_bswap16_i64:
1151
        tcg_out8(s, 0x66);
1152
        tcg_out_modrm(s, 0xc1, SHIFT_ROL, args[0]);
1153
        tcg_out8(s, 8);
1154
        break;
1155
    case INDEX_op_bswap32_i32:
1156
    case INDEX_op_bswap32_i64:
1157
        tcg_out_opc(s, (0xc8 + (args[0] & 7)) | P_EXT, 0, args[0], 0);
1158
        break;
1159
    case INDEX_op_bswap64_i64:
1160
        tcg_out_opc(s, (0xc8 + (args[0] & 7)) | P_EXT | P_REXW, 0, args[0], 0);
1161
        break;
1162

    
1163
    case INDEX_op_neg_i32:
1164
        tcg_out_modrm(s, 0xf7, 3, args[0]);
1165
        break;
1166
    case INDEX_op_neg_i64:
1167
        tcg_out_modrm(s, 0xf7 | P_REXW, 3, args[0]);
1168
        break;
1169

    
1170
    case INDEX_op_not_i32:
1171
        tcg_out_modrm(s, 0xf7, 2, args[0]);
1172
        break;
1173
    case INDEX_op_not_i64:
1174
        tcg_out_modrm(s, 0xf7 | P_REXW, 2, args[0]);
1175
        break;
1176

    
1177
    case INDEX_op_ext8s_i32:
1178
        tcg_out_modrm(s, 0xbe | P_EXT | P_REXB_RM, args[0], args[1]);
1179
        break;
1180
    case INDEX_op_ext16s_i32:
1181
        tcg_out_modrm(s, 0xbf | P_EXT, args[0], args[1]);
1182
        break;
1183
    case INDEX_op_ext8s_i64:
1184
        tcg_out_modrm(s, 0xbe | P_EXT | P_REXW, args[0], args[1]);
1185
        break;
1186
    case INDEX_op_ext16s_i64:
1187
        tcg_out_modrm(s, 0xbf | P_EXT | P_REXW, args[0], args[1]);
1188
        break;
1189
    case INDEX_op_ext32s_i64:
1190
        tcg_out_modrm(s, 0x63 | P_REXW, args[0], args[1]);
1191
        break;
1192
    case INDEX_op_ext8u_i32:
1193
    case INDEX_op_ext8u_i64:
1194
        tcg_out_modrm(s, 0xb6 | P_EXT | P_REXB_RM, args[0], args[1]);
1195
        break;
1196
    case INDEX_op_ext16u_i32:
1197
    case INDEX_op_ext16u_i64:
1198
        tcg_out_modrm(s, 0xb7 | P_EXT, args[0], args[1]);
1199
        break;
1200
    case INDEX_op_ext32u_i64:
1201
        tcg_out_modrm(s, 0x8b, args[0], args[1]);
1202
        break;
1203

    
1204
    case INDEX_op_qemu_ld8u:
1205
        tcg_out_qemu_ld(s, args, 0);
1206
        break;
1207
    case INDEX_op_qemu_ld8s:
1208
        tcg_out_qemu_ld(s, args, 0 | 4);
1209
        break;
1210
    case INDEX_op_qemu_ld16u:
1211
        tcg_out_qemu_ld(s, args, 1);
1212
        break;
1213
    case INDEX_op_qemu_ld16s:
1214
        tcg_out_qemu_ld(s, args, 1 | 4);
1215
        break;
1216
    case INDEX_op_qemu_ld32u:
1217
        tcg_out_qemu_ld(s, args, 2);
1218
        break;
1219
    case INDEX_op_qemu_ld32s:
1220
        tcg_out_qemu_ld(s, args, 2 | 4);
1221
        break;
1222
    case INDEX_op_qemu_ld64:
1223
        tcg_out_qemu_ld(s, args, 3);
1224
        break;
1225
        
1226
    case INDEX_op_qemu_st8:
1227
        tcg_out_qemu_st(s, args, 0);
1228
        break;
1229
    case INDEX_op_qemu_st16:
1230
        tcg_out_qemu_st(s, args, 1);
1231
        break;
1232
    case INDEX_op_qemu_st32:
1233
        tcg_out_qemu_st(s, args, 2);
1234
        break;
1235
    case INDEX_op_qemu_st64:
1236
        tcg_out_qemu_st(s, args, 3);
1237
        break;
1238

    
1239
    default:
1240
        tcg_abort();
1241
    }
1242
}
1243

    
1244
static int tcg_target_callee_save_regs[] = {
1245
    TCG_REG_RBP,
1246
    TCG_REG_RBX,
1247
    TCG_REG_R12,
1248
    TCG_REG_R13,
1249
    /*    TCG_REG_R14, */ /* currently used for the global env, so no
1250
                             need to save */
1251
    TCG_REG_R15,
1252
};
1253

    
1254
static inline void tcg_out_push(TCGContext *s, int reg)
1255
{
1256
    tcg_out_opc(s, (0x50 + (reg & 7)), 0, reg, 0);
1257
}
1258

    
1259
static inline void tcg_out_pop(TCGContext *s, int reg)
1260
{
1261
    tcg_out_opc(s, (0x58 + (reg & 7)), 0, reg, 0);
1262
}
1263

    
1264
/* Generate global QEMU prologue and epilogue code */
1265
void tcg_target_qemu_prologue(TCGContext *s)
1266
{
1267
    int i, frame_size, push_size, stack_addend;
1268

    
1269
    /* TB prologue */
1270
    /* save all callee saved registers */
1271
    for(i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
1272
        tcg_out_push(s, tcg_target_callee_save_regs[i]);
1273

    
1274
    }
1275
    /* reserve some stack space */
1276
    push_size = 8 + ARRAY_SIZE(tcg_target_callee_save_regs) * 8;
1277
    frame_size = push_size + TCG_STATIC_CALL_ARGS_SIZE;
1278
    frame_size = (frame_size + TCG_TARGET_STACK_ALIGN - 1) & 
1279
        ~(TCG_TARGET_STACK_ALIGN - 1);
1280
    stack_addend = frame_size - push_size;
1281
    tcg_out_addi(s, TCG_REG_RSP, -stack_addend);
1282

    
1283
    tcg_out_modrm(s, 0xff, 4, TCG_REG_RDI); /* jmp *%rdi */
1284
    
1285
    /* TB epilogue */
1286
    tb_ret_addr = s->code_ptr;
1287
    tcg_out_addi(s, TCG_REG_RSP, stack_addend);
1288
    for(i = ARRAY_SIZE(tcg_target_callee_save_regs) - 1; i >= 0; i--) {
1289
        tcg_out_pop(s, tcg_target_callee_save_regs[i]);
1290
    }
1291
    tcg_out8(s, 0xc3); /* ret */
1292
}
1293

    
1294
static const TCGTargetOpDef x86_64_op_defs[] = {
1295
    { INDEX_op_exit_tb, { } },
1296
    { INDEX_op_goto_tb, { } },
1297
    { INDEX_op_call, { "ri" } }, /* XXX: might need a specific constant constraint */
1298
    { INDEX_op_jmp, { "ri" } }, /* XXX: might need a specific constant constraint */
1299
    { INDEX_op_br, { } },
1300

    
1301
    { INDEX_op_mov_i32, { "r", "r" } },
1302
    { INDEX_op_movi_i32, { "r" } },
1303
    { INDEX_op_ld8u_i32, { "r", "r" } },
1304
    { INDEX_op_ld8s_i32, { "r", "r" } },
1305
    { INDEX_op_ld16u_i32, { "r", "r" } },
1306
    { INDEX_op_ld16s_i32, { "r", "r" } },
1307
    { INDEX_op_ld_i32, { "r", "r" } },
1308
    { INDEX_op_st8_i32, { "r", "r" } },
1309
    { INDEX_op_st16_i32, { "r", "r" } },
1310
    { INDEX_op_st_i32, { "r", "r" } },
1311

    
1312
    { INDEX_op_add_i32, { "r", "0", "ri" } },
1313
    { INDEX_op_mul_i32, { "r", "0", "ri" } },
1314
    { INDEX_op_div2_i32, { "a", "d", "0", "1", "r" } },
1315
    { INDEX_op_divu2_i32, { "a", "d", "0", "1", "r" } },
1316
    { INDEX_op_sub_i32, { "r", "0", "ri" } },
1317
    { INDEX_op_and_i32, { "r", "0", "ri" } },
1318
    { INDEX_op_or_i32, { "r", "0", "ri" } },
1319
    { INDEX_op_xor_i32, { "r", "0", "ri" } },
1320

    
1321
    { INDEX_op_shl_i32, { "r", "0", "ci" } },
1322
    { INDEX_op_shr_i32, { "r", "0", "ci" } },
1323
    { INDEX_op_sar_i32, { "r", "0", "ci" } },
1324
    { INDEX_op_rotl_i32, { "r", "0", "ci" } },
1325
    { INDEX_op_rotr_i32, { "r", "0", "ci" } },
1326

    
1327
    { INDEX_op_brcond_i32, { "r", "ri" } },
1328

    
1329
    { INDEX_op_mov_i64, { "r", "r" } },
1330
    { INDEX_op_movi_i64, { "r" } },
1331
    { INDEX_op_ld8u_i64, { "r", "r" } },
1332
    { INDEX_op_ld8s_i64, { "r", "r" } },
1333
    { INDEX_op_ld16u_i64, { "r", "r" } },
1334
    { INDEX_op_ld16s_i64, { "r", "r" } },
1335
    { INDEX_op_ld32u_i64, { "r", "r" } },
1336
    { INDEX_op_ld32s_i64, { "r", "r" } },
1337
    { INDEX_op_ld_i64, { "r", "r" } },
1338
    { INDEX_op_st8_i64, { "r", "r" } },
1339
    { INDEX_op_st16_i64, { "r", "r" } },
1340
    { INDEX_op_st32_i64, { "r", "r" } },
1341
    { INDEX_op_st_i64, { "r", "r" } },
1342

    
1343
    { INDEX_op_add_i64, { "r", "0", "re" } },
1344
    { INDEX_op_mul_i64, { "r", "0", "re" } },
1345
    { INDEX_op_div2_i64, { "a", "d", "0", "1", "r" } },
1346
    { INDEX_op_divu2_i64, { "a", "d", "0", "1", "r" } },
1347
    { INDEX_op_sub_i64, { "r", "0", "re" } },
1348
    { INDEX_op_and_i64, { "r", "0", "reZ" } },
1349
    { INDEX_op_or_i64, { "r", "0", "re" } },
1350
    { INDEX_op_xor_i64, { "r", "0", "re" } },
1351

    
1352
    { INDEX_op_shl_i64, { "r", "0", "ci" } },
1353
    { INDEX_op_shr_i64, { "r", "0", "ci" } },
1354
    { INDEX_op_sar_i64, { "r", "0", "ci" } },
1355
    { INDEX_op_rotl_i64, { "r", "0", "ci" } },
1356
    { INDEX_op_rotr_i64, { "r", "0", "ci" } },
1357

    
1358
    { INDEX_op_brcond_i64, { "r", "re" } },
1359

    
1360
    { INDEX_op_bswap16_i32, { "r", "0" } },
1361
    { INDEX_op_bswap16_i64, { "r", "0" } },
1362
    { INDEX_op_bswap32_i32, { "r", "0" } },
1363
    { INDEX_op_bswap32_i64, { "r", "0" } },
1364
    { INDEX_op_bswap64_i64, { "r", "0" } },
1365

    
1366
    { INDEX_op_neg_i32, { "r", "0" } },
1367
    { INDEX_op_neg_i64, { "r", "0" } },
1368

    
1369
    { INDEX_op_not_i32, { "r", "0" } },
1370
    { INDEX_op_not_i64, { "r", "0" } },
1371

    
1372
    { INDEX_op_ext8s_i32, { "r", "r"} },
1373
    { INDEX_op_ext16s_i32, { "r", "r"} },
1374
    { INDEX_op_ext8s_i64, { "r", "r"} },
1375
    { INDEX_op_ext16s_i64, { "r", "r"} },
1376
    { INDEX_op_ext32s_i64, { "r", "r"} },
1377
    { INDEX_op_ext8u_i32, { "r", "r"} },
1378
    { INDEX_op_ext16u_i32, { "r", "r"} },
1379
    { INDEX_op_ext8u_i64, { "r", "r"} },
1380
    { INDEX_op_ext16u_i64, { "r", "r"} },
1381
    { INDEX_op_ext32u_i64, { "r", "r"} },
1382

    
1383
    { INDEX_op_qemu_ld8u, { "r", "L" } },
1384
    { INDEX_op_qemu_ld8s, { "r", "L" } },
1385
    { INDEX_op_qemu_ld16u, { "r", "L" } },
1386
    { INDEX_op_qemu_ld16s, { "r", "L" } },
1387
    { INDEX_op_qemu_ld32u, { "r", "L" } },
1388
    { INDEX_op_qemu_ld32s, { "r", "L" } },
1389
    { INDEX_op_qemu_ld64, { "r", "L" } },
1390

    
1391
    { INDEX_op_qemu_st8, { "L", "L" } },
1392
    { INDEX_op_qemu_st16, { "L", "L" } },
1393
    { INDEX_op_qemu_st32, { "L", "L" } },
1394
    { INDEX_op_qemu_st64, { "L", "L" } },
1395

    
1396
    { -1 },
1397
};
1398

    
1399
void tcg_target_init(TCGContext *s)
1400
{
1401
    /* fail safe */
1402
    if ((1 << CPU_TLB_ENTRY_BITS) != sizeof(CPUTLBEntry))
1403
        tcg_abort();
1404

    
1405
    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffff);
1406
    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffff);
1407
    tcg_regset_set32(tcg_target_call_clobber_regs, 0,
1408
                     (1 << TCG_REG_RDI) | 
1409
                     (1 << TCG_REG_RSI) | 
1410
                     (1 << TCG_REG_RDX) |
1411
                     (1 << TCG_REG_RCX) |
1412
                     (1 << TCG_REG_R8) |
1413
                     (1 << TCG_REG_R9) |
1414
                     (1 << TCG_REG_RAX) |
1415
                     (1 << TCG_REG_R10) |
1416
                     (1 << TCG_REG_R11));
1417
    
1418
    tcg_regset_clear(s->reserved_regs);
1419
    tcg_regset_set_reg(s->reserved_regs, TCG_REG_RSP);
1420

    
1421
    tcg_add_target_add_op_defs(x86_64_op_defs);
1422
}