Statistics
| Branch: | Revision:

root / target-i386 / kvm.c @ db1669bc

History | View | Annotate | Download (52.6 kB)

1
/*
2
 * QEMU KVM support
3
 *
4
 * Copyright (C) 2006-2008 Qumranet Technologies
5
 * Copyright IBM, Corp. 2008
6
 *
7
 * Authors:
8
 *  Anthony Liguori   <aliguori@us.ibm.com>
9
 *
10
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
11
 * See the COPYING file in the top-level directory.
12
 *
13
 */
14

    
15
#include <sys/types.h>
16
#include <sys/ioctl.h>
17
#include <sys/mman.h>
18
#include <sys/utsname.h>
19

    
20
#include <linux/kvm.h>
21

    
22
#include "qemu-common.h"
23
#include "sysemu.h"
24
#include "kvm.h"
25
#include "cpu.h"
26
#include "gdbstub.h"
27
#include "host-utils.h"
28
#include "hw/pc.h"
29
#include "hw/apic.h"
30
#include "ioport.h"
31
#include "kvm_x86.h"
32

    
33
#ifdef CONFIG_KVM_PARA
34
#include <linux/kvm_para.h>
35
#endif
36
//
37
//#define DEBUG_KVM
38

    
39
#ifdef DEBUG_KVM
40
#define DPRINTF(fmt, ...) \
41
    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
42
#else
43
#define DPRINTF(fmt, ...) \
44
    do { } while (0)
45
#endif
46

    
47
#define MSR_KVM_WALL_CLOCK  0x11
48
#define MSR_KVM_SYSTEM_TIME 0x12
49

    
50
#ifndef BUS_MCEERR_AR
51
#define BUS_MCEERR_AR 4
52
#endif
53
#ifndef BUS_MCEERR_AO
54
#define BUS_MCEERR_AO 5
55
#endif
56

    
57
const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
58
    KVM_CAP_INFO(SET_TSS_ADDR),
59
    KVM_CAP_INFO(EXT_CPUID),
60
    KVM_CAP_INFO(MP_STATE),
61
    KVM_CAP_LAST_INFO
62
};
63

    
64
static bool has_msr_star;
65
static bool has_msr_hsave_pa;
66
#if defined(CONFIG_KVM_PARA) && defined(KVM_CAP_ASYNC_PF)
67
static bool has_msr_async_pf_en;
68
#endif
69
static int lm_capable_kernel;
70

    
71
static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
72
{
73
    struct kvm_cpuid2 *cpuid;
74
    int r, size;
75

    
76
    size = sizeof(*cpuid) + max * sizeof(*cpuid->entries);
77
    cpuid = (struct kvm_cpuid2 *)qemu_mallocz(size);
78
    cpuid->nent = max;
79
    r = kvm_ioctl(s, KVM_GET_SUPPORTED_CPUID, cpuid);
80
    if (r == 0 && cpuid->nent >= max) {
81
        r = -E2BIG;
82
    }
83
    if (r < 0) {
84
        if (r == -E2BIG) {
85
            qemu_free(cpuid);
86
            return NULL;
87
        } else {
88
            fprintf(stderr, "KVM_GET_SUPPORTED_CPUID failed: %s\n",
89
                    strerror(-r));
90
            exit(1);
91
        }
92
    }
93
    return cpuid;
94
}
95

    
96
uint32_t kvm_arch_get_supported_cpuid(CPUState *env, uint32_t function,
97
                                      uint32_t index, int reg)
98
{
99
    struct kvm_cpuid2 *cpuid;
100
    int i, max;
101
    uint32_t ret = 0;
102
    uint32_t cpuid_1_edx;
103

    
104
    max = 1;
105
    while ((cpuid = try_get_cpuid(env->kvm_state, max)) == NULL) {
106
        max *= 2;
107
    }
108

    
109
    for (i = 0; i < cpuid->nent; ++i) {
110
        if (cpuid->entries[i].function == function &&
111
            cpuid->entries[i].index == index) {
112
            switch (reg) {
113
            case R_EAX:
114
                ret = cpuid->entries[i].eax;
115
                break;
116
            case R_EBX:
117
                ret = cpuid->entries[i].ebx;
118
                break;
119
            case R_ECX:
120
                ret = cpuid->entries[i].ecx;
121
                break;
122
            case R_EDX:
123
                ret = cpuid->entries[i].edx;
124
                switch (function) {
125
                case 1:
126
                    /* KVM before 2.6.30 misreports the following features */
127
                    ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
128
                    break;
129
                case 0x80000001:
130
                    /* On Intel, kvm returns cpuid according to the Intel spec,
131
                     * so add missing bits according to the AMD spec:
132
                     */
133
                    cpuid_1_edx = kvm_arch_get_supported_cpuid(env, 1, 0, R_EDX);
134
                    ret |= cpuid_1_edx & 0x183f7ff;
135
                    break;
136
                }
137
                break;
138
            }
139
        }
140
    }
141

    
142
    qemu_free(cpuid);
143

    
144
    return ret;
145
}
146

    
147
#ifdef CONFIG_KVM_PARA
148
struct kvm_para_features {
149
    int cap;
150
    int feature;
151
} para_features[] = {
152
    { KVM_CAP_CLOCKSOURCE, KVM_FEATURE_CLOCKSOURCE },
153
    { KVM_CAP_NOP_IO_DELAY, KVM_FEATURE_NOP_IO_DELAY },
154
    { KVM_CAP_PV_MMU, KVM_FEATURE_MMU_OP },
155
#ifdef KVM_CAP_ASYNC_PF
156
    { KVM_CAP_ASYNC_PF, KVM_FEATURE_ASYNC_PF },
157
#endif
158
    { -1, -1 }
159
};
160

    
161
static int get_para_features(CPUState *env)
162
{
163
    int i, features = 0;
164

    
165
    for (i = 0; i < ARRAY_SIZE(para_features) - 1; i++) {
166
        if (kvm_check_extension(env->kvm_state, para_features[i].cap)) {
167
            features |= (1 << para_features[i].feature);
168
        }
169
    }
170
#ifdef KVM_CAP_ASYNC_PF
171
    has_msr_async_pf_en = features & (1 << KVM_FEATURE_ASYNC_PF);
172
#endif
173
    return features;
174
}
175
#endif
176

    
177
#ifdef KVM_CAP_MCE
178
static int kvm_get_mce_cap_supported(KVMState *s, uint64_t *mce_cap,
179
                                     int *max_banks)
180
{
181
    int r;
182

    
183
    r = kvm_check_extension(s, KVM_CAP_MCE);
184
    if (r > 0) {
185
        *max_banks = r;
186
        return kvm_ioctl(s, KVM_X86_GET_MCE_CAP_SUPPORTED, mce_cap);
187
    }
188
    return -ENOSYS;
189
}
190

    
191
static int kvm_setup_mce(CPUState *env, uint64_t *mcg_cap)
192
{
193
    return kvm_vcpu_ioctl(env, KVM_X86_SETUP_MCE, mcg_cap);
194
}
195

    
196
static int kvm_set_mce(CPUState *env, struct kvm_x86_mce *m)
197
{
198
    return kvm_vcpu_ioctl(env, KVM_X86_SET_MCE, m);
199
}
200

    
201
static int kvm_get_msr(CPUState *env, struct kvm_msr_entry *msrs, int n)
202
{
203
    struct kvm_msrs *kmsrs = qemu_malloc(sizeof *kmsrs + n * sizeof *msrs);
204
    int r;
205

    
206
    kmsrs->nmsrs = n;
207
    memcpy(kmsrs->entries, msrs, n * sizeof *msrs);
208
    r = kvm_vcpu_ioctl(env, KVM_GET_MSRS, kmsrs);
209
    memcpy(msrs, kmsrs->entries, n * sizeof *msrs);
210
    free(kmsrs);
211
    return r;
212
}
213

    
214
/* FIXME: kill this and kvm_get_msr, use env->mcg_status instead */
215
static int kvm_mce_in_progress(CPUState *env)
216
{
217
    struct kvm_msr_entry msr_mcg_status = {
218
        .index = MSR_MCG_STATUS,
219
    };
220
    int r;
221

    
222
    r = kvm_get_msr(env, &msr_mcg_status, 1);
223
    if (r == -1 || r == 0) {
224
        fprintf(stderr, "Failed to get MCE status\n");
225
        return 0;
226
    }
227
    return !!(msr_mcg_status.data & MCG_STATUS_MCIP);
228
}
229

    
230
struct kvm_x86_mce_data
231
{
232
    CPUState *env;
233
    struct kvm_x86_mce *mce;
234
    int abort_on_error;
235
};
236

    
237
static void kvm_do_inject_x86_mce(void *_data)
238
{
239
    struct kvm_x86_mce_data *data = _data;
240
    int r;
241

    
242
    /* If there is an MCE exception being processed, ignore this SRAO MCE */
243
    if ((data->env->mcg_cap & MCG_SER_P) &&
244
        !(data->mce->status & MCI_STATUS_AR)) {
245
        if (kvm_mce_in_progress(data->env)) {
246
            return;
247
        }
248
    }
249

    
250
    r = kvm_set_mce(data->env, data->mce);
251
    if (r < 0) {
252
        perror("kvm_set_mce FAILED");
253
        if (data->abort_on_error) {
254
            abort();
255
        }
256
    }
257
}
258

    
259
static void kvm_inject_x86_mce_on(CPUState *env, struct kvm_x86_mce *mce,
260
                                  int flag)
261
{
262
    struct kvm_x86_mce_data data = {
263
        .env = env,
264
        .mce = mce,
265
        .abort_on_error = (flag & ABORT_ON_ERROR),
266
    };
267

    
268
    if (!env->mcg_cap) {
269
        fprintf(stderr, "MCE support is not enabled!\n");
270
        return;
271
    }
272

    
273
    run_on_cpu(env, kvm_do_inject_x86_mce, &data);
274
}
275

    
276
static void kvm_mce_broadcast_rest(CPUState *env);
277
#endif
278

    
279
void kvm_inject_x86_mce(CPUState *cenv, int bank, uint64_t status,
280
                        uint64_t mcg_status, uint64_t addr, uint64_t misc,
281
                        int flag)
282
{
283
#ifdef KVM_CAP_MCE
284
    struct kvm_x86_mce mce = {
285
        .bank = bank,
286
        .status = status,
287
        .mcg_status = mcg_status,
288
        .addr = addr,
289
        .misc = misc,
290
    };
291

    
292
    if (flag & MCE_BROADCAST) {
293
        kvm_mce_broadcast_rest(cenv);
294
    }
295

    
296
    kvm_inject_x86_mce_on(cenv, &mce, flag);
297
#else
298
    if (flag & ABORT_ON_ERROR) {
299
        abort();
300
    }
301
#endif
302
}
303

    
304
static void cpu_update_state(void *opaque, int running, int reason)
305
{
306
    CPUState *env = opaque;
307

    
308
    if (running) {
309
        env->tsc_valid = false;
310
    }
311
}
312

    
313
int kvm_arch_init_vcpu(CPUState *env)
314
{
315
    struct {
316
        struct kvm_cpuid2 cpuid;
317
        struct kvm_cpuid_entry2 entries[100];
318
    } __attribute__((packed)) cpuid_data;
319
    uint32_t limit, i, j, cpuid_i;
320
    uint32_t unused;
321
    struct kvm_cpuid_entry2 *c;
322
#ifdef CONFIG_KVM_PARA
323
    uint32_t signature[3];
324
#endif
325

    
326
    env->cpuid_features &= kvm_arch_get_supported_cpuid(env, 1, 0, R_EDX);
327

    
328
    i = env->cpuid_ext_features & CPUID_EXT_HYPERVISOR;
329
    env->cpuid_ext_features &= kvm_arch_get_supported_cpuid(env, 1, 0, R_ECX);
330
    env->cpuid_ext_features |= i;
331

    
332
    env->cpuid_ext2_features &= kvm_arch_get_supported_cpuid(env, 0x80000001,
333
                                                             0, R_EDX);
334
    env->cpuid_ext3_features &= kvm_arch_get_supported_cpuid(env, 0x80000001,
335
                                                             0, R_ECX);
336
    env->cpuid_svm_features  &= kvm_arch_get_supported_cpuid(env, 0x8000000A,
337
                                                             0, R_EDX);
338

    
339

    
340
    cpuid_i = 0;
341

    
342
#ifdef CONFIG_KVM_PARA
343
    /* Paravirtualization CPUIDs */
344
    memcpy(signature, "KVMKVMKVM\0\0\0", 12);
345
    c = &cpuid_data.entries[cpuid_i++];
346
    memset(c, 0, sizeof(*c));
347
    c->function = KVM_CPUID_SIGNATURE;
348
    c->eax = 0;
349
    c->ebx = signature[0];
350
    c->ecx = signature[1];
351
    c->edx = signature[2];
352

    
353
    c = &cpuid_data.entries[cpuid_i++];
354
    memset(c, 0, sizeof(*c));
355
    c->function = KVM_CPUID_FEATURES;
356
    c->eax = env->cpuid_kvm_features & get_para_features(env);
357
#endif
358

    
359
    cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
360

    
361
    for (i = 0; i <= limit; i++) {
362
        c = &cpuid_data.entries[cpuid_i++];
363

    
364
        switch (i) {
365
        case 2: {
366
            /* Keep reading function 2 till all the input is received */
367
            int times;
368

    
369
            c->function = i;
370
            c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC |
371
                       KVM_CPUID_FLAG_STATE_READ_NEXT;
372
            cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
373
            times = c->eax & 0xff;
374

    
375
            for (j = 1; j < times; ++j) {
376
                c = &cpuid_data.entries[cpuid_i++];
377
                c->function = i;
378
                c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC;
379
                cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
380
            }
381
            break;
382
        }
383
        case 4:
384
        case 0xb:
385
        case 0xd:
386
            for (j = 0; ; j++) {
387
                c->function = i;
388
                c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
389
                c->index = j;
390
                cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
391

    
392
                if (i == 4 && c->eax == 0) {
393
                    break;
394
                }
395
                if (i == 0xb && !(c->ecx & 0xff00)) {
396
                    break;
397
                }
398
                if (i == 0xd && c->eax == 0) {
399
                    break;
400
                }
401
                c = &cpuid_data.entries[cpuid_i++];
402
            }
403
            break;
404
        default:
405
            c->function = i;
406
            c->flags = 0;
407
            cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
408
            break;
409
        }
410
    }
411
    cpu_x86_cpuid(env, 0x80000000, 0, &limit, &unused, &unused, &unused);
412

    
413
    for (i = 0x80000000; i <= limit; i++) {
414
        c = &cpuid_data.entries[cpuid_i++];
415

    
416
        c->function = i;
417
        c->flags = 0;
418
        cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
419
    }
420

    
421
    cpuid_data.cpuid.nent = cpuid_i;
422

    
423
#ifdef KVM_CAP_MCE
424
    if (((env->cpuid_version >> 8)&0xF) >= 6
425
        && (env->cpuid_features&(CPUID_MCE|CPUID_MCA)) == (CPUID_MCE|CPUID_MCA)
426
        && kvm_check_extension(env->kvm_state, KVM_CAP_MCE) > 0) {
427
        uint64_t mcg_cap;
428
        int banks;
429

    
430
        if (kvm_get_mce_cap_supported(env->kvm_state, &mcg_cap, &banks)) {
431
            perror("kvm_get_mce_cap_supported FAILED");
432
        } else {
433
            if (banks > MCE_BANKS_DEF)
434
                banks = MCE_BANKS_DEF;
435
            mcg_cap &= MCE_CAP_DEF;
436
            mcg_cap |= banks;
437
            if (kvm_setup_mce(env, &mcg_cap)) {
438
                perror("kvm_setup_mce FAILED");
439
            } else {
440
                env->mcg_cap = mcg_cap;
441
            }
442
        }
443
    }
444
#endif
445

    
446
    qemu_add_vm_change_state_handler(cpu_update_state, env);
447

    
448
    return kvm_vcpu_ioctl(env, KVM_SET_CPUID2, &cpuid_data);
449
}
450

    
451
void kvm_arch_reset_vcpu(CPUState *env)
452
{
453
    env->exception_injected = -1;
454
    env->interrupt_injected = -1;
455
    env->xcr0 = 1;
456
    if (kvm_irqchip_in_kernel()) {
457
        env->mp_state = cpu_is_bsp(env) ? KVM_MP_STATE_RUNNABLE :
458
                                          KVM_MP_STATE_UNINITIALIZED;
459
    } else {
460
        env->mp_state = KVM_MP_STATE_RUNNABLE;
461
    }
462
}
463

    
464
static int kvm_get_supported_msrs(KVMState *s)
465
{
466
    static int kvm_supported_msrs;
467
    int ret = 0;
468

    
469
    /* first time */
470
    if (kvm_supported_msrs == 0) {
471
        struct kvm_msr_list msr_list, *kvm_msr_list;
472

    
473
        kvm_supported_msrs = -1;
474

    
475
        /* Obtain MSR list from KVM.  These are the MSRs that we must
476
         * save/restore */
477
        msr_list.nmsrs = 0;
478
        ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, &msr_list);
479
        if (ret < 0 && ret != -E2BIG) {
480
            return ret;
481
        }
482
        /* Old kernel modules had a bug and could write beyond the provided
483
           memory. Allocate at least a safe amount of 1K. */
484
        kvm_msr_list = qemu_mallocz(MAX(1024, sizeof(msr_list) +
485
                                              msr_list.nmsrs *
486
                                              sizeof(msr_list.indices[0])));
487

    
488
        kvm_msr_list->nmsrs = msr_list.nmsrs;
489
        ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, kvm_msr_list);
490
        if (ret >= 0) {
491
            int i;
492

    
493
            for (i = 0; i < kvm_msr_list->nmsrs; i++) {
494
                if (kvm_msr_list->indices[i] == MSR_STAR) {
495
                    has_msr_star = true;
496
                    continue;
497
                }
498
                if (kvm_msr_list->indices[i] == MSR_VM_HSAVE_PA) {
499
                    has_msr_hsave_pa = true;
500
                    continue;
501
                }
502
            }
503
        }
504

    
505
        free(kvm_msr_list);
506
    }
507

    
508
    return ret;
509
}
510

    
511
int kvm_arch_init(KVMState *s)
512
{
513
    uint64_t identity_base = 0xfffbc000;
514
    int ret;
515
    struct utsname utsname;
516

    
517
    ret = kvm_get_supported_msrs(s);
518
    if (ret < 0) {
519
        return ret;
520
    }
521

    
522
    uname(&utsname);
523
    lm_capable_kernel = strcmp(utsname.machine, "x86_64") == 0;
524

    
525
    /*
526
     * On older Intel CPUs, KVM uses vm86 mode to emulate 16-bit code directly.
527
     * In order to use vm86 mode, an EPT identity map and a TSS  are needed.
528
     * Since these must be part of guest physical memory, we need to allocate
529
     * them, both by setting their start addresses in the kernel and by
530
     * creating a corresponding e820 entry. We need 4 pages before the BIOS.
531
     *
532
     * Older KVM versions may not support setting the identity map base. In
533
     * that case we need to stick with the default, i.e. a 256K maximum BIOS
534
     * size.
535
     */
536
#ifdef KVM_CAP_SET_IDENTITY_MAP_ADDR
537
    if (kvm_check_extension(s, KVM_CAP_SET_IDENTITY_MAP_ADDR)) {
538
        /* Allows up to 16M BIOSes. */
539
        identity_base = 0xfeffc000;
540

    
541
        ret = kvm_vm_ioctl(s, KVM_SET_IDENTITY_MAP_ADDR, &identity_base);
542
        if (ret < 0) {
543
            return ret;
544
        }
545
    }
546
#endif
547
    /* Set TSS base one page after EPT identity map. */
548
    ret = kvm_vm_ioctl(s, KVM_SET_TSS_ADDR, identity_base + 0x1000);
549
    if (ret < 0) {
550
        return ret;
551
    }
552

    
553
    /* Tell fw_cfg to notify the BIOS to reserve the range. */
554
    ret = e820_add_entry(identity_base, 0x4000, E820_RESERVED);
555
    if (ret < 0) {
556
        fprintf(stderr, "e820_add_entry() table is full\n");
557
        return ret;
558
    }
559

    
560
    return 0;
561
}
562

    
563
static void set_v8086_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
564
{
565
    lhs->selector = rhs->selector;
566
    lhs->base = rhs->base;
567
    lhs->limit = rhs->limit;
568
    lhs->type = 3;
569
    lhs->present = 1;
570
    lhs->dpl = 3;
571
    lhs->db = 0;
572
    lhs->s = 1;
573
    lhs->l = 0;
574
    lhs->g = 0;
575
    lhs->avl = 0;
576
    lhs->unusable = 0;
577
}
578

    
579
static void set_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
580
{
581
    unsigned flags = rhs->flags;
582
    lhs->selector = rhs->selector;
583
    lhs->base = rhs->base;
584
    lhs->limit = rhs->limit;
585
    lhs->type = (flags >> DESC_TYPE_SHIFT) & 15;
586
    lhs->present = (flags & DESC_P_MASK) != 0;
587
    lhs->dpl = (flags >> DESC_DPL_SHIFT) & 3;
588
    lhs->db = (flags >> DESC_B_SHIFT) & 1;
589
    lhs->s = (flags & DESC_S_MASK) != 0;
590
    lhs->l = (flags >> DESC_L_SHIFT) & 1;
591
    lhs->g = (flags & DESC_G_MASK) != 0;
592
    lhs->avl = (flags & DESC_AVL_MASK) != 0;
593
    lhs->unusable = 0;
594
}
595

    
596
static void get_seg(SegmentCache *lhs, const struct kvm_segment *rhs)
597
{
598
    lhs->selector = rhs->selector;
599
    lhs->base = rhs->base;
600
    lhs->limit = rhs->limit;
601
    lhs->flags = (rhs->type << DESC_TYPE_SHIFT) |
602
                 (rhs->present * DESC_P_MASK) |
603
                 (rhs->dpl << DESC_DPL_SHIFT) |
604
                 (rhs->db << DESC_B_SHIFT) |
605
                 (rhs->s * DESC_S_MASK) |
606
                 (rhs->l << DESC_L_SHIFT) |
607
                 (rhs->g * DESC_G_MASK) |
608
                 (rhs->avl * DESC_AVL_MASK);
609
}
610

    
611
static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
612
{
613
    if (set) {
614
        *kvm_reg = *qemu_reg;
615
    } else {
616
        *qemu_reg = *kvm_reg;
617
    }
618
}
619

    
620
static int kvm_getput_regs(CPUState *env, int set)
621
{
622
    struct kvm_regs regs;
623
    int ret = 0;
624

    
625
    if (!set) {
626
        ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
627
        if (ret < 0) {
628
            return ret;
629
        }
630
    }
631

    
632
    kvm_getput_reg(&regs.rax, &env->regs[R_EAX], set);
633
    kvm_getput_reg(&regs.rbx, &env->regs[R_EBX], set);
634
    kvm_getput_reg(&regs.rcx, &env->regs[R_ECX], set);
635
    kvm_getput_reg(&regs.rdx, &env->regs[R_EDX], set);
636
    kvm_getput_reg(&regs.rsi, &env->regs[R_ESI], set);
637
    kvm_getput_reg(&regs.rdi, &env->regs[R_EDI], set);
638
    kvm_getput_reg(&regs.rsp, &env->regs[R_ESP], set);
639
    kvm_getput_reg(&regs.rbp, &env->regs[R_EBP], set);
640
#ifdef TARGET_X86_64
641
    kvm_getput_reg(&regs.r8, &env->regs[8], set);
642
    kvm_getput_reg(&regs.r9, &env->regs[9], set);
643
    kvm_getput_reg(&regs.r10, &env->regs[10], set);
644
    kvm_getput_reg(&regs.r11, &env->regs[11], set);
645
    kvm_getput_reg(&regs.r12, &env->regs[12], set);
646
    kvm_getput_reg(&regs.r13, &env->regs[13], set);
647
    kvm_getput_reg(&regs.r14, &env->regs[14], set);
648
    kvm_getput_reg(&regs.r15, &env->regs[15], set);
649
#endif
650

    
651
    kvm_getput_reg(&regs.rflags, &env->eflags, set);
652
    kvm_getput_reg(&regs.rip, &env->eip, set);
653

    
654
    if (set) {
655
        ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
656
    }
657

    
658
    return ret;
659
}
660

    
661
static int kvm_put_fpu(CPUState *env)
662
{
663
    struct kvm_fpu fpu;
664
    int i;
665

    
666
    memset(&fpu, 0, sizeof fpu);
667
    fpu.fsw = env->fpus & ~(7 << 11);
668
    fpu.fsw |= (env->fpstt & 7) << 11;
669
    fpu.fcw = env->fpuc;
670
    for (i = 0; i < 8; ++i) {
671
        fpu.ftwx |= (!env->fptags[i]) << i;
672
    }
673
    memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs);
674
    memcpy(fpu.xmm, env->xmm_regs, sizeof env->xmm_regs);
675
    fpu.mxcsr = env->mxcsr;
676

    
677
    return kvm_vcpu_ioctl(env, KVM_SET_FPU, &fpu);
678
}
679

    
680
#ifdef KVM_CAP_XSAVE
681
#define XSAVE_CWD_RIP     2
682
#define XSAVE_CWD_RDP     4
683
#define XSAVE_MXCSR       6
684
#define XSAVE_ST_SPACE    8
685
#define XSAVE_XMM_SPACE   40
686
#define XSAVE_XSTATE_BV   128
687
#define XSAVE_YMMH_SPACE  144
688
#endif
689

    
690
static int kvm_put_xsave(CPUState *env)
691
{
692
#ifdef KVM_CAP_XSAVE
693
    int i, r;
694
    struct kvm_xsave* xsave;
695
    uint16_t cwd, swd, twd, fop;
696

    
697
    if (!kvm_has_xsave()) {
698
        return kvm_put_fpu(env);
699
    }
700

    
701
    xsave = qemu_memalign(4096, sizeof(struct kvm_xsave));
702
    memset(xsave, 0, sizeof(struct kvm_xsave));
703
    cwd = swd = twd = fop = 0;
704
    swd = env->fpus & ~(7 << 11);
705
    swd |= (env->fpstt & 7) << 11;
706
    cwd = env->fpuc;
707
    for (i = 0; i < 8; ++i) {
708
        twd |= (!env->fptags[i]) << i;
709
    }
710
    xsave->region[0] = (uint32_t)(swd << 16) + cwd;
711
    xsave->region[1] = (uint32_t)(fop << 16) + twd;
712
    memcpy(&xsave->region[XSAVE_ST_SPACE], env->fpregs,
713
            sizeof env->fpregs);
714
    memcpy(&xsave->region[XSAVE_XMM_SPACE], env->xmm_regs,
715
            sizeof env->xmm_regs);
716
    xsave->region[XSAVE_MXCSR] = env->mxcsr;
717
    *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV] = env->xstate_bv;
718
    memcpy(&xsave->region[XSAVE_YMMH_SPACE], env->ymmh_regs,
719
            sizeof env->ymmh_regs);
720
    r = kvm_vcpu_ioctl(env, KVM_SET_XSAVE, xsave);
721
    qemu_free(xsave);
722
    return r;
723
#else
724
    return kvm_put_fpu(env);
725
#endif
726
}
727

    
728
static int kvm_put_xcrs(CPUState *env)
729
{
730
#ifdef KVM_CAP_XCRS
731
    struct kvm_xcrs xcrs;
732

    
733
    if (!kvm_has_xcrs()) {
734
        return 0;
735
    }
736

    
737
    xcrs.nr_xcrs = 1;
738
    xcrs.flags = 0;
739
    xcrs.xcrs[0].xcr = 0;
740
    xcrs.xcrs[0].value = env->xcr0;
741
    return kvm_vcpu_ioctl(env, KVM_SET_XCRS, &xcrs);
742
#else
743
    return 0;
744
#endif
745
}
746

    
747
static int kvm_put_sregs(CPUState *env)
748
{
749
    struct kvm_sregs sregs;
750

    
751
    memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
752
    if (env->interrupt_injected >= 0) {
753
        sregs.interrupt_bitmap[env->interrupt_injected / 64] |=
754
                (uint64_t)1 << (env->interrupt_injected % 64);
755
    }
756

    
757
    if ((env->eflags & VM_MASK)) {
758
        set_v8086_seg(&sregs.cs, &env->segs[R_CS]);
759
        set_v8086_seg(&sregs.ds, &env->segs[R_DS]);
760
        set_v8086_seg(&sregs.es, &env->segs[R_ES]);
761
        set_v8086_seg(&sregs.fs, &env->segs[R_FS]);
762
        set_v8086_seg(&sregs.gs, &env->segs[R_GS]);
763
        set_v8086_seg(&sregs.ss, &env->segs[R_SS]);
764
    } else {
765
        set_seg(&sregs.cs, &env->segs[R_CS]);
766
        set_seg(&sregs.ds, &env->segs[R_DS]);
767
        set_seg(&sregs.es, &env->segs[R_ES]);
768
        set_seg(&sregs.fs, &env->segs[R_FS]);
769
        set_seg(&sregs.gs, &env->segs[R_GS]);
770
        set_seg(&sregs.ss, &env->segs[R_SS]);
771
    }
772

    
773
    set_seg(&sregs.tr, &env->tr);
774
    set_seg(&sregs.ldt, &env->ldt);
775

    
776
    sregs.idt.limit = env->idt.limit;
777
    sregs.idt.base = env->idt.base;
778
    sregs.gdt.limit = env->gdt.limit;
779
    sregs.gdt.base = env->gdt.base;
780

    
781
    sregs.cr0 = env->cr[0];
782
    sregs.cr2 = env->cr[2];
783
    sregs.cr3 = env->cr[3];
784
    sregs.cr4 = env->cr[4];
785

    
786
    sregs.cr8 = cpu_get_apic_tpr(env->apic_state);
787
    sregs.apic_base = cpu_get_apic_base(env->apic_state);
788

    
789
    sregs.efer = env->efer;
790

    
791
    return kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
792
}
793

    
794
static void kvm_msr_entry_set(struct kvm_msr_entry *entry,
795
                              uint32_t index, uint64_t value)
796
{
797
    entry->index = index;
798
    entry->data = value;
799
}
800

    
801
static int kvm_put_msrs(CPUState *env, int level)
802
{
803
    struct {
804
        struct kvm_msrs info;
805
        struct kvm_msr_entry entries[100];
806
    } msr_data;
807
    struct kvm_msr_entry *msrs = msr_data.entries;
808
    int n = 0;
809

    
810
    kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_CS, env->sysenter_cs);
811
    kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
812
    kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_EIP, env->sysenter_eip);
813
    if (has_msr_star) {
814
        kvm_msr_entry_set(&msrs[n++], MSR_STAR, env->star);
815
    }
816
    if (has_msr_hsave_pa) {
817
        kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
818
    }
819
#ifdef TARGET_X86_64
820
    if (lm_capable_kernel) {
821
        kvm_msr_entry_set(&msrs[n++], MSR_CSTAR, env->cstar);
822
        kvm_msr_entry_set(&msrs[n++], MSR_KERNELGSBASE, env->kernelgsbase);
823
        kvm_msr_entry_set(&msrs[n++], MSR_FMASK, env->fmask);
824
        kvm_msr_entry_set(&msrs[n++], MSR_LSTAR, env->lstar);
825
    }
826
#endif
827
    if (level == KVM_PUT_FULL_STATE) {
828
        /*
829
         * KVM is yet unable to synchronize TSC values of multiple VCPUs on
830
         * writeback. Until this is fixed, we only write the offset to SMP
831
         * guests after migration, desynchronizing the VCPUs, but avoiding
832
         * huge jump-backs that would occur without any writeback at all.
833
         */
834
        if (smp_cpus == 1 || env->tsc != 0) {
835
            kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSC, env->tsc);
836
        }
837
    }
838
    /*
839
     * The following paravirtual MSRs have side effects on the guest or are
840
     * too heavy for normal writeback. Limit them to reset or full state
841
     * updates.
842
     */
843
    if (level >= KVM_PUT_RESET_STATE) {
844
        kvm_msr_entry_set(&msrs[n++], MSR_KVM_SYSTEM_TIME,
845
                          env->system_time_msr);
846
        kvm_msr_entry_set(&msrs[n++], MSR_KVM_WALL_CLOCK, env->wall_clock_msr);
847
#if defined(CONFIG_KVM_PARA) && defined(KVM_CAP_ASYNC_PF)
848
        if (has_msr_async_pf_en) {
849
            kvm_msr_entry_set(&msrs[n++], MSR_KVM_ASYNC_PF_EN,
850
                              env->async_pf_en_msr);
851
        }
852
#endif
853
    }
854
#ifdef KVM_CAP_MCE
855
    if (env->mcg_cap) {
856
        int i;
857

    
858
        if (level == KVM_PUT_RESET_STATE) {
859
            kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
860
        } else if (level == KVM_PUT_FULL_STATE) {
861
            kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
862
            kvm_msr_entry_set(&msrs[n++], MSR_MCG_CTL, env->mcg_ctl);
863
            for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
864
                kvm_msr_entry_set(&msrs[n++], MSR_MC0_CTL + i, env->mce_banks[i]);
865
            }
866
        }
867
    }
868
#endif
869

    
870
    msr_data.info.nmsrs = n;
871

    
872
    return kvm_vcpu_ioctl(env, KVM_SET_MSRS, &msr_data);
873

    
874
}
875

    
876

    
877
static int kvm_get_fpu(CPUState *env)
878
{
879
    struct kvm_fpu fpu;
880
    int i, ret;
881

    
882
    ret = kvm_vcpu_ioctl(env, KVM_GET_FPU, &fpu);
883
    if (ret < 0) {
884
        return ret;
885
    }
886

    
887
    env->fpstt = (fpu.fsw >> 11) & 7;
888
    env->fpus = fpu.fsw;
889
    env->fpuc = fpu.fcw;
890
    for (i = 0; i < 8; ++i) {
891
        env->fptags[i] = !((fpu.ftwx >> i) & 1);
892
    }
893
    memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs);
894
    memcpy(env->xmm_regs, fpu.xmm, sizeof env->xmm_regs);
895
    env->mxcsr = fpu.mxcsr;
896

    
897
    return 0;
898
}
899

    
900
static int kvm_get_xsave(CPUState *env)
901
{
902
#ifdef KVM_CAP_XSAVE
903
    struct kvm_xsave* xsave;
904
    int ret, i;
905
    uint16_t cwd, swd, twd, fop;
906

    
907
    if (!kvm_has_xsave()) {
908
        return kvm_get_fpu(env);
909
    }
910

    
911
    xsave = qemu_memalign(4096, sizeof(struct kvm_xsave));
912
    ret = kvm_vcpu_ioctl(env, KVM_GET_XSAVE, xsave);
913
    if (ret < 0) {
914
        qemu_free(xsave);
915
        return ret;
916
    }
917

    
918
    cwd = (uint16_t)xsave->region[0];
919
    swd = (uint16_t)(xsave->region[0] >> 16);
920
    twd = (uint16_t)xsave->region[1];
921
    fop = (uint16_t)(xsave->region[1] >> 16);
922
    env->fpstt = (swd >> 11) & 7;
923
    env->fpus = swd;
924
    env->fpuc = cwd;
925
    for (i = 0; i < 8; ++i) {
926
        env->fptags[i] = !((twd >> i) & 1);
927
    }
928
    env->mxcsr = xsave->region[XSAVE_MXCSR];
929
    memcpy(env->fpregs, &xsave->region[XSAVE_ST_SPACE],
930
            sizeof env->fpregs);
931
    memcpy(env->xmm_regs, &xsave->region[XSAVE_XMM_SPACE],
932
            sizeof env->xmm_regs);
933
    env->xstate_bv = *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV];
934
    memcpy(env->ymmh_regs, &xsave->region[XSAVE_YMMH_SPACE],
935
            sizeof env->ymmh_regs);
936
    qemu_free(xsave);
937
    return 0;
938
#else
939
    return kvm_get_fpu(env);
940
#endif
941
}
942

    
943
static int kvm_get_xcrs(CPUState *env)
944
{
945
#ifdef KVM_CAP_XCRS
946
    int i, ret;
947
    struct kvm_xcrs xcrs;
948

    
949
    if (!kvm_has_xcrs()) {
950
        return 0;
951
    }
952

    
953
    ret = kvm_vcpu_ioctl(env, KVM_GET_XCRS, &xcrs);
954
    if (ret < 0) {
955
        return ret;
956
    }
957

    
958
    for (i = 0; i < xcrs.nr_xcrs; i++) {
959
        /* Only support xcr0 now */
960
        if (xcrs.xcrs[0].xcr == 0) {
961
            env->xcr0 = xcrs.xcrs[0].value;
962
            break;
963
        }
964
    }
965
    return 0;
966
#else
967
    return 0;
968
#endif
969
}
970

    
971
static int kvm_get_sregs(CPUState *env)
972
{
973
    struct kvm_sregs sregs;
974
    uint32_t hflags;
975
    int bit, i, ret;
976

    
977
    ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
978
    if (ret < 0) {
979
        return ret;
980
    }
981

    
982
    /* There can only be one pending IRQ set in the bitmap at a time, so try
983
       to find it and save its number instead (-1 for none). */
984
    env->interrupt_injected = -1;
985
    for (i = 0; i < ARRAY_SIZE(sregs.interrupt_bitmap); i++) {
986
        if (sregs.interrupt_bitmap[i]) {
987
            bit = ctz64(sregs.interrupt_bitmap[i]);
988
            env->interrupt_injected = i * 64 + bit;
989
            break;
990
        }
991
    }
992

    
993
    get_seg(&env->segs[R_CS], &sregs.cs);
994
    get_seg(&env->segs[R_DS], &sregs.ds);
995
    get_seg(&env->segs[R_ES], &sregs.es);
996
    get_seg(&env->segs[R_FS], &sregs.fs);
997
    get_seg(&env->segs[R_GS], &sregs.gs);
998
    get_seg(&env->segs[R_SS], &sregs.ss);
999

    
1000
    get_seg(&env->tr, &sregs.tr);
1001
    get_seg(&env->ldt, &sregs.ldt);
1002

    
1003
    env->idt.limit = sregs.idt.limit;
1004
    env->idt.base = sregs.idt.base;
1005
    env->gdt.limit = sregs.gdt.limit;
1006
    env->gdt.base = sregs.gdt.base;
1007

    
1008
    env->cr[0] = sregs.cr0;
1009
    env->cr[2] = sregs.cr2;
1010
    env->cr[3] = sregs.cr3;
1011
    env->cr[4] = sregs.cr4;
1012

    
1013
    cpu_set_apic_base(env->apic_state, sregs.apic_base);
1014

    
1015
    env->efer = sregs.efer;
1016
    //cpu_set_apic_tpr(env->apic_state, sregs.cr8);
1017

    
1018
#define HFLAG_COPY_MASK \
1019
    ~( HF_CPL_MASK | HF_PE_MASK | HF_MP_MASK | HF_EM_MASK | \
1020
       HF_TS_MASK | HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK | \
1021
       HF_OSFXSR_MASK | HF_LMA_MASK | HF_CS32_MASK | \
1022
       HF_SS32_MASK | HF_CS64_MASK | HF_ADDSEG_MASK)
1023

    
1024
    hflags = (env->segs[R_CS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
1025
    hflags |= (env->cr[0] & CR0_PE_MASK) << (HF_PE_SHIFT - CR0_PE_SHIFT);
1026
    hflags |= (env->cr[0] << (HF_MP_SHIFT - CR0_MP_SHIFT)) &
1027
                (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK);
1028
    hflags |= (env->eflags & (HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK));
1029
    hflags |= (env->cr[4] & CR4_OSFXSR_MASK) <<
1030
                (HF_OSFXSR_SHIFT - CR4_OSFXSR_SHIFT);
1031

    
1032
    if (env->efer & MSR_EFER_LMA) {
1033
        hflags |= HF_LMA_MASK;
1034
    }
1035

    
1036
    if ((hflags & HF_LMA_MASK) && (env->segs[R_CS].flags & DESC_L_MASK)) {
1037
        hflags |= HF_CS32_MASK | HF_SS32_MASK | HF_CS64_MASK;
1038
    } else {
1039
        hflags |= (env->segs[R_CS].flags & DESC_B_MASK) >>
1040
                    (DESC_B_SHIFT - HF_CS32_SHIFT);
1041
        hflags |= (env->segs[R_SS].flags & DESC_B_MASK) >>
1042
                    (DESC_B_SHIFT - HF_SS32_SHIFT);
1043
        if (!(env->cr[0] & CR0_PE_MASK) || (env->eflags & VM_MASK) ||
1044
            !(hflags & HF_CS32_MASK)) {
1045
            hflags |= HF_ADDSEG_MASK;
1046
        } else {
1047
            hflags |= ((env->segs[R_DS].base | env->segs[R_ES].base |
1048
                        env->segs[R_SS].base) != 0) << HF_ADDSEG_SHIFT;
1049
        }
1050
    }
1051
    env->hflags = (env->hflags & HFLAG_COPY_MASK) | hflags;
1052

    
1053
    return 0;
1054
}
1055

    
1056
static int kvm_get_msrs(CPUState *env)
1057
{
1058
    struct {
1059
        struct kvm_msrs info;
1060
        struct kvm_msr_entry entries[100];
1061
    } msr_data;
1062
    struct kvm_msr_entry *msrs = msr_data.entries;
1063
    int ret, i, n;
1064

    
1065
    n = 0;
1066
    msrs[n++].index = MSR_IA32_SYSENTER_CS;
1067
    msrs[n++].index = MSR_IA32_SYSENTER_ESP;
1068
    msrs[n++].index = MSR_IA32_SYSENTER_EIP;
1069
    if (has_msr_star) {
1070
        msrs[n++].index = MSR_STAR;
1071
    }
1072
    if (has_msr_hsave_pa) {
1073
        msrs[n++].index = MSR_VM_HSAVE_PA;
1074
    }
1075

    
1076
    if (!env->tsc_valid) {
1077
        msrs[n++].index = MSR_IA32_TSC;
1078
        env->tsc_valid = !vm_running;
1079
    }
1080

    
1081
#ifdef TARGET_X86_64
1082
    if (lm_capable_kernel) {
1083
        msrs[n++].index = MSR_CSTAR;
1084
        msrs[n++].index = MSR_KERNELGSBASE;
1085
        msrs[n++].index = MSR_FMASK;
1086
        msrs[n++].index = MSR_LSTAR;
1087
    }
1088
#endif
1089
    msrs[n++].index = MSR_KVM_SYSTEM_TIME;
1090
    msrs[n++].index = MSR_KVM_WALL_CLOCK;
1091
#if defined(CONFIG_KVM_PARA) && defined(KVM_CAP_ASYNC_PF)
1092
    if (has_msr_async_pf_en) {
1093
        msrs[n++].index = MSR_KVM_ASYNC_PF_EN;
1094
    }
1095
#endif
1096

    
1097
#ifdef KVM_CAP_MCE
1098
    if (env->mcg_cap) {
1099
        msrs[n++].index = MSR_MCG_STATUS;
1100
        msrs[n++].index = MSR_MCG_CTL;
1101
        for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
1102
            msrs[n++].index = MSR_MC0_CTL + i;
1103
        }
1104
    }
1105
#endif
1106

    
1107
    msr_data.info.nmsrs = n;
1108
    ret = kvm_vcpu_ioctl(env, KVM_GET_MSRS, &msr_data);
1109
    if (ret < 0) {
1110
        return ret;
1111
    }
1112

    
1113
    for (i = 0; i < ret; i++) {
1114
        switch (msrs[i].index) {
1115
        case MSR_IA32_SYSENTER_CS:
1116
            env->sysenter_cs = msrs[i].data;
1117
            break;
1118
        case MSR_IA32_SYSENTER_ESP:
1119
            env->sysenter_esp = msrs[i].data;
1120
            break;
1121
        case MSR_IA32_SYSENTER_EIP:
1122
            env->sysenter_eip = msrs[i].data;
1123
            break;
1124
        case MSR_STAR:
1125
            env->star = msrs[i].data;
1126
            break;
1127
#ifdef TARGET_X86_64
1128
        case MSR_CSTAR:
1129
            env->cstar = msrs[i].data;
1130
            break;
1131
        case MSR_KERNELGSBASE:
1132
            env->kernelgsbase = msrs[i].data;
1133
            break;
1134
        case MSR_FMASK:
1135
            env->fmask = msrs[i].data;
1136
            break;
1137
        case MSR_LSTAR:
1138
            env->lstar = msrs[i].data;
1139
            break;
1140
#endif
1141
        case MSR_IA32_TSC:
1142
            env->tsc = msrs[i].data;
1143
            break;
1144
        case MSR_VM_HSAVE_PA:
1145
            env->vm_hsave = msrs[i].data;
1146
            break;
1147
        case MSR_KVM_SYSTEM_TIME:
1148
            env->system_time_msr = msrs[i].data;
1149
            break;
1150
        case MSR_KVM_WALL_CLOCK:
1151
            env->wall_clock_msr = msrs[i].data;
1152
            break;
1153
#ifdef KVM_CAP_MCE
1154
        case MSR_MCG_STATUS:
1155
            env->mcg_status = msrs[i].data;
1156
            break;
1157
        case MSR_MCG_CTL:
1158
            env->mcg_ctl = msrs[i].data;
1159
            break;
1160
#endif
1161
        default:
1162
#ifdef KVM_CAP_MCE
1163
            if (msrs[i].index >= MSR_MC0_CTL &&
1164
                msrs[i].index < MSR_MC0_CTL + (env->mcg_cap & 0xff) * 4) {
1165
                env->mce_banks[msrs[i].index - MSR_MC0_CTL] = msrs[i].data;
1166
            }
1167
#endif
1168
            break;
1169
#if defined(CONFIG_KVM_PARA) && defined(KVM_CAP_ASYNC_PF)
1170
        case MSR_KVM_ASYNC_PF_EN:
1171
            env->async_pf_en_msr = msrs[i].data;
1172
            break;
1173
#endif
1174
        }
1175
    }
1176

    
1177
    return 0;
1178
}
1179

    
1180
static int kvm_put_mp_state(CPUState *env)
1181
{
1182
    struct kvm_mp_state mp_state = { .mp_state = env->mp_state };
1183

    
1184
    return kvm_vcpu_ioctl(env, KVM_SET_MP_STATE, &mp_state);
1185
}
1186

    
1187
static int kvm_get_mp_state(CPUState *env)
1188
{
1189
    struct kvm_mp_state mp_state;
1190
    int ret;
1191

    
1192
    ret = kvm_vcpu_ioctl(env, KVM_GET_MP_STATE, &mp_state);
1193
    if (ret < 0) {
1194
        return ret;
1195
    }
1196
    env->mp_state = mp_state.mp_state;
1197
    if (kvm_irqchip_in_kernel()) {
1198
        env->halted = (mp_state.mp_state == KVM_MP_STATE_HALTED);
1199
    }
1200
    return 0;
1201
}
1202

    
1203
static int kvm_put_vcpu_events(CPUState *env, int level)
1204
{
1205
#ifdef KVM_CAP_VCPU_EVENTS
1206
    struct kvm_vcpu_events events;
1207

    
1208
    if (!kvm_has_vcpu_events()) {
1209
        return 0;
1210
    }
1211

    
1212
    events.exception.injected = (env->exception_injected >= 0);
1213
    events.exception.nr = env->exception_injected;
1214
    events.exception.has_error_code = env->has_error_code;
1215
    events.exception.error_code = env->error_code;
1216

    
1217
    events.interrupt.injected = (env->interrupt_injected >= 0);
1218
    events.interrupt.nr = env->interrupt_injected;
1219
    events.interrupt.soft = env->soft_interrupt;
1220

    
1221
    events.nmi.injected = env->nmi_injected;
1222
    events.nmi.pending = env->nmi_pending;
1223
    events.nmi.masked = !!(env->hflags2 & HF2_NMI_MASK);
1224

    
1225
    events.sipi_vector = env->sipi_vector;
1226

    
1227
    events.flags = 0;
1228
    if (level >= KVM_PUT_RESET_STATE) {
1229
        events.flags |=
1230
            KVM_VCPUEVENT_VALID_NMI_PENDING | KVM_VCPUEVENT_VALID_SIPI_VECTOR;
1231
    }
1232

    
1233
    return kvm_vcpu_ioctl(env, KVM_SET_VCPU_EVENTS, &events);
1234
#else
1235
    return 0;
1236
#endif
1237
}
1238

    
1239
static int kvm_get_vcpu_events(CPUState *env)
1240
{
1241
#ifdef KVM_CAP_VCPU_EVENTS
1242
    struct kvm_vcpu_events events;
1243
    int ret;
1244

    
1245
    if (!kvm_has_vcpu_events()) {
1246
        return 0;
1247
    }
1248

    
1249
    ret = kvm_vcpu_ioctl(env, KVM_GET_VCPU_EVENTS, &events);
1250
    if (ret < 0) {
1251
       return ret;
1252
    }
1253
    env->exception_injected =
1254
       events.exception.injected ? events.exception.nr : -1;
1255
    env->has_error_code = events.exception.has_error_code;
1256
    env->error_code = events.exception.error_code;
1257

    
1258
    env->interrupt_injected =
1259
        events.interrupt.injected ? events.interrupt.nr : -1;
1260
    env->soft_interrupt = events.interrupt.soft;
1261

    
1262
    env->nmi_injected = events.nmi.injected;
1263
    env->nmi_pending = events.nmi.pending;
1264
    if (events.nmi.masked) {
1265
        env->hflags2 |= HF2_NMI_MASK;
1266
    } else {
1267
        env->hflags2 &= ~HF2_NMI_MASK;
1268
    }
1269

    
1270
    env->sipi_vector = events.sipi_vector;
1271
#endif
1272

    
1273
    return 0;
1274
}
1275

    
1276
static int kvm_guest_debug_workarounds(CPUState *env)
1277
{
1278
    int ret = 0;
1279
#ifdef KVM_CAP_SET_GUEST_DEBUG
1280
    unsigned long reinject_trap = 0;
1281

    
1282
    if (!kvm_has_vcpu_events()) {
1283
        if (env->exception_injected == 1) {
1284
            reinject_trap = KVM_GUESTDBG_INJECT_DB;
1285
        } else if (env->exception_injected == 3) {
1286
            reinject_trap = KVM_GUESTDBG_INJECT_BP;
1287
        }
1288
        env->exception_injected = -1;
1289
    }
1290

    
1291
    /*
1292
     * Kernels before KVM_CAP_X86_ROBUST_SINGLESTEP overwrote flags.TF
1293
     * injected via SET_GUEST_DEBUG while updating GP regs. Work around this
1294
     * by updating the debug state once again if single-stepping is on.
1295
     * Another reason to call kvm_update_guest_debug here is a pending debug
1296
     * trap raise by the guest. On kernels without SET_VCPU_EVENTS we have to
1297
     * reinject them via SET_GUEST_DEBUG.
1298
     */
1299
    if (reinject_trap ||
1300
        (!kvm_has_robust_singlestep() && env->singlestep_enabled)) {
1301
        ret = kvm_update_guest_debug(env, reinject_trap);
1302
    }
1303
#endif /* KVM_CAP_SET_GUEST_DEBUG */
1304
    return ret;
1305
}
1306

    
1307
static int kvm_put_debugregs(CPUState *env)
1308
{
1309
#ifdef KVM_CAP_DEBUGREGS
1310
    struct kvm_debugregs dbgregs;
1311
    int i;
1312

    
1313
    if (!kvm_has_debugregs()) {
1314
        return 0;
1315
    }
1316

    
1317
    for (i = 0; i < 4; i++) {
1318
        dbgregs.db[i] = env->dr[i];
1319
    }
1320
    dbgregs.dr6 = env->dr[6];
1321
    dbgregs.dr7 = env->dr[7];
1322
    dbgregs.flags = 0;
1323

    
1324
    return kvm_vcpu_ioctl(env, KVM_SET_DEBUGREGS, &dbgregs);
1325
#else
1326
    return 0;
1327
#endif
1328
}
1329

    
1330
static int kvm_get_debugregs(CPUState *env)
1331
{
1332
#ifdef KVM_CAP_DEBUGREGS
1333
    struct kvm_debugregs dbgregs;
1334
    int i, ret;
1335

    
1336
    if (!kvm_has_debugregs()) {
1337
        return 0;
1338
    }
1339

    
1340
    ret = kvm_vcpu_ioctl(env, KVM_GET_DEBUGREGS, &dbgregs);
1341
    if (ret < 0) {
1342
        return ret;
1343
    }
1344
    for (i = 0; i < 4; i++) {
1345
        env->dr[i] = dbgregs.db[i];
1346
    }
1347
    env->dr[4] = env->dr[6] = dbgregs.dr6;
1348
    env->dr[5] = env->dr[7] = dbgregs.dr7;
1349
#endif
1350

    
1351
    return 0;
1352
}
1353

    
1354
int kvm_arch_put_registers(CPUState *env, int level)
1355
{
1356
    int ret;
1357

    
1358
    assert(cpu_is_stopped(env) || qemu_cpu_self(env));
1359

    
1360
    ret = kvm_getput_regs(env, 1);
1361
    if (ret < 0) {
1362
        return ret;
1363
    }
1364
    ret = kvm_put_xsave(env);
1365
    if (ret < 0) {
1366
        return ret;
1367
    }
1368
    ret = kvm_put_xcrs(env);
1369
    if (ret < 0) {
1370
        return ret;
1371
    }
1372
    ret = kvm_put_sregs(env);
1373
    if (ret < 0) {
1374
        return ret;
1375
    }
1376
    ret = kvm_put_msrs(env, level);
1377
    if (ret < 0) {
1378
        return ret;
1379
    }
1380
    if (level >= KVM_PUT_RESET_STATE) {
1381
        ret = kvm_put_mp_state(env);
1382
        if (ret < 0) {
1383
            return ret;
1384
        }
1385
    }
1386
    ret = kvm_put_vcpu_events(env, level);
1387
    if (ret < 0) {
1388
        return ret;
1389
    }
1390
    ret = kvm_put_debugregs(env);
1391
    if (ret < 0) {
1392
        return ret;
1393
    }
1394
    /* must be last */
1395
    ret = kvm_guest_debug_workarounds(env);
1396
    if (ret < 0) {
1397
        return ret;
1398
    }
1399
    return 0;
1400
}
1401

    
1402
int kvm_arch_get_registers(CPUState *env)
1403
{
1404
    int ret;
1405

    
1406
    assert(cpu_is_stopped(env) || qemu_cpu_self(env));
1407

    
1408
    ret = kvm_getput_regs(env, 0);
1409
    if (ret < 0) {
1410
        return ret;
1411
    }
1412
    ret = kvm_get_xsave(env);
1413
    if (ret < 0) {
1414
        return ret;
1415
    }
1416
    ret = kvm_get_xcrs(env);
1417
    if (ret < 0) {
1418
        return ret;
1419
    }
1420
    ret = kvm_get_sregs(env);
1421
    if (ret < 0) {
1422
        return ret;
1423
    }
1424
    ret = kvm_get_msrs(env);
1425
    if (ret < 0) {
1426
        return ret;
1427
    }
1428
    ret = kvm_get_mp_state(env);
1429
    if (ret < 0) {
1430
        return ret;
1431
    }
1432
    ret = kvm_get_vcpu_events(env);
1433
    if (ret < 0) {
1434
        return ret;
1435
    }
1436
    ret = kvm_get_debugregs(env);
1437
    if (ret < 0) {
1438
        return ret;
1439
    }
1440
    return 0;
1441
}
1442

    
1443
int kvm_arch_pre_run(CPUState *env, struct kvm_run *run)
1444
{
1445
    /* Inject NMI */
1446
    if (env->interrupt_request & CPU_INTERRUPT_NMI) {
1447
        env->interrupt_request &= ~CPU_INTERRUPT_NMI;
1448
        DPRINTF("injected NMI\n");
1449
        kvm_vcpu_ioctl(env, KVM_NMI);
1450
    }
1451

    
1452
    if (!kvm_irqchip_in_kernel()) {
1453
        /* Force the VCPU out of its inner loop to process the INIT request */
1454
        if (env->interrupt_request & CPU_INTERRUPT_INIT) {
1455
            env->exit_request = 1;
1456
        }
1457

    
1458
        /* Try to inject an interrupt if the guest can accept it */
1459
        if (run->ready_for_interrupt_injection &&
1460
            (env->interrupt_request & CPU_INTERRUPT_HARD) &&
1461
            (env->eflags & IF_MASK)) {
1462
            int irq;
1463

    
1464
            env->interrupt_request &= ~CPU_INTERRUPT_HARD;
1465
            irq = cpu_get_pic_interrupt(env);
1466
            if (irq >= 0) {
1467
                struct kvm_interrupt intr;
1468

    
1469
                intr.irq = irq;
1470
                /* FIXME: errors */
1471
                DPRINTF("injected interrupt %d\n", irq);
1472
                kvm_vcpu_ioctl(env, KVM_INTERRUPT, &intr);
1473
            }
1474
        }
1475

    
1476
        /* If we have an interrupt but the guest is not ready to receive an
1477
         * interrupt, request an interrupt window exit.  This will
1478
         * cause a return to userspace as soon as the guest is ready to
1479
         * receive interrupts. */
1480
        if ((env->interrupt_request & CPU_INTERRUPT_HARD)) {
1481
            run->request_interrupt_window = 1;
1482
        } else {
1483
            run->request_interrupt_window = 0;
1484
        }
1485

    
1486
        DPRINTF("setting tpr\n");
1487
        run->cr8 = cpu_get_apic_tpr(env->apic_state);
1488
    }
1489

    
1490
    return 0;
1491
}
1492

    
1493
int kvm_arch_post_run(CPUState *env, struct kvm_run *run)
1494
{
1495
    if (run->if_flag) {
1496
        env->eflags |= IF_MASK;
1497
    } else {
1498
        env->eflags &= ~IF_MASK;
1499
    }
1500
    cpu_set_apic_tpr(env->apic_state, run->cr8);
1501
    cpu_set_apic_base(env->apic_state, run->apic_base);
1502

    
1503
    return 0;
1504
}
1505

    
1506
int kvm_arch_process_irqchip_events(CPUState *env)
1507
{
1508
    if (kvm_irqchip_in_kernel()) {
1509
        return 0;
1510
    }
1511

    
1512
    if (env->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI)) {
1513
        env->halted = 0;
1514
    }
1515
    if (env->interrupt_request & CPU_INTERRUPT_INIT) {
1516
        kvm_cpu_synchronize_state(env);
1517
        do_cpu_init(env);
1518
    }
1519
    if (env->interrupt_request & CPU_INTERRUPT_SIPI) {
1520
        kvm_cpu_synchronize_state(env);
1521
        do_cpu_sipi(env);
1522
    }
1523

    
1524
    return env->halted;
1525
}
1526

    
1527
static int kvm_handle_halt(CPUState *env)
1528
{
1529
    if (!((env->interrupt_request & CPU_INTERRUPT_HARD) &&
1530
          (env->eflags & IF_MASK)) &&
1531
        !(env->interrupt_request & CPU_INTERRUPT_NMI)) {
1532
        env->halted = 1;
1533
        return 0;
1534
    }
1535

    
1536
    return 1;
1537
}
1538

    
1539
static bool host_supports_vmx(void)
1540
{
1541
    uint32_t ecx, unused;
1542

    
1543
    host_cpuid(1, 0, &unused, &unused, &ecx, &unused);
1544
    return ecx & CPUID_EXT_VMX;
1545
}
1546

    
1547
#define VMX_INVALID_GUEST_STATE 0x80000021
1548

    
1549
int kvm_arch_handle_exit(CPUState *env, struct kvm_run *run)
1550
{
1551
    uint64_t code;
1552
    int ret = 0;
1553

    
1554
    switch (run->exit_reason) {
1555
    case KVM_EXIT_HLT:
1556
        DPRINTF("handle_hlt\n");
1557
        ret = kvm_handle_halt(env);
1558
        break;
1559
    case KVM_EXIT_SET_TPR:
1560
        ret = 1;
1561
        break;
1562
    case KVM_EXIT_FAIL_ENTRY:
1563
        code = run->fail_entry.hardware_entry_failure_reason;
1564
        fprintf(stderr, "KVM: entry failed, hardware error 0x%" PRIx64 "\n",
1565
                code);
1566
        if (host_supports_vmx() && code == VMX_INVALID_GUEST_STATE) {
1567
            fprintf(stderr,
1568
                    "\nIf you're runnning a guest on an Intel machine without "
1569
                        "unrestricted mode\n"
1570
                    "support, the failure can be most likely due to the guest "
1571
                        "entering an invalid\n"
1572
                    "state for Intel VT. For example, the guest maybe running "
1573
                        "in big real mode\n"
1574
                    "which is not supported on less recent Intel processors."
1575
                        "\n\n");
1576
        }
1577
        ret = -1;
1578
        break;
1579
    case KVM_EXIT_EXCEPTION:
1580
        fprintf(stderr, "KVM: exception %d exit (error code 0x%x)\n",
1581
                run->ex.exception, run->ex.error_code);
1582
        ret = -1;
1583
        break;
1584
    default:
1585
        fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
1586
        ret = -1;
1587
        break;
1588
    }
1589

    
1590
    return ret;
1591
}
1592

    
1593
#ifdef KVM_CAP_SET_GUEST_DEBUG
1594
int kvm_arch_insert_sw_breakpoint(CPUState *env, struct kvm_sw_breakpoint *bp)
1595
{
1596
    static const uint8_t int3 = 0xcc;
1597

    
1598
    if (cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 0) ||
1599
        cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&int3, 1, 1)) {
1600
        return -EINVAL;
1601
    }
1602
    return 0;
1603
}
1604

    
1605
int kvm_arch_remove_sw_breakpoint(CPUState *env, struct kvm_sw_breakpoint *bp)
1606
{
1607
    uint8_t int3;
1608

    
1609
    if (cpu_memory_rw_debug(env, bp->pc, &int3, 1, 0) || int3 != 0xcc ||
1610
        cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 1)) {
1611
        return -EINVAL;
1612
    }
1613
    return 0;
1614
}
1615

    
1616
static struct {
1617
    target_ulong addr;
1618
    int len;
1619
    int type;
1620
} hw_breakpoint[4];
1621

    
1622
static int nb_hw_breakpoint;
1623

    
1624
static int find_hw_breakpoint(target_ulong addr, int len, int type)
1625
{
1626
    int n;
1627

    
1628
    for (n = 0; n < nb_hw_breakpoint; n++) {
1629
        if (hw_breakpoint[n].addr == addr && hw_breakpoint[n].type == type &&
1630
            (hw_breakpoint[n].len == len || len == -1)) {
1631
            return n;
1632
        }
1633
    }
1634
    return -1;
1635
}
1636

    
1637
int kvm_arch_insert_hw_breakpoint(target_ulong addr,
1638
                                  target_ulong len, int type)
1639
{
1640
    switch (type) {
1641
    case GDB_BREAKPOINT_HW:
1642
        len = 1;
1643
        break;
1644
    case GDB_WATCHPOINT_WRITE:
1645
    case GDB_WATCHPOINT_ACCESS:
1646
        switch (len) {
1647
        case 1:
1648
            break;
1649
        case 2:
1650
        case 4:
1651
        case 8:
1652
            if (addr & (len - 1)) {
1653
                return -EINVAL;
1654
            }
1655
            break;
1656
        default:
1657
            return -EINVAL;
1658
        }
1659
        break;
1660
    default:
1661
        return -ENOSYS;
1662
    }
1663

    
1664
    if (nb_hw_breakpoint == 4) {
1665
        return -ENOBUFS;
1666
    }
1667
    if (find_hw_breakpoint(addr, len, type) >= 0) {
1668
        return -EEXIST;
1669
    }
1670
    hw_breakpoint[nb_hw_breakpoint].addr = addr;
1671
    hw_breakpoint[nb_hw_breakpoint].len = len;
1672
    hw_breakpoint[nb_hw_breakpoint].type = type;
1673
    nb_hw_breakpoint++;
1674

    
1675
    return 0;
1676
}
1677

    
1678
int kvm_arch_remove_hw_breakpoint(target_ulong addr,
1679
                                  target_ulong len, int type)
1680
{
1681
    int n;
1682

    
1683
    n = find_hw_breakpoint(addr, (type == GDB_BREAKPOINT_HW) ? 1 : len, type);
1684
    if (n < 0) {
1685
        return -ENOENT;
1686
    }
1687
    nb_hw_breakpoint--;
1688
    hw_breakpoint[n] = hw_breakpoint[nb_hw_breakpoint];
1689

    
1690
    return 0;
1691
}
1692

    
1693
void kvm_arch_remove_all_hw_breakpoints(void)
1694
{
1695
    nb_hw_breakpoint = 0;
1696
}
1697

    
1698
static CPUWatchpoint hw_watchpoint;
1699

    
1700
int kvm_arch_debug(struct kvm_debug_exit_arch *arch_info)
1701
{
1702
    int handle = 0;
1703
    int n;
1704

    
1705
    if (arch_info->exception == 1) {
1706
        if (arch_info->dr6 & (1 << 14)) {
1707
            if (cpu_single_env->singlestep_enabled) {
1708
                handle = 1;
1709
            }
1710
        } else {
1711
            for (n = 0; n < 4; n++) {
1712
                if (arch_info->dr6 & (1 << n)) {
1713
                    switch ((arch_info->dr7 >> (16 + n*4)) & 0x3) {
1714
                    case 0x0:
1715
                        handle = 1;
1716
                        break;
1717
                    case 0x1:
1718
                        handle = 1;
1719
                        cpu_single_env->watchpoint_hit = &hw_watchpoint;
1720
                        hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1721
                        hw_watchpoint.flags = BP_MEM_WRITE;
1722
                        break;
1723
                    case 0x3:
1724
                        handle = 1;
1725
                        cpu_single_env->watchpoint_hit = &hw_watchpoint;
1726
                        hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1727
                        hw_watchpoint.flags = BP_MEM_ACCESS;
1728
                        break;
1729
                    }
1730
                }
1731
            }
1732
        }
1733
    } else if (kvm_find_sw_breakpoint(cpu_single_env, arch_info->pc)) {
1734
        handle = 1;
1735
    }
1736
    if (!handle) {
1737
        cpu_synchronize_state(cpu_single_env);
1738
        assert(cpu_single_env->exception_injected == -1);
1739

    
1740
        cpu_single_env->exception_injected = arch_info->exception;
1741
        cpu_single_env->has_error_code = 0;
1742
    }
1743

    
1744
    return handle;
1745
}
1746

    
1747
void kvm_arch_update_guest_debug(CPUState *env, struct kvm_guest_debug *dbg)
1748
{
1749
    const uint8_t type_code[] = {
1750
        [GDB_BREAKPOINT_HW] = 0x0,
1751
        [GDB_WATCHPOINT_WRITE] = 0x1,
1752
        [GDB_WATCHPOINT_ACCESS] = 0x3
1753
    };
1754
    const uint8_t len_code[] = {
1755
        [1] = 0x0, [2] = 0x1, [4] = 0x3, [8] = 0x2
1756
    };
1757
    int n;
1758

    
1759
    if (kvm_sw_breakpoints_active(env)) {
1760
        dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
1761
    }
1762
    if (nb_hw_breakpoint > 0) {
1763
        dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP;
1764
        dbg->arch.debugreg[7] = 0x0600;
1765
        for (n = 0; n < nb_hw_breakpoint; n++) {
1766
            dbg->arch.debugreg[n] = hw_breakpoint[n].addr;
1767
            dbg->arch.debugreg[7] |= (2 << (n * 2)) |
1768
                (type_code[hw_breakpoint[n].type] << (16 + n*4)) |
1769
                ((uint32_t)len_code[hw_breakpoint[n].len] << (18 + n*4));
1770
        }
1771
    }
1772
}
1773
#endif /* KVM_CAP_SET_GUEST_DEBUG */
1774

    
1775
bool kvm_arch_stop_on_emulation_error(CPUState *env)
1776
{
1777
    return !(env->cr[0] & CR0_PE_MASK) ||
1778
           ((env->segs[R_CS].selector  & 3) != 3);
1779
}
1780

    
1781
static void hardware_memory_error(void)
1782
{
1783
    fprintf(stderr, "Hardware memory error!\n");
1784
    exit(1);
1785
}
1786

    
1787
#ifdef KVM_CAP_MCE
1788
static void kvm_mce_broadcast_rest(CPUState *env)
1789
{
1790
    struct kvm_x86_mce mce = {
1791
        .bank = 1,
1792
        .status = MCI_STATUS_VAL | MCI_STATUS_UC,
1793
        .mcg_status = MCG_STATUS_MCIP | MCG_STATUS_RIPV,
1794
        .addr = 0,
1795
        .misc = 0,
1796
    };
1797
    CPUState *cenv;
1798

    
1799
    /* Broadcast MCA signal for processor version 06H_EH and above */
1800
    if (cpu_x86_support_mca_broadcast(env)) {
1801
        for (cenv = first_cpu; cenv != NULL; cenv = cenv->next_cpu) {
1802
            if (cenv == env) {
1803
                continue;
1804
            }
1805
            kvm_inject_x86_mce_on(cenv, &mce, ABORT_ON_ERROR);
1806
        }
1807
    }
1808
}
1809

    
1810
static void kvm_mce_inj_srar_dataload(CPUState *env, target_phys_addr_t paddr)
1811
{
1812
    struct kvm_x86_mce mce = {
1813
        .bank = 9,
1814
        .status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
1815
                  | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
1816
                  | MCI_STATUS_AR | 0x134,
1817
        .mcg_status = MCG_STATUS_MCIP | MCG_STATUS_EIPV,
1818
        .addr = paddr,
1819
        .misc = (MCM_ADDR_PHYS << 6) | 0xc,
1820
    };
1821
    int r;
1822

    
1823
    r = kvm_set_mce(env, &mce);
1824
    if (r < 0) {
1825
        fprintf(stderr, "kvm_set_mce: %s\n", strerror(errno));
1826
        abort();
1827
    }
1828
    kvm_mce_broadcast_rest(env);
1829
}
1830

    
1831
static void kvm_mce_inj_srao_memscrub(CPUState *env, target_phys_addr_t paddr)
1832
{
1833
    struct kvm_x86_mce mce = {
1834
        .bank = 9,
1835
        .status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
1836
                  | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
1837
                  | 0xc0,
1838
        .mcg_status = MCG_STATUS_MCIP | MCG_STATUS_RIPV,
1839
        .addr = paddr,
1840
        .misc = (MCM_ADDR_PHYS << 6) | 0xc,
1841
    };
1842
    int r;
1843

    
1844
    r = kvm_set_mce(env, &mce);
1845
    if (r < 0) {
1846
        fprintf(stderr, "kvm_set_mce: %s\n", strerror(errno));
1847
        abort();
1848
    }
1849
    kvm_mce_broadcast_rest(env);
1850
}
1851

    
1852
static void kvm_mce_inj_srao_memscrub2(CPUState *env, target_phys_addr_t paddr)
1853
{
1854
    struct kvm_x86_mce mce = {
1855
        .bank = 9,
1856
        .status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
1857
                  | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
1858
                  | 0xc0,
1859
        .mcg_status = MCG_STATUS_MCIP | MCG_STATUS_RIPV,
1860
        .addr = paddr,
1861
        .misc = (MCM_ADDR_PHYS << 6) | 0xc,
1862
    };
1863

    
1864
    kvm_inject_x86_mce_on(env, &mce, ABORT_ON_ERROR);
1865
    kvm_mce_broadcast_rest(env);
1866
}
1867

    
1868
#endif
1869

    
1870
int kvm_arch_on_sigbus_vcpu(CPUState *env, int code, void *addr)
1871
{
1872
#if defined(KVM_CAP_MCE)
1873
    void *vaddr;
1874
    ram_addr_t ram_addr;
1875
    target_phys_addr_t paddr;
1876

    
1877
    if ((env->mcg_cap & MCG_SER_P) && addr
1878
        && (code == BUS_MCEERR_AR
1879
            || code == BUS_MCEERR_AO)) {
1880
        vaddr = (void *)addr;
1881
        if (qemu_ram_addr_from_host(vaddr, &ram_addr) ||
1882
            !kvm_physical_memory_addr_from_ram(env->kvm_state, ram_addr, &paddr)) {
1883
            fprintf(stderr, "Hardware memory error for memory used by "
1884
                    "QEMU itself instead of guest system!\n");
1885
            /* Hope we are lucky for AO MCE */
1886
            if (code == BUS_MCEERR_AO) {
1887
                return 0;
1888
            } else {
1889
                hardware_memory_error();
1890
            }
1891
        }
1892

    
1893
        if (code == BUS_MCEERR_AR) {
1894
            /* Fake an Intel architectural Data Load SRAR UCR */
1895
            kvm_mce_inj_srar_dataload(env, paddr);
1896
        } else {
1897
            /*
1898
             * If there is an MCE excpetion being processed, ignore
1899
             * this SRAO MCE
1900
             */
1901
            if (!kvm_mce_in_progress(env)) {
1902
                /* Fake an Intel architectural Memory scrubbing UCR */
1903
                kvm_mce_inj_srao_memscrub(env, paddr);
1904
            }
1905
        }
1906
    } else
1907
#endif
1908
    {
1909
        if (code == BUS_MCEERR_AO) {
1910
            return 0;
1911
        } else if (code == BUS_MCEERR_AR) {
1912
            hardware_memory_error();
1913
        } else {
1914
            return 1;
1915
        }
1916
    }
1917
    return 0;
1918
}
1919

    
1920
int kvm_arch_on_sigbus(int code, void *addr)
1921
{
1922
#if defined(KVM_CAP_MCE)
1923
    if ((first_cpu->mcg_cap & MCG_SER_P) && addr && code == BUS_MCEERR_AO) {
1924
        void *vaddr;
1925
        ram_addr_t ram_addr;
1926
        target_phys_addr_t paddr;
1927

    
1928
        /* Hope we are lucky for AO MCE */
1929
        vaddr = addr;
1930
        if (qemu_ram_addr_from_host(vaddr, &ram_addr) ||
1931
            !kvm_physical_memory_addr_from_ram(first_cpu->kvm_state, ram_addr, &paddr)) {
1932
            fprintf(stderr, "Hardware memory error for memory used by "
1933
                    "QEMU itself instead of guest system!: %p\n", addr);
1934
            return 0;
1935
        }
1936
        kvm_mce_inj_srao_memscrub2(first_cpu, paddr);
1937
    } else
1938
#endif
1939
    {
1940
        if (code == BUS_MCEERR_AO) {
1941
            return 0;
1942
        } else if (code == BUS_MCEERR_AR) {
1943
            hardware_memory_error();
1944
        } else {
1945
            return 1;
1946
        }
1947
    }
1948
    return 0;
1949
}