arch/x86/kvm/svm.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * AMD SVM support
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   8  *
   9  * Authors:
  10  *   Yaniv Kamay  <yaniv@qumranet.com>
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17 #include <linux/kvm_host.h>
  18
  19 #include "irq.h"
  20 #include "mmu.h"
  21 #include "kvm_cache_regs.h"
  22 #include "x86.h"
  23 #include "cpuid.h"
  24 #include "pmu.h"
  25
  26 #include <linux/module.h>
  27 #include <linux/mod_devicetable.h>
  28 #include <linux/kernel.h>
  29 #include <linux/vmalloc.h>
  30 #include <linux/highmem.h>
  31 #include <linux/sched.h>
  32 #include <linux/trace_events.h>
  33 #include <linux/slab.h>
  34
  35 #include <asm/perf_event.h>
  36 #include <asm/tlbflush.h>
  37 #include <asm/desc.h>
  38 #include <asm/debugreg.h>
  39 #include <asm/kvm_para.h>
  40
  41 #include <asm/virtext.h>
  42 #include "trace.h"
  43
  44 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  45
  46 MODULE_AUTHOR("Qumranet");
  47 MODULE_LICENSE("GPL");
  48
  49 static const struct x86_cpu_id svm_cpu_id[] = {
  50         X86_FEATURE_MATCH(X86_FEATURE_SVM),
  51         {}
  52 };
  53 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
  54
  55 #define IOPM_ALLOC_ORDER 2
  56 #define MSRPM_ALLOC_ORDER 1
  57
  58 #define SEG_TYPE_LDT 2
  59 #define SEG_TYPE_BUSY_TSS16 3
  60
  61 #define SVM_FEATURE_NPT            (1 <<  0)
  62 #define SVM_FEATURE_LBRV           (1 <<  1)
  63 #define SVM_FEATURE_SVML           (1 <<  2)
  64 #define SVM_FEATURE_NRIP           (1 <<  3)
  65 #define SVM_FEATURE_TSC_RATE       (1 <<  4)
  66 #define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
  67 #define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
  68 #define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
  69 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
  70
  71 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
  72 #define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
  73 #define NESTED_EXIT_CONTINUE    2       /* Further checks needed      */
  74
  75 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
  76
  77 #define TSC_RATIO_RSVD          0xffffff0000000000ULL
  78 #define TSC_RATIO_MIN           0x0000000000000001ULL
  79 #define TSC_RATIO_MAX           0x000000ffffffffffULL
  80
  81 static bool erratum_383_found __read_mostly;
  82
  83 static const u32 host_save_user_msrs[] = {
  84 #ifdef CONFIG_X86_64
  85         MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
  86         MSR_FS_BASE,
  87 #endif
  88         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
  89         MSR_TSC_AUX,
  90 };
  91
  92 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
  93
  94 struct kvm_vcpu;
  95
  96 struct nested_state {
  97         struct vmcb *hsave;
  98         u64 hsave_msr;
  99         u64 vm_cr_msr;
 100         u64 vmcb;
 101
 102         /* These are the merged vectors */
 103         u32 *msrpm;
 104
 105         /* gpa pointers to the real vectors */
 106         u64 vmcb_msrpm;
 107         u64 vmcb_iopm;
 108
 109         /* A VMEXIT is required but not yet emulated */
 110         bool exit_required;
 111
 112         /* cache for intercepts of the guest */
 113         u32 intercept_cr;
 114         u32 intercept_dr;
 115         u32 intercept_exceptions;
 116         u64 intercept;
 117
 118         /* Nested Paging related state */
 119         u64 nested_cr3;
 120 };
 121
 122 #define MSRPM_OFFSETS   16
 123 static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
 124
 125 /*
 126  * Set osvw_len to higher value when updated Revision Guides
 127  * are published and we know what the new status bits are
 128  */
 129 static uint64_t osvw_len = 4, osvw_status;
 130
 131 struct vcpu_svm {
 132         struct kvm_vcpu vcpu;
 133         struct vmcb *vmcb;
 134         unsigned long vmcb_pa;
 135         struct svm_cpu_data *svm_data;
 136         uint64_t asid_generation;
 137         uint64_t sysenter_esp;
 138         uint64_t sysenter_eip;
 139         uint64_t tsc_aux;
 140
 141         u64 next_rip;
 142
 143         u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
 144         struct {
 145                 u16 fs;
 146                 u16 gs;
 147                 u16 ldt;
 148                 u64 gs_base;
 149         } host;
 150
 151         u32 *msrpm;
 152
 153         ulong nmi_iret_rip;
 154
 155         struct nested_state nested;
 156
 157         bool nmi_singlestep;
 158
 159         unsigned int3_injected;
 160         unsigned long int3_rip;
 161         u32 apf_reason;
 162
 163         /* cached guest cpuid flags for faster access */
 164         bool nrips_enabled      : 1;
 165 };
 166
 167 static DEFINE_PER_CPU(u64, current_tsc_ratio);
 168 #define TSC_RATIO_DEFAULT       0x0100000000ULL
 169
 170 #define MSR_INVALID                     0xffffffffU
 171
 172 static const struct svm_direct_access_msrs {
 173         u32 index;   /* Index of the MSR */
 174         bool always; /* True if intercept is always on */
 175 } direct_access_msrs[] = {
 176         { .index = MSR_STAR,                            .always = true  },
 177         { .index = MSR_IA32_SYSENTER_CS,                .always = true  },
 178 #ifdef CONFIG_X86_64
 179         { .index = MSR_GS_BASE,                         .always = true  },
 180         { .index = MSR_FS_BASE,                         .always = true  },
 181         { .index = MSR_KERNEL_GS_BASE,                  .always = true  },
 182         { .index = MSR_LSTAR,                           .always = true  },
 183         { .index = MSR_CSTAR,                           .always = true  },
 184         { .index = MSR_SYSCALL_MASK,                    .always = true  },
 185 #endif
 186         { .index = MSR_IA32_LASTBRANCHFROMIP,           .always = false },
 187         { .index = MSR_IA32_LASTBRANCHTOIP,             .always = false },
 188         { .index = MSR_IA32_LASTINTFROMIP,              .always = false },
 189         { .index = MSR_IA32_LASTINTTOIP,                .always = false },
 190         { .index = MSR_INVALID,                         .always = false },
 191 };
 192
 193 /* enable NPT for AMD64 and X86 with PAE */
 194 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 195 static bool npt_enabled = true;
 196 #else
 197 static bool npt_enabled;
 198 #endif
 199
 200 /* allow nested paging (virtualized MMU) for all guests */
 201 static int npt = true;
 202 module_param(npt, int, S_IRUGO);
 203
 204 /* allow nested virtualization in KVM/SVM */
 205 static int nested = true;
 206 module_param(nested, int, S_IRUGO);
 207
 208 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
 209 static void svm_flush_tlb(struct kvm_vcpu *vcpu);
 210 static void svm_complete_interrupts(struct vcpu_svm *svm);
 211
 212 static int nested_svm_exit_handled(struct vcpu_svm *svm);
 213 static int nested_svm_intercept(struct vcpu_svm *svm);
 214 static int nested_svm_vmexit(struct vcpu_svm *svm);
 215 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 216                                       bool has_error_code, u32 error_code);
 217
 218 enum {
 219         VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
 220                             pause filter count */
 221         VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
 222         VMCB_ASID,       /* ASID */
 223         VMCB_INTR,       /* int_ctl, int_vector */
 224         VMCB_NPT,        /* npt_en, nCR3, gPAT */
 225         VMCB_CR,         /* CR0, CR3, CR4, EFER */
 226         VMCB_DR,         /* DR6, DR7 */
 227         VMCB_DT,         /* GDT, IDT */
 228         VMCB_SEG,        /* CS, DS, SS, ES, CPL */
 229         VMCB_CR2,        /* CR2 only */
 230         VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
 231         VMCB_DIRTY_MAX,
 232 };
 233
 234 /* TPR and CR2 are always written before VMRUN */
 235 #define VMCB_ALWAYS_DIRTY_MASK  ((1U << VMCB_INTR) | (1U << VMCB_CR2))
 236
 237 static inline void mark_all_dirty(struct vmcb *vmcb)
 238 {
 239         vmcb->control.clean = 0;
 240 }
 241
 242 static inline void mark_all_clean(struct vmcb *vmcb)
 243 {
 244         vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
 245                                & ~VMCB_ALWAYS_DIRTY_MASK;
 246 }
 247
 248 static inline void mark_dirty(struct vmcb *vmcb, int bit)
 249 {
 250         vmcb->control.clean &= ~(1 << bit);
 251 }
 252
 253 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 254 {
 255         return container_of(vcpu, struct vcpu_svm, vcpu);
 256 }
 257
 258 static void recalc_intercepts(struct vcpu_svm *svm)
 259 {
 260         struct vmcb_control_area *c, *h;
 261         struct nested_state *g;
 262
 263         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 264
 265         if (!is_guest_mode(&svm->vcpu))
 266                 return;
 267
 268         c = &svm->vmcb->control;
 269         h = &svm->nested.hsave->control;
 270         g = &svm->nested;
 271
 272         c->intercept_cr = h->intercept_cr | g->intercept_cr;
 273         c->intercept_dr = h->intercept_dr | g->intercept_dr;
 274         c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
 275         c->intercept = h->intercept | g->intercept;
 276 }
 277
 278 static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
 279 {
 280         if (is_guest_mode(&svm->vcpu))
 281                 return svm->nested.hsave;
 282         else
 283                 return svm->vmcb;
 284 }
 285
 286 static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
 287 {
 288         struct vmcb *vmcb = get_host_vmcb(svm);
 289
 290         vmcb->control.intercept_cr |= (1U << bit);
 291
 292         recalc_intercepts(svm);
 293 }
 294
 295 static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
 296 {
 297         struct vmcb *vmcb = get_host_vmcb(svm);
 298
 299         vmcb->control.intercept_cr &= ~(1U << bit);
 300
 301         recalc_intercepts(svm);
 302 }
 303
 304 static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
 305 {
 306         struct vmcb *vmcb = get_host_vmcb(svm);
 307
 308         return vmcb->control.intercept_cr & (1U << bit);
 309 }
 310
 311 static inline void set_dr_intercepts(struct vcpu_svm *svm)
 312 {
 313         struct vmcb *vmcb = get_host_vmcb(svm);
 314
 315         vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
 316                 | (1 << INTERCEPT_DR1_READ)
 317                 | (1 << INTERCEPT_DR2_READ)
 318                 | (1 << INTERCEPT_DR3_READ)
 319                 | (1 << INTERCEPT_DR4_READ)
 320                 | (1 << INTERCEPT_DR5_READ)
 321                 | (1 << INTERCEPT_DR6_READ)
 322                 | (1 << INTERCEPT_DR7_READ)
 323                 | (1 << INTERCEPT_DR0_WRITE)
 324                 | (1 << INTERCEPT_DR1_WRITE)
 325                 | (1 << INTERCEPT_DR2_WRITE)
 326                 | (1 << INTERCEPT_DR3_WRITE)
 327                 | (1 << INTERCEPT_DR4_WRITE)
 328                 | (1 << INTERCEPT_DR5_WRITE)
 329                 | (1 << INTERCEPT_DR6_WRITE)
 330                 | (1 << INTERCEPT_DR7_WRITE);
 331
 332         recalc_intercepts(svm);
 333 }
 334
 335 static inline void clr_dr_intercepts(struct vcpu_svm *svm)
 336 {
 337         struct vmcb *vmcb = get_host_vmcb(svm);
 338
 339         vmcb->control.intercept_dr = 0;
 340
 341         recalc_intercepts(svm);
 342 }
 343
 344 static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
 345 {
 346         struct vmcb *vmcb = get_host_vmcb(svm);
 347
 348         vmcb->control.intercept_exceptions |= (1U << bit);
 349
 350         recalc_intercepts(svm);
 351 }
 352
 353 static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
 354 {
 355         struct vmcb *vmcb = get_host_vmcb(svm);
 356
 357         vmcb->control.intercept_exceptions &= ~(1U << bit);
 358
 359         recalc_intercepts(svm);
 360 }
 361
 362 static inline void set_intercept(struct vcpu_svm *svm, int bit)
 363 {
 364         struct vmcb *vmcb = get_host_vmcb(svm);
 365
 366         vmcb->control.intercept |= (1ULL << bit);
 367
 368         recalc_intercepts(svm);
 369 }
 370
 371 static inline void clr_intercept(struct vcpu_svm *svm, int bit)
 372 {
 373         struct vmcb *vmcb = get_host_vmcb(svm);
 374
 375         vmcb->control.intercept &= ~(1ULL << bit);
 376
 377         recalc_intercepts(svm);
 378 }
 379
 380 static inline void enable_gif(struct vcpu_svm *svm)
 381 {
 382         svm->vcpu.arch.hflags |= HF_GIF_MASK;
 383 }
 384
 385 static inline void disable_gif(struct vcpu_svm *svm)
 386 {
 387         svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
 388 }
 389
 390 static inline bool gif_set(struct vcpu_svm *svm)
 391 {
 392         return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
 393 }
 394
 395 static unsigned long iopm_base;
 396
 397 struct kvm_ldttss_desc {
 398         u16 limit0;
 399         u16 base0;
 400         unsigned base1:8, type:5, dpl:2, p:1;
 401         unsigned limit1:4, zero0:3, g:1, base2:8;
 402         u32 base3;
 403         u32 zero1;
 404 } __attribute__((packed));
 405
 406 struct svm_cpu_data {
 407         int cpu;
 408
 409         u64 asid_generation;
 410         u32 max_asid;
 411         u32 next_asid;
 412         struct kvm_ldttss_desc *tss_desc;
 413
 414         struct page *save_area;
 415 };
 416
 417 static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
 418
 419 struct svm_init_data {
 420         int cpu;
 421         int r;
 422 };
 423
 424 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
 425
 426 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
 427 #define MSRS_RANGE_SIZE 2048
 428 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
 429
 430 static u32 svm_msrpm_offset(u32 msr)
 431 {
 432         u32 offset;
 433         int i;
 434
 435         for (i = 0; i < NUM_MSR_MAPS; i++) {
 436                 if (msr < msrpm_ranges[i] ||
 437                     msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
 438                         continue;
 439
 440                 offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
 441                 offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
 442
 443                 /* Now we have the u8 offset - but need the u32 offset */
 444                 return offset / 4;
 445         }
 446
 447         /* MSR not in any range */
 448         return MSR_INVALID;
 449 }
 450
 451 #define MAX_INST_SIZE 15
 452
 453 static inline void clgi(void)
 454 {
 455         asm volatile (__ex(SVM_CLGI));
 456 }
 457
 458 static inline void stgi(void)
 459 {
 460         asm volatile (__ex(SVM_STGI));
 461 }
 462
 463 static inline void invlpga(unsigned long addr, u32 asid)
 464 {
 465         asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
 466 }
 467
 468 static int get_npt_level(void)
 469 {
 470 #ifdef CONFIG_X86_64
 471         return PT64_ROOT_LEVEL;
 472 #else
 473         return PT32E_ROOT_LEVEL;
 474 #endif
 475 }
 476
 477 static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 478 {
 479         vcpu->arch.efer = efer;
 480         if (!npt_enabled && !(efer & EFER_LMA))
 481                 efer &= ~EFER_LME;
 482
 483         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
 484         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 485 }
 486
 487 static int is_external_interrupt(u32 info)
 488 {
 489         info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
 490         return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
 491 }
 492
 493 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 494 {
 495         struct vcpu_svm *svm = to_svm(vcpu);
 496         u32 ret = 0;
 497
 498         if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
 499                 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
 500         return ret;
 501 }
 502
 503 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 504 {
 505         struct vcpu_svm *svm = to_svm(vcpu);
 506
 507         if (mask == 0)
 508                 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
 509         else
 510                 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
 511
 512 }
 513
 514 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
 515 {
 516         struct vcpu_svm *svm = to_svm(vcpu);
 517
 518         if (svm->vmcb->control.next_rip != 0) {
 519                 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
 520                 svm->next_rip = svm->vmcb->control.next_rip;
 521         }
 522
 523         if (!svm->next_rip) {
 524                 if (emulate_instruction(vcpu, EMULTYPE_SKIP) !=
 525                                 EMULATE_DONE)
 526                         printk(KERN_DEBUG "%s: NOP\n", __func__);
 527                 return;
 528         }
 529         if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
 530                 printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
 531                        __func__, kvm_rip_read(vcpu), svm->next_rip);
 532
 533         kvm_rip_write(vcpu, svm->next_rip);
 534         svm_set_interrupt_shadow(vcpu, 0);
 535 }
 536
 537 static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
 538                                 bool has_error_code, u32 error_code,
 539                                 bool reinject)
 540 {
 541         struct vcpu_svm *svm = to_svm(vcpu);
 542
 543         /*
 544          * If we are within a nested VM we'd better #VMEXIT and let the guest
 545          * handle the exception
 546          */
 547         if (!reinject &&
 548             nested_svm_check_exception(svm, nr, has_error_code, error_code))
 549                 return;
 550
 551         if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) {
 552                 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
 553
 554                 /*
 555                  * For guest debugging where we have to reinject #BP if some
 556                  * INT3 is guest-owned:
 557                  * Emulate nRIP by moving RIP forward. Will fail if injection
 558                  * raises a fault that is not intercepted. Still better than
 559                  * failing in all cases.
 560                  */
 561                 skip_emulated_instruction(&svm->vcpu);
 562                 rip = kvm_rip_read(&svm->vcpu);
 563                 svm->int3_rip = rip + svm->vmcb->save.cs.base;
 564                 svm->int3_injected = rip - old_rip;
 565         }
 566
 567         svm->vmcb->control.event_inj = nr
 568                 | SVM_EVTINJ_VALID
 569                 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
 570                 | SVM_EVTINJ_TYPE_EXEPT;
 571         svm->vmcb->control.event_inj_err = error_code;
 572 }
 573
 574 static void svm_init_erratum_383(void)
 575 {
 576         u32 low, high;
 577         int err;
 578         u64 val;
 579
 580         if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
 581                 return;
 582
 583         /* Use _safe variants to not break nested virtualization */
 584         val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
 585         if (err)
 586                 return;
 587
 588         val |= (1ULL << 47);
 589
 590         low  = lower_32_bits(val);
 591         high = upper_32_bits(val);
 592
 593         native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
 594
 595         erratum_383_found = true;
 596 }
 597
 598 static void svm_init_osvw(struct kvm_vcpu *vcpu)
 599 {
 600         /*
 601          * Guests should see errata 400 and 415 as fixed (assuming that
 602          * HLT and IO instructions are intercepted).
 603          */
 604         vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
 605         vcpu->arch.osvw.status = osvw_status & ~(6ULL);
 606
 607         /*
 608          * By increasing VCPU's osvw.length to 3 we are telling the guest that
 609          * all osvw.status bits inside that length, including bit 0 (which is
 610          * reserved for erratum 298), are valid. However, if host processor's
 611          * osvw_len is 0 then osvw_status[0] carries no information. We need to
 612          * be conservative here and therefore we tell the guest that erratum 298
 613          * is present (because we really don't know).
 614          */
 615         if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
 616                 vcpu->arch.osvw.status |= 1;
 617 }
 618
 619 static int has_svm(void)
 620 {
 621         const char *msg;
 622
 623         if (!cpu_has_svm(&msg)) {
 624                 printk(KERN_INFO "has_svm: %s\n", msg);
 625                 return 0;
 626         }
 627
 628         return 1;
 629 }
 630
 631 static void svm_hardware_disable(void)
 632 {
 633         /* Make sure we clean up behind us */
 634         if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
 635                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 636
 637         cpu_svm_disable();
 638
 639         amd_pmu_disable_virt();
 640 }
 641
 642 static int svm_hardware_enable(void)
 643 {
 644
 645         struct svm_cpu_data *sd;
 646         uint64_t efer;
 647         struct desc_ptr gdt_descr;
 648         struct desc_struct *gdt;
 649         int me = raw_smp_processor_id();
 650
 651         rdmsrl(MSR_EFER, efer);
 652         if (efer & EFER_SVME)
 653                 return -EBUSY;
 654
 655         if (!has_svm()) {
 656                 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
 657                 return -EINVAL;
 658         }
 659         sd = per_cpu(svm_data, me);
 660         if (!sd) {
 661                 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
 662                 return -EINVAL;
 663         }
 664
 665         sd->asid_generation = 1;
 666         sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 667         sd->next_asid = sd->max_asid + 1;
 668
 669         native_store_gdt(&gdt_descr);
 670         gdt = (struct desc_struct *)gdt_descr.address;
 671         sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 672
 673         wrmsrl(MSR_EFER, efer | EFER_SVME);
 674
 675         wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
 676
 677         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 678                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 679                 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
 680         }
 681
 682
 683         /*
 684          * Get OSVW bits.
 685          *
 686          * Note that it is possible to have a system with mixed processor
 687          * revisions and therefore different OSVW bits. If bits are not the same
 688          * on different processors then choose the worst case (i.e. if erratum
 689          * is present on one processor and not on another then assume that the
 690          * erratum is present everywhere).
 691          */
 692         if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
 693                 uint64_t len, status = 0;
 694                 int err;
 695
 696                 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
 697                 if (!err)
 698                         status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
 699                                                       &err);
 700
 701                 if (err)
 702                         osvw_status = osvw_len = 0;
 703                 else {
 704                         if (len < osvw_len)
 705                                 osvw_len = len;
 706                         osvw_status |= status;
 707                         osvw_status &= (1ULL << osvw_len) - 1;
 708                 }
 709         } else
 710                 osvw_status = osvw_len = 0;
 711
 712         svm_init_erratum_383();
 713
 714         amd_pmu_enable_virt();
 715
 716         return 0;
 717 }
 718
 719 static void svm_cpu_uninit(int cpu)
 720 {
 721         struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
 722
 723         if (!sd)
 724                 return;
 725
 726         per_cpu(svm_data, raw_smp_processor_id()) = NULL;
 727         __free_page(sd->save_area);
 728         kfree(sd);
 729 }
 730
 731 static int svm_cpu_init(int cpu)
 732 {
 733         struct svm_cpu_data *sd;
 734         int r;
 735
 736         sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
 737         if (!sd)
 738                 return -ENOMEM;
 739         sd->cpu = cpu;
 740         sd->save_area = alloc_page(GFP_KERNEL);
 741         r = -ENOMEM;
 742         if (!sd->save_area)
 743                 goto err_1;
 744
 745         per_cpu(svm_data, cpu) = sd;
 746
 747         return 0;
 748
 749 err_1:
 750         kfree(sd);
 751         return r;
 752
 753 }
 754
 755 static bool valid_msr_intercept(u32 index)
 756 {
 757         int i;
 758
 759         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
 760                 if (direct_access_msrs[i].index == index)
 761                         return true;
 762
 763         return false;
 764 }
 765
 766 static void set_msr_interception(u32 *msrpm, unsigned msr,
 767                                  int read, int write)
 768 {
 769         u8 bit_read, bit_write;
 770         unsigned long tmp;
 771         u32 offset;
 772
 773         /*
 774          * If this warning triggers extend the direct_access_msrs list at the
 775          * beginning of the file
 776          */
 777         WARN_ON(!valid_msr_intercept(msr));
 778
 779         offset    = svm_msrpm_offset(msr);
 780         bit_read  = 2 * (msr & 0x0f);
 781         bit_write = 2 * (msr & 0x0f) + 1;
 782         tmp       = msrpm[offset];
 783
 784         BUG_ON(offset == MSR_INVALID);
 785
 786         read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
 787         write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
 788
 789         msrpm[offset] = tmp;
 790 }
 791
 792 static void svm_vcpu_init_msrpm(u32 *msrpm)
 793 {
 794         int i;
 795
 796         memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
 797
 798         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 799                 if (!direct_access_msrs[i].always)
 800                         continue;
 801
 802                 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
 803         }
 804 }
 805
 806 static void add_msr_offset(u32 offset)
 807 {
 808         int i;
 809
 810         for (i = 0; i < MSRPM_OFFSETS; ++i) {
 811
 812                 /* Offset already in list? */
 813                 if (msrpm_offsets[i] == offset)
 814                         return;
 815
 816                 /* Slot used by another offset? */
 817                 if (msrpm_offsets[i] != MSR_INVALID)
 818                         continue;
 819
 820                 /* Add offset to list */
 821                 msrpm_offsets[i] = offset;
 822
 823                 return;
 824         }
 825
 826         /*
 827          * If this BUG triggers the msrpm_offsets table has an overflow. Just
 828          * increase MSRPM_OFFSETS in this case.
 829          */
 830         BUG();
 831 }
 832
 833 static void init_msrpm_offsets(void)
 834 {
 835         int i;
 836
 837         memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
 838
 839         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 840                 u32 offset;
 841
 842                 offset = svm_msrpm_offset(direct_access_msrs[i].index);
 843                 BUG_ON(offset == MSR_INVALID);
 844
 845                 add_msr_offset(offset);
 846         }
 847 }
 848
 849 static void svm_enable_lbrv(struct vcpu_svm *svm)
 850 {
 851         u32 *msrpm = svm->msrpm;
 852
 853         svm->vmcb->control.lbr_ctl = 1;
 854         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
 855         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
 856         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
 857         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
 858 }
 859
 860 static void svm_disable_lbrv(struct vcpu_svm *svm)
 861 {
 862         u32 *msrpm = svm->msrpm;
 863
 864         svm->vmcb->control.lbr_ctl = 0;
 865         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
 866         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
 867         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
 868         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
 869 }
 870
 871 static __init int svm_hardware_setup(void)
 872 {
 873         int cpu;
 874         struct page *iopm_pages;
 875         void *iopm_va;
 876         int r;
 877
 878         iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
 879
 880         if (!iopm_pages)
 881                 return -ENOMEM;
 882
 883         iopm_va = page_address(iopm_pages);
 884         memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
 885         iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
 886
 887         init_msrpm_offsets();
 888
 889         if (boot_cpu_has(X86_FEATURE_NX))
 890                 kvm_enable_efer_bits(EFER_NX);
 891
 892         if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
 893                 kvm_enable_efer_bits(EFER_FFXSR);
 894
 895         if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 896                 kvm_has_tsc_control = true;
 897                 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
 898                 kvm_tsc_scaling_ratio_frac_bits = 32;
 899         }
 900
 901         if (nested) {
 902                 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
 903                 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
 904         }
 905
 906         for_each_possible_cpu(cpu) {
 907                 r = svm_cpu_init(cpu);
 908                 if (r)
 909                         goto err;
 910         }
 911
 912         if (!boot_cpu_has(X86_FEATURE_NPT))
 913                 npt_enabled = false;
 914
 915         if (npt_enabled && !npt) {
 916                 printk(KERN_INFO "kvm: Nested Paging disabled\n");
 917                 npt_enabled = false;
 918         }
 919
 920         if (npt_enabled) {
 921                 printk(KERN_INFO "kvm: Nested Paging enabled\n");
 922                 kvm_enable_tdp();
 923         } else
 924                 kvm_disable_tdp();
 925
 926         return 0;
 927
 928 err:
 929         __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
 930         iopm_base = 0;
 931         return r;
 932 }
 933
 934 static __exit void svm_hardware_unsetup(void)
 935 {
 936         int cpu;
 937
 938         for_each_possible_cpu(cpu)
 939                 svm_cpu_uninit(cpu);
 940
 941         __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
 942         iopm_base = 0;
 943 }
 944
 945 static void init_seg(struct vmcb_seg *seg)
 946 {
 947         seg->selector = 0;
 948         seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
 949                       SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
 950         seg->limit = 0xffff;
 951         seg->base = 0;
 952 }
 953
 954 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
 955 {
 956         seg->selector = 0;
 957         seg->attrib = SVM_SELECTOR_P_MASK | type;
 958         seg->limit = 0xffff;
 959         seg->base = 0;
 960 }
 961
 962 static u64 svm_read_tsc_offset(struct kvm_vcpu *vcpu)
 963 {
 964         struct vcpu_svm *svm = to_svm(vcpu);
 965
 966         return svm->vmcb->control.tsc_offset;
 967 }
 968
 969 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
 970 {
 971         struct vcpu_svm *svm = to_svm(vcpu);
 972         u64 g_tsc_offset = 0;
 973
 974         if (is_guest_mode(vcpu)) {
 975                 g_tsc_offset = svm->vmcb->control.tsc_offset -
 976                                svm->nested.hsave->control.tsc_offset;
 977                 svm->nested.hsave->control.tsc_offset = offset;
 978         } else
 979                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
 980                                            svm->vmcb->control.tsc_offset,
 981                                            offset);
 982
 983         svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
 984
 985         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 986 }
 987
 988 static void svm_adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, s64 adjustment)
 989 {
 990         struct vcpu_svm *svm = to_svm(vcpu);
 991
 992         svm->vmcb->control.tsc_offset += adjustment;
 993         if (is_guest_mode(vcpu))
 994                 svm->nested.hsave->control.tsc_offset += adjustment;
 995         else
 996                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
 997                                      svm->vmcb->control.tsc_offset - adjustment,
 998                                      svm->vmcb->control.tsc_offset);
 999
1000         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1001 }
1002
1003 static void init_vmcb(struct vcpu_svm *svm)
1004 {
1005         struct vmcb_control_area *control = &svm->vmcb->control;
1006         struct vmcb_save_area *save = &svm->vmcb->save;
1007
1008         svm->vcpu.fpu_active = 1;
1009         svm->vcpu.arch.hflags = 0;
1010
1011         set_cr_intercept(svm, INTERCEPT_CR0_READ);
1012         set_cr_intercept(svm, INTERCEPT_CR3_READ);
1013         set_cr_intercept(svm, INTERCEPT_CR4_READ);
1014         set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1015         set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1016         set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
1017         set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
1018
1019         set_dr_intercepts(svm);
1020
1021         set_exception_intercept(svm, PF_VECTOR);
1022         set_exception_intercept(svm, UD_VECTOR);
1023         set_exception_intercept(svm, MC_VECTOR);
1024         set_exception_intercept(svm, AC_VECTOR);
1025         set_exception_intercept(svm, DB_VECTOR);
1026
1027         set_intercept(svm, INTERCEPT_INTR);
1028         set_intercept(svm, INTERCEPT_NMI);
1029         set_intercept(svm, INTERCEPT_SMI);
1030         set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1031         set_intercept(svm, INTERCEPT_RDPMC);
1032         set_intercept(svm, INTERCEPT_CPUID);
1033         set_intercept(svm, INTERCEPT_INVD);
1034         set_intercept(svm, INTERCEPT_HLT);
1035         set_intercept(svm, INTERCEPT_INVLPG);
1036         set_intercept(svm, INTERCEPT_INVLPGA);
1037         set_intercept(svm, INTERCEPT_IOIO_PROT);
1038         set_intercept(svm, INTERCEPT_MSR_PROT);
1039         set_intercept(svm, INTERCEPT_TASK_SWITCH);
1040         set_intercept(svm, INTERCEPT_SHUTDOWN);
1041         set_intercept(svm, INTERCEPT_VMRUN);
1042         set_intercept(svm, INTERCEPT_VMMCALL);
1043         set_intercept(svm, INTERCEPT_VMLOAD);
1044         set_intercept(svm, INTERCEPT_VMSAVE);
1045         set_intercept(svm, INTERCEPT_STGI);
1046         set_intercept(svm, INTERCEPT_CLGI);
1047         set_intercept(svm, INTERCEPT_SKINIT);
1048         set_intercept(svm, INTERCEPT_WBINVD);
1049         set_intercept(svm, INTERCEPT_MONITOR);
1050         set_intercept(svm, INTERCEPT_MWAIT);
1051         set_intercept(svm, INTERCEPT_XSETBV);
1052
1053         control->iopm_base_pa = iopm_base;
1054         control->msrpm_base_pa = __pa(svm->msrpm);
1055         control->int_ctl = V_INTR_MASKING_MASK;
1056
1057         init_seg(&save->es);
1058         init_seg(&save->ss);
1059         init_seg(&save->ds);
1060         init_seg(&save->fs);
1061         init_seg(&save->gs);
1062
1063         save->cs.selector = 0xf000;
1064         save->cs.base = 0xffff0000;
1065         /* Executable/Readable Code Segment */
1066         save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1067                 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1068         save->cs.limit = 0xffff;
1069
1070         save->gdtr.limit = 0xffff;
1071         save->idtr.limit = 0xffff;
1072
1073         init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1074         init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1075
1076         svm_set_efer(&svm->vcpu, 0);
1077         save->dr6 = 0xffff0ff0;
1078         kvm_set_rflags(&svm->vcpu, 2);
1079         save->rip = 0x0000fff0;
1080         svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1081
1082         /*
1083          * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1084          * It also updates the guest-visible cr0 value.
1085          */
1086         svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1087         kvm_mmu_reset_context(&svm->vcpu);
1088
1089         save->cr4 = X86_CR4_PAE;
1090         /* rdx = ?? */
1091
1092         if (npt_enabled) {
1093                 /* Setup VMCB for Nested Paging */
1094                 control->nested_ctl = 1;
1095                 clr_intercept(svm, INTERCEPT_INVLPG);
1096                 clr_exception_intercept(svm, PF_VECTOR);
1097                 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
1098                 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1099                 save->g_pat = svm->vcpu.arch.pat;
1100                 save->cr3 = 0;
1101                 save->cr4 = 0;
1102         }
1103         svm->asid_generation = 0;
1104
1105         svm->nested.vmcb = 0;
1106         svm->vcpu.arch.hflags = 0;
1107
1108         if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1109                 control->pause_filter_count = 3000;
1110                 set_intercept(svm, INTERCEPT_PAUSE);
1111         }
1112
1113         mark_all_dirty(svm->vmcb);
1114
1115         enable_gif(svm);
1116 }
1117
1118 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1119 {
1120         struct vcpu_svm *svm = to_svm(vcpu);
1121         u32 dummy;
1122         u32 eax = 1;
1123
1124         if (!init_event) {
1125                 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
1126                                            MSR_IA32_APICBASE_ENABLE;
1127                 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
1128                         svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
1129         }
1130         init_vmcb(svm);
1131
1132         kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy);
1133         kvm_register_write(vcpu, VCPU_REGS_RDX, eax);
1134 }
1135
1136 static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
1137 {
1138         struct vcpu_svm *svm;
1139         struct page *page;
1140         struct page *msrpm_pages;
1141         struct page *hsave_page;
1142         struct page *nested_msrpm_pages;
1143         int err;
1144
1145         svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1146         if (!svm) {
1147                 err = -ENOMEM;
1148                 goto out;
1149         }
1150
1151         err = kvm_vcpu_init(&svm->vcpu, kvm, id);
1152         if (err)
1153                 goto free_svm;
1154
1155         err = -ENOMEM;
1156         page = alloc_page(GFP_KERNEL);
1157         if (!page)
1158                 goto uninit;
1159
1160         msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
1161         if (!msrpm_pages)
1162                 goto free_page1;
1163
1164         nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
1165         if (!nested_msrpm_pages)
1166                 goto free_page2;
1167
1168         hsave_page = alloc_page(GFP_KERNEL);
1169         if (!hsave_page)
1170                 goto free_page3;
1171
1172         svm->nested.hsave = page_address(hsave_page);
1173
1174         svm->msrpm = page_address(msrpm_pages);
1175         svm_vcpu_init_msrpm(svm->msrpm);
1176
1177         svm->nested.msrpm = page_address(nested_msrpm_pages);
1178         svm_vcpu_init_msrpm(svm->nested.msrpm);
1179
1180         svm->vmcb = page_address(page);
1181         clear_page(svm->vmcb);
1182         svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
1183         svm->asid_generation = 0;
1184         init_vmcb(svm);
1185
1186         svm_init_osvw(&svm->vcpu);
1187
1188         return &svm->vcpu;
1189
1190 free_page3:
1191         __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
1192 free_page2:
1193         __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
1194 free_page1:
1195         __free_page(page);
1196 uninit:
1197         kvm_vcpu_uninit(&svm->vcpu);
1198 free_svm:
1199         kmem_cache_free(kvm_vcpu_cache, svm);
1200 out:
1201         return ERR_PTR(err);
1202 }
1203
1204 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
1205 {
1206         struct vcpu_svm *svm = to_svm(vcpu);
1207
1208         __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
1209         __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
1210         __free_page(virt_to_page(svm->nested.hsave));
1211         __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
1212         kvm_vcpu_uninit(vcpu);
1213         kmem_cache_free(kvm_vcpu_cache, svm);
1214 }
1215
1216 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1217 {
1218         struct vcpu_svm *svm = to_svm(vcpu);
1219         int i;
1220
1221         if (unlikely(cpu != vcpu->cpu)) {
1222                 svm->asid_generation = 0;
1223                 mark_all_dirty(svm->vmcb);
1224         }
1225
1226 #ifdef CONFIG_X86_64
1227         rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
1228 #endif
1229         savesegment(fs, svm->host.fs);
1230         savesegment(gs, svm->host.gs);
1231         svm->host.ldt = kvm_read_ldt();
1232
1233         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1234                 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1235
1236         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1237                 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
1238                 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
1239                         __this_cpu_write(current_tsc_ratio, tsc_ratio);
1240                         wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
1241                 }
1242         }
1243         /* This assumes that the kernel never uses MSR_TSC_AUX */
1244         if (static_cpu_has(X86_FEATURE_RDTSCP))
1245                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
1246 }
1247
1248 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1249 {
1250         struct vcpu_svm *svm = to_svm(vcpu);
1251         int i;
1252
1253         ++vcpu->stat.host_state_reload;
1254         kvm_load_ldt(svm->host.ldt);
1255 #ifdef CONFIG_X86_64
1256         loadsegment(fs, svm->host.fs);
1257         wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs);
1258         load_gs_index(svm->host.gs);
1259 #else
1260 #ifdef CONFIG_X86_32_LAZY_GS
1261         loadsegment(gs, svm->host.gs);
1262 #endif
1263 #endif
1264         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1265                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1266 }
1267
1268 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1269 {
1270         return to_svm(vcpu)->vmcb->save.rflags;
1271 }
1272
1273 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1274 {
1275        /*
1276         * Any change of EFLAGS.VM is accompained by a reload of SS
1277         * (caused by either a task switch or an inter-privilege IRET),
1278         * so we do not need to update the CPL here.
1279         */
1280         to_svm(vcpu)->vmcb->save.rflags = rflags;
1281 }
1282
1283 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1284 {
1285         switch (reg) {
1286         case VCPU_EXREG_PDPTR:
1287                 BUG_ON(!npt_enabled);
1288                 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
1289                 break;
1290         default:
1291                 BUG();
1292         }
1293 }
1294
1295 static void svm_set_vintr(struct vcpu_svm *svm)
1296 {
1297         set_intercept(svm, INTERCEPT_VINTR);
1298 }
1299
1300 static void svm_clear_vintr(struct vcpu_svm *svm)
1301 {
1302         clr_intercept(svm, INTERCEPT_VINTR);
1303 }
1304
1305 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1306 {
1307         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1308
1309         switch (seg) {
1310         case VCPU_SREG_CS: return &save->cs;
1311         case VCPU_SREG_DS: return &save->ds;
1312         case VCPU_SREG_ES: return &save->es;
1313         case VCPU_SREG_FS: return &save->fs;
1314         case VCPU_SREG_GS: return &save->gs;
1315         case VCPU_SREG_SS: return &save->ss;
1316         case VCPU_SREG_TR: return &save->tr;
1317         case VCPU_SREG_LDTR: return &save->ldtr;
1318         }
1319         BUG();
1320         return NULL;
1321 }
1322
1323 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1324 {
1325         struct vmcb_seg *s = svm_seg(vcpu, seg);
1326
1327         return s->base;
1328 }
1329
1330 static void svm_get_segment(struct kvm_vcpu *vcpu,
1331                             struct kvm_segment *var, int seg)
1332 {
1333         struct vmcb_seg *s = svm_seg(vcpu, seg);
1334
1335         var->base = s->base;
1336         var->limit = s->limit;
1337         var->selector = s->selector;
1338         var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1339         var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1340         var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1341         var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1342         var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1343         var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1344         var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1345
1346         /*
1347          * AMD CPUs circa 2014 track the G bit for all segments except CS.
1348          * However, the SVM spec states that the G bit is not observed by the
1349          * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1350          * So let's synthesize a legal G bit for all segments, this helps
1351          * running KVM nested. It also helps cross-vendor migration, because
1352          * Intel's vmentry has a check on the 'G' bit.
1353          */
1354         var->g = s->limit > 0xfffff;
1355
1356         /*
1357          * AMD's VMCB does not have an explicit unusable field, so emulate it
1358          * for cross vendor migration purposes by "not present"
1359          */
1360         var->unusable = !var->present || (var->type == 0);
1361
1362         switch (seg) {
1363         case VCPU_SREG_TR:
1364                 /*
1365                  * Work around a bug where the busy flag in the tr selector
1366                  * isn't exposed
1367                  */
1368                 var->type |= 0x2;
1369                 break;
1370         case VCPU_SREG_DS:
1371         case VCPU_SREG_ES:
1372         case VCPU_SREG_FS:
1373         case VCPU_SREG_GS:
1374                 /*
1375                  * The accessed bit must always be set in the segment
1376                  * descriptor cache, although it can be cleared in the
1377                  * descriptor, the cached bit always remains at 1. Since
1378                  * Intel has a check on this, set it here to support
1379                  * cross-vendor migration.
1380                  */
1381                 if (!var->unusable)
1382                         var->type |= 0x1;
1383                 break;
1384         case VCPU_SREG_SS:
1385                 /*
1386                  * On AMD CPUs sometimes the DB bit in the segment
1387                  * descriptor is left as 1, although the whole segment has
1388                  * been made unusable. Clear it here to pass an Intel VMX
1389                  * entry check when cross vendor migrating.
1390                  */
1391                 if (var->unusable)
1392                         var->db = 0;
1393                 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1394                 break;
1395         }
1396 }
1397
1398 static int svm_get_cpl(struct kvm_vcpu *vcpu)
1399 {
1400         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1401
1402         return save->cpl;
1403 }
1404
1405 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1406 {
1407         struct vcpu_svm *svm = to_svm(vcpu);
1408
1409         dt->size = svm->vmcb->save.idtr.limit;
1410         dt->address = svm->vmcb->save.idtr.base;
1411 }
1412
1413 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1414 {
1415         struct vcpu_svm *svm = to_svm(vcpu);
1416
1417         svm->vmcb->save.idtr.limit = dt->size;
1418         svm->vmcb->save.idtr.base = dt->address ;
1419         mark_dirty(svm->vmcb, VMCB_DT);
1420 }
1421
1422 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1423 {
1424         struct vcpu_svm *svm = to_svm(vcpu);
1425
1426         dt->size = svm->vmcb->save.gdtr.limit;
1427         dt->address = svm->vmcb->save.gdtr.base;
1428 }
1429
1430 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1431 {
1432         struct vcpu_svm *svm = to_svm(vcpu);
1433
1434         svm->vmcb->save.gdtr.limit = dt->size;
1435         svm->vmcb->save.gdtr.base = dt->address ;
1436         mark_dirty(svm->vmcb, VMCB_DT);
1437 }
1438
1439 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
1440 {
1441 }
1442
1443 static void svm_decache_cr3(struct kvm_vcpu *vcpu)
1444 {
1445 }
1446
1447 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
1448 {
1449 }
1450
1451 static void update_cr0_intercept(struct vcpu_svm *svm)
1452 {
1453         ulong gcr0 = svm->vcpu.arch.cr0;
1454         u64 *hcr0 = &svm->vmcb->save.cr0;
1455
1456         if (!svm->vcpu.fpu_active)
1457                 *hcr0 |= SVM_CR0_SELECTIVE_MASK;
1458         else
1459                 *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
1460                         | (gcr0 & SVM_CR0_SELECTIVE_MASK);
1461
1462         mark_dirty(svm->vmcb, VMCB_CR);
1463
1464         if (gcr0 == *hcr0 && svm->vcpu.fpu_active) {
1465                 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
1466                 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1467         } else {
1468                 set_cr_intercept(svm, INTERCEPT_CR0_READ);
1469                 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1470         }
1471 }
1472
1473 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1474 {
1475         struct vcpu_svm *svm = to_svm(vcpu);
1476
1477 #ifdef CONFIG_X86_64
1478         if (vcpu->arch.efer & EFER_LME) {
1479                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1480                         vcpu->arch.efer |= EFER_LMA;
1481                         svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1482                 }
1483
1484                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1485                         vcpu->arch.efer &= ~EFER_LMA;
1486                         svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1487                 }
1488         }
1489 #endif
1490         vcpu->arch.cr0 = cr0;
1491
1492         if (!npt_enabled)
1493                 cr0 |= X86_CR0_PG | X86_CR0_WP;
1494
1495         if (!vcpu->fpu_active)
1496                 cr0 |= X86_CR0_TS;
1497         /*
1498          * re-enable caching here because the QEMU bios
1499          * does not do it - this results in some delay at
1500          * reboot
1501          */
1502         if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1503                 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1504         svm->vmcb->save.cr0 = cr0;
1505         mark_dirty(svm->vmcb, VMCB_CR);
1506         update_cr0_intercept(svm);
1507 }
1508
1509 static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1510 {
1511         unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1512         unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
1513
1514         if (cr4 & X86_CR4_VMXE)
1515                 return 1;
1516
1517         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1518                 svm_flush_tlb(vcpu);
1519
1520         vcpu->arch.cr4 = cr4;
1521         if (!npt_enabled)
1522                 cr4 |= X86_CR4_PAE;
1523         cr4 |= host_cr4_mce;
1524         to_svm(vcpu)->vmcb->save.cr4 = cr4;
1525         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1526         return 0;
1527 }
1528
1529 static void svm_set_segment(struct kvm_vcpu *vcpu,
1530                             struct kvm_segment *var, int seg)
1531 {
1532         struct vcpu_svm *svm = to_svm(vcpu);
1533         struct vmcb_seg *s = svm_seg(vcpu, seg);
1534
1535         s->base = var->base;
1536         s->limit = var->limit;
1537         s->selector = var->selector;
1538         if (var->unusable)
1539                 s->attrib = 0;
1540         else {
1541                 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1542                 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1543                 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1544                 s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
1545                 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1546                 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1547                 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1548                 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1549         }
1550
1551         /*
1552          * This is always accurate, except if SYSRET returned to a segment
1553          * with SS.DPL != 3.  Intel does not have this quirk, and always
1554          * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1555          * would entail passing the CPL to userspace and back.
1556          */
1557         if (seg == VCPU_SREG_SS)
1558                 svm->vmcb->save.cpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1559
1560         mark_dirty(svm->vmcb, VMCB_SEG);
1561 }
1562
1563 static void update_bp_intercept(struct kvm_vcpu *vcpu)
1564 {
1565         struct vcpu_svm *svm = to_svm(vcpu);
1566
1567         clr_exception_intercept(svm, BP_VECTOR);
1568
1569         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1570                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1571                         set_exception_intercept(svm, BP_VECTOR);
1572         } else
1573                 vcpu->guest_debug = 0;
1574 }
1575
1576 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1577 {
1578         if (sd->next_asid > sd->max_asid) {
1579                 ++sd->asid_generation;
1580                 sd->next_asid = 1;
1581                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
1582         }
1583
1584         svm->asid_generation = sd->asid_generation;
1585         svm->vmcb->control.asid = sd->next_asid++;
1586
1587         mark_dirty(svm->vmcb, VMCB_ASID);
1588 }
1589
1590 static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
1591 {
1592         return to_svm(vcpu)->vmcb->save.dr6;
1593 }
1594
1595 static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
1596 {
1597         struct vcpu_svm *svm = to_svm(vcpu);
1598
1599         svm->vmcb->save.dr6 = value;
1600         mark_dirty(svm->vmcb, VMCB_DR);
1601 }
1602
1603 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
1604 {
1605         struct vcpu_svm *svm = to_svm(vcpu);
1606
1607         get_debugreg(vcpu->arch.db[0], 0);
1608         get_debugreg(vcpu->arch.db[1], 1);
1609         get_debugreg(vcpu->arch.db[2], 2);
1610         get_debugreg(vcpu->arch.db[3], 3);
1611         vcpu->arch.dr6 = svm_get_dr6(vcpu);
1612         vcpu->arch.dr7 = svm->vmcb->save.dr7;
1613
1614         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
1615         set_dr_intercepts(svm);
1616 }
1617
1618 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
1619 {
1620         struct vcpu_svm *svm = to_svm(vcpu);
1621
1622         svm->vmcb->save.dr7 = value;
1623         mark_dirty(svm->vmcb, VMCB_DR);
1624 }
1625
1626 static int pf_interception(struct vcpu_svm *svm)
1627 {
1628         u64 fault_address = svm->vmcb->control.exit_info_2;
1629         u32 error_code;
1630         int r = 1;
1631
1632         switch (svm->apf_reason) {
1633         default:
1634                 error_code = svm->vmcb->control.exit_info_1;
1635
1636                 trace_kvm_page_fault(fault_address, error_code);
1637                 if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
1638                         kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
1639                 r = kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
1640                         svm->vmcb->control.insn_bytes,
1641                         svm->vmcb->control.insn_len);
1642                 break;
1643         case KVM_PV_REASON_PAGE_NOT_PRESENT:
1644                 svm->apf_reason = 0;
1645                 local_irq_disable();
1646                 kvm_async_pf_task_wait(fault_address);
1647                 local_irq_enable();
1648                 break;
1649         case KVM_PV_REASON_PAGE_READY:
1650                 svm->apf_reason = 0;
1651                 local_irq_disable();
1652                 kvm_async_pf_task_wake(fault_address);
1653                 local_irq_enable();
1654                 break;
1655         }
1656         return r;
1657 }
1658
1659 static int db_interception(struct vcpu_svm *svm)
1660 {
1661         struct kvm_run *kvm_run = svm->vcpu.run;
1662
1663         if (!(svm->vcpu.guest_debug &
1664               (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
1665                 !svm->nmi_singlestep) {
1666                 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
1667                 return 1;
1668         }
1669
1670         if (svm->nmi_singlestep) {
1671                 svm->nmi_singlestep = false;
1672                 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
1673                         svm->vmcb->save.rflags &=
1674                                 ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1675         }
1676
1677         if (svm->vcpu.guest_debug &
1678             (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
1679                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1680                 kvm_run->debug.arch.pc =
1681                         svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1682                 kvm_run->debug.arch.exception = DB_VECTOR;
1683                 return 0;
1684         }
1685
1686         return 1;
1687 }
1688
1689 static int bp_interception(struct vcpu_svm *svm)
1690 {
1691         struct kvm_run *kvm_run = svm->vcpu.run;
1692
1693         kvm_run->exit_reason = KVM_EXIT_DEBUG;
1694         kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1695         kvm_run->debug.arch.exception = BP_VECTOR;
1696         return 0;
1697 }
1698
1699 static int ud_interception(struct vcpu_svm *svm)
1700 {
1701         int er;
1702
1703         er = emulate_instruction(&svm->vcpu, EMULTYPE_TRAP_UD);
1704         if (er != EMULATE_DONE)
1705                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
1706         return 1;
1707 }
1708
1709 static int ac_interception(struct vcpu_svm *svm)
1710 {
1711         kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
1712         return 1;
1713 }
1714
1715 static void svm_fpu_activate(struct kvm_vcpu *vcpu)
1716 {
1717         struct vcpu_svm *svm = to_svm(vcpu);
1718
1719         clr_exception_intercept(svm, NM_VECTOR);
1720
1721         svm->vcpu.fpu_active = 1;
1722         update_cr0_intercept(svm);
1723 }
1724
1725 static int nm_interception(struct vcpu_svm *svm)
1726 {
1727         svm_fpu_activate(&svm->vcpu);
1728         return 1;
1729 }
1730
1731 static bool is_erratum_383(void)
1732 {
1733         int err, i;
1734         u64 value;
1735
1736         if (!erratum_383_found)
1737                 return false;
1738
1739         value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
1740         if (err)
1741                 return false;
1742
1743         /* Bit 62 may or may not be set for this mce */
1744         value &= ~(1ULL << 62);
1745
1746         if (value != 0xb600000000010015ULL)
1747                 return false;
1748
1749         /* Clear MCi_STATUS registers */
1750         for (i = 0; i < 6; ++i)
1751                 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
1752
1753         value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
1754         if (!err) {
1755                 u32 low, high;
1756
1757                 value &= ~(1ULL << 2);
1758                 low    = lower_32_bits(value);
1759                 high   = upper_32_bits(value);
1760
1761                 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
1762         }
1763
1764         /* Flush tlb to evict multi-match entries */
1765         __flush_tlb_all();
1766
1767         return true;
1768 }
1769
1770 static void svm_handle_mce(struct vcpu_svm *svm)
1771 {
1772         if (is_erratum_383()) {
1773                 /*
1774                  * Erratum 383 triggered. Guest state is corrupt so kill the
1775                  * guest.
1776                  */
1777                 pr_err("KVM: Guest triggered AMD Erratum 383\n");
1778
1779                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
1780
1781                 return;
1782         }
1783
1784         /*
1785          * On an #MC intercept the MCE handler is not called automatically in
1786          * the host. So do it by hand here.
1787          */
1788         asm volatile (
1789                 "int $0x12\n");
1790         /* not sure if we ever come back to this point */
1791
1792         return;
1793 }
1794
1795 static int mc_interception(struct vcpu_svm *svm)
1796 {
1797         return 1;
1798 }
1799
1800 static int shutdown_interception(struct vcpu_svm *svm)
1801 {
1802         struct kvm_run *kvm_run = svm->vcpu.run;
1803
1804         /*
1805          * VMCB is undefined after a SHUTDOWN intercept
1806          * so reinitialize it.
1807          */
1808         clear_page(svm->vmcb);
1809         init_vmcb(svm);
1810
1811         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
1812         return 0;
1813 }
1814
1815 static int io_interception(struct vcpu_svm *svm)
1816 {
1817         struct kvm_vcpu *vcpu = &svm->vcpu;
1818         u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
1819         int size, in, string;
1820         unsigned port;
1821
1822         ++svm->vcpu.stat.io_exits;
1823         string = (io_info & SVM_IOIO_STR_MASK) != 0;
1824         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
1825         if (string || in)
1826                 return emulate_instruction(vcpu, 0) == EMULATE_DONE;
1827
1828         port = io_info >> 16;
1829         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
1830         svm->next_rip = svm->vmcb->control.exit_info_2;
1831         skip_emulated_instruction(&svm->vcpu);
1832
1833         return kvm_fast_pio_out(vcpu, size, port);
1834 }
1835
1836 static int nmi_interception(struct vcpu_svm *svm)
1837 {
1838         return 1;
1839 }
1840
1841 static int intr_interception(struct vcpu_svm *svm)
1842 {
1843         ++svm->vcpu.stat.irq_exits;
1844         return 1;
1845 }
1846
1847 static int nop_on_interception(struct vcpu_svm *svm)
1848 {
1849         return 1;
1850 }
1851
1852 static int halt_interception(struct vcpu_svm *svm)
1853 {
1854         svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
1855         return kvm_emulate_halt(&svm->vcpu);
1856 }
1857
1858 static int vmmcall_interception(struct vcpu_svm *svm)
1859 {
1860         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
1861         kvm_emulate_hypercall(&svm->vcpu);
1862         return 1;
1863 }
1864
1865 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
1866 {
1867         struct vcpu_svm *svm = to_svm(vcpu);
1868
1869         return svm->nested.nested_cr3;
1870 }
1871
1872 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
1873 {
1874         struct vcpu_svm *svm = to_svm(vcpu);
1875         u64 cr3 = svm->nested.nested_cr3;
1876         u64 pdpte;
1877         int ret;
1878
1879         ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte,
1880                                        offset_in_page(cr3) + index * 8, 8);
1881         if (ret)
1882                 return 0;
1883         return pdpte;
1884 }
1885
1886 static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
1887                                    unsigned long root)
1888 {
1889         struct vcpu_svm *svm = to_svm(vcpu);
1890
1891         svm->vmcb->control.nested_cr3 = root;
1892         mark_dirty(svm->vmcb, VMCB_NPT);
1893         svm_flush_tlb(vcpu);
1894 }
1895
1896 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
1897                                        struct x86_exception *fault)
1898 {
1899         struct vcpu_svm *svm = to_svm(vcpu);
1900
1901         if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
1902                 /*
1903                  * TODO: track the cause of the nested page fault, and
1904                  * correctly fill in the high bits of exit_info_1.
1905                  */
1906                 svm->vmcb->control.exit_code = SVM_EXIT_NPF;
1907                 svm->vmcb->control.exit_code_hi = 0;
1908                 svm->vmcb->control.exit_info_1 = (1ULL << 32);
1909                 svm->vmcb->control.exit_info_2 = fault->address;
1910         }
1911
1912         svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
1913         svm->vmcb->control.exit_info_1 |= fault->error_code;
1914
1915         /*
1916          * The present bit is always zero for page structure faults on real
1917          * hardware.
1918          */
1919         if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
1920                 svm->vmcb->control.exit_info_1 &= ~1;
1921
1922         nested_svm_vmexit(svm);
1923 }
1924
1925 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
1926 {
1927         WARN_ON(mmu_is_nested(vcpu));
1928         kvm_init_shadow_mmu(vcpu);
1929         vcpu->arch.mmu.set_cr3           = nested_svm_set_tdp_cr3;
1930         vcpu->arch.mmu.get_cr3           = nested_svm_get_tdp_cr3;
1931         vcpu->arch.mmu.get_pdptr         = nested_svm_get_tdp_pdptr;
1932         vcpu->arch.mmu.inject_page_fault = nested_svm_inject_npf_exit;
1933         vcpu->arch.mmu.shadow_root_level = get_npt_level();
1934         reset_shadow_zero_bits_mask(vcpu, &vcpu->arch.mmu);
1935         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
1936 }
1937
1938 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
1939 {
1940         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
1941 }
1942
1943 static int nested_svm_check_permissions(struct vcpu_svm *svm)
1944 {
1945         if (!(svm->vcpu.arch.efer & EFER_SVME)
1946             || !is_paging(&svm->vcpu)) {
1947                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
1948                 return 1;
1949         }
1950
1951         if (svm->vmcb->save.cpl) {
1952                 kvm_inject_gp(&svm->vcpu, 0);
1953                 return 1;
1954         }
1955
1956        return 0;
1957 }
1958
1959 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
1960                                       bool has_error_code, u32 error_code)
1961 {
1962         int vmexit;
1963
1964         if (!is_guest_mode(&svm->vcpu))
1965                 return 0;
1966
1967         svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
1968         svm->vmcb->control.exit_code_hi = 0;
1969         svm->vmcb->control.exit_info_1 = error_code;
1970         svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
1971
1972         vmexit = nested_svm_intercept(svm);
1973         if (vmexit == NESTED_EXIT_DONE)
1974                 svm->nested.exit_required = true;
1975
1976         return vmexit;
1977 }
1978
1979 /* This function returns true if it is save to enable the irq window */
1980 static inline bool nested_svm_intr(struct vcpu_svm *svm)
1981 {
1982         if (!is_guest_mode(&svm->vcpu))
1983                 return true;
1984
1985         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
1986                 return true;
1987
1988         if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
1989                 return false;
1990
1991         /*
1992          * if vmexit was already requested (by intercepted exception
1993          * for instance) do not overwrite it with "external interrupt"
1994          * vmexit.
1995          */
1996         if (svm->nested.exit_required)
1997                 return false;
1998
1999         svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
2000         svm->vmcb->control.exit_info_1 = 0;
2001         svm->vmcb->control.exit_info_2 = 0;
2002
2003         if (svm->nested.intercept & 1ULL) {
2004                 /*
2005                  * The #vmexit can't be emulated here directly because this
2006                  * code path runs with irqs and preemption disabled. A
2007                  * #vmexit emulation might sleep. Only signal request for
2008                  * the #vmexit here.
2009                  */
2010                 svm->nested.exit_required = true;
2011                 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
2012                 return false;
2013         }
2014
2015         return true;
2016 }
2017
2018 /* This function returns true if it is save to enable the nmi window */
2019 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
2020 {
2021         if (!is_guest_mode(&svm->vcpu))
2022                 return true;
2023
2024         if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
2025                 return true;
2026
2027         svm->vmcb->control.exit_code = SVM_EXIT_NMI;
2028         svm->nested.exit_required = true;
2029
2030         return false;
2031 }
2032
2033 static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
2034 {
2035         struct page *page;
2036
2037         might_sleep();
2038
2039         page = kvm_vcpu_gfn_to_page(&svm->vcpu, gpa >> PAGE_SHIFT);
2040         if (is_error_page(page))
2041                 goto error;
2042
2043         *_page = page;
2044
2045         return kmap(page);
2046
2047 error:
2048         kvm_inject_gp(&svm->vcpu, 0);
2049
2050         return NULL;
2051 }
2052
2053 static void nested_svm_unmap(struct page *page)
2054 {
2055         kunmap(page);
2056         kvm_release_page_dirty(page);
2057 }
2058
2059 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
2060 {
2061         unsigned port, size, iopm_len;
2062         u16 val, mask;
2063         u8 start_bit;
2064         u64 gpa;
2065
2066         if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
2067                 return NESTED_EXIT_HOST;
2068
2069         port = svm->vmcb->control.exit_info_1 >> 16;
2070         size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
2071                 SVM_IOIO_SIZE_SHIFT;
2072         gpa  = svm->nested.vmcb_iopm + (port / 8);
2073         start_bit = port % 8;
2074         iopm_len = (start_bit + size > 8) ? 2 : 1;
2075         mask = (0xf >> (4 - size)) << start_bit;
2076         val = 0;
2077
2078         if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
2079                 return NESTED_EXIT_DONE;
2080
2081         return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2082 }
2083
2084 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
2085 {
2086         u32 offset, msr, value;
2087         int write, mask;
2088
2089         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2090                 return NESTED_EXIT_HOST;
2091
2092         msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
2093         offset = svm_msrpm_offset(msr);
2094         write  = svm->vmcb->control.exit_info_1 & 1;
2095         mask   = 1 << ((2 * (msr & 0xf)) + write);
2096
2097         if (offset == MSR_INVALID)
2098                 return NESTED_EXIT_DONE;
2099
2100         /* Offset is in 32 bit units but need in 8 bit units */
2101         offset *= 4;
2102
2103         if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
2104                 return NESTED_EXIT_DONE;
2105
2106         return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2107 }
2108
2109 static int nested_svm_exit_special(struct vcpu_svm *svm)
2110 {
2111         u32 exit_code = svm->vmcb->control.exit_code;
2112
2113         switch (exit_code) {
2114         case SVM_EXIT_INTR:
2115         case SVM_EXIT_NMI:
2116         case SVM_EXIT_EXCP_BASE + MC_VECTOR:
2117                 return NESTED_EXIT_HOST;
2118         case SVM_EXIT_NPF:
2119                 /* For now we are always handling NPFs when using them */
2120                 if (npt_enabled)
2121                         return NESTED_EXIT_HOST;
2122                 break;
2123         case SVM_EXIT_EXCP_BASE + PF_VECTOR:
2124                 /* When we're shadowing, trap PFs, but not async PF */
2125                 if (!npt_enabled && svm->apf_reason == 0)
2126                         return NESTED_EXIT_HOST;
2127                 break;
2128         case SVM_EXIT_EXCP_BASE + NM_VECTOR:
2129                 nm_interception(svm);
2130                 break;
2131         default:
2132                 break;
2133         }
2134
2135         return NESTED_EXIT_CONTINUE;
2136 }
2137
2138 /*
2139  * If this function returns true, this #vmexit was already handled
2140  */
2141 static int nested_svm_intercept(struct vcpu_svm *svm)
2142 {
2143         u32 exit_code = svm->vmcb->control.exit_code;
2144         int vmexit = NESTED_EXIT_HOST;
2145
2146         switch (exit_code) {
2147         case SVM_EXIT_MSR:
2148                 vmexit = nested_svm_exit_handled_msr(svm);
2149                 break;
2150         case SVM_EXIT_IOIO:
2151                 vmexit = nested_svm_intercept_ioio(svm);
2152                 break;
2153         case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
2154                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
2155                 if (svm->nested.intercept_cr & bit)
2156                         vmexit = NESTED_EXIT_DONE;
2157                 break;
2158         }
2159         case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
2160                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
2161                 if (svm->nested.intercept_dr & bit)
2162                         vmexit = NESTED_EXIT_DONE;
2163                 break;
2164         }
2165         case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
2166                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
2167                 if (svm->nested.intercept_exceptions & excp_bits)
2168                         vmexit = NESTED_EXIT_DONE;
2169                 /* async page fault always cause vmexit */
2170                 else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
2171                          svm->apf_reason != 0)
2172                         vmexit = NESTED_EXIT_DONE;
2173                 break;
2174         }
2175         case SVM_EXIT_ERR: {
2176                 vmexit = NESTED_EXIT_DONE;
2177                 break;
2178         }
2179         default: {
2180                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
2181                 if (svm->nested.intercept & exit_bits)
2182                         vmexit = NESTED_EXIT_DONE;
2183         }
2184         }
2185
2186         return vmexit;
2187 }
2188
2189 static int nested_svm_exit_handled(struct vcpu_svm *svm)
2190 {
2191         int vmexit;
2192
2193         vmexit = nested_svm_intercept(svm);
2194
2195         if (vmexit == NESTED_EXIT_DONE)
2196                 nested_svm_vmexit(svm);
2197
2198         return vmexit;
2199 }
2200
2201 static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
2202 {
2203         struct vmcb_control_area *dst  = &dst_vmcb->control;
2204         struct vmcb_control_area *from = &from_vmcb->control;
2205
2206         dst->intercept_cr         = from->intercept_cr;
2207         dst->intercept_dr         = from->intercept_dr;
2208         dst->intercept_exceptions = from->intercept_exceptions;
2209         dst->intercept            = from->intercept;
2210         dst->iopm_base_pa         = from->iopm_base_pa;
2211         dst->msrpm_base_pa        = from->msrpm_base_pa;
2212         dst->tsc_offset           = from->tsc_offset;
2213         dst->asid                 = from->asid;
2214         dst->tlb_ctl              = from->tlb_ctl;
2215         dst->int_ctl              = from->int_ctl;
2216         dst->int_vector           = from->int_vector;
2217         dst->int_state            = from->int_state;
2218         dst->exit_code            = from->exit_code;
2219         dst->exit_code_hi         = from->exit_code_hi;
2220         dst->exit_info_1          = from->exit_info_1;
2221         dst->exit_info_2          = from->exit_info_2;
2222         dst->exit_int_info        = from->exit_int_info;
2223         dst->exit_int_info_err    = from->exit_int_info_err;
2224         dst->nested_ctl           = from->nested_ctl;
2225         dst->event_inj            = from->event_inj;
2226         dst->event_inj_err        = from->event_inj_err;
2227         dst->nested_cr3           = from->nested_cr3;
2228         dst->lbr_ctl              = from->lbr_ctl;
2229 }
2230
2231 static int nested_svm_vmexit(struct vcpu_svm *svm)
2232 {
2233         struct vmcb *nested_vmcb;
2234         struct vmcb *hsave = svm->nested.hsave;
2235         struct vmcb *vmcb = svm->vmcb;
2236         struct page *page;
2237
2238         trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
2239                                        vmcb->control.exit_info_1,
2240                                        vmcb->control.exit_info_2,
2241                                        vmcb->control.exit_int_info,
2242                                        vmcb->control.exit_int_info_err,
2243                                        KVM_ISA_SVM);
2244
2245         nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
2246         if (!nested_vmcb)
2247                 return 1;
2248
2249         /* Exit Guest-Mode */
2250         leave_guest_mode(&svm->vcpu);
2251         svm->nested.vmcb = 0;
2252
2253         /* Give the current vmcb to the guest */
2254         disable_gif(svm);
2255
2256         nested_vmcb->save.es     = vmcb->save.es;
2257         nested_vmcb->save.cs     = vmcb->save.cs;
2258         nested_vmcb->save.ss     = vmcb->save.ss;
2259         nested_vmcb->save.ds     = vmcb->save.ds;
2260         nested_vmcb->save.gdtr   = vmcb->save.gdtr;
2261         nested_vmcb->save.idtr   = vmcb->save.idtr;
2262         nested_vmcb->save.efer   = svm->vcpu.arch.efer;
2263         nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
2264         nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
2265         nested_vmcb->save.cr2    = vmcb->save.cr2;
2266         nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
2267         nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
2268         nested_vmcb->save.rip    = vmcb->save.rip;
2269         nested_vmcb->save.rsp    = vmcb->save.rsp;
2270         nested_vmcb->save.rax    = vmcb->save.rax;
2271         nested_vmcb->save.dr7    = vmcb->save.dr7;
2272         nested_vmcb->save.dr6    = vmcb->save.dr6;
2273         nested_vmcb->save.cpl    = vmcb->save.cpl;
2274
2275         nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
2276         nested_vmcb->control.int_vector        = vmcb->control.int_vector;
2277         nested_vmcb->control.int_state         = vmcb->control.int_state;
2278         nested_vmcb->control.exit_code         = vmcb->control.exit_code;
2279         nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
2280         nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
2281         nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
2282         nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
2283         nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
2284
2285         if (svm->nrips_enabled)
2286                 nested_vmcb->control.next_rip  = vmcb->control.next_rip;
2287
2288         /*
2289          * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
2290          * to make sure that we do not lose injected events. So check event_inj
2291          * here and copy it to exit_int_info if it is valid.
2292          * Exit_int_info and event_inj can't be both valid because the case
2293          * below only happens on a VMRUN instruction intercept which has
2294          * no valid exit_int_info set.
2295          */
2296         if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
2297                 struct vmcb_control_area *nc = &nested_vmcb->control;
2298
2299                 nc->exit_int_info     = vmcb->control.event_inj;
2300                 nc->exit_int_info_err = vmcb->control.event_inj_err;
2301         }
2302
2303         nested_vmcb->control.tlb_ctl           = 0;
2304         nested_vmcb->control.event_inj         = 0;
2305         nested_vmcb->control.event_inj_err     = 0;
2306
2307         /* We always set V_INTR_MASKING and remember the old value in hflags */
2308         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
2309                 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
2310
2311         /* Restore the original control entries */
2312         copy_vmcb_control_area(vmcb, hsave);
2313
2314         kvm_clear_exception_queue(&svm->vcpu);
2315         kvm_clear_interrupt_queue(&svm->vcpu);
2316
2317         svm->nested.nested_cr3 = 0;
2318
2319         /* Restore selected save entries */
2320         svm->vmcb->save.es = hsave->save.es;
2321         svm->vmcb->save.cs = hsave->save.cs;
2322         svm->vmcb->save.ss = hsave->save.ss;
2323         svm->vmcb->save.ds = hsave->save.ds;
2324         svm->vmcb->save.gdtr = hsave->save.gdtr;
2325         svm->vmcb->save.idtr = hsave->save.idtr;
2326         kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
2327         svm_set_efer(&svm->vcpu, hsave->save.efer);
2328         svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
2329         svm_set_cr4(&svm->vcpu, hsave->save.cr4);
2330         if (npt_enabled) {
2331                 svm->vmcb->save.cr3 = hsave->save.cr3;
2332                 svm->vcpu.arch.cr3 = hsave->save.cr3;
2333         } else {
2334                 (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
2335         }
2336         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
2337         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
2338         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
2339         svm->vmcb->save.dr7 = 0;
2340         svm->vmcb->save.cpl = 0;
2341         svm->vmcb->control.exit_int_info = 0;
2342
2343         mark_all_dirty(svm->vmcb);
2344
2345         nested_svm_unmap(page);
2346
2347         nested_svm_uninit_mmu_context(&svm->vcpu);
2348         kvm_mmu_reset_context(&svm->vcpu);
2349         kvm_mmu_load(&svm->vcpu);
2350
2351         return 0;
2352 }
2353
2354 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
2355 {
2356         /*
2357          * This function merges the msr permission bitmaps of kvm and the
2358          * nested vmcb. It is optimized in that it only merges the parts where
2359          * the kvm msr permission bitmap may contain zero bits
2360          */
2361         int i;
2362
2363         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2364                 return true;
2365
2366         for (i = 0; i < MSRPM_OFFSETS; i++) {
2367                 u32 value, p;
2368                 u64 offset;
2369
2370                 if (msrpm_offsets[i] == 0xffffffff)
2371                         break;
2372
2373                 p      = msrpm_offsets[i];
2374                 offset = svm->nested.vmcb_msrpm + (p * 4);
2375
2376                 if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
2377                         return false;
2378
2379                 svm->nested.msrpm[p] = svm->msrpm[p] | value;
2380         }
2381
2382         svm->vmcb->control.msrpm_base_pa = __pa(svm->nested.msrpm);
2383
2384         return true;
2385 }
2386
2387 static bool nested_vmcb_checks(struct vmcb *vmcb)
2388 {
2389         if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
2390                 return false;
2391
2392         if (vmcb->control.asid == 0)
2393                 return false;
2394
2395         if (vmcb->control.nested_ctl && !npt_enabled)
2396                 return false;
2397
2398         return true;
2399 }
2400
2401 static bool nested_svm_vmrun(struct vcpu_svm *svm)
2402 {
2403         struct vmcb *nested_vmcb;
2404         struct vmcb *hsave = svm->nested.hsave;
2405         struct vmcb *vmcb = svm->vmcb;
2406         struct page *page;
2407         u64 vmcb_gpa;
2408
2409         vmcb_gpa = svm->vmcb->save.rax;
2410
2411         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2412         if (!nested_vmcb)
2413                 return false;
2414
2415         if (!nested_vmcb_checks(nested_vmcb)) {
2416                 nested_vmcb->control.exit_code    = SVM_EXIT_ERR;
2417                 nested_vmcb->control.exit_code_hi = 0;
2418                 nested_vmcb->control.exit_info_1  = 0;
2419                 nested_vmcb->control.exit_info_2  = 0;
2420
2421                 nested_svm_unmap(page);
2422
2423                 return false;
2424         }
2425
2426         trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
2427                                nested_vmcb->save.rip,
2428                                nested_vmcb->control.int_ctl,
2429                                nested_vmcb->control.event_inj,
2430                                nested_vmcb->control.nested_ctl);
2431
2432         trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
2433                                     nested_vmcb->control.intercept_cr >> 16,
2434                                     nested_vmcb->control.intercept_exceptions,
2435                                     nested_vmcb->control.intercept);
2436
2437         /* Clear internal status */
2438         kvm_clear_exception_queue(&svm->vcpu);
2439         kvm_clear_interrupt_queue(&svm->vcpu);
2440
2441         /*
2442          * Save the old vmcb, so we don't need to pick what we save, but can
2443          * restore everything when a VMEXIT occurs
2444          */
2445         hsave->save.es     = vmcb->save.es;
2446         hsave->save.cs     = vmcb->save.cs;
2447         hsave->save.ss     = vmcb->save.ss;
2448         hsave->save.ds     = vmcb->save.ds;
2449         hsave->save.gdtr   = vmcb->save.gdtr;
2450         hsave->save.idtr   = vmcb->save.idtr;
2451         hsave->save.efer   = svm->vcpu.arch.efer;
2452         hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
2453         hsave->save.cr4    = svm->vcpu.arch.cr4;
2454         hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
2455         hsave->save.rip    = kvm_rip_read(&svm->vcpu);
2456         hsave->save.rsp    = vmcb->save.rsp;
2457         hsave->save.rax    = vmcb->save.rax;
2458         if (npt_enabled)
2459                 hsave->save.cr3    = vmcb->save.cr3;
2460         else
2461                 hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);
2462
2463         copy_vmcb_control_area(hsave, vmcb);
2464
2465         if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
2466                 svm->vcpu.arch.hflags |= HF_HIF_MASK;
2467         else
2468                 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
2469
2470         if (nested_vmcb->control.nested_ctl) {
2471                 kvm_mmu_unload(&svm->vcpu);
2472                 svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
2473                 nested_svm_init_mmu_context(&svm->vcpu);
2474         }
2475
2476         /* Load the nested guest state */
2477         svm->vmcb->save.es = nested_vmcb->save.es;
2478         svm->vmcb->save.cs = nested_vmcb->save.cs;
2479         svm->vmcb->save.ss = nested_vmcb->save.ss;
2480         svm->vmcb->save.ds = nested_vmcb->save.ds;
2481         svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
2482         svm->vmcb->save.idtr = nested_vmcb->save.idtr;
2483         kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
2484         svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
2485         svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
2486         svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
2487         if (npt_enabled) {
2488                 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
2489                 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
2490         } else
2491                 (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
2492
2493         /* Guest paging mode is active - reset mmu */
2494         kvm_mmu_reset_context(&svm->vcpu);
2495
2496         svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
2497         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
2498         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
2499         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
2500
2501         /* In case we don't even reach vcpu_run, the fields are not updated */
2502         svm->vmcb->save.rax = nested_vmcb->save.rax;
2503         svm->vmcb->save.rsp = nested_vmcb->save.rsp;
2504         svm->vmcb->save.rip = nested_vmcb->save.rip;
2505         svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
2506         svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
2507         svm->vmcb->save.cpl = nested_vmcb->save.cpl;
2508
2509         svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
2510         svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
2511
2512         /* cache intercepts */
2513         svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
2514         svm->nested.intercept_dr         = nested_vmcb->control.intercept_dr;
2515         svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
2516         svm->nested.intercept            = nested_vmcb->control.intercept;
2517
2518         svm_flush_tlb(&svm->vcpu);
2519         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
2520         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
2521                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
2522         else
2523                 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
2524
2525         if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
2526                 /* We only want the cr8 intercept bits of the guest */
2527                 clr_cr_intercept(svm, INTERCEPT_CR8_READ);
2528                 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
2529         }
2530
2531         /* We don't want to see VMMCALLs from a nested guest */
2532         clr_intercept(svm, INTERCEPT_VMMCALL);
2533
2534         svm->vmcb->control.lbr_ctl = nested_vmcb->control.lbr_ctl;
2535         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
2536         svm->vmcb->control.int_state = nested_vmcb->control.int_state;
2537         svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
2538         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
2539         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
2540
2541         nested_svm_unmap(page);
2542
2543         /* Enter Guest-Mode */
2544         enter_guest_mode(&svm->vcpu);
2545
2546         /*
2547          * Merge guest and host intercepts - must be called  with vcpu in
2548          * guest-mode to take affect here
2549          */
2550         recalc_intercepts(svm);
2551
2552         svm->nested.vmcb = vmcb_gpa;
2553
2554         enable_gif(svm);
2555
2556         mark_all_dirty(svm->vmcb);
2557
2558         return true;
2559 }
2560
2561 static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
2562 {
2563         to_vmcb->save.fs = from_vmcb->save.fs;
2564         to_vmcb->save.gs = from_vmcb->save.gs;
2565         to_vmcb->save.tr = from_vmcb->save.tr;
2566         to_vmcb->save.ldtr = from_vmcb->save.ldtr;
2567         to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
2568         to_vmcb->save.star = from_vmcb->save.star;
2569         to_vmcb->save.lstar = from_vmcb->save.lstar;
2570         to_vmcb->save.cstar = from_vmcb->save.cstar;
2571         to_vmcb->save.sfmask = from_vmcb->save.sfmask;
2572         to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
2573         to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
2574         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
2575 }
2576
2577 static int vmload_interception(struct vcpu_svm *svm)
2578 {
2579         struct vmcb *nested_vmcb;
2580         struct page *page;
2581
2582         if (nested_svm_check_permissions(svm))
2583                 return 1;
2584
2585         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2586         if (!nested_vmcb)
2587                 return 1;
2588
2589         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2590         skip_emulated_instruction(&svm->vcpu);
2591
2592         nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
2593         nested_svm_unmap(page);
2594
2595         return 1;
2596 }
2597
2598 static int vmsave_interception(struct vcpu_svm *svm)
2599 {
2600         struct vmcb *nested_vmcb;
2601         struct page *page;
2602
2603         if (nested_svm_check_permissions(svm))
2604                 return 1;
2605
2606         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2607         if (!nested_vmcb)
2608                 return 1;
2609
2610         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2611         skip_emulated_instruction(&svm->vcpu);
2612
2613         nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
2614         nested_svm_unmap(page);
2615
2616         return 1;
2617 }
2618
2619 static int vmrun_interception(struct vcpu_svm *svm)
2620 {
2621         if (nested_svm_check_permissions(svm))
2622                 return 1;
2623
2624         /* Save rip after vmrun instruction */
2625         kvm_rip_write(&svm->vcpu, kvm_rip_read(&svm->vcpu) + 3);
2626
2627         if (!nested_svm_vmrun(svm))
2628                 return 1;
2629
2630         if (!nested_svm_vmrun_msrpm(svm))
2631                 goto failed;
2632
2633         return 1;
2634
2635 failed:
2636
2637         svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
2638         svm->vmcb->control.exit_code_hi = 0;
2639         svm->vmcb->control.exit_info_1  = 0;
2640         svm->vmcb->control.exit_info_2  = 0;
2641
2642         nested_svm_vmexit(svm);
2643
2644         return 1;
2645 }
2646
2647 static int stgi_interception(struct vcpu_svm *svm)
2648 {
2649         if (nested_svm_check_permissions(svm))
2650                 return 1;
2651
2652         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2653         skip_emulated_instruction(&svm->vcpu);
2654         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2655
2656         enable_gif(svm);
2657
2658         return 1;
2659 }
2660
2661 static int clgi_interception(struct vcpu_svm *svm)
2662 {
2663         if (nested_svm_check_permissions(svm))
2664                 return 1;
2665
2666         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2667         skip_emulated_instruction(&svm->vcpu);
2668
2669         disable_gif(svm);
2670
2671         /* After a CLGI no interrupts should come */
2672         svm_clear_vintr(svm);
2673         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
2674
2675         mark_dirty(svm->vmcb, VMCB_INTR);
2676
2677         return 1;
2678 }
2679
2680 static int invlpga_interception(struct vcpu_svm *svm)
2681 {
2682         struct kvm_vcpu *vcpu = &svm->vcpu;
2683
2684         trace_kvm_invlpga(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RCX),
2685                           kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2686
2687         /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2688         kvm_mmu_invlpg(vcpu, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2689
2690         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2691         skip_emulated_instruction(&svm->vcpu);
2692         return 1;
2693 }
2694
2695 static int skinit_interception(struct vcpu_svm *svm)
2696 {
2697         trace_kvm_skinit(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2698
2699         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2700         return 1;
2701 }
2702
2703 static int wbinvd_interception(struct vcpu_svm *svm)
2704 {
2705         kvm_emulate_wbinvd(&svm->vcpu);
2706         return 1;
2707 }
2708
2709 static int xsetbv_interception(struct vcpu_svm *svm)
2710 {
2711         u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
2712         u32 index = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
2713
2714         if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
2715                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2716                 skip_emulated_instruction(&svm->vcpu);
2717         }
2718
2719         return 1;
2720 }
2721
2722 static int task_switch_interception(struct vcpu_svm *svm)
2723 {
2724         u16 tss_selector;
2725         int reason;
2726         int int_type = svm->vmcb->control.exit_int_info &
2727                 SVM_EXITINTINFO_TYPE_MASK;
2728         int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2729         uint32_t type =
2730                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2731         uint32_t idt_v =
2732                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2733         bool has_error_code = false;
2734         u32 error_code = 0;
2735
2736         tss_selector = (u16)svm->vmcb->control.exit_info_1;
2737
2738         if (svm->vmcb->control.exit_info_2 &
2739             (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2740                 reason = TASK_SWITCH_IRET;
2741         else if (svm->vmcb->control.exit_info_2 &
2742                  (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2743                 reason = TASK_SWITCH_JMP;
2744         else if (idt_v)
2745                 reason = TASK_SWITCH_GATE;
2746         else
2747                 reason = TASK_SWITCH_CALL;
2748
2749         if (reason == TASK_SWITCH_GATE) {
2750                 switch (type) {
2751                 case SVM_EXITINTINFO_TYPE_NMI:
2752                         svm->vcpu.arch.nmi_injected = false;
2753                         break;
2754                 case SVM_EXITINTINFO_TYPE_EXEPT:
2755                         if (svm->vmcb->control.exit_info_2 &
2756                             (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2757                                 has_error_code = true;
2758                                 error_code =
2759                                         (u32)svm->vmcb->control.exit_info_2;
2760                         }
2761                         kvm_clear_exception_queue(&svm->vcpu);
2762                         break;
2763                 case SVM_EXITINTINFO_TYPE_INTR:
2764                         kvm_clear_interrupt_queue(&svm->vcpu);
2765                         break;
2766                 default:
2767                         break;
2768                 }
2769         }
2770
2771         if (reason != TASK_SWITCH_GATE ||
2772             int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2773             (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2774              (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
2775                 skip_emulated_instruction(&svm->vcpu);
2776
2777         if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2778                 int_vec = -1;
2779
2780         if (kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
2781                                 has_error_code, error_code) == EMULATE_FAIL) {
2782                 svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
2783                 svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
2784                 svm->vcpu.run->internal.ndata = 0;
2785                 return 0;
2786         }
2787         return 1;
2788 }
2789
2790 static int cpuid_interception(struct vcpu_svm *svm)
2791 {
2792         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
2793         kvm_emulate_cpuid(&svm->vcpu);
2794         return 1;
2795 }
2796
2797 static int iret_interception(struct vcpu_svm *svm)
2798 {
2799         ++svm->vcpu.stat.nmi_window_exits;
2800         clr_intercept(svm, INTERCEPT_IRET);
2801         svm->vcpu.arch.hflags |= HF_IRET_MASK;
2802         svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
2803         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2804         return 1;
2805 }
2806
2807 static int invlpg_interception(struct vcpu_svm *svm)
2808 {
2809         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2810                 return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
2811
2812         kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
2813         skip_emulated_instruction(&svm->vcpu);
2814         return 1;
2815 }
2816
2817 static int emulate_on_interception(struct vcpu_svm *svm)
2818 {
2819         return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
2820 }
2821
2822 static int rdpmc_interception(struct vcpu_svm *svm)
2823 {
2824         int err;
2825
2826         if (!static_cpu_has(X86_FEATURE_NRIPS))
2827                 return emulate_on_interception(svm);
2828
2829         err = kvm_rdpmc(&svm->vcpu);
2830         kvm_complete_insn_gp(&svm->vcpu, err);
2831
2832         return 1;
2833 }
2834
2835 static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
2836                                             unsigned long val)
2837 {
2838         unsigned long cr0 = svm->vcpu.arch.cr0;
2839         bool ret = false;
2840         u64 intercept;
2841
2842         intercept = svm->nested.intercept;
2843
2844         if (!is_guest_mode(&svm->vcpu) ||
2845             (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))))
2846                 return false;
2847
2848         cr0 &= ~SVM_CR0_SELECTIVE_MASK;
2849         val &= ~SVM_CR0_SELECTIVE_MASK;
2850
2851         if (cr0 ^ val) {
2852                 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
2853                 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
2854         }
2855
2856         return ret;
2857 }
2858
2859 #define CR_VALID (1ULL << 63)
2860
2861 static int cr_interception(struct vcpu_svm *svm)
2862 {
2863         int reg, cr;
2864         unsigned long val;
2865         int err;
2866
2867         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2868                 return emulate_on_interception(svm);
2869
2870         if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
2871                 return emulate_on_interception(svm);
2872
2873         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2874         if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2875                 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2876         else
2877                 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2878
2879         err = 0;
2880         if (cr >= 16) { /* mov to cr */
2881                 cr -= 16;
2882                 val = kvm_register_read(&svm->vcpu, reg);
2883                 switch (cr) {
2884                 case 0:
2885                         if (!check_selective_cr0_intercepted(svm, val))
2886                                 err = kvm_set_cr0(&svm->vcpu, val);
2887                         else
2888                                 return 1;
2889
2890                         break;
2891                 case 3:
2892                         err = kvm_set_cr3(&svm->vcpu, val);
2893                         break;
2894                 case 4:
2895                         err = kvm_set_cr4(&svm->vcpu, val);
2896                         break;
2897                 case 8:
2898                         err = kvm_set_cr8(&svm->vcpu, val);
2899                         break;
2900                 default:
2901                         WARN(1, "unhandled write to CR%d", cr);
2902                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2903                         return 1;
2904                 }
2905         } else { /* mov from cr */
2906                 switch (cr) {
2907                 case 0:
2908                         val = kvm_read_cr0(&svm->vcpu);
2909                         break;
2910                 case 2:
2911                         val = svm->vcpu.arch.cr2;
2912                         break;
2913                 case 3:
2914                         val = kvm_read_cr3(&svm->vcpu);
2915                         break;
2916                 case 4:
2917                         val = kvm_read_cr4(&svm->vcpu);
2918                         break;
2919                 case 8:
2920                         val = kvm_get_cr8(&svm->vcpu);
2921                         break;
2922                 default:
2923                         WARN(1, "unhandled read from CR%d", cr);
2924                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2925                         return 1;
2926                 }
2927                 kvm_register_write(&svm->vcpu, reg, val);
2928         }
2929         kvm_complete_insn_gp(&svm->vcpu, err);
2930
2931         return 1;
2932 }
2933
2934 static int dr_interception(struct vcpu_svm *svm)
2935 {
2936         int reg, dr;
2937         unsigned long val;
2938
2939         if (svm->vcpu.guest_debug == 0) {
2940                 /*
2941                  * No more DR vmexits; force a reload of the debug registers
2942                  * and reenter on this instruction.  The next vmexit will
2943                  * retrieve the full state of the debug registers.
2944                  */
2945                 clr_dr_intercepts(svm);
2946                 svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
2947                 return 1;
2948         }
2949
2950         if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
2951                 return emulate_on_interception(svm);
2952
2953         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2954         dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2955
2956         if (dr >= 16) { /* mov to DRn */
2957                 if (!kvm_require_dr(&svm->vcpu, dr - 16))
2958                         return 1;
2959                 val = kvm_register_read(&svm->vcpu, reg);
2960                 kvm_set_dr(&svm->vcpu, dr - 16, val);
2961         } else {
2962                 if (!kvm_require_dr(&svm->vcpu, dr))
2963                         return 1;
2964                 kvm_get_dr(&svm->vcpu, dr, &val);
2965                 kvm_register_write(&svm->vcpu, reg, val);
2966         }
2967
2968         skip_emulated_instruction(&svm->vcpu);
2969
2970         return 1;
2971 }
2972
2973 static int cr8_write_interception(struct vcpu_svm *svm)
2974 {
2975         struct kvm_run *kvm_run = svm->vcpu.run;
2976         int r;
2977
2978         u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
2979         /* instruction emulation calls kvm_set_cr8() */
2980         r = cr_interception(svm);
2981         if (lapic_in_kernel(&svm->vcpu))
2982                 return r;
2983         if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
2984                 return r;
2985         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
2986         return 0;
2987 }
2988
2989 static u64 svm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
2990 {
2991         struct vmcb *vmcb = get_host_vmcb(to_svm(vcpu));
2992         return vmcb->control.tsc_offset + host_tsc;
2993 }
2994
2995 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2996 {
2997         struct vcpu_svm *svm = to_svm(vcpu);
2998
2999         switch (msr_info->index) {
3000         case MSR_IA32_TSC: {
3001                 msr_info->data = svm->vmcb->control.tsc_offset +
3002                         kvm_scale_tsc(vcpu, rdtsc());
3003
3004                 break;
3005         }
3006         case MSR_STAR:
3007                 msr_info->data = svm->vmcb->save.star;
3008                 break;
3009 #ifdef CONFIG_X86_64
3010         case MSR_LSTAR:
3011                 msr_info->data = svm->vmcb->save.lstar;
3012                 break;
3013         case MSR_CSTAR:
3014                 msr_info->data = svm->vmcb->save.cstar;
3015                 break;
3016         case MSR_KERNEL_GS_BASE:
3017                 msr_info->data = svm->vmcb->save.kernel_gs_base;
3018                 break;
3019         case MSR_SYSCALL_MASK:
3020                 msr_info->data = svm->vmcb->save.sfmask;
3021                 break;
3022 #endif
3023         case MSR_IA32_SYSENTER_CS:
3024                 msr_info->data = svm->vmcb->save.sysenter_cs;
3025                 break;
3026         case MSR_IA32_SYSENTER_EIP:
3027                 msr_info->data = svm->sysenter_eip;
3028                 break;
3029         case MSR_IA32_SYSENTER_ESP:
3030                 msr_info->data = svm->sysenter_esp;
3031                 break;
3032         case MSR_TSC_AUX:
3033                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
3034                         return 1;
3035                 msr_info->data = svm->tsc_aux;
3036                 break;
3037         /*
3038          * Nobody will change the following 5 values in the VMCB so we can
3039          * safely return them on rdmsr. They will always be 0 until LBRV is
3040          * implemented.
3041          */
3042         case MSR_IA32_DEBUGCTLMSR:
3043                 msr_info->data = svm->vmcb->save.dbgctl;
3044                 break;
3045         case MSR_IA32_LASTBRANCHFROMIP:
3046                 msr_info->data = svm->vmcb->save.br_from;
3047                 break;
3048         case MSR_IA32_LASTBRANCHTOIP:
3049                 msr_info->data = svm->vmcb->save.br_to;
3050                 break;
3051         case MSR_IA32_LASTINTFROMIP:
3052                 msr_info->data = svm->vmcb->save.last_excp_from;
3053                 break;
3054         case MSR_IA32_LASTINTTOIP:
3055                 msr_info->data = svm->vmcb->save.last_excp_to;
3056                 break;
3057         case MSR_VM_HSAVE_PA:
3058                 msr_info->data = svm->nested.hsave_msr;
3059                 break;
3060         case MSR_VM_CR:
3061                 msr_info->data = svm->nested.vm_cr_msr;
3062                 break;
3063         case MSR_IA32_UCODE_REV:
3064                 msr_info->data = 0x01000065;
3065                 break;
3066         case MSR_F15H_IC_CFG: {
3067
3068                 int family, model;
3069
3070                 family = guest_cpuid_family(vcpu);
3071                 model  = guest_cpuid_model(vcpu);
3072
3073                 if (family < 0 || model < 0)
3074                         return kvm_get_msr_common(vcpu, msr_info);
3075
3076                 msr_info->data = 0;
3077
3078                 if (family == 0x15 &&
3079                     (model >= 0x2 && model < 0x20))
3080                         msr_info->data = 0x1E;
3081                 }
3082                 break;
3083         default:
3084                 return kvm_get_msr_common(vcpu, msr_info);
3085         }
3086         return 0;
3087 }
3088
3089 static int rdmsr_interception(struct vcpu_svm *svm)
3090 {
3091         u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3092         struct msr_data msr_info;
3093
3094         msr_info.index = ecx;
3095         msr_info.host_initiated = false;
3096         if (svm_get_msr(&svm->vcpu, &msr_info)) {
3097                 trace_kvm_msr_read_ex(ecx);
3098                 kvm_inject_gp(&svm->vcpu, 0);
3099         } else {
3100                 trace_kvm_msr_read(ecx, msr_info.data);
3101
3102                 kvm_register_write(&svm->vcpu, VCPU_REGS_RAX,
3103                                    msr_info.data & 0xffffffff);
3104                 kvm_register_write(&svm->vcpu, VCPU_REGS_RDX,
3105                                    msr_info.data >> 32);
3106                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3107                 skip_emulated_instruction(&svm->vcpu);
3108         }
3109         return 1;
3110 }
3111
3112 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
3113 {
3114         struct vcpu_svm *svm = to_svm(vcpu);
3115         int svm_dis, chg_mask;
3116
3117         if (data & ~SVM_VM_CR_VALID_MASK)
3118                 return 1;
3119
3120         chg_mask = SVM_VM_CR_VALID_MASK;
3121
3122         if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
3123                 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
3124
3125         svm->nested.vm_cr_msr &= ~chg_mask;
3126         svm->nested.vm_cr_msr |= (data & chg_mask);
3127
3128         svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
3129
3130         /* check for svm_disable while efer.svme is set */
3131         if (svm_dis && (vcpu->arch.efer & EFER_SVME))
3132                 return 1;
3133
3134         return 0;
3135 }
3136
3137 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
3138 {
3139         struct vcpu_svm *svm = to_svm(vcpu);
3140
3141         u32 ecx = msr->index;
3142         u64 data = msr->data;
3143         switch (ecx) {
3144         case MSR_IA32_TSC:
3145                 kvm_write_tsc(vcpu, msr);
3146                 break;
3147         case MSR_STAR:
3148                 svm->vmcb->save.star = data;
3149                 break;
3150 #ifdef CONFIG_X86_64
3151         case MSR_LSTAR:
3152                 svm->vmcb->save.lstar = data;
3153                 break;
3154         case MSR_CSTAR:
3155                 svm->vmcb->save.cstar = data;
3156                 break;
3157         case MSR_KERNEL_GS_BASE:
3158                 svm->vmcb->save.kernel_gs_base = data;
3159                 break;
3160         case MSR_SYSCALL_MASK:
3161                 svm->vmcb->save.sfmask = data;
3162                 break;
3163 #endif
3164         case MSR_IA32_SYSENTER_CS:
3165                 svm->vmcb->save.sysenter_cs = data;
3166                 break;
3167         case MSR_IA32_SYSENTER_EIP:
3168                 svm->sysenter_eip = data;
3169                 svm->vmcb->save.sysenter_eip = data;
3170                 break;
3171         case MSR_IA32_SYSENTER_ESP:
3172                 svm->sysenter_esp = data;
3173                 svm->vmcb->save.sysenter_esp = data;
3174                 break;
3175         case MSR_TSC_AUX:
3176                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
3177                         return 1;
3178
3179                 /*
3180                  * This is rare, so we update the MSR here instead of using
3181                  * direct_access_msrs.  Doing that would require a rdmsr in
3182                  * svm_vcpu_put.
3183                  */
3184                 svm->tsc_aux = data;
3185                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
3186                 break;
3187         case MSR_IA32_DEBUGCTLMSR:
3188                 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
3189                         vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
3190                                     __func__, data);
3191                         break;
3192                 }
3193                 if (data & DEBUGCTL_RESERVED_BITS)
3194                         return 1;
3195
3196                 svm->vmcb->save.dbgctl = data;
3197                 mark_dirty(svm->vmcb, VMCB_LBR);
3198                 if (data & (1ULL<<0))
3199                         svm_enable_lbrv(svm);
3200                 else
3201                         svm_disable_lbrv(svm);
3202                 break;
3203         case MSR_VM_HSAVE_PA:
3204                 svm->nested.hsave_msr = data;
3205                 break;
3206         case MSR_VM_CR:
3207                 return svm_set_vm_cr(vcpu, data);
3208         case MSR_VM_IGNNE:
3209                 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
3210                 break;
3211         default:
3212                 return kvm_set_msr_common(vcpu, msr);
3213         }
3214         return 0;
3215 }
3216
3217 static int wrmsr_interception(struct vcpu_svm *svm)
3218 {
3219         struct msr_data msr;
3220         u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3221         u64 data = kvm_read_edx_eax(&svm->vcpu);
3222
3223         msr.data = data;
3224         msr.index = ecx;
3225         msr.host_initiated = false;
3226
3227         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3228         if (kvm_set_msr(&svm->vcpu, &msr)) {
3229                 trace_kvm_msr_write_ex(ecx, data);
3230                 kvm_inject_gp(&svm->vcpu, 0);
3231         } else {
3232                 trace_kvm_msr_write(ecx, data);
3233                 skip_emulated_instruction(&svm->vcpu);
3234         }
3235         return 1;
3236 }
3237
3238 static int msr_interception(struct vcpu_svm *svm)
3239 {
3240         if (svm->vmcb->control.exit_info_1)
3241                 return wrmsr_interception(svm);
3242         else
3243                 return rdmsr_interception(svm);
3244 }
3245
3246 static int interrupt_window_interception(struct vcpu_svm *svm)
3247 {
3248         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3249         svm_clear_vintr(svm);
3250         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
3251         mark_dirty(svm->vmcb, VMCB_INTR);
3252         ++svm->vcpu.stat.irq_window_exits;
3253         return 1;
3254 }
3255
3256 static int pause_interception(struct vcpu_svm *svm)
3257 {
3258         kvm_vcpu_on_spin(&(svm->vcpu));
3259         return 1;
3260 }
3261
3262 static int nop_interception(struct vcpu_svm *svm)
3263 {
3264         skip_emulated_instruction(&(svm->vcpu));
3265         return 1;
3266 }
3267
3268 static int monitor_interception(struct vcpu_svm *svm)
3269 {
3270         printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
3271         return nop_interception(svm);
3272 }
3273
3274 static int mwait_interception(struct vcpu_svm *svm)
3275 {
3276         printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
3277         return nop_interception(svm);
3278 }
3279
3280 static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
3281         [SVM_EXIT_READ_CR0]                     = cr_interception,
3282         [SVM_EXIT_READ_CR3]                     = cr_interception,
3283         [SVM_EXIT_READ_CR4]                     = cr_interception,
3284         [SVM_EXIT_READ_CR8]                     = cr_interception,
3285         [SVM_EXIT_CR0_SEL_WRITE]                = cr_interception,
3286         [SVM_EXIT_WRITE_CR0]                    = cr_interception,
3287         [SVM_EXIT_WRITE_CR3]                    = cr_interception,
3288         [SVM_EXIT_WRITE_CR4]                    = cr_interception,
3289         [SVM_EXIT_WRITE_CR8]                    = cr8_write_interception,
3290         [SVM_EXIT_READ_DR0]                     = dr_interception,
3291         [SVM_EXIT_READ_DR1]                     = dr_interception,
3292         [SVM_EXIT_READ_DR2]                     = dr_interception,
3293         [SVM_EXIT_READ_DR3]                     = dr_interception,
3294         [SVM_EXIT_READ_DR4]                     = dr_interception,
3295         [SVM_EXIT_READ_DR5]                     = dr_interception,
3296         [SVM_EXIT_READ_DR6]                     = dr_interception,
3297         [SVM_EXIT_READ_DR7]                     = dr_interception,
3298         [SVM_EXIT_WRITE_DR0]                    = dr_interception,
3299         [SVM_EXIT_WRITE_DR1]                    = dr_interception,
3300         [SVM_EXIT_WRITE_DR2]                    = dr_interception,
3301         [SVM_EXIT_WRITE_DR3]                    = dr_interception,
3302         [SVM_EXIT_WRITE_DR4]                    = dr_interception,
3303         [SVM_EXIT_WRITE_DR5]                    = dr_interception,
3304         [SVM_EXIT_WRITE_DR6]                    = dr_interception,
3305         [SVM_EXIT_WRITE_DR7]                    = dr_interception,
3306         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
3307         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
3308         [SVM_EXIT_EXCP_BASE + UD_VECTOR]        = ud_interception,
3309         [SVM_EXIT_EXCP_BASE + PF_VECTOR]        = pf_interception,
3310         [SVM_EXIT_EXCP_BASE + NM_VECTOR]        = nm_interception,
3311         [SVM_EXIT_EXCP_BASE + MC_VECTOR]        = mc_interception,
3312         [SVM_EXIT_EXCP_BASE + AC_VECTOR]        = ac_interception,
3313         [SVM_EXIT_INTR]                         = intr_interception,
3314         [SVM_EXIT_NMI]                          = nmi_interception,
3315         [SVM_EXIT_SMI]                          = nop_on_interception,
3316         [SVM_EXIT_INIT]                         = nop_on_interception,
3317         [SVM_EXIT_VINTR]                        = interrupt_window_interception,
3318         [SVM_EXIT_RDPMC]                        = rdpmc_interception,
3319         [SVM_EXIT_CPUID]                        = cpuid_interception,
3320         [SVM_EXIT_IRET]                         = iret_interception,
3321         [SVM_EXIT_INVD]                         = emulate_on_interception,
3322         [SVM_EXIT_PAUSE]                        = pause_interception,
3323         [SVM_EXIT_HLT]                          = halt_interception,
3324         [SVM_EXIT_INVLPG]                       = invlpg_interception,
3325         [SVM_EXIT_INVLPGA]                      = invlpga_interception,
3326         [SVM_EXIT_IOIO]                         = io_interception,
3327         [SVM_EXIT_MSR]                          = msr_interception,
3328         [SVM_EXIT_TASK_SWITCH]                  = task_switch_interception,
3329         [SVM_EXIT_SHUTDOWN]                     = shutdown_interception,
3330         [SVM_EXIT_VMRUN]                        = vmrun_interception,
3331         [SVM_EXIT_VMMCALL]                      = vmmcall_interception,
3332         [SVM_EXIT_VMLOAD]                       = vmload_interception,
3333         [SVM_EXIT_VMSAVE]                       = vmsave_interception,
3334         [SVM_EXIT_STGI]                         = stgi_interception,
3335         [SVM_EXIT_CLGI]                         = clgi_interception,
3336         [SVM_EXIT_SKINIT]                       = skinit_interception,
3337         [SVM_EXIT_WBINVD]                       = wbinvd_interception,
3338         [SVM_EXIT_MONITOR]                      = monitor_interception,
3339         [SVM_EXIT_MWAIT]                        = mwait_interception,
3340         [SVM_EXIT_XSETBV]                       = xsetbv_interception,
3341         [SVM_EXIT_NPF]                          = pf_interception,
3342         [SVM_EXIT_RSM]                          = emulate_on_interception,
3343 };
3344
3345 static void dump_vmcb(struct kvm_vcpu *vcpu)
3346 {
3347         struct vcpu_svm *svm = to_svm(vcpu);
3348         struct vmcb_control_area *control = &svm->vmcb->control;
3349         struct vmcb_save_area *save = &svm->vmcb->save;
3350
3351         pr_err("VMCB Control Area:\n");
3352         pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff);
3353         pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16);
3354         pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff);
3355         pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16);
3356         pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions);
3357         pr_err("%-20s%016llx\n", "intercepts:", control->intercept);
3358         pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
3359         pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
3360         pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
3361         pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
3362         pr_err("%-20s%d\n", "asid:", control->asid);
3363         pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
3364         pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
3365         pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
3366         pr_err("%-20s%08x\n", "int_state:", control->int_state);
3367         pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
3368         pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
3369         pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
3370         pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
3371         pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
3372         pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
3373         pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
3374         pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
3375         pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
3376         pr_err("%-20s%lld\n", "lbr_ctl:", control->lbr_ctl);
3377         pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
3378         pr_err("VMCB State Save Area:\n");
3379         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3380                "es:",
3381                save->es.selector, save->es.attrib,
3382                save->es.limit, save->es.base);
3383         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3384                "cs:",
3385                save->cs.selector, save->cs.attrib,
3386                save->cs.limit, save->cs.base);
3387         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3388                "ss:",
3389                save->ss.selector, save->ss.attrib,
3390                save->ss.limit, save->ss.base);
3391         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3392                "ds:",
3393                save->ds.selector, save->ds.attrib,
3394                save->ds.limit, save->ds.base);
3395         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3396                "fs:",
3397                save->fs.selector, save->fs.attrib,
3398                save->fs.limit, save->fs.base);
3399         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3400                "gs:",
3401                save->gs.selector, save->gs.attrib,
3402                save->gs.limit, save->gs.base);
3403         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3404                "gdtr:",
3405                save->gdtr.selector, save->gdtr.attrib,
3406                save->gdtr.limit, save->gdtr.base);
3407         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3408                "ldtr:",
3409                save->ldtr.selector, save->ldtr.attrib,
3410                save->ldtr.limit, save->ldtr.base);
3411         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3412                "idtr:",
3413                save->idtr.selector, save->idtr.attrib,
3414                save->idtr.limit, save->idtr.base);
3415         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3416                "tr:",
3417                save->tr.selector, save->tr.attrib,
3418                save->tr.limit, save->tr.base);
3419         pr_err("cpl:            %d                efer:         %016llx\n",
3420                 save->cpl, save->efer);
3421         pr_err("%-15s %016llx %-13s %016llx\n",
3422                "cr0:", save->cr0, "cr2:", save->cr2);
3423         pr_err("%-15s %016llx %-13s %016llx\n",
3424                "cr3:", save->cr3, "cr4:", save->cr4);
3425         pr_err("%-15s %016llx %-13s %016llx\n",
3426                "dr6:", save->dr6, "dr7:", save->dr7);
3427         pr_err("%-15s %016llx %-13s %016llx\n",
3428                "rip:", save->rip, "rflags:", save->rflags);
3429         pr_err("%-15s %016llx %-13s %016llx\n",
3430                "rsp:", save->rsp, "rax:", save->rax);
3431         pr_err("%-15s %016llx %-13s %016llx\n",
3432                "star:", save->star, "lstar:", save->lstar);
3433         pr_err("%-15s %016llx %-13s %016llx\n",
3434                "cstar:", save->cstar, "sfmask:", save->sfmask);
3435         pr_err("%-15s %016llx %-13s %016llx\n",
3436                "kernel_gs_base:", save->kernel_gs_base,
3437                "sysenter_cs:", save->sysenter_cs);
3438         pr_err("%-15s %016llx %-13s %016llx\n",
3439                "sysenter_esp:", save->sysenter_esp,
3440                "sysenter_eip:", save->sysenter_eip);
3441         pr_err("%-15s %016llx %-13s %016llx\n",
3442                "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3443         pr_err("%-15s %016llx %-13s %016llx\n",
3444                "br_from:", save->br_from, "br_to:", save->br_to);
3445         pr_err("%-15s %016llx %-13s %016llx\n",
3446                "excp_from:", save->last_excp_from,
3447                "excp_to:", save->last_excp_to);
3448 }
3449
3450 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
3451 {
3452         struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3453
3454         *info1 = control->exit_info_1;
3455         *info2 = control->exit_info_2;
3456 }
3457
3458 static int handle_exit(struct kvm_vcpu *vcpu)
3459 {
3460         struct vcpu_svm *svm = to_svm(vcpu);
3461         struct kvm_run *kvm_run = vcpu->run;
3462         u32 exit_code = svm->vmcb->control.exit_code;
3463
3464         trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
3465
3466         if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
3467                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
3468         if (npt_enabled)
3469                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
3470
3471         if (unlikely(svm->nested.exit_required)) {
3472                 nested_svm_vmexit(svm);
3473                 svm->nested.exit_required = false;
3474
3475                 return 1;
3476         }
3477
3478         if (is_guest_mode(vcpu)) {
3479                 int vmexit;
3480
3481                 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
3482                                         svm->vmcb->control.exit_info_1,
3483                                         svm->vmcb->control.exit_info_2,
3484                                         svm->vmcb->control.exit_int_info,
3485                                         svm->vmcb->control.exit_int_info_err,
3486                                         KVM_ISA_SVM);
3487
3488                 vmexit = nested_svm_exit_special(svm);
3489
3490                 if (vmexit == NESTED_EXIT_CONTINUE)
3491                         vmexit = nested_svm_exit_handled(svm);
3492
3493                 if (vmexit == NESTED_EXIT_DONE)
3494                         return 1;
3495         }
3496
3497         svm_complete_interrupts(svm);
3498
3499         if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3500                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3501                 kvm_run->fail_entry.hardware_entry_failure_reason
3502                         = svm->vmcb->control.exit_code;
3503                 pr_err("KVM: FAILED VMRUN WITH VMCB:\n");
3504                 dump_vmcb(vcpu);
3505                 return 0;
3506         }
3507
3508         if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
3509             exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
3510             exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
3511             exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
3512                 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
3513                        "exit_code 0x%x\n",
3514                        __func__, svm->vmcb->control.exit_int_info,
3515                        exit_code);
3516
3517         if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
3518             || !svm_exit_handlers[exit_code]) {
3519                 WARN_ONCE(1, "svm: unexpected exit reason 0x%x\n", exit_code);
3520                 kvm_queue_exception(vcpu, UD_VECTOR);
3521                 return 1;
3522         }
3523
3524         return svm_exit_handlers[exit_code](svm);
3525 }
3526
3527 static void reload_tss(struct kvm_vcpu *vcpu)
3528 {
3529         int cpu = raw_smp_processor_id();
3530
3531         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
3532         sd->tss_desc->type = 9; /* available 32/64-bit TSS */
3533         load_TR_desc();
3534 }
3535
3536 static void pre_svm_run(struct vcpu_svm *svm)
3537 {
3538         int cpu = raw_smp_processor_id();
3539
3540         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
3541
3542         /* FIXME: handle wraparound of asid_generation */
3543         if (svm->asid_generation != sd->asid_generation)
3544                 new_asid(svm, sd);
3545 }
3546
3547 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3548 {
3549         struct vcpu_svm *svm = to_svm(vcpu);
3550
3551         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3552         vcpu->arch.hflags |= HF_NMI_MASK;
3553         set_intercept(svm, INTERCEPT_IRET);
3554         ++vcpu->stat.nmi_injections;
3555 }
3556
3557 static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
3558 {
3559         struct vmcb_control_area *control;
3560
3561         control = &svm->vmcb->control;
3562         control->int_vector = irq;
3563         control->int_ctl &= ~V_INTR_PRIO_MASK;
3564         control->int_ctl |= V_IRQ_MASK |
3565                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
3566         mark_dirty(svm->vmcb, VMCB_INTR);
3567 }
3568
3569 static void svm_set_irq(struct kvm_vcpu *vcpu)
3570 {
3571         struct vcpu_svm *svm = to_svm(vcpu);
3572
3573         BUG_ON(!(gif_set(svm)));
3574
3575         trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
3576         ++vcpu->stat.irq_injections;
3577
3578         svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3579                 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
3580 }
3581
3582 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3583 {
3584         struct vcpu_svm *svm = to_svm(vcpu);
3585
3586         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3587                 return;
3588
3589         clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3590
3591         if (irr == -1)
3592                 return;
3593
3594         if (tpr >= irr)
3595                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3596 }
3597
3598 static void svm_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
3599 {
3600         return;
3601 }
3602
3603 static bool svm_get_enable_apicv(void)
3604 {
3605         return false;
3606 }
3607
3608 static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
3609 {
3610 }
3611
3612 static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
3613 {
3614         return;
3615 }
3616
3617 static void svm_sync_pir_to_irr(struct kvm_vcpu *vcpu)
3618 {
3619         return;
3620 }
3621
3622 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
3623 {
3624         struct vcpu_svm *svm = to_svm(vcpu);
3625         struct vmcb *vmcb = svm->vmcb;
3626         int ret;
3627         ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
3628               !(svm->vcpu.arch.hflags & HF_NMI_MASK);
3629         ret = ret && gif_set(svm) && nested_svm_nmi(svm);
3630
3631         return ret;
3632 }
3633
3634 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3635 {
3636         struct vcpu_svm *svm = to_svm(vcpu);
3637
3638         return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
3639 }
3640
3641 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3642 {
3643         struct vcpu_svm *svm = to_svm(vcpu);
3644
3645         if (masked) {
3646                 svm->vcpu.arch.hflags |= HF_NMI_MASK;
3647                 set_intercept(svm, INTERCEPT_IRET);
3648         } else {
3649                 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
3650                 clr_intercept(svm, INTERCEPT_IRET);
3651         }
3652 }
3653
3654 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
3655 {
3656         struct vcpu_svm *svm = to_svm(vcpu);
3657         struct vmcb *vmcb = svm->vmcb;
3658         int ret;
3659
3660         if (!gif_set(svm) ||
3661              (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
3662                 return 0;
3663
3664         ret = !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF);
3665
3666         if (is_guest_mode(vcpu))
3667                 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
3668
3669         return ret;
3670 }
3671
3672 static void enable_irq_window(struct kvm_vcpu *vcpu)
3673 {
3674         struct vcpu_svm *svm = to_svm(vcpu);
3675
3676         /*
3677          * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3678          * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3679          * get that intercept, this function will be called again though and
3680          * we'll get the vintr intercept.
3681          */
3682         if (gif_set(svm) && nested_svm_intr(svm)) {
3683                 svm_set_vintr(svm);
3684                 svm_inject_irq(svm, 0x0);
3685         }
3686 }
3687
3688 static void enable_nmi_window(struct kvm_vcpu *vcpu)
3689 {
3690         struct vcpu_svm *svm = to_svm(vcpu);
3691
3692         if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
3693             == HF_NMI_MASK)
3694                 return; /* IRET will cause a vm exit */
3695
3696         /*
3697          * Something prevents NMI from been injected. Single step over possible
3698          * problem (IRET or exception injection or interrupt shadow)
3699          */
3700         svm->nmi_singlestep = true;
3701         svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3702 }
3703
3704 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
3705 {
3706         return 0;
3707 }
3708
3709 static void svm_flush_tlb(struct kvm_vcpu *vcpu)
3710 {
3711         struct vcpu_svm *svm = to_svm(vcpu);
3712
3713         if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3714                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3715         else
3716                 svm->asid_generation--;
3717 }
3718
3719 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
3720 {
3721 }
3722
3723 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3724 {
3725         struct vcpu_svm *svm = to_svm(vcpu);
3726
3727         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3728                 return;
3729
3730         if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
3731                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3732                 kvm_set_cr8(vcpu, cr8);
3733         }
3734 }
3735
3736 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
3737 {
3738         struct vcpu_svm *svm = to_svm(vcpu);
3739         u64 cr8;
3740
3741         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3742                 return;
3743
3744         cr8 = kvm_get_cr8(vcpu);
3745         svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
3746         svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
3747 }
3748
3749 static void svm_complete_interrupts(struct vcpu_svm *svm)
3750 {
3751         u8 vector;
3752         int type;
3753         u32 exitintinfo = svm->vmcb->control.exit_int_info;
3754         unsigned int3_injected = svm->int3_injected;
3755
3756         svm->int3_injected = 0;
3757
3758         /*
3759          * If we've made progress since setting HF_IRET_MASK, we've
3760          * executed an IRET and can allow NMI injection.
3761          */
3762         if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
3763             && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
3764                 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
3765                 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3766         }
3767
3768         svm->vcpu.arch.nmi_injected = false;
3769         kvm_clear_exception_queue(&svm->vcpu);
3770         kvm_clear_interrupt_queue(&svm->vcpu);
3771
3772         if (!(exitintinfo & SVM_EXITINTINFO_VALID))
3773                 return;
3774
3775         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3776
3777         vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
3778         type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
3779
3780         switch (type) {
3781         case SVM_EXITINTINFO_TYPE_NMI:
3782                 svm->vcpu.arch.nmi_injected = true;
3783                 break;
3784         case SVM_EXITINTINFO_TYPE_EXEPT:
3785                 /*
3786                  * In case of software exceptions, do not reinject the vector,
3787                  * but re-execute the instruction instead. Rewind RIP first
3788                  * if we emulated INT3 before.
3789                  */
3790                 if (kvm_exception_is_soft(vector)) {
3791                         if (vector == BP_VECTOR && int3_injected &&
3792                             kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
3793                                 kvm_rip_write(&svm->vcpu,
3794                                               kvm_rip_read(&svm->vcpu) -
3795                                               int3_injected);
3796                         break;
3797                 }
3798                 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
3799                         u32 err = svm->vmcb->control.exit_int_info_err;
3800                         kvm_requeue_exception_e(&svm->vcpu, vector, err);
3801
3802                 } else
3803                         kvm_requeue_exception(&svm->vcpu, vector);
3804                 break;
3805         case SVM_EXITINTINFO_TYPE_INTR:
3806                 kvm_queue_interrupt(&svm->vcpu, vector, false);
3807                 break;
3808         default:
3809                 break;
3810         }
3811 }
3812
3813 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
3814 {
3815         struct vcpu_svm *svm = to_svm(vcpu);
3816         struct vmcb_control_area *control = &svm->vmcb->control;
3817
3818         control->exit_int_info = control->event_inj;
3819         control->exit_int_info_err = control->event_inj_err;
3820         control->event_inj = 0;
3821         svm_complete_interrupts(svm);
3822 }
3823
3824 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
3825 {
3826         struct vcpu_svm *svm = to_svm(vcpu);
3827
3828         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
3829         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
3830         svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
3831
3832         /*
3833          * A vmexit emulation is required before the vcpu can be executed
3834          * again.
3835          */
3836         if (unlikely(svm->nested.exit_required))
3837                 return;
3838
3839         pre_svm_run(svm);
3840
3841         sync_lapic_to_cr8(vcpu);
3842
3843         svm->vmcb->save.cr2 = vcpu->arch.cr2;
3844
3845         clgi();
3846
3847         local_irq_enable();
3848
3849         asm volatile (
3850                 "push %%" _ASM_BP "; \n\t"
3851                 "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
3852                 "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
3853                 "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
3854                 "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
3855                 "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
3856                 "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
3857 #ifdef CONFIG_X86_64
3858                 "mov %c[r8](%[svm]),  %%r8  \n\t"
3859                 "mov %c[r9](%[svm]),  %%r9  \n\t"
3860                 "mov %c[r10](%[svm]), %%r10 \n\t"
3861                 "mov %c[r11](%[svm]), %%r11 \n\t"
3862                 "mov %c[r12](%[svm]), %%r12 \n\t"
3863                 "mov %c[r13](%[svm]), %%r13 \n\t"
3864                 "mov %c[r14](%[svm]), %%r14 \n\t"
3865                 "mov %c[r15](%[svm]), %%r15 \n\t"
3866 #endif
3867
3868                 /* Enter guest mode */
3869                 "push %%" _ASM_AX " \n\t"
3870                 "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
3871                 __ex(SVM_VMLOAD) "\n\t"
3872                 __ex(SVM_VMRUN) "\n\t"
3873                 __ex(SVM_VMSAVE) "\n\t"
3874                 "pop %%" _ASM_AX " \n\t"
3875
3876                 /* Save guest registers, load host registers */
3877                 "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
3878                 "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
3879                 "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
3880                 "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
3881                 "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
3882                 "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
3883 #ifdef CONFIG_X86_64
3884                 "mov %%r8,  %c[r8](%[svm]) \n\t"
3885                 "mov %%r9,  %c[r9](%[svm]) \n\t"
3886                 "mov %%r10, %c[r10](%[svm]) \n\t"
3887                 "mov %%r11, %c[r11](%[svm]) \n\t"
3888                 "mov %%r12, %c[r12](%[svm]) \n\t"
3889                 "mov %%r13, %c[r13](%[svm]) \n\t"
3890                 "mov %%r14, %c[r14](%[svm]) \n\t"
3891                 "mov %%r15, %c[r15](%[svm]) \n\t"
3892 #endif
3893                 "pop %%" _ASM_BP
3894                 :
3895                 : [svm]"a"(svm),
3896                   [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
3897                   [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
3898                   [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
3899                   [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
3900                   [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
3901                   [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
3902                   [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
3903 #ifdef CONFIG_X86_64
3904                   , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
3905                   [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
3906                   [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
3907                   [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
3908                   [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
3909                   [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
3910                   [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
3911                   [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
3912 #endif
3913                 : "cc", "memory"
3914 #ifdef CONFIG_X86_64
3915                 , "rbx", "rcx", "rdx", "rsi", "rdi"
3916                 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
3917 #else
3918                 , "ebx", "ecx", "edx", "esi", "edi"
3919 #endif
3920                 );
3921
3922 #ifdef CONFIG_X86_64
3923         wrmsrl(MSR_GS_BASE, svm->host.gs_base);
3924 #else
3925         loadsegment(fs, svm->host.fs);
3926 #ifndef CONFIG_X86_32_LAZY_GS
3927         loadsegment(gs, svm->host.gs);
3928 #endif
3929 #endif
3930
3931         reload_tss(vcpu);
3932
3933         local_irq_disable();
3934
3935         vcpu->arch.cr2 = svm->vmcb->save.cr2;
3936         vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
3937         vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
3938         vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
3939
3940         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3941                 kvm_before_handle_nmi(&svm->vcpu);
3942
3943         stgi();
3944
3945         /* Any pending NMI will happen here */
3946
3947         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
3948                 kvm_after_handle_nmi(&svm->vcpu);
3949
3950         sync_cr8_to_lapic(vcpu);
3951
3952         svm->next_rip = 0;
3953
3954         svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
3955
3956         /* if exit due to PF check for async PF */
3957         if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
3958                 svm->apf_reason = kvm_read_and_reset_pf_reason();
3959
3960         if (npt_enabled) {
3961                 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
3962                 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
3963         }
3964
3965         /*
3966          * We need to handle MC intercepts here before the vcpu has a chance to
3967          * change the physical cpu
3968          */
3969         if (unlikely(svm->vmcb->control.exit_code ==
3970                      SVM_EXIT_EXCP_BASE + MC_VECTOR))
3971                 svm_handle_mce(svm);
3972
3973         mark_all_clean(svm->vmcb);
3974 }
3975
3976 static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
3977 {
3978         struct vcpu_svm *svm = to_svm(vcpu);
3979
3980         svm->vmcb->save.cr3 = root;
3981         mark_dirty(svm->vmcb, VMCB_CR);
3982         svm_flush_tlb(vcpu);
3983 }
3984
3985 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
3986 {
3987         struct vcpu_svm *svm = to_svm(vcpu);
3988
3989         svm->vmcb->control.nested_cr3 = root;
3990         mark_dirty(svm->vmcb, VMCB_NPT);
3991
3992         /* Also sync guest cr3 here in case we live migrate */
3993         svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
3994         mark_dirty(svm->vmcb, VMCB_CR);
3995
3996         svm_flush_tlb(vcpu);
3997 }
3998
3999 static int is_disabled(void)
4000 {
4001         u64 vm_cr;
4002
4003         rdmsrl(MSR_VM_CR, vm_cr);
4004         if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
4005                 return 1;
4006
4007         return 0;
4008 }
4009
4010 static void
4011 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4012 {
4013         /*
4014          * Patch in the VMMCALL instruction:
4015          */
4016         hypercall[0] = 0x0f;
4017         hypercall[1] = 0x01;
4018         hypercall[2] = 0xd9;
4019 }
4020
4021 static void svm_check_processor_compat(void *rtn)
4022 {
4023         *(int *)rtn = 0;
4024 }
4025
4026 static bool svm_cpu_has_accelerated_tpr(void)
4027 {
4028         return false;
4029 }
4030
4031 static bool svm_has_high_real_mode_segbase(void)
4032 {
4033         return true;
4034 }
4035
4036 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
4037 {
4038         return 0;
4039 }
4040
4041 static void svm_cpuid_update(struct kvm_vcpu *vcpu)
4042 {
4043         struct vcpu_svm *svm = to_svm(vcpu);
4044
4045         /* Update nrips enabled cache */
4046         svm->nrips_enabled = !!guest_cpuid_has_nrips(&svm->vcpu);
4047 }
4048
4049 static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
4050 {
4051         switch (func) {
4052         case 0x80000001:
4053                 if (nested)
4054                         entry->ecx |= (1 << 2); /* Set SVM bit */
4055                 break;
4056         case 0x8000000A:
4057                 entry->eax = 1; /* SVM revision 1 */
4058                 entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
4059                                    ASID emulation to nested SVM */
4060                 entry->ecx = 0; /* Reserved */
4061                 entry->edx = 0; /* Per default do not support any
4062                                    additional features */
4063
4064                 /* Support next_rip if host supports it */
4065                 if (boot_cpu_has(X86_FEATURE_NRIPS))
4066                         entry->edx |= SVM_FEATURE_NRIP;
4067
4068                 /* Support NPT for the guest if enabled */
4069                 if (npt_enabled)
4070                         entry->edx |= SVM_FEATURE_NPT;
4071
4072                 break;
4073         }
4074 }
4075
4076 static int svm_get_lpage_level(void)
4077 {
4078         return PT_PDPE_LEVEL;
4079 }
4080
4081 static bool svm_rdtscp_supported(void)
4082 {
4083         return boot_cpu_has(X86_FEATURE_RDTSCP);
4084 }
4085
4086 static bool svm_invpcid_supported(void)
4087 {
4088         return false;
4089 }
4090
4091 static bool svm_mpx_supported(void)
4092 {
4093         return false;
4094 }
4095
4096 static bool svm_xsaves_supported(void)
4097 {
4098         return false;
4099 }
4100
4101 static bool svm_has_wbinvd_exit(void)
4102 {
4103         return true;
4104 }
4105
4106 static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
4107 {
4108         struct vcpu_svm *svm = to_svm(vcpu);
4109
4110         set_exception_intercept(svm, NM_VECTOR);
4111         update_cr0_intercept(svm);
4112 }
4113
4114 #define PRE_EX(exit)  { .exit_code = (exit), \
4115                         .stage = X86_ICPT_PRE_EXCEPT, }
4116 #define POST_EX(exit) { .exit_code = (exit), \
4117                         .stage = X86_ICPT_POST_EXCEPT, }
4118 #define POST_MEM(exit) { .exit_code = (exit), \
4119                         .stage = X86_ICPT_POST_MEMACCESS, }
4120
4121 static const struct __x86_intercept {
4122         u32 exit_code;
4123         enum x86_intercept_stage stage;
4124 } x86_intercept_map[] = {
4125         [x86_intercept_cr_read]         = POST_EX(SVM_EXIT_READ_CR0),
4126         [x86_intercept_cr_write]        = POST_EX(SVM_EXIT_WRITE_CR0),
4127         [x86_intercept_clts]            = POST_EX(SVM_EXIT_WRITE_CR0),
4128         [x86_intercept_lmsw]            = POST_EX(SVM_EXIT_WRITE_CR0),
4129         [x86_intercept_smsw]            = POST_EX(SVM_EXIT_READ_CR0),
4130         [x86_intercept_dr_read]         = POST_EX(SVM_EXIT_READ_DR0),
4131         [x86_intercept_dr_write]        = POST_EX(SVM_EXIT_WRITE_DR0),
4132         [x86_intercept_sldt]            = POST_EX(SVM_EXIT_LDTR_READ),
4133         [x86_intercept_str]             = POST_EX(SVM_EXIT_TR_READ),
4134         [x86_intercept_lldt]            = POST_EX(SVM_EXIT_LDTR_WRITE),
4135         [x86_intercept_ltr]             = POST_EX(SVM_EXIT_TR_WRITE),
4136         [x86_intercept_sgdt]            = POST_EX(SVM_EXIT_GDTR_READ),
4137         [x86_intercept_sidt]            = POST_EX(SVM_EXIT_IDTR_READ),
4138         [x86_intercept_lgdt]            = POST_EX(SVM_EXIT_GDTR_WRITE),
4139         [x86_intercept_lidt]            = POST_EX(SVM_EXIT_IDTR_WRITE),
4140         [x86_intercept_vmrun]           = POST_EX(SVM_EXIT_VMRUN),
4141         [x86_intercept_vmmcall]         = POST_EX(SVM_EXIT_VMMCALL),
4142         [x86_intercept_vmload]          = POST_EX(SVM_EXIT_VMLOAD),
4143         [x86_intercept_vmsave]          = POST_EX(SVM_EXIT_VMSAVE),
4144         [x86_intercept_stgi]            = POST_EX(SVM_EXIT_STGI),
4145         [x86_intercept_clgi]            = POST_EX(SVM_EXIT_CLGI),
4146         [x86_intercept_skinit]          = POST_EX(SVM_EXIT_SKINIT),
4147         [x86_intercept_invlpga]         = POST_EX(SVM_EXIT_INVLPGA),
4148         [x86_intercept_rdtscp]          = POST_EX(SVM_EXIT_RDTSCP),
4149         [x86_intercept_monitor]         = POST_MEM(SVM_EXIT_MONITOR),
4150         [x86_intercept_mwait]           = POST_EX(SVM_EXIT_MWAIT),
4151         [x86_intercept_invlpg]          = POST_EX(SVM_EXIT_INVLPG),
4152         [x86_intercept_invd]            = POST_EX(SVM_EXIT_INVD),
4153         [x86_intercept_wbinvd]          = POST_EX(SVM_EXIT_WBINVD),
4154         [x86_intercept_wrmsr]           = POST_EX(SVM_EXIT_MSR),
4155         [x86_intercept_rdtsc]           = POST_EX(SVM_EXIT_RDTSC),
4156         [x86_intercept_rdmsr]           = POST_EX(SVM_EXIT_MSR),
4157         [x86_intercept_rdpmc]           = POST_EX(SVM_EXIT_RDPMC),
4158         [x86_intercept_cpuid]           = PRE_EX(SVM_EXIT_CPUID),
4159         [x86_intercept_rsm]             = PRE_EX(SVM_EXIT_RSM),
4160         [x86_intercept_pause]           = PRE_EX(SVM_EXIT_PAUSE),
4161         [x86_intercept_pushf]           = PRE_EX(SVM_EXIT_PUSHF),
4162         [x86_intercept_popf]            = PRE_EX(SVM_EXIT_POPF),
4163         [x86_intercept_intn]            = PRE_EX(SVM_EXIT_SWINT),
4164         [x86_intercept_iret]            = PRE_EX(SVM_EXIT_IRET),
4165         [x86_intercept_icebp]           = PRE_EX(SVM_EXIT_ICEBP),
4166         [x86_intercept_hlt]             = POST_EX(SVM_EXIT_HLT),
4167         [x86_intercept_in]              = POST_EX(SVM_EXIT_IOIO),
4168         [x86_intercept_ins]             = POST_EX(SVM_EXIT_IOIO),
4169         [x86_intercept_out]             = POST_EX(SVM_EXIT_IOIO),
4170         [x86_intercept_outs]            = POST_EX(SVM_EXIT_IOIO),
4171 };
4172
4173 #undef PRE_EX
4174 #undef POST_EX
4175 #undef POST_MEM
4176
4177 static int svm_check_intercept(struct kvm_vcpu *vcpu,
4178                                struct x86_instruction_info *info,
4179                                enum x86_intercept_stage stage)
4180 {
4181         struct vcpu_svm *svm = to_svm(vcpu);
4182         int vmexit, ret = X86EMUL_CONTINUE;
4183         struct __x86_intercept icpt_info;
4184         struct vmcb *vmcb = svm->vmcb;
4185
4186         if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
4187                 goto out;
4188
4189         icpt_info = x86_intercept_map[info->intercept];
4190
4191         if (stage != icpt_info.stage)
4192                 goto out;
4193
4194         switch (icpt_info.exit_code) {
4195         case SVM_EXIT_READ_CR0:
4196                 if (info->intercept == x86_intercept_cr_read)
4197                         icpt_info.exit_code += info->modrm_reg;
4198                 break;
4199         case SVM_EXIT_WRITE_CR0: {
4200                 unsigned long cr0, val;
4201                 u64 intercept;
4202
4203                 if (info->intercept == x86_intercept_cr_write)
4204                         icpt_info.exit_code += info->modrm_reg;
4205
4206                 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
4207                     info->intercept == x86_intercept_clts)
4208                         break;
4209
4210                 intercept = svm->nested.intercept;
4211
4212                 if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))
4213                         break;
4214
4215                 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
4216                 val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
4217
4218                 if (info->intercept == x86_intercept_lmsw) {
4219                         cr0 &= 0xfUL;
4220                         val &= 0xfUL;
4221                         /* lmsw can't clear PE - catch this here */
4222                         if (cr0 & X86_CR0_PE)
4223                                 val |= X86_CR0_PE;
4224                 }
4225
4226                 if (cr0 ^ val)
4227                         icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4228
4229                 break;
4230         }
4231         case SVM_EXIT_READ_DR0:
4232         case SVM_EXIT_WRITE_DR0:
4233                 icpt_info.exit_code += info->modrm_reg;
4234                 break;
4235         case SVM_EXIT_MSR:
4236                 if (info->intercept == x86_intercept_wrmsr)
4237                         vmcb->control.exit_info_1 = 1;
4238                 else
4239                         vmcb->control.exit_info_1 = 0;
4240                 break;
4241         case SVM_EXIT_PAUSE:
4242                 /*
4243                  * We get this for NOP only, but pause
4244                  * is rep not, check this here
4245                  */
4246                 if (info->rep_prefix != REPE_PREFIX)
4247                         goto out;
4248         case SVM_EXIT_IOIO: {
4249                 u64 exit_info;
4250                 u32 bytes;
4251
4252                 if (info->intercept == x86_intercept_in ||
4253                     info->intercept == x86_intercept_ins) {
4254                         exit_info = ((info->src_val & 0xffff) << 16) |
4255                                 SVM_IOIO_TYPE_MASK;
4256                         bytes = info->dst_bytes;
4257                 } else {
4258                         exit_info = (info->dst_val & 0xffff) << 16;
4259                         bytes = info->src_bytes;
4260                 }
4261
4262                 if (info->intercept == x86_intercept_outs ||
4263                     info->intercept == x86_intercept_ins)
4264                         exit_info |= SVM_IOIO_STR_MASK;
4265
4266                 if (info->rep_prefix)
4267                         exit_info |= SVM_IOIO_REP_MASK;
4268
4269                 bytes = min(bytes, 4u);
4270
4271                 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
4272
4273                 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
4274
4275                 vmcb->control.exit_info_1 = exit_info;
4276                 vmcb->control.exit_info_2 = info->next_rip;
4277
4278                 break;
4279         }
4280         default:
4281                 break;
4282         }
4283
4284         /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
4285         if (static_cpu_has(X86_FEATURE_NRIPS))
4286                 vmcb->control.next_rip  = info->next_rip;
4287         vmcb->control.exit_code = icpt_info.exit_code;
4288         vmexit = nested_svm_exit_handled(svm);
4289
4290         ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
4291                                            : X86EMUL_CONTINUE;
4292
4293 out:
4294         return ret;
4295 }
4296
4297 static void svm_handle_external_intr(struct kvm_vcpu *vcpu)
4298 {
4299         local_irq_enable();
4300 }
4301
4302 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
4303 {
4304 }
4305
4306 static struct kvm_x86_ops svm_x86_ops = {
4307         .cpu_has_kvm_support = has_svm,
4308         .disabled_by_bios = is_disabled,
4309         .hardware_setup = svm_hardware_setup,
4310         .hardware_unsetup = svm_hardware_unsetup,
4311         .check_processor_compatibility = svm_check_processor_compat,
4312         .hardware_enable = svm_hardware_enable,
4313         .hardware_disable = svm_hardware_disable,
4314         .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
4315         .cpu_has_high_real_mode_segbase = svm_has_high_real_mode_segbase,
4316
4317         .vcpu_create = svm_create_vcpu,
4318         .vcpu_free = svm_free_vcpu,
4319         .vcpu_reset = svm_vcpu_reset,
4320
4321         .prepare_guest_switch = svm_prepare_guest_switch,
4322         .vcpu_load = svm_vcpu_load,
4323         .vcpu_put = svm_vcpu_put,
4324
4325         .update_bp_intercept = update_bp_intercept,
4326         .get_msr = svm_get_msr,
4327         .set_msr = svm_set_msr,
4328         .get_segment_base = svm_get_segment_base,
4329         .get_segment = svm_get_segment,
4330         .set_segment = svm_set_segment,
4331         .get_cpl = svm_get_cpl,
4332         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
4333         .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
4334         .decache_cr3 = svm_decache_cr3,
4335         .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
4336         .set_cr0 = svm_set_cr0,
4337         .set_cr3 = svm_set_cr3,
4338         .set_cr4 = svm_set_cr4,
4339         .set_efer = svm_set_efer,
4340         .get_idt = svm_get_idt,
4341         .set_idt = svm_set_idt,
4342         .get_gdt = svm_get_gdt,
4343         .set_gdt = svm_set_gdt,
4344         .get_dr6 = svm_get_dr6,
4345         .set_dr6 = svm_set_dr6,
4346         .set_dr7 = svm_set_dr7,
4347         .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
4348         .cache_reg = svm_cache_reg,
4349         .get_rflags = svm_get_rflags,
4350         .set_rflags = svm_set_rflags,
4351         .fpu_activate = svm_fpu_activate,
4352         .fpu_deactivate = svm_fpu_deactivate,
4353
4354         .tlb_flush = svm_flush_tlb,
4355
4356         .run = svm_vcpu_run,
4357         .handle_exit = handle_exit,
4358         .skip_emulated_instruction = skip_emulated_instruction,
4359         .set_interrupt_shadow = svm_set_interrupt_shadow,
4360         .get_interrupt_shadow = svm_get_interrupt_shadow,
4361         .patch_hypercall = svm_patch_hypercall,
4362         .set_irq = svm_set_irq,
4363         .set_nmi = svm_inject_nmi,
4364         .queue_exception = svm_queue_exception,
4365         .cancel_injection = svm_cancel_injection,
4366         .interrupt_allowed = svm_interrupt_allowed,
4367         .nmi_allowed = svm_nmi_allowed,
4368         .get_nmi_mask = svm_get_nmi_mask,
4369         .set_nmi_mask = svm_set_nmi_mask,
4370         .enable_nmi_window = enable_nmi_window,
4371         .enable_irq_window = enable_irq_window,
4372         .update_cr8_intercept = update_cr8_intercept,
4373         .set_virtual_x2apic_mode = svm_set_virtual_x2apic_mode,
4374         .get_enable_apicv = svm_get_enable_apicv,
4375         .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
4376         .load_eoi_exitmap = svm_load_eoi_exitmap,
4377         .sync_pir_to_irr = svm_sync_pir_to_irr,
4378
4379         .set_tss_addr = svm_set_tss_addr,
4380         .get_tdp_level = get_npt_level,
4381         .get_mt_mask = svm_get_mt_mask,
4382
4383         .get_exit_info = svm_get_exit_info,
4384
4385         .get_lpage_level = svm_get_lpage_level,
4386
4387         .cpuid_update = svm_cpuid_update,
4388
4389         .rdtscp_supported = svm_rdtscp_supported,
4390         .invpcid_supported = svm_invpcid_supported,
4391         .mpx_supported = svm_mpx_supported,
4392         .xsaves_supported = svm_xsaves_supported,
4393
4394         .set_supported_cpuid = svm_set_supported_cpuid,
4395
4396         .has_wbinvd_exit = svm_has_wbinvd_exit,
4397
4398         .read_tsc_offset = svm_read_tsc_offset,
4399         .write_tsc_offset = svm_write_tsc_offset,
4400         .adjust_tsc_offset_guest = svm_adjust_tsc_offset_guest,
4401         .read_l1_tsc = svm_read_l1_tsc,
4402
4403         .set_tdp_cr3 = set_tdp_cr3,
4404
4405         .check_intercept = svm_check_intercept,
4406         .handle_external_intr = svm_handle_external_intr,
4407
4408         .sched_in = svm_sched_in,
4409
4410         .pmu_ops = &amd_pmu_ops,
4411 };
4412
4413 static int __init svm_init(void)
4414 {
4415         return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
4416                         __alignof__(struct vcpu_svm), THIS_MODULE);
4417 }
4418
4419 static void __exit svm_exit(void)
4420 {
4421         kvm_exit();
4422 }
4423
4424 module_init(svm_init)
4425 module_exit(svm_exit)