arch/mips/kvm/emulate.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * KVM/MIPS: Instruction/Exception emulation
   7  *
   8  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
   9  * Authors: Sanjay Lal <sanjayl@kymasys.com>
  10  */
  11
  12 #include <linux/errno.h>
  13 #include <linux/err.h>
  14 #include <linux/ktime.h>
  15 #include <linux/kvm_host.h>
  16 #include <linux/module.h>
  17 #include <linux/vmalloc.h>
  18 #include <linux/fs.h>
  19 #include <linux/bootmem.h>
  20 #include <linux/random.h>
  21 #include <asm/page.h>
  22 #include <asm/cacheflush.h>
  23 #include <asm/cacheops.h>
  24 #include <asm/cpu-info.h>
  25 #include <asm/mmu_context.h>
  26 #include <asm/tlbflush.h>
  27 #include <asm/inst.h>
  28
  29 #undef CONFIG_MIPS_MT
  30 #include <asm/r4kcache.h>
  31 #define CONFIG_MIPS_MT
  32
  33 #include "interrupt.h"
  34 #include "commpage.h"
  35
  36 #include "trace.h"
  37
  38 /*
  39  * Compute the return address and do emulate branch simulation, if required.
  40  * This function should be called only in branch delay slot active.
  41  */
  42 unsigned long kvm_compute_return_epc(struct kvm_vcpu *vcpu,
  43         unsigned long instpc)
  44 {
  45         unsigned int dspcontrol;
  46         union mips_instruction insn;
  47         struct kvm_vcpu_arch *arch = &vcpu->arch;
  48         long epc = instpc;
  49         long nextpc = KVM_INVALID_INST;
  50
  51         if (epc & 3)
  52                 goto unaligned;
  53
  54         /* Read the instruction */
  55         insn.word = kvm_get_inst((u32 *) epc, vcpu);
  56
  57         if (insn.word == KVM_INVALID_INST)
  58                 return KVM_INVALID_INST;
  59
  60         switch (insn.i_format.opcode) {
  61                 /* jr and jalr are in r_format format. */
  62         case spec_op:
  63                 switch (insn.r_format.func) {
  64                 case jalr_op:
  65                         arch->gprs[insn.r_format.rd] = epc + 8;
  66                         /* Fall through */
  67                 case jr_op:
  68                         nextpc = arch->gprs[insn.r_format.rs];
  69                         break;
  70                 }
  71                 break;
  72
  73                 /*
  74                  * This group contains:
  75                  * bltz_op, bgez_op, bltzl_op, bgezl_op,
  76                  * bltzal_op, bgezal_op, bltzall_op, bgezall_op.
  77                  */
  78         case bcond_op:
  79                 switch (insn.i_format.rt) {
  80                 case bltz_op:
  81                 case bltzl_op:
  82                         if ((long)arch->gprs[insn.i_format.rs] < 0)
  83                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
  84                         else
  85                                 epc += 8;
  86                         nextpc = epc;
  87                         break;
  88
  89                 case bgez_op:
  90                 case bgezl_op:
  91                         if ((long)arch->gprs[insn.i_format.rs] >= 0)
  92                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
  93                         else
  94                                 epc += 8;
  95                         nextpc = epc;
  96                         break;
  97
  98                 case bltzal_op:
  99                 case bltzall_op:
 100                         arch->gprs[31] = epc + 8;
 101                         if ((long)arch->gprs[insn.i_format.rs] < 0)
 102                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 103                         else
 104                                 epc += 8;
 105                         nextpc = epc;
 106                         break;
 107
 108                 case bgezal_op:
 109                 case bgezall_op:
 110                         arch->gprs[31] = epc + 8;
 111                         if ((long)arch->gprs[insn.i_format.rs] >= 0)
 112                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 113                         else
 114                                 epc += 8;
 115                         nextpc = epc;
 116                         break;
 117                 case bposge32_op:
 118                         if (!cpu_has_dsp)
 119                                 goto sigill;
 120
 121                         dspcontrol = rddsp(0x01);
 122
 123                         if (dspcontrol >= 32)
 124                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 125                         else
 126                                 epc += 8;
 127                         nextpc = epc;
 128                         break;
 129                 }
 130                 break;
 131
 132                 /* These are unconditional and in j_format. */
 133         case jal_op:
 134                 arch->gprs[31] = instpc + 8;
 135         case j_op:
 136                 epc += 4;
 137                 epc >>= 28;
 138                 epc <<= 28;
 139                 epc |= (insn.j_format.target << 2);
 140                 nextpc = epc;
 141                 break;
 142
 143                 /* These are conditional and in i_format. */
 144         case beq_op:
 145         case beql_op:
 146                 if (arch->gprs[insn.i_format.rs] ==
 147                     arch->gprs[insn.i_format.rt])
 148                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 149                 else
 150                         epc += 8;
 151                 nextpc = epc;
 152                 break;
 153
 154         case bne_op:
 155         case bnel_op:
 156                 if (arch->gprs[insn.i_format.rs] !=
 157                     arch->gprs[insn.i_format.rt])
 158                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 159                 else
 160                         epc += 8;
 161                 nextpc = epc;
 162                 break;
 163
 164         case blez_op:   /* POP06 */
 165 #ifndef CONFIG_CPU_MIPSR6
 166         case blezl_op:  /* removed in R6 */
 167 #endif
 168                 if (insn.i_format.rt != 0)
 169                         goto compact_branch;
 170                 if ((long)arch->gprs[insn.i_format.rs] <= 0)
 171                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 172                 else
 173                         epc += 8;
 174                 nextpc = epc;
 175                 break;
 176
 177         case bgtz_op:   /* POP07 */
 178 #ifndef CONFIG_CPU_MIPSR6
 179         case bgtzl_op:  /* removed in R6 */
 180 #endif
 181                 if (insn.i_format.rt != 0)
 182                         goto compact_branch;
 183                 if ((long)arch->gprs[insn.i_format.rs] > 0)
 184                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 185                 else
 186                         epc += 8;
 187                 nextpc = epc;
 188                 break;
 189
 190                 /* And now the FPA/cp1 branch instructions. */
 191         case cop1_op:
 192                 kvm_err("%s: unsupported cop1_op\n", __func__);
 193                 break;
 194
 195 #ifdef CONFIG_CPU_MIPSR6
 196         /* R6 added the following compact branches with forbidden slots */
 197         case blezl_op:  /* POP26 */
 198         case bgtzl_op:  /* POP27 */
 199                 /* only rt == 0 isn't compact branch */
 200                 if (insn.i_format.rt != 0)
 201                         goto compact_branch;
 202                 break;
 203         case pop10_op:
 204         case pop30_op:
 205                 /* only rs == rt == 0 is reserved, rest are compact branches */
 206                 if (insn.i_format.rs != 0 || insn.i_format.rt != 0)
 207                         goto compact_branch;
 208                 break;
 209         case pop66_op:
 210         case pop76_op:
 211                 /* only rs == 0 isn't compact branch */
 212                 if (insn.i_format.rs != 0)
 213                         goto compact_branch;
 214                 break;
 215 compact_branch:
 216                 /*
 217                  * If we've hit an exception on the forbidden slot, then
 218                  * the branch must not have been taken.
 219                  */
 220                 epc += 8;
 221                 nextpc = epc;
 222                 break;
 223 #else
 224 compact_branch:
 225                 /* Compact branches not supported before R6 */
 226                 break;
 227 #endif
 228         }
 229
 230         return nextpc;
 231
 232 unaligned:
 233         kvm_err("%s: unaligned epc\n", __func__);
 234         return nextpc;
 235
 236 sigill:
 237         kvm_err("%s: DSP branch but not DSP ASE\n", __func__);
 238         return nextpc;
 239 }
 240
 241 enum emulation_result update_pc(struct kvm_vcpu *vcpu, u32 cause)
 242 {
 243         unsigned long branch_pc;
 244         enum emulation_result er = EMULATE_DONE;
 245
 246         if (cause & CAUSEF_BD) {
 247                 branch_pc = kvm_compute_return_epc(vcpu, vcpu->arch.pc);
 248                 if (branch_pc == KVM_INVALID_INST) {
 249                         er = EMULATE_FAIL;
 250                 } else {
 251                         vcpu->arch.pc = branch_pc;
 252                         kvm_debug("BD update_pc(): New PC: %#lx\n",
 253                                   vcpu->arch.pc);
 254                 }
 255         } else
 256                 vcpu->arch.pc += 4;
 257
 258         kvm_debug("update_pc(): New PC: %#lx\n", vcpu->arch.pc);
 259
 260         return er;
 261 }
 262
 263 /**
 264  * kvm_mips_count_disabled() - Find whether the CP0_Count timer is disabled.
 265  * @vcpu:       Virtual CPU.
 266  *
 267  * Returns:     1 if the CP0_Count timer is disabled by either the guest
 268  *              CP0_Cause.DC bit or the count_ctl.DC bit.
 269  *              0 otherwise (in which case CP0_Count timer is running).
 270  */
 271 static inline int kvm_mips_count_disabled(struct kvm_vcpu *vcpu)
 272 {
 273         struct mips_coproc *cop0 = vcpu->arch.cop0;
 274
 275         return  (vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) ||
 276                 (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC);
 277 }
 278
 279 /**
 280  * kvm_mips_ktime_to_count() - Scale ktime_t to a 32-bit count.
 281  *
 282  * Caches the dynamic nanosecond bias in vcpu->arch.count_dyn_bias.
 283  *
 284  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 285  */
 286 static u32 kvm_mips_ktime_to_count(struct kvm_vcpu *vcpu, ktime_t now)
 287 {
 288         s64 now_ns, periods;
 289         u64 delta;
 290
 291         now_ns = ktime_to_ns(now);
 292         delta = now_ns + vcpu->arch.count_dyn_bias;
 293
 294         if (delta >= vcpu->arch.count_period) {
 295                 /* If delta is out of safe range the bias needs adjusting */
 296                 periods = div64_s64(now_ns, vcpu->arch.count_period);
 297                 vcpu->arch.count_dyn_bias = -periods * vcpu->arch.count_period;
 298                 /* Recalculate delta with new bias */
 299                 delta = now_ns + vcpu->arch.count_dyn_bias;
 300         }
 301
 302         /*
 303          * We've ensured that:
 304          *   delta < count_period
 305          *
 306          * Therefore the intermediate delta*count_hz will never overflow since
 307          * at the boundary condition:
 308          *   delta = count_period
 309          *   delta = NSEC_PER_SEC * 2^32 / count_hz
 310          *   delta * count_hz = NSEC_PER_SEC * 2^32
 311          */
 312         return div_u64(delta * vcpu->arch.count_hz, NSEC_PER_SEC);
 313 }
 314
 315 /**
 316  * kvm_mips_count_time() - Get effective current time.
 317  * @vcpu:       Virtual CPU.
 318  *
 319  * Get effective monotonic ktime. This is usually a straightforward ktime_get(),
 320  * except when the master disable bit is set in count_ctl, in which case it is
 321  * count_resume, i.e. the time that the count was disabled.
 322  *
 323  * Returns:     Effective monotonic ktime for CP0_Count.
 324  */
 325 static inline ktime_t kvm_mips_count_time(struct kvm_vcpu *vcpu)
 326 {
 327         if (unlikely(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC))
 328                 return vcpu->arch.count_resume;
 329
 330         return ktime_get();
 331 }
 332
 333 /**
 334  * kvm_mips_read_count_running() - Read the current count value as if running.
 335  * @vcpu:       Virtual CPU.
 336  * @now:        Kernel time to read CP0_Count at.
 337  *
 338  * Returns the current guest CP0_Count register at time @now and handles if the
 339  * timer interrupt is pending and hasn't been handled yet.
 340  *
 341  * Returns:     The current value of the guest CP0_Count register.
 342  */
 343 static u32 kvm_mips_read_count_running(struct kvm_vcpu *vcpu, ktime_t now)
 344 {
 345         struct mips_coproc *cop0 = vcpu->arch.cop0;
 346         ktime_t expires, threshold;
 347         u32 count, compare;
 348         int running;
 349
 350         /* Calculate the biased and scaled guest CP0_Count */
 351         count = vcpu->arch.count_bias + kvm_mips_ktime_to_count(vcpu, now);
 352         compare = kvm_read_c0_guest_compare(cop0);
 353
 354         /*
 355          * Find whether CP0_Count has reached the closest timer interrupt. If
 356          * not, we shouldn't inject it.
 357          */
 358         if ((s32)(count - compare) < 0)
 359                 return count;
 360
 361         /*
 362          * The CP0_Count we're going to return has already reached the closest
 363          * timer interrupt. Quickly check if it really is a new interrupt by
 364          * looking at whether the interval until the hrtimer expiry time is
 365          * less than 1/4 of the timer period.
 366          */
 367         expires = hrtimer_get_expires(&vcpu->arch.comparecount_timer);
 368         threshold = ktime_add_ns(now, vcpu->arch.count_period / 4);
 369         if (ktime_before(expires, threshold)) {
 370                 /*
 371                  * Cancel it while we handle it so there's no chance of
 372                  * interference with the timeout handler.
 373                  */
 374                 running = hrtimer_cancel(&vcpu->arch.comparecount_timer);
 375
 376                 /* Nothing should be waiting on the timeout */
 377                 kvm_mips_callbacks->queue_timer_int(vcpu);
 378
 379                 /*
 380                  * Restart the timer if it was running based on the expiry time
 381                  * we read, so that we don't push it back 2 periods.
 382                  */
 383                 if (running) {
 384                         expires = ktime_add_ns(expires,
 385                                                vcpu->arch.count_period);
 386                         hrtimer_start(&vcpu->arch.comparecount_timer, expires,
 387                                       HRTIMER_MODE_ABS);
 388                 }
 389         }
 390
 391         return count;
 392 }
 393
 394 /**
 395  * kvm_mips_read_count() - Read the current count value.
 396  * @vcpu:       Virtual CPU.
 397  *
 398  * Read the current guest CP0_Count value, taking into account whether the timer
 399  * is stopped.
 400  *
 401  * Returns:     The current guest CP0_Count value.
 402  */
 403 u32 kvm_mips_read_count(struct kvm_vcpu *vcpu)
 404 {
 405         struct mips_coproc *cop0 = vcpu->arch.cop0;
 406
 407         /* If count disabled just read static copy of count */
 408         if (kvm_mips_count_disabled(vcpu))
 409                 return kvm_read_c0_guest_count(cop0);
 410
 411         return kvm_mips_read_count_running(vcpu, ktime_get());
 412 }
 413
 414 /**
 415  * kvm_mips_freeze_hrtimer() - Safely stop the hrtimer.
 416  * @vcpu:       Virtual CPU.
 417  * @count:      Output pointer for CP0_Count value at point of freeze.
 418  *
 419  * Freeze the hrtimer safely and return both the ktime and the CP0_Count value
 420  * at the point it was frozen. It is guaranteed that any pending interrupts at
 421  * the point it was frozen are handled, and none after that point.
 422  *
 423  * This is useful where the time/CP0_Count is needed in the calculation of the
 424  * new parameters.
 425  *
 426  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 427  *
 428  * Returns:     The ktime at the point of freeze.
 429  */
 430 static ktime_t kvm_mips_freeze_hrtimer(struct kvm_vcpu *vcpu, u32 *count)
 431 {
 432         ktime_t now;
 433
 434         /* stop hrtimer before finding time */
 435         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 436         now = ktime_get();
 437
 438         /* find count at this point and handle pending hrtimer */
 439         *count = kvm_mips_read_count_running(vcpu, now);
 440
 441         return now;
 442 }
 443
 444 /**
 445  * kvm_mips_resume_hrtimer() - Resume hrtimer, updating expiry.
 446  * @vcpu:       Virtual CPU.
 447  * @now:        ktime at point of resume.
 448  * @count:      CP0_Count at point of resume.
 449  *
 450  * Resumes the timer and updates the timer expiry based on @now and @count.
 451  * This can be used in conjunction with kvm_mips_freeze_timer() when timer
 452  * parameters need to be changed.
 453  *
 454  * It is guaranteed that a timer interrupt immediately after resume will be
 455  * handled, but not if CP_Compare is exactly at @count. That case is already
 456  * handled by kvm_mips_freeze_timer().
 457  *
 458  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 459  */
 460 static void kvm_mips_resume_hrtimer(struct kvm_vcpu *vcpu,
 461                                     ktime_t now, u32 count)
 462 {
 463         struct mips_coproc *cop0 = vcpu->arch.cop0;
 464         u32 compare;
 465         u64 delta;
 466         ktime_t expire;
 467
 468         /* Calculate timeout (wrap 0 to 2^32) */
 469         compare = kvm_read_c0_guest_compare(cop0);
 470         delta = (u64)(u32)(compare - count - 1) + 1;
 471         delta = div_u64(delta * NSEC_PER_SEC, vcpu->arch.count_hz);
 472         expire = ktime_add_ns(now, delta);
 473
 474         /* Update hrtimer to use new timeout */
 475         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 476         hrtimer_start(&vcpu->arch.comparecount_timer, expire, HRTIMER_MODE_ABS);
 477 }
 478
 479 /**
 480  * kvm_mips_write_count() - Modify the count and update timer.
 481  * @vcpu:       Virtual CPU.
 482  * @count:      Guest CP0_Count value to set.
 483  *
 484  * Sets the CP0_Count value and updates the timer accordingly.
 485  */
 486 void kvm_mips_write_count(struct kvm_vcpu *vcpu, u32 count)
 487 {
 488         struct mips_coproc *cop0 = vcpu->arch.cop0;
 489         ktime_t now;
 490
 491         /* Calculate bias */
 492         now = kvm_mips_count_time(vcpu);
 493         vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now);
 494
 495         if (kvm_mips_count_disabled(vcpu))
 496                 /* The timer's disabled, adjust the static count */
 497                 kvm_write_c0_guest_count(cop0, count);
 498         else
 499                 /* Update timeout */
 500                 kvm_mips_resume_hrtimer(vcpu, now, count);
 501 }
 502
 503 /**
 504  * kvm_mips_init_count() - Initialise timer.
 505  * @vcpu:       Virtual CPU.
 506  *
 507  * Initialise the timer to a sensible frequency, namely 100MHz, zero it, and set
 508  * it going if it's enabled.
 509  */
 510 void kvm_mips_init_count(struct kvm_vcpu *vcpu)
 511 {
 512         /* 100 MHz */
 513         vcpu->arch.count_hz = 100*1000*1000;
 514         vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32,
 515                                           vcpu->arch.count_hz);
 516         vcpu->arch.count_dyn_bias = 0;
 517
 518         /* Starting at 0 */
 519         kvm_mips_write_count(vcpu, 0);
 520 }
 521
 522 /**
 523  * kvm_mips_set_count_hz() - Update the frequency of the timer.
 524  * @vcpu:       Virtual CPU.
 525  * @count_hz:   Frequency of CP0_Count timer in Hz.
 526  *
 527  * Change the frequency of the CP0_Count timer. This is done atomically so that
 528  * CP0_Count is continuous and no timer interrupt is lost.
 529  *
 530  * Returns:     -EINVAL if @count_hz is out of range.
 531  *              0 on success.
 532  */
 533 int kvm_mips_set_count_hz(struct kvm_vcpu *vcpu, s64 count_hz)
 534 {
 535         struct mips_coproc *cop0 = vcpu->arch.cop0;
 536         int dc;
 537         ktime_t now;
 538         u32 count;
 539
 540         /* ensure the frequency is in a sensible range... */
 541         if (count_hz <= 0 || count_hz > NSEC_PER_SEC)
 542                 return -EINVAL;
 543         /* ... and has actually changed */
 544         if (vcpu->arch.count_hz == count_hz)
 545                 return 0;
 546
 547         /* Safely freeze timer so we can keep it continuous */
 548         dc = kvm_mips_count_disabled(vcpu);
 549         if (dc) {
 550                 now = kvm_mips_count_time(vcpu);
 551                 count = kvm_read_c0_guest_count(cop0);
 552         } else {
 553                 now = kvm_mips_freeze_hrtimer(vcpu, &count);
 554         }
 555
 556         /* Update the frequency */
 557         vcpu->arch.count_hz = count_hz;
 558         vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32, count_hz);
 559         vcpu->arch.count_dyn_bias = 0;
 560
 561         /* Calculate adjusted bias so dynamic count is unchanged */
 562         vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now);
 563
 564         /* Update and resume hrtimer */
 565         if (!dc)
 566                 kvm_mips_resume_hrtimer(vcpu, now, count);
 567         return 0;
 568 }
 569
 570 /**
 571  * kvm_mips_write_compare() - Modify compare and update timer.
 572  * @vcpu:       Virtual CPU.
 573  * @compare:    New CP0_Compare value.
 574  * @ack:        Whether to acknowledge timer interrupt.
 575  *
 576  * Update CP0_Compare to a new value and update the timeout.
 577  * If @ack, atomically acknowledge any pending timer interrupt, otherwise ensure
 578  * any pending timer interrupt is preserved.
 579  */
 580 void kvm_mips_write_compare(struct kvm_vcpu *vcpu, u32 compare, bool ack)
 581 {
 582         struct mips_coproc *cop0 = vcpu->arch.cop0;
 583         int dc;
 584         u32 old_compare = kvm_read_c0_guest_compare(cop0);
 585         ktime_t now;
 586         u32 count;
 587
 588         /* if unchanged, must just be an ack */
 589         if (old_compare == compare) {
 590                 if (!ack)
 591                         return;
 592                 kvm_mips_callbacks->dequeue_timer_int(vcpu);
 593                 kvm_write_c0_guest_compare(cop0, compare);
 594                 return;
 595         }
 596
 597         /* freeze_hrtimer() takes care of timer interrupts <= count */
 598         dc = kvm_mips_count_disabled(vcpu);
 599         if (!dc)
 600                 now = kvm_mips_freeze_hrtimer(vcpu, &count);
 601
 602         if (ack)
 603                 kvm_mips_callbacks->dequeue_timer_int(vcpu);
 604
 605         kvm_write_c0_guest_compare(cop0, compare);
 606
 607         /* resume_hrtimer() takes care of timer interrupts > count */
 608         if (!dc)
 609                 kvm_mips_resume_hrtimer(vcpu, now, count);
 610 }
 611
 612 /**
 613  * kvm_mips_count_disable() - Disable count.
 614  * @vcpu:       Virtual CPU.
 615  *
 616  * Disable the CP0_Count timer. A timer interrupt on or before the final stop
 617  * time will be handled but not after.
 618  *
 619  * Assumes CP0_Count was previously enabled but now Guest.CP0_Cause.DC or
 620  * count_ctl.DC has been set (count disabled).
 621  *
 622  * Returns:     The time that the timer was stopped.
 623  */
 624 static ktime_t kvm_mips_count_disable(struct kvm_vcpu *vcpu)
 625 {
 626         struct mips_coproc *cop0 = vcpu->arch.cop0;
 627         u32 count;
 628         ktime_t now;
 629
 630         /* Stop hrtimer */
 631         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 632
 633         /* Set the static count from the dynamic count, handling pending TI */
 634         now = ktime_get();
 635         count = kvm_mips_read_count_running(vcpu, now);
 636         kvm_write_c0_guest_count(cop0, count);
 637
 638         return now;
 639 }
 640
 641 /**
 642  * kvm_mips_count_disable_cause() - Disable count using CP0_Cause.DC.
 643  * @vcpu:       Virtual CPU.
 644  *
 645  * Disable the CP0_Count timer and set CP0_Cause.DC. A timer interrupt on or
 646  * before the final stop time will be handled if the timer isn't disabled by
 647  * count_ctl.DC, but not after.
 648  *
 649  * Assumes CP0_Cause.DC is clear (count enabled).
 650  */
 651 void kvm_mips_count_disable_cause(struct kvm_vcpu *vcpu)
 652 {
 653         struct mips_coproc *cop0 = vcpu->arch.cop0;
 654
 655         kvm_set_c0_guest_cause(cop0, CAUSEF_DC);
 656         if (!(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC))
 657                 kvm_mips_count_disable(vcpu);
 658 }
 659
 660 /**
 661  * kvm_mips_count_enable_cause() - Enable count using CP0_Cause.DC.
 662  * @vcpu:       Virtual CPU.
 663  *
 664  * Enable the CP0_Count timer and clear CP0_Cause.DC. A timer interrupt after
 665  * the start time will be handled if the timer isn't disabled by count_ctl.DC,
 666  * potentially before even returning, so the caller should be careful with
 667  * ordering of CP0_Cause modifications so as not to lose it.
 668  *
 669  * Assumes CP0_Cause.DC is set (count disabled).
 670  */
 671 void kvm_mips_count_enable_cause(struct kvm_vcpu *vcpu)
 672 {
 673         struct mips_coproc *cop0 = vcpu->arch.cop0;
 674         u32 count;
 675
 676         kvm_clear_c0_guest_cause(cop0, CAUSEF_DC);
 677
 678         /*
 679          * Set the dynamic count to match the static count.
 680          * This starts the hrtimer if count_ctl.DC allows it.
 681          * Otherwise it conveniently updates the biases.
 682          */
 683         count = kvm_read_c0_guest_count(cop0);
 684         kvm_mips_write_count(vcpu, count);
 685 }
 686
 687 /**
 688  * kvm_mips_set_count_ctl() - Update the count control KVM register.
 689  * @vcpu:       Virtual CPU.
 690  * @count_ctl:  Count control register new value.
 691  *
 692  * Set the count control KVM register. The timer is updated accordingly.
 693  *
 694  * Returns:     -EINVAL if reserved bits are set.
 695  *              0 on success.
 696  */
 697 int kvm_mips_set_count_ctl(struct kvm_vcpu *vcpu, s64 count_ctl)
 698 {
 699         struct mips_coproc *cop0 = vcpu->arch.cop0;
 700         s64 changed = count_ctl ^ vcpu->arch.count_ctl;
 701         s64 delta;
 702         ktime_t expire, now;
 703         u32 count, compare;
 704
 705         /* Only allow defined bits to be changed */
 706         if (changed & ~(s64)(KVM_REG_MIPS_COUNT_CTL_DC))
 707                 return -EINVAL;
 708
 709         /* Apply new value */
 710         vcpu->arch.count_ctl = count_ctl;
 711
 712         /* Master CP0_Count disable */
 713         if (changed & KVM_REG_MIPS_COUNT_CTL_DC) {
 714                 /* Is CP0_Cause.DC already disabling CP0_Count? */
 715                 if (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC) {
 716                         if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)
 717                                 /* Just record the current time */
 718                                 vcpu->arch.count_resume = ktime_get();
 719                 } else if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) {
 720                         /* disable timer and record current time */
 721                         vcpu->arch.count_resume = kvm_mips_count_disable(vcpu);
 722                 } else {
 723                         /*
 724                          * Calculate timeout relative to static count at resume
 725                          * time (wrap 0 to 2^32).
 726                          */
 727                         count = kvm_read_c0_guest_count(cop0);
 728                         compare = kvm_read_c0_guest_compare(cop0);
 729                         delta = (u64)(u32)(compare - count - 1) + 1;
 730                         delta = div_u64(delta * NSEC_PER_SEC,
 731                                         vcpu->arch.count_hz);
 732                         expire = ktime_add_ns(vcpu->arch.count_resume, delta);
 733
 734                         /* Handle pending interrupt */
 735                         now = ktime_get();
 736                         if (ktime_compare(now, expire) >= 0)
 737                                 /* Nothing should be waiting on the timeout */
 738                                 kvm_mips_callbacks->queue_timer_int(vcpu);
 739
 740                         /* Resume hrtimer without changing bias */
 741                         count = kvm_mips_read_count_running(vcpu, now);
 742                         kvm_mips_resume_hrtimer(vcpu, now, count);
 743                 }
 744         }
 745
 746         return 0;
 747 }
 748
 749 /**
 750  * kvm_mips_set_count_resume() - Update the count resume KVM register.
 751  * @vcpu:               Virtual CPU.
 752  * @count_resume:       Count resume register new value.
 753  *
 754  * Set the count resume KVM register.
 755  *
 756  * Returns:     -EINVAL if out of valid range (0..now).
 757  *              0 on success.
 758  */
 759 int kvm_mips_set_count_resume(struct kvm_vcpu *vcpu, s64 count_resume)
 760 {
 761         /*
 762          * It doesn't make sense for the resume time to be in the future, as it
 763          * would be possible for the next interrupt to be more than a full
 764          * period in the future.
 765          */
 766         if (count_resume < 0 || count_resume > ktime_to_ns(ktime_get()))
 767                 return -EINVAL;
 768
 769         vcpu->arch.count_resume = ns_to_ktime(count_resume);
 770         return 0;
 771 }
 772
 773 /**
 774  * kvm_mips_count_timeout() - Push timer forward on timeout.
 775  * @vcpu:       Virtual CPU.
 776  *
 777  * Handle an hrtimer event by push the hrtimer forward a period.
 778  *
 779  * Returns:     The hrtimer_restart value to return to the hrtimer subsystem.
 780  */
 781 enum hrtimer_restart kvm_mips_count_timeout(struct kvm_vcpu *vcpu)
 782 {
 783         /* Add the Count period to the current expiry time */
 784         hrtimer_add_expires_ns(&vcpu->arch.comparecount_timer,
 785                                vcpu->arch.count_period);
 786         return HRTIMER_RESTART;
 787 }
 788
 789 enum emulation_result kvm_mips_emul_eret(struct kvm_vcpu *vcpu)
 790 {
 791         struct mips_coproc *cop0 = vcpu->arch.cop0;
 792         enum emulation_result er = EMULATE_DONE;
 793
 794         if (kvm_read_c0_guest_status(cop0) & ST0_EXL) {
 795                 kvm_debug("[%#lx] ERET to %#lx\n", vcpu->arch.pc,
 796                           kvm_read_c0_guest_epc(cop0));
 797                 kvm_clear_c0_guest_status(cop0, ST0_EXL);
 798                 vcpu->arch.pc = kvm_read_c0_guest_epc(cop0);
 799
 800         } else if (kvm_read_c0_guest_status(cop0) & ST0_ERL) {
 801                 kvm_clear_c0_guest_status(cop0, ST0_ERL);
 802                 vcpu->arch.pc = kvm_read_c0_guest_errorepc(cop0);
 803         } else {
 804                 kvm_err("[%#lx] ERET when MIPS_SR_EXL|MIPS_SR_ERL == 0\n",
 805                         vcpu->arch.pc);
 806                 er = EMULATE_FAIL;
 807         }
 808
 809         return er;
 810 }
 811
 812 enum emulation_result kvm_mips_emul_wait(struct kvm_vcpu *vcpu)
 813 {
 814         kvm_debug("[%#lx] !!!WAIT!!! (%#lx)\n", vcpu->arch.pc,
 815                   vcpu->arch.pending_exceptions);
 816
 817         ++vcpu->stat.wait_exits;
 818         trace_kvm_exit(vcpu, KVM_TRACE_EXIT_WAIT);
 819         if (!vcpu->arch.pending_exceptions) {
 820                 vcpu->arch.wait = 1;
 821                 kvm_vcpu_block(vcpu);
 822
 823                 /*
 824                  * We we are runnable, then definitely go off to user space to
 825                  * check if any I/O interrupts are pending.
 826                  */
 827                 if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) {
 828                         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
 829                         vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
 830                 }
 831         }
 832
 833         return EMULATE_DONE;
 834 }
 835
 836 /*
 837  * XXXKYMA: Linux doesn't seem to use TLBR, return EMULATE_FAIL for now so that
 838  * we can catch this, if things ever change
 839  */
 840 enum emulation_result kvm_mips_emul_tlbr(struct kvm_vcpu *vcpu)
 841 {
 842         struct mips_coproc *cop0 = vcpu->arch.cop0;
 843         unsigned long pc = vcpu->arch.pc;
 844
 845         kvm_err("[%#lx] COP0_TLBR [%ld]\n", pc, kvm_read_c0_guest_index(cop0));
 846         return EMULATE_FAIL;
 847 }
 848
 849 /**
 850  * kvm_mips_invalidate_guest_tlb() - Indicates a change in guest MMU map.
 851  * @vcpu:       VCPU with changed mappings.
 852  * @tlb:        TLB entry being removed.
 853  *
 854  * This is called to indicate a single change in guest MMU mappings, so that we
 855  * can arrange TLB flushes on this and other CPUs.
 856  */
 857 static void kvm_mips_invalidate_guest_tlb(struct kvm_vcpu *vcpu,
 858                                           struct kvm_mips_tlb *tlb)
 859 {
 860         int cpu, i;
 861         bool user;
 862
 863         /* No need to flush for entries which are already invalid */
 864         if (!((tlb->tlb_lo[0] | tlb->tlb_lo[1]) & ENTRYLO_V))
 865                 return;
 866         /* User address space doesn't need flushing for KSeg2/3 changes */
 867         user = tlb->tlb_hi < KVM_GUEST_KSEG0;
 868
 869         preempt_disable();
 870
 871         /*
 872          * Probe the shadow host TLB for the entry being overwritten, if one
 873          * matches, invalidate it
 874          */
 875         kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi);
 876
 877         /* Invalidate the whole ASID on other CPUs */
 878         cpu = smp_processor_id();
 879         for_each_possible_cpu(i) {
 880                 if (i == cpu)
 881                         continue;
 882                 if (user)
 883                         vcpu->arch.guest_user_asid[i] = 0;
 884                 vcpu->arch.guest_kernel_asid[i] = 0;
 885         }
 886
 887         preempt_enable();
 888 }
 889
 890 /* Write Guest TLB Entry @ Index */
 891 enum emulation_result kvm_mips_emul_tlbwi(struct kvm_vcpu *vcpu)
 892 {
 893         struct mips_coproc *cop0 = vcpu->arch.cop0;
 894         int index = kvm_read_c0_guest_index(cop0);
 895         struct kvm_mips_tlb *tlb = NULL;
 896         unsigned long pc = vcpu->arch.pc;
 897
 898         if (index < 0 || index >= KVM_MIPS_GUEST_TLB_SIZE) {
 899                 kvm_debug("%s: illegal index: %d\n", __func__, index);
 900                 kvm_debug("[%#lx] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n",
 901                           pc, index, kvm_read_c0_guest_entryhi(cop0),
 902                           kvm_read_c0_guest_entrylo0(cop0),
 903                           kvm_read_c0_guest_entrylo1(cop0),
 904                           kvm_read_c0_guest_pagemask(cop0));
 905                 index = (index & ~0x80000000) % KVM_MIPS_GUEST_TLB_SIZE;
 906         }
 907
 908         tlb = &vcpu->arch.guest_tlb[index];
 909
 910         kvm_mips_invalidate_guest_tlb(vcpu, tlb);
 911
 912         tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0);
 913         tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0);
 914         tlb->tlb_lo[0] = kvm_read_c0_guest_entrylo0(cop0);
 915         tlb->tlb_lo[1] = kvm_read_c0_guest_entrylo1(cop0);
 916
 917         kvm_debug("[%#lx] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n",
 918                   pc, index, kvm_read_c0_guest_entryhi(cop0),
 919                   kvm_read_c0_guest_entrylo0(cop0),
 920                   kvm_read_c0_guest_entrylo1(cop0),
 921                   kvm_read_c0_guest_pagemask(cop0));
 922
 923         return EMULATE_DONE;
 924 }
 925
 926 /* Write Guest TLB Entry @ Random Index */
 927 enum emulation_result kvm_mips_emul_tlbwr(struct kvm_vcpu *vcpu)
 928 {
 929         struct mips_coproc *cop0 = vcpu->arch.cop0;
 930         struct kvm_mips_tlb *tlb = NULL;
 931         unsigned long pc = vcpu->arch.pc;
 932         int index;
 933
 934         get_random_bytes(&index, sizeof(index));
 935         index &= (KVM_MIPS_GUEST_TLB_SIZE - 1);
 936
 937         tlb = &vcpu->arch.guest_tlb[index];
 938
 939         kvm_mips_invalidate_guest_tlb(vcpu, tlb);
 940
 941         tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0);
 942         tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0);
 943         tlb->tlb_lo[0] = kvm_read_c0_guest_entrylo0(cop0);
 944         tlb->tlb_lo[1] = kvm_read_c0_guest_entrylo1(cop0);
 945
 946         kvm_debug("[%#lx] COP0_TLBWR[%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx)\n",
 947                   pc, index, kvm_read_c0_guest_entryhi(cop0),
 948                   kvm_read_c0_guest_entrylo0(cop0),
 949                   kvm_read_c0_guest_entrylo1(cop0));
 950
 951         return EMULATE_DONE;
 952 }
 953
 954 enum emulation_result kvm_mips_emul_tlbp(struct kvm_vcpu *vcpu)
 955 {
 956         struct mips_coproc *cop0 = vcpu->arch.cop0;
 957         long entryhi = kvm_read_c0_guest_entryhi(cop0);
 958         unsigned long pc = vcpu->arch.pc;
 959         int index = -1;
 960
 961         index = kvm_mips_guest_tlb_lookup(vcpu, entryhi);
 962
 963         kvm_write_c0_guest_index(cop0, index);
 964
 965         kvm_debug("[%#lx] COP0_TLBP (entryhi: %#lx), index: %d\n", pc, entryhi,
 966                   index);
 967
 968         return EMULATE_DONE;
 969 }
 970
 971 /**
 972  * kvm_mips_config1_wrmask() - Find mask of writable bits in guest Config1
 973  * @vcpu:       Virtual CPU.
 974  *
 975  * Finds the mask of bits which are writable in the guest's Config1 CP0
 976  * register, by userland (currently read-only to the guest).
 977  */
 978 unsigned int kvm_mips_config1_wrmask(struct kvm_vcpu *vcpu)
 979 {
 980         unsigned int mask = 0;
 981
 982         /* Permit FPU to be present if FPU is supported */
 983         if (kvm_mips_guest_can_have_fpu(&vcpu->arch))
 984                 mask |= MIPS_CONF1_FP;
 985
 986         return mask;
 987 }
 988
 989 /**
 990  * kvm_mips_config3_wrmask() - Find mask of writable bits in guest Config3
 991  * @vcpu:       Virtual CPU.
 992  *
 993  * Finds the mask of bits which are writable in the guest's Config3 CP0
 994  * register, by userland (currently read-only to the guest).
 995  */
 996 unsigned int kvm_mips_config3_wrmask(struct kvm_vcpu *vcpu)
 997 {
 998         /* Config4 and ULRI are optional */
 999         unsigned int mask = MIPS_CONF_M | MIPS_CONF3_ULRI;
1000
1001         /* Permit MSA to be present if MSA is supported */
1002         if (kvm_mips_guest_can_have_msa(&vcpu->arch))
1003                 mask |= MIPS_CONF3_MSA;
1004
1005         return mask;
1006 }
1007
1008 /**
1009  * kvm_mips_config4_wrmask() - Find mask of writable bits in guest Config4
1010  * @vcpu:       Virtual CPU.
1011  *
1012  * Finds the mask of bits which are writable in the guest's Config4 CP0
1013  * register, by userland (currently read-only to the guest).
1014  */
1015 unsigned int kvm_mips_config4_wrmask(struct kvm_vcpu *vcpu)
1016 {
1017         /* Config5 is optional */
1018         unsigned int mask = MIPS_CONF_M;
1019
1020         /* KScrExist */
1021         mask |= (unsigned int)vcpu->arch.kscratch_enabled << 16;
1022
1023         return mask;
1024 }
1025
1026 /**
1027  * kvm_mips_config5_wrmask() - Find mask of writable bits in guest Config5
1028  * @vcpu:       Virtual CPU.
1029  *
1030  * Finds the mask of bits which are writable in the guest's Config5 CP0
1031  * register, by the guest itself.
1032  */
1033 unsigned int kvm_mips_config5_wrmask(struct kvm_vcpu *vcpu)
1034 {
1035         unsigned int mask = 0;
1036
1037         /* Permit MSAEn changes if MSA supported and enabled */
1038         if (kvm_mips_guest_has_msa(&vcpu->arch))
1039                 mask |= MIPS_CONF5_MSAEN;
1040
1041         /*
1042          * Permit guest FPU mode changes if FPU is enabled and the relevant
1043          * feature exists according to FIR register.
1044          */
1045         if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
1046                 if (cpu_has_fre)
1047                         mask |= MIPS_CONF5_FRE;
1048                 /* We don't support UFR or UFE */
1049         }
1050
1051         return mask;
1052 }
1053
1054 enum emulation_result kvm_mips_emulate_CP0(union mips_instruction inst,
1055                                            u32 *opc, u32 cause,
1056                                            struct kvm_run *run,
1057                                            struct kvm_vcpu *vcpu)
1058 {
1059         struct mips_coproc *cop0 = vcpu->arch.cop0;
1060         enum emulation_result er = EMULATE_DONE;
1061         u32 rt, rd, sel;
1062         unsigned long curr_pc;
1063         int cpu, i;
1064
1065         /*
1066          * Update PC and hold onto current PC in case there is
1067          * an error and we want to rollback the PC
1068          */
1069         curr_pc = vcpu->arch.pc;
1070         er = update_pc(vcpu, cause);
1071         if (er == EMULATE_FAIL)
1072                 return er;
1073
1074         if (inst.co_format.co) {
1075                 switch (inst.co_format.func) {
1076                 case tlbr_op:   /*  Read indexed TLB entry  */
1077                         er = kvm_mips_emul_tlbr(vcpu);
1078                         break;
1079                 case tlbwi_op:  /*  Write indexed  */
1080                         er = kvm_mips_emul_tlbwi(vcpu);
1081                         break;
1082                 case tlbwr_op:  /*  Write random  */
1083                         er = kvm_mips_emul_tlbwr(vcpu);
1084                         break;
1085                 case tlbp_op:   /* TLB Probe */
1086                         er = kvm_mips_emul_tlbp(vcpu);
1087                         break;
1088                 case rfe_op:
1089                         kvm_err("!!!COP0_RFE!!!\n");
1090                         break;
1091                 case eret_op:
1092                         er = kvm_mips_emul_eret(vcpu);
1093                         goto dont_update_pc;
1094                 case wait_op:
1095                         er = kvm_mips_emul_wait(vcpu);
1096                         break;
1097                 }
1098         } else {
1099                 rt = inst.c0r_format.rt;
1100                 rd = inst.c0r_format.rd;
1101                 sel = inst.c0r_format.sel;
1102
1103                 switch (inst.c0r_format.rs) {
1104                 case mfc_op:
1105 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
1106                         cop0->stat[rd][sel]++;
1107 #endif
1108                         /* Get reg */
1109                         if ((rd == MIPS_CP0_COUNT) && (sel == 0)) {
1110                                 vcpu->arch.gprs[rt] =
1111                                     (s32)kvm_mips_read_count(vcpu);
1112                         } else if ((rd == MIPS_CP0_ERRCTL) && (sel == 0)) {
1113                                 vcpu->arch.gprs[rt] = 0x0;
1114 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1115                                 kvm_mips_trans_mfc0(inst, opc, vcpu);
1116 #endif
1117                         } else {
1118                                 vcpu->arch.gprs[rt] = (s32)cop0->reg[rd][sel];
1119
1120 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1121                                 kvm_mips_trans_mfc0(inst, opc, vcpu);
1122 #endif
1123                         }
1124
1125                         trace_kvm_hwr(vcpu, KVM_TRACE_MFC0,
1126                                       KVM_TRACE_COP0(rd, sel),
1127                                       vcpu->arch.gprs[rt]);
1128                         break;
1129
1130                 case dmfc_op:
1131                         vcpu->arch.gprs[rt] = cop0->reg[rd][sel];
1132
1133                         trace_kvm_hwr(vcpu, KVM_TRACE_DMFC0,
1134                                       KVM_TRACE_COP0(rd, sel),
1135                                       vcpu->arch.gprs[rt]);
1136                         break;
1137
1138                 case mtc_op:
1139 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
1140                         cop0->stat[rd][sel]++;
1141 #endif
1142                         trace_kvm_hwr(vcpu, KVM_TRACE_MTC0,
1143                                       KVM_TRACE_COP0(rd, sel),
1144                                       vcpu->arch.gprs[rt]);
1145
1146                         if ((rd == MIPS_CP0_TLB_INDEX)
1147                             && (vcpu->arch.gprs[rt] >=
1148                                 KVM_MIPS_GUEST_TLB_SIZE)) {
1149                                 kvm_err("Invalid TLB Index: %ld",
1150                                         vcpu->arch.gprs[rt]);
1151                                 er = EMULATE_FAIL;
1152                                 break;
1153                         }
1154 #define C0_EBASE_CORE_MASK 0xff
1155                         if ((rd == MIPS_CP0_PRID) && (sel == 1)) {
1156                                 /* Preserve CORE number */
1157                                 kvm_change_c0_guest_ebase(cop0,
1158                                                           ~(C0_EBASE_CORE_MASK),
1159                                                           vcpu->arch.gprs[rt]);
1160                                 kvm_err("MTCz, cop0->reg[EBASE]: %#lx\n",
1161                                         kvm_read_c0_guest_ebase(cop0));
1162                         } else if (rd == MIPS_CP0_TLB_HI && sel == 0) {
1163                                 u32 nasid =
1164                                         vcpu->arch.gprs[rt] & KVM_ENTRYHI_ASID;
1165                                 if ((KSEGX(vcpu->arch.gprs[rt]) != CKSEG0) &&
1166                                     ((kvm_read_c0_guest_entryhi(cop0) &
1167                                       KVM_ENTRYHI_ASID) != nasid)) {
1168                                         trace_kvm_asid_change(vcpu,
1169                                                 kvm_read_c0_guest_entryhi(cop0)
1170                                                         & KVM_ENTRYHI_ASID,
1171                                                 nasid);
1172
1173                                         preempt_disable();
1174                                         /* Blow away the shadow host TLBs */
1175                                         kvm_mips_flush_host_tlb(1);
1176                                         cpu = smp_processor_id();
1177                                         for_each_possible_cpu(i)
1178                                                 if (i != cpu) {
1179                                                         vcpu->arch.guest_user_asid[i] = 0;
1180                                                         vcpu->arch.guest_kernel_asid[i] = 0;
1181                                                 }
1182                                         preempt_enable();
1183                                 }
1184                                 kvm_write_c0_guest_entryhi(cop0,
1185                                                            vcpu->arch.gprs[rt]);
1186                         }
1187                         /* Are we writing to COUNT */
1188                         else if ((rd == MIPS_CP0_COUNT) && (sel == 0)) {
1189                                 kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]);
1190                                 goto done;
1191                         } else if ((rd == MIPS_CP0_COMPARE) && (sel == 0)) {
1192                                 /* If we are writing to COMPARE */
1193                                 /* Clear pending timer interrupt, if any */
1194                                 kvm_mips_write_compare(vcpu,
1195                                                        vcpu->arch.gprs[rt],
1196                                                        true);
1197                         } else if ((rd == MIPS_CP0_STATUS) && (sel == 0)) {
1198                                 unsigned int old_val, val, change;
1199
1200                                 old_val = kvm_read_c0_guest_status(cop0);
1201                                 val = vcpu->arch.gprs[rt];
1202                                 change = val ^ old_val;
1203
1204                                 /* Make sure that the NMI bit is never set */
1205                                 val &= ~ST0_NMI;
1206
1207                                 /*
1208                                  * Don't allow CU1 or FR to be set unless FPU
1209                                  * capability enabled and exists in guest
1210                                  * configuration.
1211                                  */
1212                                 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1213                                         val &= ~(ST0_CU1 | ST0_FR);
1214
1215                                 /*
1216                                  * Also don't allow FR to be set if host doesn't
1217                                  * support it.
1218                                  */
1219                                 if (!(current_cpu_data.fpu_id & MIPS_FPIR_F64))
1220                                         val &= ~ST0_FR;
1221
1222
1223                                 /* Handle changes in FPU mode */
1224                                 preempt_disable();
1225
1226                                 /*
1227                                  * FPU and Vector register state is made
1228                                  * UNPREDICTABLE by a change of FR, so don't
1229                                  * even bother saving it.
1230                                  */
1231                                 if (change & ST0_FR)
1232                                         kvm_drop_fpu(vcpu);
1233
1234                                 /*
1235                                  * If MSA state is already live, it is undefined
1236                                  * how it interacts with FR=0 FPU state, and we
1237                                  * don't want to hit reserved instruction
1238                                  * exceptions trying to save the MSA state later
1239                                  * when CU=1 && FR=1, so play it safe and save
1240                                  * it first.
1241                                  */
1242                                 if (change & ST0_CU1 && !(val & ST0_FR) &&
1243                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1244                                         kvm_lose_fpu(vcpu);
1245
1246                                 /*
1247                                  * Propagate CU1 (FPU enable) changes
1248                                  * immediately if the FPU context is already
1249                                  * loaded. When disabling we leave the context
1250                                  * loaded so it can be quickly enabled again in
1251                                  * the near future.
1252                                  */
1253                                 if (change & ST0_CU1 &&
1254                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
1255                                         change_c0_status(ST0_CU1, val);
1256
1257                                 preempt_enable();
1258
1259                                 kvm_write_c0_guest_status(cop0, val);
1260
1261 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1262                                 /*
1263                                  * If FPU present, we need CU1/FR bits to take
1264                                  * effect fairly soon.
1265                                  */
1266                                 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1267                                         kvm_mips_trans_mtc0(inst, opc, vcpu);
1268 #endif
1269                         } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) {
1270                                 unsigned int old_val, val, change, wrmask;
1271
1272                                 old_val = kvm_read_c0_guest_config5(cop0);
1273                                 val = vcpu->arch.gprs[rt];
1274
1275                                 /* Only a few bits are writable in Config5 */
1276                                 wrmask = kvm_mips_config5_wrmask(vcpu);
1277                                 change = (val ^ old_val) & wrmask;
1278                                 val = old_val ^ change;
1279
1280
1281                                 /* Handle changes in FPU/MSA modes */
1282                                 preempt_disable();
1283
1284                                 /*
1285                                  * Propagate FRE changes immediately if the FPU
1286                                  * context is already loaded.
1287                                  */
1288                                 if (change & MIPS_CONF5_FRE &&
1289                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
1290                                         change_c0_config5(MIPS_CONF5_FRE, val);
1291
1292                                 /*
1293                                  * Propagate MSAEn changes immediately if the
1294                                  * MSA context is already loaded. When disabling
1295                                  * we leave the context loaded so it can be
1296                                  * quickly enabled again in the near future.
1297                                  */
1298                                 if (change & MIPS_CONF5_MSAEN &&
1299                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1300                                         change_c0_config5(MIPS_CONF5_MSAEN,
1301                                                           val);
1302
1303                                 preempt_enable();
1304
1305                                 kvm_write_c0_guest_config5(cop0, val);
1306                         } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) {
1307                                 u32 old_cause, new_cause;
1308
1309                                 old_cause = kvm_read_c0_guest_cause(cop0);
1310                                 new_cause = vcpu->arch.gprs[rt];
1311                                 /* Update R/W bits */
1312                                 kvm_change_c0_guest_cause(cop0, 0x08800300,
1313                                                           new_cause);
1314                                 /* DC bit enabling/disabling timer? */
1315                                 if ((old_cause ^ new_cause) & CAUSEF_DC) {
1316                                         if (new_cause & CAUSEF_DC)
1317                                                 kvm_mips_count_disable_cause(vcpu);
1318                                         else
1319                                                 kvm_mips_count_enable_cause(vcpu);
1320                                 }
1321                         } else if ((rd == MIPS_CP0_HWRENA) && (sel == 0)) {
1322                                 u32 mask = MIPS_HWRENA_CPUNUM |
1323                                            MIPS_HWRENA_SYNCISTEP |
1324                                            MIPS_HWRENA_CC |
1325                                            MIPS_HWRENA_CCRES;
1326
1327                                 if (kvm_read_c0_guest_config3(cop0) &
1328                                     MIPS_CONF3_ULRI)
1329                                         mask |= MIPS_HWRENA_ULR;
1330                                 cop0->reg[rd][sel] = vcpu->arch.gprs[rt] & mask;
1331                         } else {
1332                                 cop0->reg[rd][sel] = vcpu->arch.gprs[rt];
1333 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1334                                 kvm_mips_trans_mtc0(inst, opc, vcpu);
1335 #endif
1336                         }
1337                         break;
1338
1339                 case dmtc_op:
1340                         kvm_err("!!!!!!![%#lx]dmtc_op: rt: %d, rd: %d, sel: %d!!!!!!\n",
1341                                 vcpu->arch.pc, rt, rd, sel);
1342                         trace_kvm_hwr(vcpu, KVM_TRACE_DMTC0,
1343                                       KVM_TRACE_COP0(rd, sel),
1344                                       vcpu->arch.gprs[rt]);
1345                         er = EMULATE_FAIL;
1346                         break;
1347
1348                 case mfmc0_op:
1349 #ifdef KVM_MIPS_DEBUG_COP0_COUNTERS
1350                         cop0->stat[MIPS_CP0_STATUS][0]++;
1351 #endif
1352                         if (rt != 0)
1353                                 vcpu->arch.gprs[rt] =
1354                                     kvm_read_c0_guest_status(cop0);
1355                         /* EI */
1356                         if (inst.mfmc0_format.sc) {
1357                                 kvm_debug("[%#lx] mfmc0_op: EI\n",
1358                                           vcpu->arch.pc);
1359                                 kvm_set_c0_guest_status(cop0, ST0_IE);
1360                         } else {
1361                                 kvm_debug("[%#lx] mfmc0_op: DI\n",
1362                                           vcpu->arch.pc);
1363                                 kvm_clear_c0_guest_status(cop0, ST0_IE);
1364                         }
1365
1366                         break;
1367
1368                 case wrpgpr_op:
1369                         {
1370                                 u32 css = cop0->reg[MIPS_CP0_STATUS][2] & 0xf;
1371                                 u32 pss =
1372                                     (cop0->reg[MIPS_CP0_STATUS][2] >> 6) & 0xf;
1373                                 /*
1374                                  * We don't support any shadow register sets, so
1375                                  * SRSCtl[PSS] == SRSCtl[CSS] = 0
1376                                  */
1377                                 if (css || pss) {
1378                                         er = EMULATE_FAIL;
1379                                         break;
1380                                 }
1381                                 kvm_debug("WRPGPR[%d][%d] = %#lx\n", pss, rd,
1382                                           vcpu->arch.gprs[rt]);
1383                                 vcpu->arch.gprs[rd] = vcpu->arch.gprs[rt];
1384                         }
1385                         break;
1386                 default:
1387                         kvm_err("[%#lx]MachEmulateCP0: unsupported COP0, copz: 0x%x\n",
1388                                 vcpu->arch.pc, inst.c0r_format.rs);
1389                         er = EMULATE_FAIL;
1390                         break;
1391                 }
1392         }
1393
1394 done:
1395         /* Rollback PC only if emulation was unsuccessful */
1396         if (er == EMULATE_FAIL)
1397                 vcpu->arch.pc = curr_pc;
1398
1399 dont_update_pc:
1400         /*
1401          * This is for special instructions whose emulation
1402          * updates the PC, so do not overwrite the PC under
1403          * any circumstances
1404          */
1405
1406         return er;
1407 }
1408
1409 enum emulation_result kvm_mips_emulate_store(union mips_instruction inst,
1410                                              u32 cause,
1411                                              struct kvm_run *run,
1412                                              struct kvm_vcpu *vcpu)
1413 {
1414         enum emulation_result er = EMULATE_DO_MMIO;
1415         u32 rt;
1416         u32 bytes;
1417         void *data = run->mmio.data;
1418         unsigned long curr_pc;
1419
1420         /*
1421          * Update PC and hold onto current PC in case there is
1422          * an error and we want to rollback the PC
1423          */
1424         curr_pc = vcpu->arch.pc;
1425         er = update_pc(vcpu, cause);
1426         if (er == EMULATE_FAIL)
1427                 return er;
1428
1429         rt = inst.i_format.rt;
1430
1431         switch (inst.i_format.opcode) {
1432         case sb_op:
1433                 bytes = 1;
1434                 if (bytes > sizeof(run->mmio.data)) {
1435                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1436                                run->mmio.len);
1437                 }
1438                 run->mmio.phys_addr =
1439                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1440                                                    host_cp0_badvaddr);
1441                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1442                         er = EMULATE_FAIL;
1443                         break;
1444                 }
1445                 run->mmio.len = bytes;
1446                 run->mmio.is_write = 1;
1447                 vcpu->mmio_needed = 1;
1448                 vcpu->mmio_is_write = 1;
1449                 *(u8 *) data = vcpu->arch.gprs[rt];
1450                 kvm_debug("OP_SB: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1451                           vcpu->arch.host_cp0_badvaddr, vcpu->arch.gprs[rt],
1452                           *(u8 *) data);
1453
1454                 break;
1455
1456         case sw_op:
1457                 bytes = 4;
1458                 if (bytes > sizeof(run->mmio.data)) {
1459                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1460                                run->mmio.len);
1461                 }
1462                 run->mmio.phys_addr =
1463                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1464                                                    host_cp0_badvaddr);
1465                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1466                         er = EMULATE_FAIL;
1467                         break;
1468                 }
1469
1470                 run->mmio.len = bytes;
1471                 run->mmio.is_write = 1;
1472                 vcpu->mmio_needed = 1;
1473                 vcpu->mmio_is_write = 1;
1474                 *(u32 *) data = vcpu->arch.gprs[rt];
1475
1476                 kvm_debug("[%#lx] OP_SW: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1477                           vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1478                           vcpu->arch.gprs[rt], *(u32 *) data);
1479                 break;
1480
1481         case sh_op:
1482                 bytes = 2;
1483                 if (bytes > sizeof(run->mmio.data)) {
1484                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1485                                run->mmio.len);
1486                 }
1487                 run->mmio.phys_addr =
1488                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1489                                                    host_cp0_badvaddr);
1490                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1491                         er = EMULATE_FAIL;
1492                         break;
1493                 }
1494
1495                 run->mmio.len = bytes;
1496                 run->mmio.is_write = 1;
1497                 vcpu->mmio_needed = 1;
1498                 vcpu->mmio_is_write = 1;
1499                 *(u16 *) data = vcpu->arch.gprs[rt];
1500
1501                 kvm_debug("[%#lx] OP_SH: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1502                           vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1503                           vcpu->arch.gprs[rt], *(u32 *) data);
1504                 break;
1505
1506         default:
1507                 kvm_err("Store not yet supported (inst=0x%08x)\n",
1508                         inst.word);
1509                 er = EMULATE_FAIL;
1510                 break;
1511         }
1512
1513         /* Rollback PC if emulation was unsuccessful */
1514         if (er == EMULATE_FAIL)
1515                 vcpu->arch.pc = curr_pc;
1516
1517         return er;
1518 }
1519
1520 enum emulation_result kvm_mips_emulate_load(union mips_instruction inst,
1521                                             u32 cause, struct kvm_run *run,
1522                                             struct kvm_vcpu *vcpu)
1523 {
1524         enum emulation_result er = EMULATE_DO_MMIO;
1525         u32 op, rt;
1526         u32 bytes;
1527
1528         rt = inst.i_format.rt;
1529         op = inst.i_format.opcode;
1530
1531         vcpu->arch.pending_load_cause = cause;
1532         vcpu->arch.io_gpr = rt;
1533
1534         switch (op) {
1535         case lw_op:
1536                 bytes = 4;
1537                 if (bytes > sizeof(run->mmio.data)) {
1538                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1539                                run->mmio.len);
1540                         er = EMULATE_FAIL;
1541                         break;
1542                 }
1543                 run->mmio.phys_addr =
1544                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1545                                                    host_cp0_badvaddr);
1546                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1547                         er = EMULATE_FAIL;
1548                         break;
1549                 }
1550
1551                 run->mmio.len = bytes;
1552                 run->mmio.is_write = 0;
1553                 vcpu->mmio_needed = 1;
1554                 vcpu->mmio_is_write = 0;
1555                 break;
1556
1557         case lh_op:
1558         case lhu_op:
1559                 bytes = 2;
1560                 if (bytes > sizeof(run->mmio.data)) {
1561                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1562                                run->mmio.len);
1563                         er = EMULATE_FAIL;
1564                         break;
1565                 }
1566                 run->mmio.phys_addr =
1567                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1568                                                    host_cp0_badvaddr);
1569                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1570                         er = EMULATE_FAIL;
1571                         break;
1572                 }
1573
1574                 run->mmio.len = bytes;
1575                 run->mmio.is_write = 0;
1576                 vcpu->mmio_needed = 1;
1577                 vcpu->mmio_is_write = 0;
1578
1579                 if (op == lh_op)
1580                         vcpu->mmio_needed = 2;
1581                 else
1582                         vcpu->mmio_needed = 1;
1583
1584                 break;
1585
1586         case lbu_op:
1587         case lb_op:
1588                 bytes = 1;
1589                 if (bytes > sizeof(run->mmio.data)) {
1590                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1591                                run->mmio.len);
1592                         er = EMULATE_FAIL;
1593                         break;
1594                 }
1595                 run->mmio.phys_addr =
1596                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1597                                                    host_cp0_badvaddr);
1598                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1599                         er = EMULATE_FAIL;
1600                         break;
1601                 }
1602
1603                 run->mmio.len = bytes;
1604                 run->mmio.is_write = 0;
1605                 vcpu->mmio_is_write = 0;
1606
1607                 if (op == lb_op)
1608                         vcpu->mmio_needed = 2;
1609                 else
1610                         vcpu->mmio_needed = 1;
1611
1612                 break;
1613
1614         default:
1615                 kvm_err("Load not yet supported (inst=0x%08x)\n",
1616                         inst.word);
1617                 er = EMULATE_FAIL;
1618                 break;
1619         }
1620
1621         return er;
1622 }
1623
1624 enum emulation_result kvm_mips_emulate_cache(union mips_instruction inst,
1625                                              u32 *opc, u32 cause,
1626                                              struct kvm_run *run,
1627                                              struct kvm_vcpu *vcpu)
1628 {
1629         struct mips_coproc *cop0 = vcpu->arch.cop0;
1630         enum emulation_result er = EMULATE_DONE;
1631         u32 cache, op_inst, op, base;
1632         s16 offset;
1633         struct kvm_vcpu_arch *arch = &vcpu->arch;
1634         unsigned long va;
1635         unsigned long curr_pc;
1636
1637         /*
1638          * Update PC and hold onto current PC in case there is
1639          * an error and we want to rollback the PC
1640          */
1641         curr_pc = vcpu->arch.pc;
1642         er = update_pc(vcpu, cause);
1643         if (er == EMULATE_FAIL)
1644                 return er;
1645
1646         base = inst.i_format.rs;
1647         op_inst = inst.i_format.rt;
1648         if (cpu_has_mips_r6)
1649                 offset = inst.spec3_format.simmediate;
1650         else
1651                 offset = inst.i_format.simmediate;
1652         cache = op_inst & CacheOp_Cache;
1653         op = op_inst & CacheOp_Op;
1654
1655         va = arch->gprs[base] + offset;
1656
1657         kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1658                   cache, op, base, arch->gprs[base], offset);
1659
1660         /*
1661          * Treat INDEX_INV as a nop, basically issued by Linux on startup to
1662          * invalidate the caches entirely by stepping through all the
1663          * ways/indexes
1664          */
1665         if (op == Index_Writeback_Inv) {
1666                 kvm_debug("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1667                           vcpu->arch.pc, vcpu->arch.gprs[31], cache, op, base,
1668                           arch->gprs[base], offset);
1669
1670                 if (cache == Cache_D)
1671                         r4k_blast_dcache();
1672                 else if (cache == Cache_I)
1673                         r4k_blast_icache();
1674                 else {
1675                         kvm_err("%s: unsupported CACHE INDEX operation\n",
1676                                 __func__);
1677                         return EMULATE_FAIL;
1678                 }
1679
1680 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1681                 kvm_mips_trans_cache_index(inst, opc, vcpu);
1682 #endif
1683                 goto done;
1684         }
1685
1686         preempt_disable();
1687         if (KVM_GUEST_KSEGX(va) == KVM_GUEST_KSEG0) {
1688                 if (kvm_mips_host_tlb_lookup(vcpu, va) < 0 &&
1689                     kvm_mips_handle_kseg0_tlb_fault(va, vcpu)) {
1690                         kvm_err("%s: handling mapped kseg0 tlb fault for %lx, vcpu: %p, ASID: %#lx\n",
1691                                 __func__, va, vcpu, read_c0_entryhi());
1692                         er = EMULATE_FAIL;
1693                         preempt_enable();
1694                         goto done;
1695                 }
1696         } else if ((KVM_GUEST_KSEGX(va) < KVM_GUEST_KSEG0) ||
1697                    KVM_GUEST_KSEGX(va) == KVM_GUEST_KSEG23) {
1698                 int index;
1699
1700                 /* If an entry already exists then skip */
1701                 if (kvm_mips_host_tlb_lookup(vcpu, va) >= 0)
1702                         goto skip_fault;
1703
1704                 /*
1705                  * If address not in the guest TLB, then give the guest a fault,
1706                  * the resulting handler will do the right thing
1707                  */
1708                 index = kvm_mips_guest_tlb_lookup(vcpu, (va & VPN2_MASK) |
1709                                                   (kvm_read_c0_guest_entryhi
1710                                                    (cop0) & KVM_ENTRYHI_ASID));
1711
1712                 if (index < 0) {
1713                         vcpu->arch.host_cp0_badvaddr = va;
1714                         vcpu->arch.pc = curr_pc;
1715                         er = kvm_mips_emulate_tlbmiss_ld(cause, NULL, run,
1716                                                          vcpu);
1717                         preempt_enable();
1718                         goto dont_update_pc;
1719                 } else {
1720                         struct kvm_mips_tlb *tlb = &vcpu->arch.guest_tlb[index];
1721                         /*
1722                          * Check if the entry is valid, if not then setup a TLB
1723                          * invalid exception to the guest
1724                          */
1725                         if (!TLB_IS_VALID(*tlb, va)) {
1726                                 vcpu->arch.host_cp0_badvaddr = va;
1727                                 vcpu->arch.pc = curr_pc;
1728                                 er = kvm_mips_emulate_tlbinv_ld(cause, NULL,
1729                                                                 run, vcpu);
1730                                 preempt_enable();
1731                                 goto dont_update_pc;
1732                         }
1733                         /*
1734                          * We fault an entry from the guest tlb to the
1735                          * shadow host TLB
1736                          */
1737                         if (kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb)) {
1738                                 kvm_err("%s: handling mapped seg tlb fault for %lx, index: %u, vcpu: %p, ASID: %#lx\n",
1739                                         __func__, va, index, vcpu,
1740                                         read_c0_entryhi());
1741                                 er = EMULATE_FAIL;
1742                                 preempt_enable();
1743                                 goto done;
1744                         }
1745                 }
1746         } else {
1747                 kvm_err("INVALID CACHE INDEX/ADDRESS (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1748                         cache, op, base, arch->gprs[base], offset);
1749                 er = EMULATE_FAIL;
1750                 preempt_enable();
1751                 goto done;
1752
1753         }
1754
1755 skip_fault:
1756         /* XXXKYMA: Only a subset of cache ops are supported, used by Linux */
1757         if (op_inst == Hit_Writeback_Inv_D || op_inst == Hit_Invalidate_D) {
1758                 flush_dcache_line(va);
1759
1760 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1761                 /*
1762                  * Replace the CACHE instruction, with a SYNCI, not the same,
1763                  * but avoids a trap
1764                  */
1765                 kvm_mips_trans_cache_va(inst, opc, vcpu);
1766 #endif
1767         } else if (op_inst == Hit_Invalidate_I) {
1768                 flush_dcache_line(va);
1769                 flush_icache_line(va);
1770
1771 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1772                 /* Replace the CACHE instruction, with a SYNCI */
1773                 kvm_mips_trans_cache_va(inst, opc, vcpu);
1774 #endif
1775         } else {
1776                 kvm_err("NO-OP CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1777                         cache, op, base, arch->gprs[base], offset);
1778                 er = EMULATE_FAIL;
1779         }
1780
1781         preempt_enable();
1782 done:
1783         /* Rollback PC only if emulation was unsuccessful */
1784         if (er == EMULATE_FAIL)
1785                 vcpu->arch.pc = curr_pc;
1786
1787 dont_update_pc:
1788         /*
1789          * This is for exceptions whose emulation updates the PC, so do not
1790          * overwrite the PC under any circumstances
1791          */
1792
1793         return er;
1794 }
1795
1796 enum emulation_result kvm_mips_emulate_inst(u32 cause, u32 *opc,
1797                                             struct kvm_run *run,
1798                                             struct kvm_vcpu *vcpu)
1799 {
1800         union mips_instruction inst;
1801         enum emulation_result er = EMULATE_DONE;
1802
1803         /* Fetch the instruction. */
1804         if (cause & CAUSEF_BD)
1805                 opc += 1;
1806
1807         inst.word = kvm_get_inst(opc, vcpu);
1808
1809         switch (inst.r_format.opcode) {
1810         case cop0_op:
1811                 er = kvm_mips_emulate_CP0(inst, opc, cause, run, vcpu);
1812                 break;
1813         case sb_op:
1814         case sh_op:
1815         case sw_op:
1816                 er = kvm_mips_emulate_store(inst, cause, run, vcpu);
1817                 break;
1818         case lb_op:
1819         case lbu_op:
1820         case lhu_op:
1821         case lh_op:
1822         case lw_op:
1823                 er = kvm_mips_emulate_load(inst, cause, run, vcpu);
1824                 break;
1825
1826 #ifndef CONFIG_CPU_MIPSR6
1827         case cache_op:
1828                 ++vcpu->stat.cache_exits;
1829                 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
1830                 er = kvm_mips_emulate_cache(inst, opc, cause, run, vcpu);
1831                 break;
1832 #else
1833         case spec3_op:
1834                 switch (inst.spec3_format.func) {
1835                 case cache6_op:
1836                         ++vcpu->stat.cache_exits;
1837                         trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
1838                         er = kvm_mips_emulate_cache(inst, opc, cause, run,
1839                                                     vcpu);
1840                         break;
1841                 default:
1842                         goto unknown;
1843                 };
1844                 break;
1845 unknown:
1846 #endif
1847
1848         default:
1849                 kvm_err("Instruction emulation not supported (%p/%#x)\n", opc,
1850                         inst.word);
1851                 kvm_arch_vcpu_dump_regs(vcpu);
1852                 er = EMULATE_FAIL;
1853                 break;
1854         }
1855
1856         return er;
1857 }
1858
1859 enum emulation_result kvm_mips_emulate_syscall(u32 cause,
1860                                                u32 *opc,
1861                                                struct kvm_run *run,
1862                                                struct kvm_vcpu *vcpu)
1863 {
1864         struct mips_coproc *cop0 = vcpu->arch.cop0;
1865         struct kvm_vcpu_arch *arch = &vcpu->arch;
1866         enum emulation_result er = EMULATE_DONE;
1867
1868         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1869                 /* save old pc */
1870                 kvm_write_c0_guest_epc(cop0, arch->pc);
1871                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1872
1873                 if (cause & CAUSEF_BD)
1874                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1875                 else
1876                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1877
1878                 kvm_debug("Delivering SYSCALL @ pc %#lx\n", arch->pc);
1879
1880                 kvm_change_c0_guest_cause(cop0, (0xff),
1881                                           (EXCCODE_SYS << CAUSEB_EXCCODE));
1882
1883                 /* Set PC to the exception entry point */
1884                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1885
1886         } else {
1887                 kvm_err("Trying to deliver SYSCALL when EXL is already set\n");
1888                 er = EMULATE_FAIL;
1889         }
1890
1891         return er;
1892 }
1893
1894 enum emulation_result kvm_mips_emulate_tlbmiss_ld(u32 cause,
1895                                                   u32 *opc,
1896                                                   struct kvm_run *run,
1897                                                   struct kvm_vcpu *vcpu)
1898 {
1899         struct mips_coproc *cop0 = vcpu->arch.cop0;
1900         struct kvm_vcpu_arch *arch = &vcpu->arch;
1901         unsigned long entryhi = (vcpu->arch.  host_cp0_badvaddr & VPN2_MASK) |
1902                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1903
1904         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1905                 /* save old pc */
1906                 kvm_write_c0_guest_epc(cop0, arch->pc);
1907                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1908
1909                 if (cause & CAUSEF_BD)
1910                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1911                 else
1912                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1913
1914                 kvm_debug("[EXL == 0] delivering TLB MISS @ pc %#lx\n",
1915                           arch->pc);
1916
1917                 /* set pc to the exception entry point */
1918                 arch->pc = KVM_GUEST_KSEG0 + 0x0;
1919
1920         } else {
1921                 kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n",
1922                           arch->pc);
1923
1924                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1925         }
1926
1927         kvm_change_c0_guest_cause(cop0, (0xff),
1928                                   (EXCCODE_TLBL << CAUSEB_EXCCODE));
1929
1930         /* setup badvaddr, context and entryhi registers for the guest */
1931         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1932         /* XXXKYMA: is the context register used by linux??? */
1933         kvm_write_c0_guest_entryhi(cop0, entryhi);
1934         /* Blow away the shadow host TLBs */
1935         kvm_mips_flush_host_tlb(1);
1936
1937         return EMULATE_DONE;
1938 }
1939
1940 enum emulation_result kvm_mips_emulate_tlbinv_ld(u32 cause,
1941                                                  u32 *opc,
1942                                                  struct kvm_run *run,
1943                                                  struct kvm_vcpu *vcpu)
1944 {
1945         struct mips_coproc *cop0 = vcpu->arch.cop0;
1946         struct kvm_vcpu_arch *arch = &vcpu->arch;
1947         unsigned long entryhi =
1948                 (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1949                 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1950
1951         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1952                 /* save old pc */
1953                 kvm_write_c0_guest_epc(cop0, arch->pc);
1954                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1955
1956                 if (cause & CAUSEF_BD)
1957                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1958                 else
1959                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1960
1961                 kvm_debug("[EXL == 0] delivering TLB INV @ pc %#lx\n",
1962                           arch->pc);
1963
1964                 /* set pc to the exception entry point */
1965                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1966
1967         } else {
1968                 kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n",
1969                           arch->pc);
1970                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1971         }
1972
1973         kvm_change_c0_guest_cause(cop0, (0xff),
1974                                   (EXCCODE_TLBL << CAUSEB_EXCCODE));
1975
1976         /* setup badvaddr, context and entryhi registers for the guest */
1977         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1978         /* XXXKYMA: is the context register used by linux??? */
1979         kvm_write_c0_guest_entryhi(cop0, entryhi);
1980         /* Blow away the shadow host TLBs */
1981         kvm_mips_flush_host_tlb(1);
1982
1983         return EMULATE_DONE;
1984 }
1985
1986 enum emulation_result kvm_mips_emulate_tlbmiss_st(u32 cause,
1987                                                   u32 *opc,
1988                                                   struct kvm_run *run,
1989                                                   struct kvm_vcpu *vcpu)
1990 {
1991         struct mips_coproc *cop0 = vcpu->arch.cop0;
1992         struct kvm_vcpu_arch *arch = &vcpu->arch;
1993         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1994                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1995
1996         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1997                 /* save old pc */
1998                 kvm_write_c0_guest_epc(cop0, arch->pc);
1999                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2000
2001                 if (cause & CAUSEF_BD)
2002                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2003                 else
2004                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2005
2006                 kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n",
2007                           arch->pc);
2008
2009                 /* Set PC to the exception entry point */
2010                 arch->pc = KVM_GUEST_KSEG0 + 0x0;
2011         } else {
2012                 kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n",
2013                           arch->pc);
2014                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2015         }
2016
2017         kvm_change_c0_guest_cause(cop0, (0xff),
2018                                   (EXCCODE_TLBS << CAUSEB_EXCCODE));
2019
2020         /* setup badvaddr, context and entryhi registers for the guest */
2021         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2022         /* XXXKYMA: is the context register used by linux??? */
2023         kvm_write_c0_guest_entryhi(cop0, entryhi);
2024         /* Blow away the shadow host TLBs */
2025         kvm_mips_flush_host_tlb(1);
2026
2027         return EMULATE_DONE;
2028 }
2029
2030 enum emulation_result kvm_mips_emulate_tlbinv_st(u32 cause,
2031                                                  u32 *opc,
2032                                                  struct kvm_run *run,
2033                                                  struct kvm_vcpu *vcpu)
2034 {
2035         struct mips_coproc *cop0 = vcpu->arch.cop0;
2036         struct kvm_vcpu_arch *arch = &vcpu->arch;
2037         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2038                 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2039
2040         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2041                 /* save old pc */
2042                 kvm_write_c0_guest_epc(cop0, arch->pc);
2043                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2044
2045                 if (cause & CAUSEF_BD)
2046                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2047                 else
2048                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2049
2050                 kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n",
2051                           arch->pc);
2052
2053                 /* Set PC to the exception entry point */
2054                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2055         } else {
2056                 kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n",
2057                           arch->pc);
2058                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2059         }
2060
2061         kvm_change_c0_guest_cause(cop0, (0xff),
2062                                   (EXCCODE_TLBS << CAUSEB_EXCCODE));
2063
2064         /* setup badvaddr, context and entryhi registers for the guest */
2065         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2066         /* XXXKYMA: is the context register used by linux??? */
2067         kvm_write_c0_guest_entryhi(cop0, entryhi);
2068         /* Blow away the shadow host TLBs */
2069         kvm_mips_flush_host_tlb(1);
2070
2071         return EMULATE_DONE;
2072 }
2073
2074 /* TLBMOD: store into address matching TLB with Dirty bit off */
2075 enum emulation_result kvm_mips_handle_tlbmod(u32 cause, u32 *opc,
2076                                              struct kvm_run *run,
2077                                              struct kvm_vcpu *vcpu)
2078 {
2079         enum emulation_result er = EMULATE_DONE;
2080 #ifdef DEBUG
2081         struct mips_coproc *cop0 = vcpu->arch.cop0;
2082         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2083                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2084         int index;
2085
2086         /* If address not in the guest TLB, then we are in trouble */
2087         index = kvm_mips_guest_tlb_lookup(vcpu, entryhi);
2088         if (index < 0) {
2089                 /* XXXKYMA Invalidate and retry */
2090                 kvm_mips_host_tlb_inv(vcpu, vcpu->arch.host_cp0_badvaddr);
2091                 kvm_err("%s: host got TLBMOD for %#lx but entry not present in Guest TLB\n",
2092                      __func__, entryhi);
2093                 kvm_mips_dump_guest_tlbs(vcpu);
2094                 kvm_mips_dump_host_tlbs();
2095                 return EMULATE_FAIL;
2096         }
2097 #endif
2098
2099         er = kvm_mips_emulate_tlbmod(cause, opc, run, vcpu);
2100         return er;
2101 }
2102
2103 enum emulation_result kvm_mips_emulate_tlbmod(u32 cause,
2104                                               u32 *opc,
2105                                               struct kvm_run *run,
2106                                               struct kvm_vcpu *vcpu)
2107 {
2108         struct mips_coproc *cop0 = vcpu->arch.cop0;
2109         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2110                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2111         struct kvm_vcpu_arch *arch = &vcpu->arch;
2112
2113         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2114                 /* save old pc */
2115                 kvm_write_c0_guest_epc(cop0, arch->pc);
2116                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2117
2118                 if (cause & CAUSEF_BD)
2119                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2120                 else
2121                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2122
2123                 kvm_debug("[EXL == 0] Delivering TLB MOD @ pc %#lx\n",
2124                           arch->pc);
2125
2126                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2127         } else {
2128                 kvm_debug("[EXL == 1] Delivering TLB MOD @ pc %#lx\n",
2129                           arch->pc);
2130                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2131         }
2132
2133         kvm_change_c0_guest_cause(cop0, (0xff),
2134                                   (EXCCODE_MOD << CAUSEB_EXCCODE));
2135
2136         /* setup badvaddr, context and entryhi registers for the guest */
2137         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2138         /* XXXKYMA: is the context register used by linux??? */
2139         kvm_write_c0_guest_entryhi(cop0, entryhi);
2140         /* Blow away the shadow host TLBs */
2141         kvm_mips_flush_host_tlb(1);
2142
2143         return EMULATE_DONE;
2144 }
2145
2146 enum emulation_result kvm_mips_emulate_fpu_exc(u32 cause,
2147                                                u32 *opc,
2148                                                struct kvm_run *run,
2149                                                struct kvm_vcpu *vcpu)
2150 {
2151         struct mips_coproc *cop0 = vcpu->arch.cop0;
2152         struct kvm_vcpu_arch *arch = &vcpu->arch;
2153
2154         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2155                 /* save old pc */
2156                 kvm_write_c0_guest_epc(cop0, arch->pc);
2157                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2158
2159                 if (cause & CAUSEF_BD)
2160                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2161                 else
2162                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2163
2164         }
2165
2166         arch->pc = KVM_GUEST_KSEG0 + 0x180;
2167
2168         kvm_change_c0_guest_cause(cop0, (0xff),
2169                                   (EXCCODE_CPU << CAUSEB_EXCCODE));
2170         kvm_change_c0_guest_cause(cop0, (CAUSEF_CE), (0x1 << CAUSEB_CE));
2171
2172         return EMULATE_DONE;
2173 }
2174
2175 enum emulation_result kvm_mips_emulate_ri_exc(u32 cause,
2176                                               u32 *opc,
2177                                               struct kvm_run *run,
2178                                               struct kvm_vcpu *vcpu)
2179 {
2180         struct mips_coproc *cop0 = vcpu->arch.cop0;
2181         struct kvm_vcpu_arch *arch = &vcpu->arch;
2182         enum emulation_result er = EMULATE_DONE;
2183
2184         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2185                 /* save old pc */
2186                 kvm_write_c0_guest_epc(cop0, arch->pc);
2187                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2188
2189                 if (cause & CAUSEF_BD)
2190                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2191                 else
2192                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2193
2194                 kvm_debug("Delivering RI @ pc %#lx\n", arch->pc);
2195
2196                 kvm_change_c0_guest_cause(cop0, (0xff),
2197                                           (EXCCODE_RI << CAUSEB_EXCCODE));
2198
2199                 /* Set PC to the exception entry point */
2200                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2201
2202         } else {
2203                 kvm_err("Trying to deliver RI when EXL is already set\n");
2204                 er = EMULATE_FAIL;
2205         }
2206
2207         return er;
2208 }
2209
2210 enum emulation_result kvm_mips_emulate_bp_exc(u32 cause,
2211                                               u32 *opc,
2212                                               struct kvm_run *run,
2213                                               struct kvm_vcpu *vcpu)
2214 {
2215         struct mips_coproc *cop0 = vcpu->arch.cop0;
2216         struct kvm_vcpu_arch *arch = &vcpu->arch;
2217         enum emulation_result er = EMULATE_DONE;
2218
2219         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2220                 /* save old pc */
2221                 kvm_write_c0_guest_epc(cop0, arch->pc);
2222                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2223
2224                 if (cause & CAUSEF_BD)
2225                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2226                 else
2227                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2228
2229                 kvm_debug("Delivering BP @ pc %#lx\n", arch->pc);
2230
2231                 kvm_change_c0_guest_cause(cop0, (0xff),
2232                                           (EXCCODE_BP << CAUSEB_EXCCODE));
2233
2234                 /* Set PC to the exception entry point */
2235                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2236
2237         } else {
2238                 kvm_err("Trying to deliver BP when EXL is already set\n");
2239                 er = EMULATE_FAIL;
2240         }
2241
2242         return er;
2243 }
2244
2245 enum emulation_result kvm_mips_emulate_trap_exc(u32 cause,
2246                                                 u32 *opc,
2247                                                 struct kvm_run *run,
2248                                                 struct kvm_vcpu *vcpu)
2249 {
2250         struct mips_coproc *cop0 = vcpu->arch.cop0;
2251         struct kvm_vcpu_arch *arch = &vcpu->arch;
2252         enum emulation_result er = EMULATE_DONE;
2253
2254         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2255                 /* save old pc */
2256                 kvm_write_c0_guest_epc(cop0, arch->pc);
2257                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2258
2259                 if (cause & CAUSEF_BD)
2260                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2261                 else
2262                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2263
2264                 kvm_debug("Delivering TRAP @ pc %#lx\n", arch->pc);
2265
2266                 kvm_change_c0_guest_cause(cop0, (0xff),
2267                                           (EXCCODE_TR << CAUSEB_EXCCODE));
2268
2269                 /* Set PC to the exception entry point */
2270                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2271
2272         } else {
2273                 kvm_err("Trying to deliver TRAP when EXL is already set\n");
2274                 er = EMULATE_FAIL;
2275         }
2276
2277         return er;
2278 }
2279
2280 enum emulation_result kvm_mips_emulate_msafpe_exc(u32 cause,
2281                                                   u32 *opc,
2282                                                   struct kvm_run *run,
2283                                                   struct kvm_vcpu *vcpu)
2284 {
2285         struct mips_coproc *cop0 = vcpu->arch.cop0;
2286         struct kvm_vcpu_arch *arch = &vcpu->arch;
2287         enum emulation_result er = EMULATE_DONE;
2288
2289         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2290                 /* save old pc */
2291                 kvm_write_c0_guest_epc(cop0, arch->pc);
2292                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2293
2294                 if (cause & CAUSEF_BD)
2295                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2296                 else
2297                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2298
2299                 kvm_debug("Delivering MSAFPE @ pc %#lx\n", arch->pc);
2300
2301                 kvm_change_c0_guest_cause(cop0, (0xff),
2302                                           (EXCCODE_MSAFPE << CAUSEB_EXCCODE));
2303
2304                 /* Set PC to the exception entry point */
2305                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2306
2307         } else {
2308                 kvm_err("Trying to deliver MSAFPE when EXL is already set\n");
2309                 er = EMULATE_FAIL;
2310         }
2311
2312         return er;
2313 }
2314
2315 enum emulation_result kvm_mips_emulate_fpe_exc(u32 cause,
2316                                                u32 *opc,
2317                                                struct kvm_run *run,
2318                                                struct kvm_vcpu *vcpu)
2319 {
2320         struct mips_coproc *cop0 = vcpu->arch.cop0;
2321         struct kvm_vcpu_arch *arch = &vcpu->arch;
2322         enum emulation_result er = EMULATE_DONE;
2323
2324         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2325                 /* save old pc */
2326                 kvm_write_c0_guest_epc(cop0, arch->pc);
2327                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2328
2329                 if (cause & CAUSEF_BD)
2330                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2331                 else
2332                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2333
2334                 kvm_debug("Delivering FPE @ pc %#lx\n", arch->pc);
2335
2336                 kvm_change_c0_guest_cause(cop0, (0xff),
2337                                           (EXCCODE_FPE << CAUSEB_EXCCODE));
2338
2339                 /* Set PC to the exception entry point */
2340                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2341
2342         } else {
2343                 kvm_err("Trying to deliver FPE when EXL is already set\n");
2344                 er = EMULATE_FAIL;
2345         }
2346
2347         return er;
2348 }
2349
2350 enum emulation_result kvm_mips_emulate_msadis_exc(u32 cause,
2351                                                   u32 *opc,
2352                                                   struct kvm_run *run,
2353                                                   struct kvm_vcpu *vcpu)
2354 {
2355         struct mips_coproc *cop0 = vcpu->arch.cop0;
2356         struct kvm_vcpu_arch *arch = &vcpu->arch;
2357         enum emulation_result er = EMULATE_DONE;
2358
2359         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2360                 /* save old pc */
2361                 kvm_write_c0_guest_epc(cop0, arch->pc);
2362                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2363
2364                 if (cause & CAUSEF_BD)
2365                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2366                 else
2367                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2368
2369                 kvm_debug("Delivering MSADIS @ pc %#lx\n", arch->pc);
2370
2371                 kvm_change_c0_guest_cause(cop0, (0xff),
2372                                           (EXCCODE_MSADIS << CAUSEB_EXCCODE));
2373
2374                 /* Set PC to the exception entry point */
2375                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2376
2377         } else {
2378                 kvm_err("Trying to deliver MSADIS when EXL is already set\n");
2379                 er = EMULATE_FAIL;
2380         }
2381
2382         return er;
2383 }
2384
2385 enum emulation_result kvm_mips_handle_ri(u32 cause, u32 *opc,
2386                                          struct kvm_run *run,
2387                                          struct kvm_vcpu *vcpu)
2388 {
2389         struct mips_coproc *cop0 = vcpu->arch.cop0;
2390         struct kvm_vcpu_arch *arch = &vcpu->arch;
2391         enum emulation_result er = EMULATE_DONE;
2392         unsigned long curr_pc;
2393         union mips_instruction inst;
2394
2395         /*
2396          * Update PC and hold onto current PC in case there is
2397          * an error and we want to rollback the PC
2398          */
2399         curr_pc = vcpu->arch.pc;
2400         er = update_pc(vcpu, cause);
2401         if (er == EMULATE_FAIL)
2402                 return er;
2403
2404         /* Fetch the instruction. */
2405         if (cause & CAUSEF_BD)
2406                 opc += 1;
2407
2408         inst.word = kvm_get_inst(opc, vcpu);
2409
2410         if (inst.word == KVM_INVALID_INST) {
2411                 kvm_err("%s: Cannot get inst @ %p\n", __func__, opc);
2412                 return EMULATE_FAIL;
2413         }
2414
2415         if (inst.r_format.opcode == spec3_op &&
2416             inst.r_format.func == rdhwr_op &&
2417             inst.r_format.rs == 0 &&
2418             (inst.r_format.re >> 3) == 0) {
2419                 int usermode = !KVM_GUEST_KERNEL_MODE(vcpu);
2420                 int rd = inst.r_format.rd;
2421                 int rt = inst.r_format.rt;
2422                 int sel = inst.r_format.re & 0x7;
2423
2424                 /* If usermode, check RDHWR rd is allowed by guest HWREna */
2425                 if (usermode && !(kvm_read_c0_guest_hwrena(cop0) & BIT(rd))) {
2426                         kvm_debug("RDHWR %#x disallowed by HWREna @ %p\n",
2427                                   rd, opc);
2428                         goto emulate_ri;
2429                 }
2430                 switch (rd) {
2431                 case MIPS_HWR_CPUNUM:           /* CPU number */
2432                         arch->gprs[rt] = vcpu->vcpu_id;
2433                         break;
2434                 case MIPS_HWR_SYNCISTEP:        /* SYNCI length */
2435                         arch->gprs[rt] = min(current_cpu_data.dcache.linesz,
2436                                              current_cpu_data.icache.linesz);
2437                         break;
2438                 case MIPS_HWR_CC:               /* Read count register */
2439                         arch->gprs[rt] = (s32)kvm_mips_read_count(vcpu);
2440                         break;
2441                 case MIPS_HWR_CCRES:            /* Count register resolution */
2442                         switch (current_cpu_data.cputype) {
2443                         case CPU_20KC:
2444                         case CPU_25KF:
2445                                 arch->gprs[rt] = 1;
2446                                 break;
2447                         default:
2448                                 arch->gprs[rt] = 2;
2449                         }
2450                         break;
2451                 case MIPS_HWR_ULR:              /* Read UserLocal register */
2452                         arch->gprs[rt] = kvm_read_c0_guest_userlocal(cop0);
2453                         break;
2454
2455                 default:
2456                         kvm_debug("RDHWR %#x not supported @ %p\n", rd, opc);
2457                         goto emulate_ri;
2458                 }
2459
2460                 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, KVM_TRACE_HWR(rd, sel),
2461                               vcpu->arch.gprs[rt]);
2462         } else {
2463                 kvm_debug("Emulate RI not supported @ %p: %#x\n",
2464                           opc, inst.word);
2465                 goto emulate_ri;
2466         }
2467
2468         return EMULATE_DONE;
2469
2470 emulate_ri:
2471         /*
2472          * Rollback PC (if in branch delay slot then the PC already points to
2473          * branch target), and pass the RI exception to the guest OS.
2474          */
2475         vcpu->arch.pc = curr_pc;
2476         return kvm_mips_emulate_ri_exc(cause, opc, run, vcpu);
2477 }
2478
2479 enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu,
2480                                                   struct kvm_run *run)
2481 {
2482         unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
2483         enum emulation_result er = EMULATE_DONE;
2484
2485         if (run->mmio.len > sizeof(*gpr)) {
2486                 kvm_err("Bad MMIO length: %d", run->mmio.len);
2487                 er = EMULATE_FAIL;
2488                 goto done;
2489         }
2490
2491         er = update_pc(vcpu, vcpu->arch.pending_load_cause);
2492         if (er == EMULATE_FAIL)
2493                 return er;
2494
2495         switch (run->mmio.len) {
2496         case 4:
2497                 *gpr = *(s32 *) run->mmio.data;
2498                 break;
2499
2500         case 2:
2501                 if (vcpu->mmio_needed == 2)
2502                         *gpr = *(s16 *) run->mmio.data;
2503                 else
2504                         *gpr = *(u16 *)run->mmio.data;
2505
2506                 break;
2507         case 1:
2508                 if (vcpu->mmio_needed == 2)
2509                         *gpr = *(s8 *) run->mmio.data;
2510                 else
2511                         *gpr = *(u8 *) run->mmio.data;
2512                 break;
2513         }
2514
2515         if (vcpu->arch.pending_load_cause & CAUSEF_BD)
2516                 kvm_debug("[%#lx] Completing %d byte BD Load to gpr %d (0x%08lx) type %d\n",
2517                           vcpu->arch.pc, run->mmio.len, vcpu->arch.io_gpr, *gpr,
2518                           vcpu->mmio_needed);
2519
2520 done:
2521         return er;
2522 }
2523
2524 static enum emulation_result kvm_mips_emulate_exc(u32 cause,
2525                                                   u32 *opc,
2526                                                   struct kvm_run *run,
2527                                                   struct kvm_vcpu *vcpu)
2528 {
2529         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2530         struct mips_coproc *cop0 = vcpu->arch.cop0;
2531         struct kvm_vcpu_arch *arch = &vcpu->arch;
2532         enum emulation_result er = EMULATE_DONE;
2533
2534         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2535                 /* save old pc */
2536                 kvm_write_c0_guest_epc(cop0, arch->pc);
2537                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2538
2539                 if (cause & CAUSEF_BD)
2540                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2541                 else
2542                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2543
2544                 kvm_change_c0_guest_cause(cop0, (0xff),
2545                                           (exccode << CAUSEB_EXCCODE));
2546
2547                 /* Set PC to the exception entry point */
2548                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2549                 kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2550
2551                 kvm_debug("Delivering EXC %d @ pc %#lx, badVaddr: %#lx\n",
2552                           exccode, kvm_read_c0_guest_epc(cop0),
2553                           kvm_read_c0_guest_badvaddr(cop0));
2554         } else {
2555                 kvm_err("Trying to deliver EXC when EXL is already set\n");
2556                 er = EMULATE_FAIL;
2557         }
2558
2559         return er;
2560 }
2561
2562 enum emulation_result kvm_mips_check_privilege(u32 cause,
2563                                                u32 *opc,
2564                                                struct kvm_run *run,
2565                                                struct kvm_vcpu *vcpu)
2566 {
2567         enum emulation_result er = EMULATE_DONE;
2568         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2569         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
2570
2571         int usermode = !KVM_GUEST_KERNEL_MODE(vcpu);
2572
2573         if (usermode) {
2574                 switch (exccode) {
2575                 case EXCCODE_INT:
2576                 case EXCCODE_SYS:
2577                 case EXCCODE_BP:
2578                 case EXCCODE_RI:
2579                 case EXCCODE_TR:
2580                 case EXCCODE_MSAFPE:
2581                 case EXCCODE_FPE:
2582                 case EXCCODE_MSADIS:
2583                         break;
2584
2585                 case EXCCODE_CPU:
2586                         if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 0)
2587                                 er = EMULATE_PRIV_FAIL;
2588                         break;
2589
2590                 case EXCCODE_MOD:
2591                         break;
2592
2593                 case EXCCODE_TLBL:
2594                         /*
2595                          * We we are accessing Guest kernel space, then send an
2596                          * address error exception to the guest
2597                          */
2598                         if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) {
2599                                 kvm_debug("%s: LD MISS @ %#lx\n", __func__,
2600                                           badvaddr);
2601                                 cause &= ~0xff;
2602                                 cause |= (EXCCODE_ADEL << CAUSEB_EXCCODE);
2603                                 er = EMULATE_PRIV_FAIL;
2604                         }
2605                         break;
2606
2607                 case EXCCODE_TLBS:
2608                         /*
2609                          * We we are accessing Guest kernel space, then send an
2610                          * address error exception to the guest
2611                          */
2612                         if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) {
2613                                 kvm_debug("%s: ST MISS @ %#lx\n", __func__,
2614                                           badvaddr);
2615                                 cause &= ~0xff;
2616                                 cause |= (EXCCODE_ADES << CAUSEB_EXCCODE);
2617                                 er = EMULATE_PRIV_FAIL;
2618                         }
2619                         break;
2620
2621                 case EXCCODE_ADES:
2622                         kvm_debug("%s: address error ST @ %#lx\n", __func__,
2623                                   badvaddr);
2624                         if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) {
2625                                 cause &= ~0xff;
2626                                 cause |= (EXCCODE_TLBS << CAUSEB_EXCCODE);
2627                         }
2628                         er = EMULATE_PRIV_FAIL;
2629                         break;
2630                 case EXCCODE_ADEL:
2631                         kvm_debug("%s: address error LD @ %#lx\n", __func__,
2632                                   badvaddr);
2633                         if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) {
2634                                 cause &= ~0xff;
2635                                 cause |= (EXCCODE_TLBL << CAUSEB_EXCCODE);
2636                         }
2637                         er = EMULATE_PRIV_FAIL;
2638                         break;
2639                 default:
2640                         er = EMULATE_PRIV_FAIL;
2641                         break;
2642                 }
2643         }
2644
2645         if (er == EMULATE_PRIV_FAIL)
2646                 kvm_mips_emulate_exc(cause, opc, run, vcpu);
2647
2648         return er;
2649 }
2650
2651 /*
2652  * User Address (UA) fault, this could happen if
2653  * (1) TLB entry not present/valid in both Guest and shadow host TLBs, in this
2654  *     case we pass on the fault to the guest kernel and let it handle it.
2655  * (2) TLB entry is present in the Guest TLB but not in the shadow, in this
2656  *     case we inject the TLB from the Guest TLB into the shadow host TLB
2657  */
2658 enum emulation_result kvm_mips_handle_tlbmiss(u32 cause,
2659                                               u32 *opc,
2660                                               struct kvm_run *run,
2661                                               struct kvm_vcpu *vcpu)
2662 {
2663         enum emulation_result er = EMULATE_DONE;
2664         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2665         unsigned long va = vcpu->arch.host_cp0_badvaddr;
2666         int index;
2667
2668         kvm_debug("kvm_mips_handle_tlbmiss: badvaddr: %#lx\n",
2669                   vcpu->arch.host_cp0_badvaddr);
2670
2671         /*
2672          * KVM would not have got the exception if this entry was valid in the
2673          * shadow host TLB. Check the Guest TLB, if the entry is not there then
2674          * send the guest an exception. The guest exc handler should then inject
2675          * an entry into the guest TLB.
2676          */
2677         index = kvm_mips_guest_tlb_lookup(vcpu,
2678                       (va & VPN2_MASK) |
2679                       (kvm_read_c0_guest_entryhi(vcpu->arch.cop0) &
2680                        KVM_ENTRYHI_ASID));
2681         if (index < 0) {
2682                 if (exccode == EXCCODE_TLBL) {
2683                         er = kvm_mips_emulate_tlbmiss_ld(cause, opc, run, vcpu);
2684                 } else if (exccode == EXCCODE_TLBS) {
2685                         er = kvm_mips_emulate_tlbmiss_st(cause, opc, run, vcpu);
2686                 } else {
2687                         kvm_err("%s: invalid exc code: %d\n", __func__,
2688                                 exccode);
2689                         er = EMULATE_FAIL;
2690                 }
2691         } else {
2692                 struct kvm_mips_tlb *tlb = &vcpu->arch.guest_tlb[index];
2693
2694                 /*
2695                  * Check if the entry is valid, if not then setup a TLB invalid
2696                  * exception to the guest
2697                  */
2698                 if (!TLB_IS_VALID(*tlb, va)) {
2699                         if (exccode == EXCCODE_TLBL) {
2700                                 er = kvm_mips_emulate_tlbinv_ld(cause, opc, run,
2701                                                                 vcpu);
2702                         } else if (exccode == EXCCODE_TLBS) {
2703                                 er = kvm_mips_emulate_tlbinv_st(cause, opc, run,
2704                                                                 vcpu);
2705                         } else {
2706                                 kvm_err("%s: invalid exc code: %d\n", __func__,
2707                                         exccode);
2708                                 er = EMULATE_FAIL;
2709                         }
2710                 } else {
2711                         kvm_debug("Injecting hi: %#lx, lo0: %#lx, lo1: %#lx into shadow host TLB\n",
2712                                   tlb->tlb_hi, tlb->tlb_lo[0], tlb->tlb_lo[1]);
2713                         /*
2714                          * OK we have a Guest TLB entry, now inject it into the
2715                          * shadow host TLB
2716                          */
2717                         if (kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb)) {
2718                                 kvm_err("%s: handling mapped seg tlb fault for %lx, index: %u, vcpu: %p, ASID: %#lx\n",
2719                                         __func__, va, index, vcpu,
2720                                         read_c0_entryhi());
2721                                 er = EMULATE_FAIL;
2722                         }
2723                 }
2724         }
2725
2726         return er;
2727 }