arch/mips/kvm/emulate.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * KVM/MIPS: Instruction/Exception emulation
   7  *
   8  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
   9  * Authors: Sanjay Lal <sanjayl@kymasys.com>
  10  */
  11
  12 #include <linux/errno.h>
  13 #include <linux/err.h>
  14 #include <linux/ktime.h>
  15 #include <linux/kvm_host.h>
  16 #include <linux/module.h>
  17 #include <linux/vmalloc.h>
  18 #include <linux/fs.h>
  19 #include <linux/bootmem.h>
  20 #include <linux/random.h>
  21 #include <asm/page.h>
  22 #include <asm/cacheflush.h>
  23 #include <asm/cacheops.h>
  24 #include <asm/cpu-info.h>
  25 #include <asm/mmu_context.h>
  26 #include <asm/tlbflush.h>
  27 #include <asm/inst.h>
  28
  29 #undef CONFIG_MIPS_MT
  30 #include <asm/r4kcache.h>
  31 #define CONFIG_MIPS_MT
  32
  33 #include "interrupt.h"
  34 #include "commpage.h"
  35
  36 #include "trace.h"
  37
  38 /*
  39  * Compute the return address and do emulate branch simulation, if required.
  40  * This function should be called only in branch delay slot active.
  41  */
  42 unsigned long kvm_compute_return_epc(struct kvm_vcpu *vcpu,
  43         unsigned long instpc)
  44 {
  45         unsigned int dspcontrol;
  46         union mips_instruction insn;
  47         struct kvm_vcpu_arch *arch = &vcpu->arch;
  48         long epc = instpc;
  49         long nextpc = KVM_INVALID_INST;
  50
  51         if (epc & 3)
  52                 goto unaligned;
  53
  54         /* Read the instruction */
  55         insn.word = kvm_get_inst((u32 *) epc, vcpu);
  56
  57         if (insn.word == KVM_INVALID_INST)
  58                 return KVM_INVALID_INST;
  59
  60         switch (insn.i_format.opcode) {
  61                 /* jr and jalr are in r_format format. */
  62         case spec_op:
  63                 switch (insn.r_format.func) {
  64                 case jalr_op:
  65                         arch->gprs[insn.r_format.rd] = epc + 8;
  66                         /* Fall through */
  67                 case jr_op:
  68                         nextpc = arch->gprs[insn.r_format.rs];
  69                         break;
  70                 }
  71                 break;
  72
  73                 /*
  74                  * This group contains:
  75                  * bltz_op, bgez_op, bltzl_op, bgezl_op,
  76                  * bltzal_op, bgezal_op, bltzall_op, bgezall_op.
  77                  */
  78         case bcond_op:
  79                 switch (insn.i_format.rt) {
  80                 case bltz_op:
  81                 case bltzl_op:
  82                         if ((long)arch->gprs[insn.i_format.rs] < 0)
  83                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
  84                         else
  85                                 epc += 8;
  86                         nextpc = epc;
  87                         break;
  88
  89                 case bgez_op:
  90                 case bgezl_op:
  91                         if ((long)arch->gprs[insn.i_format.rs] >= 0)
  92                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
  93                         else
  94                                 epc += 8;
  95                         nextpc = epc;
  96                         break;
  97
  98                 case bltzal_op:
  99                 case bltzall_op:
 100                         arch->gprs[31] = epc + 8;
 101                         if ((long)arch->gprs[insn.i_format.rs] < 0)
 102                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 103                         else
 104                                 epc += 8;
 105                         nextpc = epc;
 106                         break;
 107
 108                 case bgezal_op:
 109                 case bgezall_op:
 110                         arch->gprs[31] = epc + 8;
 111                         if ((long)arch->gprs[insn.i_format.rs] >= 0)
 112                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 113                         else
 114                                 epc += 8;
 115                         nextpc = epc;
 116                         break;
 117                 case bposge32_op:
 118                         if (!cpu_has_dsp)
 119                                 goto sigill;
 120
 121                         dspcontrol = rddsp(0x01);
 122
 123                         if (dspcontrol >= 32)
 124                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 125                         else
 126                                 epc += 8;
 127                         nextpc = epc;
 128                         break;
 129                 }
 130                 break;
 131
 132                 /* These are unconditional and in j_format. */
 133         case jal_op:
 134                 arch->gprs[31] = instpc + 8;
 135         case j_op:
 136                 epc += 4;
 137                 epc >>= 28;
 138                 epc <<= 28;
 139                 epc |= (insn.j_format.target << 2);
 140                 nextpc = epc;
 141                 break;
 142
 143                 /* These are conditional and in i_format. */
 144         case beq_op:
 145         case beql_op:
 146                 if (arch->gprs[insn.i_format.rs] ==
 147                     arch->gprs[insn.i_format.rt])
 148                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 149                 else
 150                         epc += 8;
 151                 nextpc = epc;
 152                 break;
 153
 154         case bne_op:
 155         case bnel_op:
 156                 if (arch->gprs[insn.i_format.rs] !=
 157                     arch->gprs[insn.i_format.rt])
 158                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 159                 else
 160                         epc += 8;
 161                 nextpc = epc;
 162                 break;
 163
 164         case blez_op:   /* POP06 */
 165 #ifndef CONFIG_CPU_MIPSR6
 166         case blezl_op:  /* removed in R6 */
 167 #endif
 168                 if (insn.i_format.rt != 0)
 169                         goto compact_branch;
 170                 if ((long)arch->gprs[insn.i_format.rs] <= 0)
 171                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 172                 else
 173                         epc += 8;
 174                 nextpc = epc;
 175                 break;
 176
 177         case bgtz_op:   /* POP07 */
 178 #ifndef CONFIG_CPU_MIPSR6
 179         case bgtzl_op:  /* removed in R6 */
 180 #endif
 181                 if (insn.i_format.rt != 0)
 182                         goto compact_branch;
 183                 if ((long)arch->gprs[insn.i_format.rs] > 0)
 184                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 185                 else
 186                         epc += 8;
 187                 nextpc = epc;
 188                 break;
 189
 190                 /* And now the FPA/cp1 branch instructions. */
 191         case cop1_op:
 192                 kvm_err("%s: unsupported cop1_op\n", __func__);
 193                 break;
 194
 195 #ifdef CONFIG_CPU_MIPSR6
 196         /* R6 added the following compact branches with forbidden slots */
 197         case blezl_op:  /* POP26 */
 198         case bgtzl_op:  /* POP27 */
 199                 /* only rt == 0 isn't compact branch */
 200                 if (insn.i_format.rt != 0)
 201                         goto compact_branch;
 202                 break;
 203         case pop10_op:
 204         case pop30_op:
 205                 /* only rs == rt == 0 is reserved, rest are compact branches */
 206                 if (insn.i_format.rs != 0 || insn.i_format.rt != 0)
 207                         goto compact_branch;
 208                 break;
 209         case pop66_op:
 210         case pop76_op:
 211                 /* only rs == 0 isn't compact branch */
 212                 if (insn.i_format.rs != 0)
 213                         goto compact_branch;
 214                 break;
 215 compact_branch:
 216                 /*
 217                  * If we've hit an exception on the forbidden slot, then
 218                  * the branch must not have been taken.
 219                  */
 220                 epc += 8;
 221                 nextpc = epc;
 222                 break;
 223 #else
 224 compact_branch:
 225                 /* Compact branches not supported before R6 */
 226                 break;
 227 #endif
 228         }
 229
 230         return nextpc;
 231
 232 unaligned:
 233         kvm_err("%s: unaligned epc\n", __func__);
 234         return nextpc;
 235
 236 sigill:
 237         kvm_err("%s: DSP branch but not DSP ASE\n", __func__);
 238         return nextpc;
 239 }
 240
 241 enum emulation_result update_pc(struct kvm_vcpu *vcpu, u32 cause)
 242 {
 243         unsigned long branch_pc;
 244         enum emulation_result er = EMULATE_DONE;
 245
 246         if (cause & CAUSEF_BD) {
 247                 branch_pc = kvm_compute_return_epc(vcpu, vcpu->arch.pc);
 248                 if (branch_pc == KVM_INVALID_INST) {
 249                         er = EMULATE_FAIL;
 250                 } else {
 251                         vcpu->arch.pc = branch_pc;
 252                         kvm_debug("BD update_pc(): New PC: %#lx\n",
 253                                   vcpu->arch.pc);
 254                 }
 255         } else
 256                 vcpu->arch.pc += 4;
 257
 258         kvm_debug("update_pc(): New PC: %#lx\n", vcpu->arch.pc);
 259
 260         return er;
 261 }
 262
 263 /**
 264  * kvm_mips_count_disabled() - Find whether the CP0_Count timer is disabled.
 265  * @vcpu:       Virtual CPU.
 266  *
 267  * Returns:     1 if the CP0_Count timer is disabled by either the guest
 268  *              CP0_Cause.DC bit or the count_ctl.DC bit.
 269  *              0 otherwise (in which case CP0_Count timer is running).
 270  */
 271 static inline int kvm_mips_count_disabled(struct kvm_vcpu *vcpu)
 272 {
 273         struct mips_coproc *cop0 = vcpu->arch.cop0;
 274
 275         return  (vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) ||
 276                 (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC);
 277 }
 278
 279 /**
 280  * kvm_mips_ktime_to_count() - Scale ktime_t to a 32-bit count.
 281  *
 282  * Caches the dynamic nanosecond bias in vcpu->arch.count_dyn_bias.
 283  *
 284  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 285  */
 286 static u32 kvm_mips_ktime_to_count(struct kvm_vcpu *vcpu, ktime_t now)
 287 {
 288         s64 now_ns, periods;
 289         u64 delta;
 290
 291         now_ns = ktime_to_ns(now);
 292         delta = now_ns + vcpu->arch.count_dyn_bias;
 293
 294         if (delta >= vcpu->arch.count_period) {
 295                 /* If delta is out of safe range the bias needs adjusting */
 296                 periods = div64_s64(now_ns, vcpu->arch.count_period);
 297                 vcpu->arch.count_dyn_bias = -periods * vcpu->arch.count_period;
 298                 /* Recalculate delta with new bias */
 299                 delta = now_ns + vcpu->arch.count_dyn_bias;
 300         }
 301
 302         /*
 303          * We've ensured that:
 304          *   delta < count_period
 305          *
 306          * Therefore the intermediate delta*count_hz will never overflow since
 307          * at the boundary condition:
 308          *   delta = count_period
 309          *   delta = NSEC_PER_SEC * 2^32 / count_hz
 310          *   delta * count_hz = NSEC_PER_SEC * 2^32
 311          */
 312         return div_u64(delta * vcpu->arch.count_hz, NSEC_PER_SEC);
 313 }
 314
 315 /**
 316  * kvm_mips_count_time() - Get effective current time.
 317  * @vcpu:       Virtual CPU.
 318  *
 319  * Get effective monotonic ktime. This is usually a straightforward ktime_get(),
 320  * except when the master disable bit is set in count_ctl, in which case it is
 321  * count_resume, i.e. the time that the count was disabled.
 322  *
 323  * Returns:     Effective monotonic ktime for CP0_Count.
 324  */
 325 static inline ktime_t kvm_mips_count_time(struct kvm_vcpu *vcpu)
 326 {
 327         if (unlikely(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC))
 328                 return vcpu->arch.count_resume;
 329
 330         return ktime_get();
 331 }
 332
 333 /**
 334  * kvm_mips_read_count_running() - Read the current count value as if running.
 335  * @vcpu:       Virtual CPU.
 336  * @now:        Kernel time to read CP0_Count at.
 337  *
 338  * Returns the current guest CP0_Count register at time @now and handles if the
 339  * timer interrupt is pending and hasn't been handled yet.
 340  *
 341  * Returns:     The current value of the guest CP0_Count register.
 342  */
 343 static u32 kvm_mips_read_count_running(struct kvm_vcpu *vcpu, ktime_t now)
 344 {
 345         struct mips_coproc *cop0 = vcpu->arch.cop0;
 346         ktime_t expires, threshold;
 347         u32 count, compare;
 348         int running;
 349
 350         /* Calculate the biased and scaled guest CP0_Count */
 351         count = vcpu->arch.count_bias + kvm_mips_ktime_to_count(vcpu, now);
 352         compare = kvm_read_c0_guest_compare(cop0);
 353
 354         /*
 355          * Find whether CP0_Count has reached the closest timer interrupt. If
 356          * not, we shouldn't inject it.
 357          */
 358         if ((s32)(count - compare) < 0)
 359                 return count;
 360
 361         /*
 362          * The CP0_Count we're going to return has already reached the closest
 363          * timer interrupt. Quickly check if it really is a new interrupt by
 364          * looking at whether the interval until the hrtimer expiry time is
 365          * less than 1/4 of the timer period.
 366          */
 367         expires = hrtimer_get_expires(&vcpu->arch.comparecount_timer);
 368         threshold = ktime_add_ns(now, vcpu->arch.count_period / 4);
 369         if (ktime_before(expires, threshold)) {
 370                 /*
 371                  * Cancel it while we handle it so there's no chance of
 372                  * interference with the timeout handler.
 373                  */
 374                 running = hrtimer_cancel(&vcpu->arch.comparecount_timer);
 375
 376                 /* Nothing should be waiting on the timeout */
 377                 kvm_mips_callbacks->queue_timer_int(vcpu);
 378
 379                 /*
 380                  * Restart the timer if it was running based on the expiry time
 381                  * we read, so that we don't push it back 2 periods.
 382                  */
 383                 if (running) {
 384                         expires = ktime_add_ns(expires,
 385                                                vcpu->arch.count_period);
 386                         hrtimer_start(&vcpu->arch.comparecount_timer, expires,
 387                                       HRTIMER_MODE_ABS);
 388                 }
 389         }
 390
 391         return count;
 392 }
 393
 394 /**
 395  * kvm_mips_read_count() - Read the current count value.
 396  * @vcpu:       Virtual CPU.
 397  *
 398  * Read the current guest CP0_Count value, taking into account whether the timer
 399  * is stopped.
 400  *
 401  * Returns:     The current guest CP0_Count value.
 402  */
 403 u32 kvm_mips_read_count(struct kvm_vcpu *vcpu)
 404 {
 405         struct mips_coproc *cop0 = vcpu->arch.cop0;
 406
 407         /* If count disabled just read static copy of count */
 408         if (kvm_mips_count_disabled(vcpu))
 409                 return kvm_read_c0_guest_count(cop0);
 410
 411         return kvm_mips_read_count_running(vcpu, ktime_get());
 412 }
 413
 414 /**
 415  * kvm_mips_freeze_hrtimer() - Safely stop the hrtimer.
 416  * @vcpu:       Virtual CPU.
 417  * @count:      Output pointer for CP0_Count value at point of freeze.
 418  *
 419  * Freeze the hrtimer safely and return both the ktime and the CP0_Count value
 420  * at the point it was frozen. It is guaranteed that any pending interrupts at
 421  * the point it was frozen are handled, and none after that point.
 422  *
 423  * This is useful where the time/CP0_Count is needed in the calculation of the
 424  * new parameters.
 425  *
 426  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 427  *
 428  * Returns:     The ktime at the point of freeze.
 429  */
 430 static ktime_t kvm_mips_freeze_hrtimer(struct kvm_vcpu *vcpu, u32 *count)
 431 {
 432         ktime_t now;
 433
 434         /* stop hrtimer before finding time */
 435         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 436         now = ktime_get();
 437
 438         /* find count at this point and handle pending hrtimer */
 439         *count = kvm_mips_read_count_running(vcpu, now);
 440
 441         return now;
 442 }
 443
 444 /**
 445  * kvm_mips_resume_hrtimer() - Resume hrtimer, updating expiry.
 446  * @vcpu:       Virtual CPU.
 447  * @now:        ktime at point of resume.
 448  * @count:      CP0_Count at point of resume.
 449  *
 450  * Resumes the timer and updates the timer expiry based on @now and @count.
 451  * This can be used in conjunction with kvm_mips_freeze_timer() when timer
 452  * parameters need to be changed.
 453  *
 454  * It is guaranteed that a timer interrupt immediately after resume will be
 455  * handled, but not if CP_Compare is exactly at @count. That case is already
 456  * handled by kvm_mips_freeze_timer().
 457  *
 458  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 459  */
 460 static void kvm_mips_resume_hrtimer(struct kvm_vcpu *vcpu,
 461                                     ktime_t now, u32 count)
 462 {
 463         struct mips_coproc *cop0 = vcpu->arch.cop0;
 464         u32 compare;
 465         u64 delta;
 466         ktime_t expire;
 467
 468         /* Calculate timeout (wrap 0 to 2^32) */
 469         compare = kvm_read_c0_guest_compare(cop0);
 470         delta = (u64)(u32)(compare - count - 1) + 1;
 471         delta = div_u64(delta * NSEC_PER_SEC, vcpu->arch.count_hz);
 472         expire = ktime_add_ns(now, delta);
 473
 474         /* Update hrtimer to use new timeout */
 475         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 476         hrtimer_start(&vcpu->arch.comparecount_timer, expire, HRTIMER_MODE_ABS);
 477 }
 478
 479 /**
 480  * kvm_mips_write_count() - Modify the count and update timer.
 481  * @vcpu:       Virtual CPU.
 482  * @count:      Guest CP0_Count value to set.
 483  *
 484  * Sets the CP0_Count value and updates the timer accordingly.
 485  */
 486 void kvm_mips_write_count(struct kvm_vcpu *vcpu, u32 count)
 487 {
 488         struct mips_coproc *cop0 = vcpu->arch.cop0;
 489         ktime_t now;
 490
 491         /* Calculate bias */
 492         now = kvm_mips_count_time(vcpu);
 493         vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now);
 494
 495         if (kvm_mips_count_disabled(vcpu))
 496                 /* The timer's disabled, adjust the static count */
 497                 kvm_write_c0_guest_count(cop0, count);
 498         else
 499                 /* Update timeout */
 500                 kvm_mips_resume_hrtimer(vcpu, now, count);
 501 }
 502
 503 /**
 504  * kvm_mips_init_count() - Initialise timer.
 505  * @vcpu:       Virtual CPU.
 506  *
 507  * Initialise the timer to a sensible frequency, namely 100MHz, zero it, and set
 508  * it going if it's enabled.
 509  */
 510 void kvm_mips_init_count(struct kvm_vcpu *vcpu)
 511 {
 512         /* 100 MHz */
 513         vcpu->arch.count_hz = 100*1000*1000;
 514         vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32,
 515                                           vcpu->arch.count_hz);
 516         vcpu->arch.count_dyn_bias = 0;
 517
 518         /* Starting at 0 */
 519         kvm_mips_write_count(vcpu, 0);
 520 }
 521
 522 /**
 523  * kvm_mips_set_count_hz() - Update the frequency of the timer.
 524  * @vcpu:       Virtual CPU.
 525  * @count_hz:   Frequency of CP0_Count timer in Hz.
 526  *
 527  * Change the frequency of the CP0_Count timer. This is done atomically so that
 528  * CP0_Count is continuous and no timer interrupt is lost.
 529  *
 530  * Returns:     -EINVAL if @count_hz is out of range.
 531  *              0 on success.
 532  */
 533 int kvm_mips_set_count_hz(struct kvm_vcpu *vcpu, s64 count_hz)
 534 {
 535         struct mips_coproc *cop0 = vcpu->arch.cop0;
 536         int dc;
 537         ktime_t now;
 538         u32 count;
 539
 540         /* ensure the frequency is in a sensible range... */
 541         if (count_hz <= 0 || count_hz > NSEC_PER_SEC)
 542                 return -EINVAL;
 543         /* ... and has actually changed */
 544         if (vcpu->arch.count_hz == count_hz)
 545                 return 0;
 546
 547         /* Safely freeze timer so we can keep it continuous */
 548         dc = kvm_mips_count_disabled(vcpu);
 549         if (dc) {
 550                 now = kvm_mips_count_time(vcpu);
 551                 count = kvm_read_c0_guest_count(cop0);
 552         } else {
 553                 now = kvm_mips_freeze_hrtimer(vcpu, &count);
 554         }
 555
 556         /* Update the frequency */
 557         vcpu->arch.count_hz = count_hz;
 558         vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32, count_hz);
 559         vcpu->arch.count_dyn_bias = 0;
 560
 561         /* Calculate adjusted bias so dynamic count is unchanged */
 562         vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now);
 563
 564         /* Update and resume hrtimer */
 565         if (!dc)
 566                 kvm_mips_resume_hrtimer(vcpu, now, count);
 567         return 0;
 568 }
 569
 570 /**
 571  * kvm_mips_write_compare() - Modify compare and update timer.
 572  * @vcpu:       Virtual CPU.
 573  * @compare:    New CP0_Compare value.
 574  * @ack:        Whether to acknowledge timer interrupt.
 575  *
 576  * Update CP0_Compare to a new value and update the timeout.
 577  * If @ack, atomically acknowledge any pending timer interrupt, otherwise ensure
 578  * any pending timer interrupt is preserved.
 579  */
 580 void kvm_mips_write_compare(struct kvm_vcpu *vcpu, u32 compare, bool ack)
 581 {
 582         struct mips_coproc *cop0 = vcpu->arch.cop0;
 583         int dc;
 584         u32 old_compare = kvm_read_c0_guest_compare(cop0);
 585         ktime_t now;
 586         u32 count;
 587
 588         /* if unchanged, must just be an ack */
 589         if (old_compare == compare) {
 590                 if (!ack)
 591                         return;
 592                 kvm_mips_callbacks->dequeue_timer_int(vcpu);
 593                 kvm_write_c0_guest_compare(cop0, compare);
 594                 return;
 595         }
 596
 597         /* freeze_hrtimer() takes care of timer interrupts <= count */
 598         dc = kvm_mips_count_disabled(vcpu);
 599         if (!dc)
 600                 now = kvm_mips_freeze_hrtimer(vcpu, &count);
 601
 602         if (ack)
 603                 kvm_mips_callbacks->dequeue_timer_int(vcpu);
 604
 605         kvm_write_c0_guest_compare(cop0, compare);
 606
 607         /* resume_hrtimer() takes care of timer interrupts > count */
 608         if (!dc)
 609                 kvm_mips_resume_hrtimer(vcpu, now, count);
 610 }
 611
 612 /**
 613  * kvm_mips_count_disable() - Disable count.
 614  * @vcpu:       Virtual CPU.
 615  *
 616  * Disable the CP0_Count timer. A timer interrupt on or before the final stop
 617  * time will be handled but not after.
 618  *
 619  * Assumes CP0_Count was previously enabled but now Guest.CP0_Cause.DC or
 620  * count_ctl.DC has been set (count disabled).
 621  *
 622  * Returns:     The time that the timer was stopped.
 623  */
 624 static ktime_t kvm_mips_count_disable(struct kvm_vcpu *vcpu)
 625 {
 626         struct mips_coproc *cop0 = vcpu->arch.cop0;
 627         u32 count;
 628         ktime_t now;
 629
 630         /* Stop hrtimer */
 631         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 632
 633         /* Set the static count from the dynamic count, handling pending TI */
 634         now = ktime_get();
 635         count = kvm_mips_read_count_running(vcpu, now);
 636         kvm_write_c0_guest_count(cop0, count);
 637
 638         return now;
 639 }
 640
 641 /**
 642  * kvm_mips_count_disable_cause() - Disable count using CP0_Cause.DC.
 643  * @vcpu:       Virtual CPU.
 644  *
 645  * Disable the CP0_Count timer and set CP0_Cause.DC. A timer interrupt on or
 646  * before the final stop time will be handled if the timer isn't disabled by
 647  * count_ctl.DC, but not after.
 648  *
 649  * Assumes CP0_Cause.DC is clear (count enabled).
 650  */
 651 void kvm_mips_count_disable_cause(struct kvm_vcpu *vcpu)
 652 {
 653         struct mips_coproc *cop0 = vcpu->arch.cop0;
 654
 655         kvm_set_c0_guest_cause(cop0, CAUSEF_DC);
 656         if (!(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC))
 657                 kvm_mips_count_disable(vcpu);
 658 }
 659
 660 /**
 661  * kvm_mips_count_enable_cause() - Enable count using CP0_Cause.DC.
 662  * @vcpu:       Virtual CPU.
 663  *
 664  * Enable the CP0_Count timer and clear CP0_Cause.DC. A timer interrupt after
 665  * the start time will be handled if the timer isn't disabled by count_ctl.DC,
 666  * potentially before even returning, so the caller should be careful with
 667  * ordering of CP0_Cause modifications so as not to lose it.
 668  *
 669  * Assumes CP0_Cause.DC is set (count disabled).
 670  */
 671 void kvm_mips_count_enable_cause(struct kvm_vcpu *vcpu)
 672 {
 673         struct mips_coproc *cop0 = vcpu->arch.cop0;
 674         u32 count;
 675
 676         kvm_clear_c0_guest_cause(cop0, CAUSEF_DC);
 677
 678         /*
 679          * Set the dynamic count to match the static count.
 680          * This starts the hrtimer if count_ctl.DC allows it.
 681          * Otherwise it conveniently updates the biases.
 682          */
 683         count = kvm_read_c0_guest_count(cop0);
 684         kvm_mips_write_count(vcpu, count);
 685 }
 686
 687 /**
 688  * kvm_mips_set_count_ctl() - Update the count control KVM register.
 689  * @vcpu:       Virtual CPU.
 690  * @count_ctl:  Count control register new value.
 691  *
 692  * Set the count control KVM register. The timer is updated accordingly.
 693  *
 694  * Returns:     -EINVAL if reserved bits are set.
 695  *              0 on success.
 696  */
 697 int kvm_mips_set_count_ctl(struct kvm_vcpu *vcpu, s64 count_ctl)
 698 {
 699         struct mips_coproc *cop0 = vcpu->arch.cop0;
 700         s64 changed = count_ctl ^ vcpu->arch.count_ctl;
 701         s64 delta;
 702         ktime_t expire, now;
 703         u32 count, compare;
 704
 705         /* Only allow defined bits to be changed */
 706         if (changed & ~(s64)(KVM_REG_MIPS_COUNT_CTL_DC))
 707                 return -EINVAL;
 708
 709         /* Apply new value */
 710         vcpu->arch.count_ctl = count_ctl;
 711
 712         /* Master CP0_Count disable */
 713         if (changed & KVM_REG_MIPS_COUNT_CTL_DC) {
 714                 /* Is CP0_Cause.DC already disabling CP0_Count? */
 715                 if (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC) {
 716                         if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)
 717                                 /* Just record the current time */
 718                                 vcpu->arch.count_resume = ktime_get();
 719                 } else if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) {
 720                         /* disable timer and record current time */
 721                         vcpu->arch.count_resume = kvm_mips_count_disable(vcpu);
 722                 } else {
 723                         /*
 724                          * Calculate timeout relative to static count at resume
 725                          * time (wrap 0 to 2^32).
 726                          */
 727                         count = kvm_read_c0_guest_count(cop0);
 728                         compare = kvm_read_c0_guest_compare(cop0);
 729                         delta = (u64)(u32)(compare - count - 1) + 1;
 730                         delta = div_u64(delta * NSEC_PER_SEC,
 731                                         vcpu->arch.count_hz);
 732                         expire = ktime_add_ns(vcpu->arch.count_resume, delta);
 733
 734                         /* Handle pending interrupt */
 735                         now = ktime_get();
 736                         if (ktime_compare(now, expire) >= 0)
 737                                 /* Nothing should be waiting on the timeout */
 738                                 kvm_mips_callbacks->queue_timer_int(vcpu);
 739
 740                         /* Resume hrtimer without changing bias */
 741                         count = kvm_mips_read_count_running(vcpu, now);
 742                         kvm_mips_resume_hrtimer(vcpu, now, count);
 743                 }
 744         }
 745
 746         return 0;
 747 }
 748
 749 /**
 750  * kvm_mips_set_count_resume() - Update the count resume KVM register.
 751  * @vcpu:               Virtual CPU.
 752  * @count_resume:       Count resume register new value.
 753  *
 754  * Set the count resume KVM register.
 755  *
 756  * Returns:     -EINVAL if out of valid range (0..now).
 757  *              0 on success.
 758  */
 759 int kvm_mips_set_count_resume(struct kvm_vcpu *vcpu, s64 count_resume)
 760 {
 761         /*
 762          * It doesn't make sense for the resume time to be in the future, as it
 763          * would be possible for the next interrupt to be more than a full
 764          * period in the future.
 765          */
 766         if (count_resume < 0 || count_resume > ktime_to_ns(ktime_get()))
 767                 return -EINVAL;
 768
 769         vcpu->arch.count_resume = ns_to_ktime(count_resume);
 770         return 0;
 771 }
 772
 773 /**
 774  * kvm_mips_count_timeout() - Push timer forward on timeout.
 775  * @vcpu:       Virtual CPU.
 776  *
 777  * Handle an hrtimer event by push the hrtimer forward a period.
 778  *
 779  * Returns:     The hrtimer_restart value to return to the hrtimer subsystem.
 780  */
 781 enum hrtimer_restart kvm_mips_count_timeout(struct kvm_vcpu *vcpu)
 782 {
 783         /* Add the Count period to the current expiry time */
 784         hrtimer_add_expires_ns(&vcpu->arch.comparecount_timer,
 785                                vcpu->arch.count_period);
 786         return HRTIMER_RESTART;
 787 }
 788
 789 enum emulation_result kvm_mips_emul_eret(struct kvm_vcpu *vcpu)
 790 {
 791         struct mips_coproc *cop0 = vcpu->arch.cop0;
 792         enum emulation_result er = EMULATE_DONE;
 793
 794         if (kvm_read_c0_guest_status(cop0) & ST0_EXL) {
 795                 kvm_debug("[%#lx] ERET to %#lx\n", vcpu->arch.pc,
 796                           kvm_read_c0_guest_epc(cop0));
 797                 kvm_clear_c0_guest_status(cop0, ST0_EXL);
 798                 vcpu->arch.pc = kvm_read_c0_guest_epc(cop0);
 799
 800         } else if (kvm_read_c0_guest_status(cop0) & ST0_ERL) {
 801                 kvm_clear_c0_guest_status(cop0, ST0_ERL);
 802                 vcpu->arch.pc = kvm_read_c0_guest_errorepc(cop0);
 803         } else {
 804                 kvm_err("[%#lx] ERET when MIPS_SR_EXL|MIPS_SR_ERL == 0\n",
 805                         vcpu->arch.pc);
 806                 er = EMULATE_FAIL;
 807         }
 808
 809         return er;
 810 }
 811
 812 enum emulation_result kvm_mips_emul_wait(struct kvm_vcpu *vcpu)
 813 {
 814         kvm_debug("[%#lx] !!!WAIT!!! (%#lx)\n", vcpu->arch.pc,
 815                   vcpu->arch.pending_exceptions);
 816
 817         ++vcpu->stat.wait_exits;
 818         trace_kvm_exit(vcpu, KVM_TRACE_EXIT_WAIT);
 819         if (!vcpu->arch.pending_exceptions) {
 820                 vcpu->arch.wait = 1;
 821                 kvm_vcpu_block(vcpu);
 822
 823                 /*
 824                  * We we are runnable, then definitely go off to user space to
 825                  * check if any I/O interrupts are pending.
 826                  */
 827                 if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) {
 828                         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
 829                         vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
 830                 }
 831         }
 832
 833         return EMULATE_DONE;
 834 }
 835
 836 /*
 837  * XXXKYMA: Linux doesn't seem to use TLBR, return EMULATE_FAIL for now so that
 838  * we can catch this, if things ever change
 839  */
 840 enum emulation_result kvm_mips_emul_tlbr(struct kvm_vcpu *vcpu)
 841 {
 842         struct mips_coproc *cop0 = vcpu->arch.cop0;
 843         unsigned long pc = vcpu->arch.pc;
 844
 845         kvm_err("[%#lx] COP0_TLBR [%ld]\n", pc, kvm_read_c0_guest_index(cop0));
 846         return EMULATE_FAIL;
 847 }
 848
 849 /* Write Guest TLB Entry @ Index */
 850 enum emulation_result kvm_mips_emul_tlbwi(struct kvm_vcpu *vcpu)
 851 {
 852         struct mips_coproc *cop0 = vcpu->arch.cop0;
 853         int index = kvm_read_c0_guest_index(cop0);
 854         struct kvm_mips_tlb *tlb = NULL;
 855         unsigned long pc = vcpu->arch.pc;
 856
 857         if (index < 0 || index >= KVM_MIPS_GUEST_TLB_SIZE) {
 858                 kvm_debug("%s: illegal index: %d\n", __func__, index);
 859                 kvm_debug("[%#lx] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n",
 860                           pc, index, kvm_read_c0_guest_entryhi(cop0),
 861                           kvm_read_c0_guest_entrylo0(cop0),
 862                           kvm_read_c0_guest_entrylo1(cop0),
 863                           kvm_read_c0_guest_pagemask(cop0));
 864                 index = (index & ~0x80000000) % KVM_MIPS_GUEST_TLB_SIZE;
 865         }
 866
 867         tlb = &vcpu->arch.guest_tlb[index];
 868         /*
 869          * Probe the shadow host TLB for the entry being overwritten, if one
 870          * matches, invalidate it
 871          */
 872         kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi);
 873
 874         tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0);
 875         tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0);
 876         tlb->tlb_lo[0] = kvm_read_c0_guest_entrylo0(cop0);
 877         tlb->tlb_lo[1] = kvm_read_c0_guest_entrylo1(cop0);
 878
 879         kvm_debug("[%#lx] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n",
 880                   pc, index, kvm_read_c0_guest_entryhi(cop0),
 881                   kvm_read_c0_guest_entrylo0(cop0),
 882                   kvm_read_c0_guest_entrylo1(cop0),
 883                   kvm_read_c0_guest_pagemask(cop0));
 884
 885         return EMULATE_DONE;
 886 }
 887
 888 /* Write Guest TLB Entry @ Random Index */
 889 enum emulation_result kvm_mips_emul_tlbwr(struct kvm_vcpu *vcpu)
 890 {
 891         struct mips_coproc *cop0 = vcpu->arch.cop0;
 892         struct kvm_mips_tlb *tlb = NULL;
 893         unsigned long pc = vcpu->arch.pc;
 894         int index;
 895
 896         get_random_bytes(&index, sizeof(index));
 897         index &= (KVM_MIPS_GUEST_TLB_SIZE - 1);
 898
 899         tlb = &vcpu->arch.guest_tlb[index];
 900
 901         /*
 902          * Probe the shadow host TLB for the entry being overwritten, if one
 903          * matches, invalidate it
 904          */
 905         kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi);
 906
 907         tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0);
 908         tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0);
 909         tlb->tlb_lo[0] = kvm_read_c0_guest_entrylo0(cop0);
 910         tlb->tlb_lo[1] = kvm_read_c0_guest_entrylo1(cop0);
 911
 912         kvm_debug("[%#lx] COP0_TLBWR[%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx)\n",
 913                   pc, index, kvm_read_c0_guest_entryhi(cop0),
 914                   kvm_read_c0_guest_entrylo0(cop0),
 915                   kvm_read_c0_guest_entrylo1(cop0));
 916
 917         return EMULATE_DONE;
 918 }
 919
 920 enum emulation_result kvm_mips_emul_tlbp(struct kvm_vcpu *vcpu)
 921 {
 922         struct mips_coproc *cop0 = vcpu->arch.cop0;
 923         long entryhi = kvm_read_c0_guest_entryhi(cop0);
 924         unsigned long pc = vcpu->arch.pc;
 925         int index = -1;
 926
 927         index = kvm_mips_guest_tlb_lookup(vcpu, entryhi);
 928
 929         kvm_write_c0_guest_index(cop0, index);
 930
 931         kvm_debug("[%#lx] COP0_TLBP (entryhi: %#lx), index: %d\n", pc, entryhi,
 932                   index);
 933
 934         return EMULATE_DONE;
 935 }
 936
 937 /**
 938  * kvm_mips_config1_wrmask() - Find mask of writable bits in guest Config1
 939  * @vcpu:       Virtual CPU.
 940  *
 941  * Finds the mask of bits which are writable in the guest's Config1 CP0
 942  * register, by userland (currently read-only to the guest).
 943  */
 944 unsigned int kvm_mips_config1_wrmask(struct kvm_vcpu *vcpu)
 945 {
 946         unsigned int mask = 0;
 947
 948         /* Permit FPU to be present if FPU is supported */
 949         if (kvm_mips_guest_can_have_fpu(&vcpu->arch))
 950                 mask |= MIPS_CONF1_FP;
 951
 952         return mask;
 953 }
 954
 955 /**
 956  * kvm_mips_config3_wrmask() - Find mask of writable bits in guest Config3
 957  * @vcpu:       Virtual CPU.
 958  *
 959  * Finds the mask of bits which are writable in the guest's Config3 CP0
 960  * register, by userland (currently read-only to the guest).
 961  */
 962 unsigned int kvm_mips_config3_wrmask(struct kvm_vcpu *vcpu)
 963 {
 964         /* Config4 and ULRI are optional */
 965         unsigned int mask = MIPS_CONF_M | MIPS_CONF3_ULRI;
 966
 967         /* Permit MSA to be present if MSA is supported */
 968         if (kvm_mips_guest_can_have_msa(&vcpu->arch))
 969                 mask |= MIPS_CONF3_MSA;
 970
 971         return mask;
 972 }
 973
 974 /**
 975  * kvm_mips_config4_wrmask() - Find mask of writable bits in guest Config4
 976  * @vcpu:       Virtual CPU.
 977  *
 978  * Finds the mask of bits which are writable in the guest's Config4 CP0
 979  * register, by userland (currently read-only to the guest).
 980  */
 981 unsigned int kvm_mips_config4_wrmask(struct kvm_vcpu *vcpu)
 982 {
 983         /* Config5 is optional */
 984         unsigned int mask = MIPS_CONF_M;
 985
 986         /* KScrExist */
 987         mask |= (unsigned int)vcpu->arch.kscratch_enabled << 16;
 988
 989         return mask;
 990 }
 991
 992 /**
 993  * kvm_mips_config5_wrmask() - Find mask of writable bits in guest Config5
 994  * @vcpu:       Virtual CPU.
 995  *
 996  * Finds the mask of bits which are writable in the guest's Config5 CP0
 997  * register, by the guest itself.
 998  */
 999 unsigned int kvm_mips_config5_wrmask(struct kvm_vcpu *vcpu)
1000 {
1001         unsigned int mask = 0;
1002
1003         /* Permit MSAEn changes if MSA supported and enabled */
1004         if (kvm_mips_guest_has_msa(&vcpu->arch))
1005                 mask |= MIPS_CONF5_MSAEN;
1006
1007         /*
1008          * Permit guest FPU mode changes if FPU is enabled and the relevant
1009          * feature exists according to FIR register.
1010          */
1011         if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
1012                 if (cpu_has_fre)
1013                         mask |= MIPS_CONF5_FRE;
1014                 /* We don't support UFR or UFE */
1015         }
1016
1017         return mask;
1018 }
1019
1020 enum emulation_result kvm_mips_emulate_CP0(union mips_instruction inst,
1021                                            u32 *opc, u32 cause,
1022                                            struct kvm_run *run,
1023                                            struct kvm_vcpu *vcpu)
1024 {
1025         struct mips_coproc *cop0 = vcpu->arch.cop0;
1026         enum emulation_result er = EMULATE_DONE;
1027         u32 rt, rd, sel;
1028         unsigned long curr_pc;
1029
1030         /*
1031          * Update PC and hold onto current PC in case there is
1032          * an error and we want to rollback the PC
1033          */
1034         curr_pc = vcpu->arch.pc;
1035         er = update_pc(vcpu, cause);
1036         if (er == EMULATE_FAIL)
1037                 return er;
1038
1039         if (inst.co_format.co) {
1040                 switch (inst.co_format.func) {
1041                 case tlbr_op:   /*  Read indexed TLB entry  */
1042                         er = kvm_mips_emul_tlbr(vcpu);
1043                         break;
1044                 case tlbwi_op:  /*  Write indexed  */
1045                         er = kvm_mips_emul_tlbwi(vcpu);
1046                         break;
1047                 case tlbwr_op:  /*  Write random  */
1048                         er = kvm_mips_emul_tlbwr(vcpu);
1049                         break;
1050                 case tlbp_op:   /* TLB Probe */
1051                         er = kvm_mips_emul_tlbp(vcpu);
1052                         break;
1053                 case rfe_op:
1054                         kvm_err("!!!COP0_RFE!!!\n");
1055                         break;
1056                 case eret_op:
1057                         er = kvm_mips_emul_eret(vcpu);
1058                         goto dont_update_pc;
1059                 case wait_op:
1060                         er = kvm_mips_emul_wait(vcpu);
1061                         break;
1062                 }
1063         } else {
1064                 rt = inst.c0r_format.rt;
1065                 rd = inst.c0r_format.rd;
1066                 sel = inst.c0r_format.sel;
1067
1068                 switch (inst.c0r_format.rs) {
1069                 case mfc_op:
1070 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
1071                         cop0->stat[rd][sel]++;
1072 #endif
1073                         /* Get reg */
1074                         if ((rd == MIPS_CP0_COUNT) && (sel == 0)) {
1075                                 vcpu->arch.gprs[rt] = kvm_mips_read_count(vcpu);
1076                         } else if ((rd == MIPS_CP0_ERRCTL) && (sel == 0)) {
1077                                 vcpu->arch.gprs[rt] = 0x0;
1078 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1079                                 kvm_mips_trans_mfc0(inst, opc, vcpu);
1080 #endif
1081                         } else {
1082                                 vcpu->arch.gprs[rt] = cop0->reg[rd][sel];
1083
1084 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1085                                 kvm_mips_trans_mfc0(inst, opc, vcpu);
1086 #endif
1087                         }
1088
1089                         trace_kvm_hwr(vcpu, KVM_TRACE_MFC0,
1090                                       KVM_TRACE_COP0(rd, sel),
1091                                       vcpu->arch.gprs[rt]);
1092                         break;
1093
1094                 case dmfc_op:
1095                         vcpu->arch.gprs[rt] = cop0->reg[rd][sel];
1096
1097                         trace_kvm_hwr(vcpu, KVM_TRACE_DMFC0,
1098                                       KVM_TRACE_COP0(rd, sel),
1099                                       vcpu->arch.gprs[rt]);
1100                         break;
1101
1102                 case mtc_op:
1103 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
1104                         cop0->stat[rd][sel]++;
1105 #endif
1106                         trace_kvm_hwr(vcpu, KVM_TRACE_MTC0,
1107                                       KVM_TRACE_COP0(rd, sel),
1108                                       vcpu->arch.gprs[rt]);
1109
1110                         if ((rd == MIPS_CP0_TLB_INDEX)
1111                             && (vcpu->arch.gprs[rt] >=
1112                                 KVM_MIPS_GUEST_TLB_SIZE)) {
1113                                 kvm_err("Invalid TLB Index: %ld",
1114                                         vcpu->arch.gprs[rt]);
1115                                 er = EMULATE_FAIL;
1116                                 break;
1117                         }
1118 #define C0_EBASE_CORE_MASK 0xff
1119                         if ((rd == MIPS_CP0_PRID) && (sel == 1)) {
1120                                 /* Preserve CORE number */
1121                                 kvm_change_c0_guest_ebase(cop0,
1122                                                           ~(C0_EBASE_CORE_MASK),
1123                                                           vcpu->arch.gprs[rt]);
1124                                 kvm_err("MTCz, cop0->reg[EBASE]: %#lx\n",
1125                                         kvm_read_c0_guest_ebase(cop0));
1126                         } else if (rd == MIPS_CP0_TLB_HI && sel == 0) {
1127                                 u32 nasid =
1128                                         vcpu->arch.gprs[rt] & KVM_ENTRYHI_ASID;
1129                                 if ((KSEGX(vcpu->arch.gprs[rt]) != CKSEG0) &&
1130                                     ((kvm_read_c0_guest_entryhi(cop0) &
1131                                       KVM_ENTRYHI_ASID) != nasid)) {
1132                                         trace_kvm_asid_change(vcpu,
1133                                                 kvm_read_c0_guest_entryhi(cop0)
1134                                                         & KVM_ENTRYHI_ASID,
1135                                                 nasid);
1136
1137                                         /* Blow away the shadow host TLBs */
1138                                         kvm_mips_flush_host_tlb(1);
1139                                 }
1140                                 kvm_write_c0_guest_entryhi(cop0,
1141                                                            vcpu->arch.gprs[rt]);
1142                         }
1143                         /* Are we writing to COUNT */
1144                         else if ((rd == MIPS_CP0_COUNT) && (sel == 0)) {
1145                                 kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]);
1146                                 goto done;
1147                         } else if ((rd == MIPS_CP0_COMPARE) && (sel == 0)) {
1148                                 /* If we are writing to COMPARE */
1149                                 /* Clear pending timer interrupt, if any */
1150                                 kvm_mips_write_compare(vcpu,
1151                                                        vcpu->arch.gprs[rt],
1152                                                        true);
1153                         } else if ((rd == MIPS_CP0_STATUS) && (sel == 0)) {
1154                                 unsigned int old_val, val, change;
1155
1156                                 old_val = kvm_read_c0_guest_status(cop0);
1157                                 val = vcpu->arch.gprs[rt];
1158                                 change = val ^ old_val;
1159
1160                                 /* Make sure that the NMI bit is never set */
1161                                 val &= ~ST0_NMI;
1162
1163                                 /*
1164                                  * Don't allow CU1 or FR to be set unless FPU
1165                                  * capability enabled and exists in guest
1166                                  * configuration.
1167                                  */
1168                                 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1169                                         val &= ~(ST0_CU1 | ST0_FR);
1170
1171                                 /*
1172                                  * Also don't allow FR to be set if host doesn't
1173                                  * support it.
1174                                  */
1175                                 if (!(current_cpu_data.fpu_id & MIPS_FPIR_F64))
1176                                         val &= ~ST0_FR;
1177
1178
1179                                 /* Handle changes in FPU mode */
1180                                 preempt_disable();
1181
1182                                 /*
1183                                  * FPU and Vector register state is made
1184                                  * UNPREDICTABLE by a change of FR, so don't
1185                                  * even bother saving it.
1186                                  */
1187                                 if (change & ST0_FR)
1188                                         kvm_drop_fpu(vcpu);
1189
1190                                 /*
1191                                  * If MSA state is already live, it is undefined
1192                                  * how it interacts with FR=0 FPU state, and we
1193                                  * don't want to hit reserved instruction
1194                                  * exceptions trying to save the MSA state later
1195                                  * when CU=1 && FR=1, so play it safe and save
1196                                  * it first.
1197                                  */
1198                                 if (change & ST0_CU1 && !(val & ST0_FR) &&
1199                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1200                                         kvm_lose_fpu(vcpu);
1201
1202                                 /*
1203                                  * Propagate CU1 (FPU enable) changes
1204                                  * immediately if the FPU context is already
1205                                  * loaded. When disabling we leave the context
1206                                  * loaded so it can be quickly enabled again in
1207                                  * the near future.
1208                                  */
1209                                 if (change & ST0_CU1 &&
1210                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
1211                                         change_c0_status(ST0_CU1, val);
1212
1213                                 preempt_enable();
1214
1215                                 kvm_write_c0_guest_status(cop0, val);
1216
1217 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1218                                 /*
1219                                  * If FPU present, we need CU1/FR bits to take
1220                                  * effect fairly soon.
1221                                  */
1222                                 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1223                                         kvm_mips_trans_mtc0(inst, opc, vcpu);
1224 #endif
1225                         } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) {
1226                                 unsigned int old_val, val, change, wrmask;
1227
1228                                 old_val = kvm_read_c0_guest_config5(cop0);
1229                                 val = vcpu->arch.gprs[rt];
1230
1231                                 /* Only a few bits are writable in Config5 */
1232                                 wrmask = kvm_mips_config5_wrmask(vcpu);
1233                                 change = (val ^ old_val) & wrmask;
1234                                 val = old_val ^ change;
1235
1236
1237                                 /* Handle changes in FPU/MSA modes */
1238                                 preempt_disable();
1239
1240                                 /*
1241                                  * Propagate FRE changes immediately if the FPU
1242                                  * context is already loaded.
1243                                  */
1244                                 if (change & MIPS_CONF5_FRE &&
1245                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
1246                                         change_c0_config5(MIPS_CONF5_FRE, val);
1247
1248                                 /*
1249                                  * Propagate MSAEn changes immediately if the
1250                                  * MSA context is already loaded. When disabling
1251                                  * we leave the context loaded so it can be
1252                                  * quickly enabled again in the near future.
1253                                  */
1254                                 if (change & MIPS_CONF5_MSAEN &&
1255                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1256                                         change_c0_config5(MIPS_CONF5_MSAEN,
1257                                                           val);
1258
1259                                 preempt_enable();
1260
1261                                 kvm_write_c0_guest_config5(cop0, val);
1262                         } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) {
1263                                 u32 old_cause, new_cause;
1264
1265                                 old_cause = kvm_read_c0_guest_cause(cop0);
1266                                 new_cause = vcpu->arch.gprs[rt];
1267                                 /* Update R/W bits */
1268                                 kvm_change_c0_guest_cause(cop0, 0x08800300,
1269                                                           new_cause);
1270                                 /* DC bit enabling/disabling timer? */
1271                                 if ((old_cause ^ new_cause) & CAUSEF_DC) {
1272                                         if (new_cause & CAUSEF_DC)
1273                                                 kvm_mips_count_disable_cause(vcpu);
1274                                         else
1275                                                 kvm_mips_count_enable_cause(vcpu);
1276                                 }
1277                         } else if ((rd == MIPS_CP0_HWRENA) && (sel == 0)) {
1278                                 u32 mask = MIPS_HWRENA_CPUNUM |
1279                                            MIPS_HWRENA_SYNCISTEP |
1280                                            MIPS_HWRENA_CC |
1281                                            MIPS_HWRENA_CCRES;
1282
1283                                 if (kvm_read_c0_guest_config3(cop0) &
1284                                     MIPS_CONF3_ULRI)
1285                                         mask |= MIPS_HWRENA_ULR;
1286                                 cop0->reg[rd][sel] = vcpu->arch.gprs[rt] & mask;
1287                         } else {
1288                                 cop0->reg[rd][sel] = vcpu->arch.gprs[rt];
1289 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1290                                 kvm_mips_trans_mtc0(inst, opc, vcpu);
1291 #endif
1292                         }
1293                         break;
1294
1295                 case dmtc_op:
1296                         kvm_err("!!!!!!![%#lx]dmtc_op: rt: %d, rd: %d, sel: %d!!!!!!\n",
1297                                 vcpu->arch.pc, rt, rd, sel);
1298                         trace_kvm_hwr(vcpu, KVM_TRACE_DMTC0,
1299                                       KVM_TRACE_COP0(rd, sel),
1300                                       vcpu->arch.gprs[rt]);
1301                         er = EMULATE_FAIL;
1302                         break;
1303
1304                 case mfmc0_op:
1305 #ifdef KVM_MIPS_DEBUG_COP0_COUNTERS
1306                         cop0->stat[MIPS_CP0_STATUS][0]++;
1307 #endif
1308                         if (rt != 0)
1309                                 vcpu->arch.gprs[rt] =
1310                                     kvm_read_c0_guest_status(cop0);
1311                         /* EI */
1312                         if (inst.mfmc0_format.sc) {
1313                                 kvm_debug("[%#lx] mfmc0_op: EI\n",
1314                                           vcpu->arch.pc);
1315                                 kvm_set_c0_guest_status(cop0, ST0_IE);
1316                         } else {
1317                                 kvm_debug("[%#lx] mfmc0_op: DI\n",
1318                                           vcpu->arch.pc);
1319                                 kvm_clear_c0_guest_status(cop0, ST0_IE);
1320                         }
1321
1322                         break;
1323
1324                 case wrpgpr_op:
1325                         {
1326                                 u32 css = cop0->reg[MIPS_CP0_STATUS][2] & 0xf;
1327                                 u32 pss =
1328                                     (cop0->reg[MIPS_CP0_STATUS][2] >> 6) & 0xf;
1329                                 /*
1330                                  * We don't support any shadow register sets, so
1331                                  * SRSCtl[PSS] == SRSCtl[CSS] = 0
1332                                  */
1333                                 if (css || pss) {
1334                                         er = EMULATE_FAIL;
1335                                         break;
1336                                 }
1337                                 kvm_debug("WRPGPR[%d][%d] = %#lx\n", pss, rd,
1338                                           vcpu->arch.gprs[rt]);
1339                                 vcpu->arch.gprs[rd] = vcpu->arch.gprs[rt];
1340                         }
1341                         break;
1342                 default:
1343                         kvm_err("[%#lx]MachEmulateCP0: unsupported COP0, copz: 0x%x\n",
1344                                 vcpu->arch.pc, inst.c0r_format.rs);
1345                         er = EMULATE_FAIL;
1346                         break;
1347                 }
1348         }
1349
1350 done:
1351         /* Rollback PC only if emulation was unsuccessful */
1352         if (er == EMULATE_FAIL)
1353                 vcpu->arch.pc = curr_pc;
1354
1355 dont_update_pc:
1356         /*
1357          * This is for special instructions whose emulation
1358          * updates the PC, so do not overwrite the PC under
1359          * any circumstances
1360          */
1361
1362         return er;
1363 }
1364
1365 enum emulation_result kvm_mips_emulate_store(union mips_instruction inst,
1366                                              u32 cause,
1367                                              struct kvm_run *run,
1368                                              struct kvm_vcpu *vcpu)
1369 {
1370         enum emulation_result er = EMULATE_DO_MMIO;
1371         u32 rt;
1372         u32 bytes;
1373         void *data = run->mmio.data;
1374         unsigned long curr_pc;
1375
1376         /*
1377          * Update PC and hold onto current PC in case there is
1378          * an error and we want to rollback the PC
1379          */
1380         curr_pc = vcpu->arch.pc;
1381         er = update_pc(vcpu, cause);
1382         if (er == EMULATE_FAIL)
1383                 return er;
1384
1385         rt = inst.i_format.rt;
1386
1387         switch (inst.i_format.opcode) {
1388         case sb_op:
1389                 bytes = 1;
1390                 if (bytes > sizeof(run->mmio.data)) {
1391                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1392                                run->mmio.len);
1393                 }
1394                 run->mmio.phys_addr =
1395                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1396                                                    host_cp0_badvaddr);
1397                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1398                         er = EMULATE_FAIL;
1399                         break;
1400                 }
1401                 run->mmio.len = bytes;
1402                 run->mmio.is_write = 1;
1403                 vcpu->mmio_needed = 1;
1404                 vcpu->mmio_is_write = 1;
1405                 *(u8 *) data = vcpu->arch.gprs[rt];
1406                 kvm_debug("OP_SB: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1407                           vcpu->arch.host_cp0_badvaddr, vcpu->arch.gprs[rt],
1408                           *(u8 *) data);
1409
1410                 break;
1411
1412         case sw_op:
1413                 bytes = 4;
1414                 if (bytes > sizeof(run->mmio.data)) {
1415                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1416                                run->mmio.len);
1417                 }
1418                 run->mmio.phys_addr =
1419                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1420                                                    host_cp0_badvaddr);
1421                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1422                         er = EMULATE_FAIL;
1423                         break;
1424                 }
1425
1426                 run->mmio.len = bytes;
1427                 run->mmio.is_write = 1;
1428                 vcpu->mmio_needed = 1;
1429                 vcpu->mmio_is_write = 1;
1430                 *(u32 *) data = vcpu->arch.gprs[rt];
1431
1432                 kvm_debug("[%#lx] OP_SW: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1433                           vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1434                           vcpu->arch.gprs[rt], *(u32 *) data);
1435                 break;
1436
1437         case sh_op:
1438                 bytes = 2;
1439                 if (bytes > sizeof(run->mmio.data)) {
1440                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1441                                run->mmio.len);
1442                 }
1443                 run->mmio.phys_addr =
1444                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1445                                                    host_cp0_badvaddr);
1446                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1447                         er = EMULATE_FAIL;
1448                         break;
1449                 }
1450
1451                 run->mmio.len = bytes;
1452                 run->mmio.is_write = 1;
1453                 vcpu->mmio_needed = 1;
1454                 vcpu->mmio_is_write = 1;
1455                 *(u16 *) data = vcpu->arch.gprs[rt];
1456
1457                 kvm_debug("[%#lx] OP_SH: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1458                           vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1459                           vcpu->arch.gprs[rt], *(u32 *) data);
1460                 break;
1461
1462         default:
1463                 kvm_err("Store not yet supported (inst=0x%08x)\n",
1464                         inst.word);
1465                 er = EMULATE_FAIL;
1466                 break;
1467         }
1468
1469         /* Rollback PC if emulation was unsuccessful */
1470         if (er == EMULATE_FAIL)
1471                 vcpu->arch.pc = curr_pc;
1472
1473         return er;
1474 }
1475
1476 enum emulation_result kvm_mips_emulate_load(union mips_instruction inst,
1477                                             u32 cause, struct kvm_run *run,
1478                                             struct kvm_vcpu *vcpu)
1479 {
1480         enum emulation_result er = EMULATE_DO_MMIO;
1481         u32 op, rt;
1482         u32 bytes;
1483
1484         rt = inst.i_format.rt;
1485         op = inst.i_format.opcode;
1486
1487         vcpu->arch.pending_load_cause = cause;
1488         vcpu->arch.io_gpr = rt;
1489
1490         switch (op) {
1491         case lw_op:
1492                 bytes = 4;
1493                 if (bytes > sizeof(run->mmio.data)) {
1494                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1495                                run->mmio.len);
1496                         er = EMULATE_FAIL;
1497                         break;
1498                 }
1499                 run->mmio.phys_addr =
1500                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1501                                                    host_cp0_badvaddr);
1502                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1503                         er = EMULATE_FAIL;
1504                         break;
1505                 }
1506
1507                 run->mmio.len = bytes;
1508                 run->mmio.is_write = 0;
1509                 vcpu->mmio_needed = 1;
1510                 vcpu->mmio_is_write = 0;
1511                 break;
1512
1513         case lh_op:
1514         case lhu_op:
1515                 bytes = 2;
1516                 if (bytes > sizeof(run->mmio.data)) {
1517                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1518                                run->mmio.len);
1519                         er = EMULATE_FAIL;
1520                         break;
1521                 }
1522                 run->mmio.phys_addr =
1523                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1524                                                    host_cp0_badvaddr);
1525                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1526                         er = EMULATE_FAIL;
1527                         break;
1528                 }
1529
1530                 run->mmio.len = bytes;
1531                 run->mmio.is_write = 0;
1532                 vcpu->mmio_needed = 1;
1533                 vcpu->mmio_is_write = 0;
1534
1535                 if (op == lh_op)
1536                         vcpu->mmio_needed = 2;
1537                 else
1538                         vcpu->mmio_needed = 1;
1539
1540                 break;
1541
1542         case lbu_op:
1543         case lb_op:
1544                 bytes = 1;
1545                 if (bytes > sizeof(run->mmio.data)) {
1546                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1547                                run->mmio.len);
1548                         er = EMULATE_FAIL;
1549                         break;
1550                 }
1551                 run->mmio.phys_addr =
1552                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1553                                                    host_cp0_badvaddr);
1554                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1555                         er = EMULATE_FAIL;
1556                         break;
1557                 }
1558
1559                 run->mmio.len = bytes;
1560                 run->mmio.is_write = 0;
1561                 vcpu->mmio_is_write = 0;
1562
1563                 if (op == lb_op)
1564                         vcpu->mmio_needed = 2;
1565                 else
1566                         vcpu->mmio_needed = 1;
1567
1568                 break;
1569
1570         default:
1571                 kvm_err("Load not yet supported (inst=0x%08x)\n",
1572                         inst.word);
1573                 er = EMULATE_FAIL;
1574                 break;
1575         }
1576
1577         return er;
1578 }
1579
1580 enum emulation_result kvm_mips_emulate_cache(union mips_instruction inst,
1581                                              u32 *opc, u32 cause,
1582                                              struct kvm_run *run,
1583                                              struct kvm_vcpu *vcpu)
1584 {
1585         struct mips_coproc *cop0 = vcpu->arch.cop0;
1586         enum emulation_result er = EMULATE_DONE;
1587         u32 cache, op_inst, op, base;
1588         s16 offset;
1589         struct kvm_vcpu_arch *arch = &vcpu->arch;
1590         unsigned long va;
1591         unsigned long curr_pc;
1592
1593         /*
1594          * Update PC and hold onto current PC in case there is
1595          * an error and we want to rollback the PC
1596          */
1597         curr_pc = vcpu->arch.pc;
1598         er = update_pc(vcpu, cause);
1599         if (er == EMULATE_FAIL)
1600                 return er;
1601
1602         base = inst.i_format.rs;
1603         op_inst = inst.i_format.rt;
1604         offset = inst.i_format.simmediate;
1605         cache = op_inst & CacheOp_Cache;
1606         op = op_inst & CacheOp_Op;
1607
1608         va = arch->gprs[base] + offset;
1609
1610         kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1611                   cache, op, base, arch->gprs[base], offset);
1612
1613         /*
1614          * Treat INDEX_INV as a nop, basically issued by Linux on startup to
1615          * invalidate the caches entirely by stepping through all the
1616          * ways/indexes
1617          */
1618         if (op == Index_Writeback_Inv) {
1619                 kvm_debug("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1620                           vcpu->arch.pc, vcpu->arch.gprs[31], cache, op, base,
1621                           arch->gprs[base], offset);
1622
1623                 if (cache == Cache_D)
1624                         r4k_blast_dcache();
1625                 else if (cache == Cache_I)
1626                         r4k_blast_icache();
1627                 else {
1628                         kvm_err("%s: unsupported CACHE INDEX operation\n",
1629                                 __func__);
1630                         return EMULATE_FAIL;
1631                 }
1632
1633 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1634                 kvm_mips_trans_cache_index(inst, opc, vcpu);
1635 #endif
1636                 goto done;
1637         }
1638
1639         preempt_disable();
1640         if (KVM_GUEST_KSEGX(va) == KVM_GUEST_KSEG0) {
1641                 if (kvm_mips_host_tlb_lookup(vcpu, va) < 0)
1642                         kvm_mips_handle_kseg0_tlb_fault(va, vcpu);
1643         } else if ((KVM_GUEST_KSEGX(va) < KVM_GUEST_KSEG0) ||
1644                    KVM_GUEST_KSEGX(va) == KVM_GUEST_KSEG23) {
1645                 int index;
1646
1647                 /* If an entry already exists then skip */
1648                 if (kvm_mips_host_tlb_lookup(vcpu, va) >= 0)
1649                         goto skip_fault;
1650
1651                 /*
1652                  * If address not in the guest TLB, then give the guest a fault,
1653                  * the resulting handler will do the right thing
1654                  */
1655                 index = kvm_mips_guest_tlb_lookup(vcpu, (va & VPN2_MASK) |
1656                                                   (kvm_read_c0_guest_entryhi
1657                                                    (cop0) & KVM_ENTRYHI_ASID));
1658
1659                 if (index < 0) {
1660                         vcpu->arch.host_cp0_badvaddr = va;
1661                         vcpu->arch.pc = curr_pc;
1662                         er = kvm_mips_emulate_tlbmiss_ld(cause, NULL, run,
1663                                                          vcpu);
1664                         preempt_enable();
1665                         goto dont_update_pc;
1666                 } else {
1667                         struct kvm_mips_tlb *tlb = &vcpu->arch.guest_tlb[index];
1668                         /*
1669                          * Check if the entry is valid, if not then setup a TLB
1670                          * invalid exception to the guest
1671                          */
1672                         if (!TLB_IS_VALID(*tlb, va)) {
1673                                 vcpu->arch.host_cp0_badvaddr = va;
1674                                 vcpu->arch.pc = curr_pc;
1675                                 er = kvm_mips_emulate_tlbinv_ld(cause, NULL,
1676                                                                 run, vcpu);
1677                                 preempt_enable();
1678                                 goto dont_update_pc;
1679                         } else {
1680                                 /*
1681                                  * We fault an entry from the guest tlb to the
1682                                  * shadow host TLB
1683                                  */
1684                                 kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb);
1685                         }
1686                 }
1687         } else {
1688                 kvm_err("INVALID CACHE INDEX/ADDRESS (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1689                         cache, op, base, arch->gprs[base], offset);
1690                 er = EMULATE_FAIL;
1691                 preempt_enable();
1692                 goto done;
1693
1694         }
1695
1696 skip_fault:
1697         /* XXXKYMA: Only a subset of cache ops are supported, used by Linux */
1698         if (op_inst == Hit_Writeback_Inv_D || op_inst == Hit_Invalidate_D) {
1699                 flush_dcache_line(va);
1700
1701 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1702                 /*
1703                  * Replace the CACHE instruction, with a SYNCI, not the same,
1704                  * but avoids a trap
1705                  */
1706                 kvm_mips_trans_cache_va(inst, opc, vcpu);
1707 #endif
1708         } else if (op_inst == Hit_Invalidate_I) {
1709                 flush_dcache_line(va);
1710                 flush_icache_line(va);
1711
1712 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1713                 /* Replace the CACHE instruction, with a SYNCI */
1714                 kvm_mips_trans_cache_va(inst, opc, vcpu);
1715 #endif
1716         } else {
1717                 kvm_err("NO-OP CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1718                         cache, op, base, arch->gprs[base], offset);
1719                 er = EMULATE_FAIL;
1720         }
1721
1722         preempt_enable();
1723 done:
1724         /* Rollback PC only if emulation was unsuccessful */
1725         if (er == EMULATE_FAIL)
1726                 vcpu->arch.pc = curr_pc;
1727
1728 dont_update_pc:
1729         /*
1730          * This is for exceptions whose emulation updates the PC, so do not
1731          * overwrite the PC under any circumstances
1732          */
1733
1734         return er;
1735 }
1736
1737 enum emulation_result kvm_mips_emulate_inst(u32 cause, u32 *opc,
1738                                             struct kvm_run *run,
1739                                             struct kvm_vcpu *vcpu)
1740 {
1741         union mips_instruction inst;
1742         enum emulation_result er = EMULATE_DONE;
1743
1744         /* Fetch the instruction. */
1745         if (cause & CAUSEF_BD)
1746                 opc += 1;
1747
1748         inst.word = kvm_get_inst(opc, vcpu);
1749
1750         switch (inst.r_format.opcode) {
1751         case cop0_op:
1752                 er = kvm_mips_emulate_CP0(inst, opc, cause, run, vcpu);
1753                 break;
1754         case sb_op:
1755         case sh_op:
1756         case sw_op:
1757                 er = kvm_mips_emulate_store(inst, cause, run, vcpu);
1758                 break;
1759         case lb_op:
1760         case lbu_op:
1761         case lhu_op:
1762         case lh_op:
1763         case lw_op:
1764                 er = kvm_mips_emulate_load(inst, cause, run, vcpu);
1765                 break;
1766
1767         case cache_op:
1768                 ++vcpu->stat.cache_exits;
1769                 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
1770                 er = kvm_mips_emulate_cache(inst, opc, cause, run, vcpu);
1771                 break;
1772
1773         default:
1774                 kvm_err("Instruction emulation not supported (%p/%#x)\n", opc,
1775                         inst.word);
1776                 kvm_arch_vcpu_dump_regs(vcpu);
1777                 er = EMULATE_FAIL;
1778                 break;
1779         }
1780
1781         return er;
1782 }
1783
1784 enum emulation_result kvm_mips_emulate_syscall(u32 cause,
1785                                                u32 *opc,
1786                                                struct kvm_run *run,
1787                                                struct kvm_vcpu *vcpu)
1788 {
1789         struct mips_coproc *cop0 = vcpu->arch.cop0;
1790         struct kvm_vcpu_arch *arch = &vcpu->arch;
1791         enum emulation_result er = EMULATE_DONE;
1792
1793         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1794                 /* save old pc */
1795                 kvm_write_c0_guest_epc(cop0, arch->pc);
1796                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1797
1798                 if (cause & CAUSEF_BD)
1799                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1800                 else
1801                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1802
1803                 kvm_debug("Delivering SYSCALL @ pc %#lx\n", arch->pc);
1804
1805                 kvm_change_c0_guest_cause(cop0, (0xff),
1806                                           (EXCCODE_SYS << CAUSEB_EXCCODE));
1807
1808                 /* Set PC to the exception entry point */
1809                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1810
1811         } else {
1812                 kvm_err("Trying to deliver SYSCALL when EXL is already set\n");
1813                 er = EMULATE_FAIL;
1814         }
1815
1816         return er;
1817 }
1818
1819 enum emulation_result kvm_mips_emulate_tlbmiss_ld(u32 cause,
1820                                                   u32 *opc,
1821                                                   struct kvm_run *run,
1822                                                   struct kvm_vcpu *vcpu)
1823 {
1824         struct mips_coproc *cop0 = vcpu->arch.cop0;
1825         struct kvm_vcpu_arch *arch = &vcpu->arch;
1826         unsigned long entryhi = (vcpu->arch.  host_cp0_badvaddr & VPN2_MASK) |
1827                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1828
1829         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1830                 /* save old pc */
1831                 kvm_write_c0_guest_epc(cop0, arch->pc);
1832                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1833
1834                 if (cause & CAUSEF_BD)
1835                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1836                 else
1837                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1838
1839                 kvm_debug("[EXL == 0] delivering TLB MISS @ pc %#lx\n",
1840                           arch->pc);
1841
1842                 /* set pc to the exception entry point */
1843                 arch->pc = KVM_GUEST_KSEG0 + 0x0;
1844
1845         } else {
1846                 kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n",
1847                           arch->pc);
1848
1849                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1850         }
1851
1852         kvm_change_c0_guest_cause(cop0, (0xff),
1853                                   (EXCCODE_TLBL << CAUSEB_EXCCODE));
1854
1855         /* setup badvaddr, context and entryhi registers for the guest */
1856         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1857         /* XXXKYMA: is the context register used by linux??? */
1858         kvm_write_c0_guest_entryhi(cop0, entryhi);
1859         /* Blow away the shadow host TLBs */
1860         kvm_mips_flush_host_tlb(1);
1861
1862         return EMULATE_DONE;
1863 }
1864
1865 enum emulation_result kvm_mips_emulate_tlbinv_ld(u32 cause,
1866                                                  u32 *opc,
1867                                                  struct kvm_run *run,
1868                                                  struct kvm_vcpu *vcpu)
1869 {
1870         struct mips_coproc *cop0 = vcpu->arch.cop0;
1871         struct kvm_vcpu_arch *arch = &vcpu->arch;
1872         unsigned long entryhi =
1873                 (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1874                 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1875
1876         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1877                 /* save old pc */
1878                 kvm_write_c0_guest_epc(cop0, arch->pc);
1879                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1880
1881                 if (cause & CAUSEF_BD)
1882                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1883                 else
1884                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1885
1886                 kvm_debug("[EXL == 0] delivering TLB INV @ pc %#lx\n",
1887                           arch->pc);
1888
1889                 /* set pc to the exception entry point */
1890                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1891
1892         } else {
1893                 kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n",
1894                           arch->pc);
1895                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1896         }
1897
1898         kvm_change_c0_guest_cause(cop0, (0xff),
1899                                   (EXCCODE_TLBL << CAUSEB_EXCCODE));
1900
1901         /* setup badvaddr, context and entryhi registers for the guest */
1902         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1903         /* XXXKYMA: is the context register used by linux??? */
1904         kvm_write_c0_guest_entryhi(cop0, entryhi);
1905         /* Blow away the shadow host TLBs */
1906         kvm_mips_flush_host_tlb(1);
1907
1908         return EMULATE_DONE;
1909 }
1910
1911 enum emulation_result kvm_mips_emulate_tlbmiss_st(u32 cause,
1912                                                   u32 *opc,
1913                                                   struct kvm_run *run,
1914                                                   struct kvm_vcpu *vcpu)
1915 {
1916         struct mips_coproc *cop0 = vcpu->arch.cop0;
1917         struct kvm_vcpu_arch *arch = &vcpu->arch;
1918         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1919                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1920
1921         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1922                 /* save old pc */
1923                 kvm_write_c0_guest_epc(cop0, arch->pc);
1924                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1925
1926                 if (cause & CAUSEF_BD)
1927                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1928                 else
1929                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1930
1931                 kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n",
1932                           arch->pc);
1933
1934                 /* Set PC to the exception entry point */
1935                 arch->pc = KVM_GUEST_KSEG0 + 0x0;
1936         } else {
1937                 kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n",
1938                           arch->pc);
1939                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1940         }
1941
1942         kvm_change_c0_guest_cause(cop0, (0xff),
1943                                   (EXCCODE_TLBS << CAUSEB_EXCCODE));
1944
1945         /* setup badvaddr, context and entryhi registers for the guest */
1946         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1947         /* XXXKYMA: is the context register used by linux??? */
1948         kvm_write_c0_guest_entryhi(cop0, entryhi);
1949         /* Blow away the shadow host TLBs */
1950         kvm_mips_flush_host_tlb(1);
1951
1952         return EMULATE_DONE;
1953 }
1954
1955 enum emulation_result kvm_mips_emulate_tlbinv_st(u32 cause,
1956                                                  u32 *opc,
1957                                                  struct kvm_run *run,
1958                                                  struct kvm_vcpu *vcpu)
1959 {
1960         struct mips_coproc *cop0 = vcpu->arch.cop0;
1961         struct kvm_vcpu_arch *arch = &vcpu->arch;
1962         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1963                 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1964
1965         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1966                 /* save old pc */
1967                 kvm_write_c0_guest_epc(cop0, arch->pc);
1968                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1969
1970                 if (cause & CAUSEF_BD)
1971                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1972                 else
1973                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1974
1975                 kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n",
1976                           arch->pc);
1977
1978                 /* Set PC to the exception entry point */
1979                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1980         } else {
1981                 kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n",
1982                           arch->pc);
1983                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
1984         }
1985
1986         kvm_change_c0_guest_cause(cop0, (0xff),
1987                                   (EXCCODE_TLBS << CAUSEB_EXCCODE));
1988
1989         /* setup badvaddr, context and entryhi registers for the guest */
1990         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1991         /* XXXKYMA: is the context register used by linux??? */
1992         kvm_write_c0_guest_entryhi(cop0, entryhi);
1993         /* Blow away the shadow host TLBs */
1994         kvm_mips_flush_host_tlb(1);
1995
1996         return EMULATE_DONE;
1997 }
1998
1999 /* TLBMOD: store into address matching TLB with Dirty bit off */
2000 enum emulation_result kvm_mips_handle_tlbmod(u32 cause, u32 *opc,
2001                                              struct kvm_run *run,
2002                                              struct kvm_vcpu *vcpu)
2003 {
2004         enum emulation_result er = EMULATE_DONE;
2005 #ifdef DEBUG
2006         struct mips_coproc *cop0 = vcpu->arch.cop0;
2007         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2008                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2009         int index;
2010
2011         /* If address not in the guest TLB, then we are in trouble */
2012         index = kvm_mips_guest_tlb_lookup(vcpu, entryhi);
2013         if (index < 0) {
2014                 /* XXXKYMA Invalidate and retry */
2015                 kvm_mips_host_tlb_inv(vcpu, vcpu->arch.host_cp0_badvaddr);
2016                 kvm_err("%s: host got TLBMOD for %#lx but entry not present in Guest TLB\n",
2017                      __func__, entryhi);
2018                 kvm_mips_dump_guest_tlbs(vcpu);
2019                 kvm_mips_dump_host_tlbs();
2020                 return EMULATE_FAIL;
2021         }
2022 #endif
2023
2024         er = kvm_mips_emulate_tlbmod(cause, opc, run, vcpu);
2025         return er;
2026 }
2027
2028 enum emulation_result kvm_mips_emulate_tlbmod(u32 cause,
2029                                               u32 *opc,
2030                                               struct kvm_run *run,
2031                                               struct kvm_vcpu *vcpu)
2032 {
2033         struct mips_coproc *cop0 = vcpu->arch.cop0;
2034         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2035                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2036         struct kvm_vcpu_arch *arch = &vcpu->arch;
2037
2038         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2039                 /* save old pc */
2040                 kvm_write_c0_guest_epc(cop0, arch->pc);
2041                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2042
2043                 if (cause & CAUSEF_BD)
2044                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2045                 else
2046                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2047
2048                 kvm_debug("[EXL == 0] Delivering TLB MOD @ pc %#lx\n",
2049                           arch->pc);
2050
2051                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2052         } else {
2053                 kvm_debug("[EXL == 1] Delivering TLB MOD @ pc %#lx\n",
2054                           arch->pc);
2055                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2056         }
2057
2058         kvm_change_c0_guest_cause(cop0, (0xff),
2059                                   (EXCCODE_MOD << CAUSEB_EXCCODE));
2060
2061         /* setup badvaddr, context and entryhi registers for the guest */
2062         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2063         /* XXXKYMA: is the context register used by linux??? */
2064         kvm_write_c0_guest_entryhi(cop0, entryhi);
2065         /* Blow away the shadow host TLBs */
2066         kvm_mips_flush_host_tlb(1);
2067
2068         return EMULATE_DONE;
2069 }
2070
2071 enum emulation_result kvm_mips_emulate_fpu_exc(u32 cause,
2072                                                u32 *opc,
2073                                                struct kvm_run *run,
2074                                                struct kvm_vcpu *vcpu)
2075 {
2076         struct mips_coproc *cop0 = vcpu->arch.cop0;
2077         struct kvm_vcpu_arch *arch = &vcpu->arch;
2078
2079         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2080                 /* save old pc */
2081                 kvm_write_c0_guest_epc(cop0, arch->pc);
2082                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2083
2084                 if (cause & CAUSEF_BD)
2085                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2086                 else
2087                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2088
2089         }
2090
2091         arch->pc = KVM_GUEST_KSEG0 + 0x180;
2092
2093         kvm_change_c0_guest_cause(cop0, (0xff),
2094                                   (EXCCODE_CPU << CAUSEB_EXCCODE));
2095         kvm_change_c0_guest_cause(cop0, (CAUSEF_CE), (0x1 << CAUSEB_CE));
2096
2097         return EMULATE_DONE;
2098 }
2099
2100 enum emulation_result kvm_mips_emulate_ri_exc(u32 cause,
2101                                               u32 *opc,
2102                                               struct kvm_run *run,
2103                                               struct kvm_vcpu *vcpu)
2104 {
2105         struct mips_coproc *cop0 = vcpu->arch.cop0;
2106         struct kvm_vcpu_arch *arch = &vcpu->arch;
2107         enum emulation_result er = EMULATE_DONE;
2108
2109         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2110                 /* save old pc */
2111                 kvm_write_c0_guest_epc(cop0, arch->pc);
2112                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2113
2114                 if (cause & CAUSEF_BD)
2115                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2116                 else
2117                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2118
2119                 kvm_debug("Delivering RI @ pc %#lx\n", arch->pc);
2120
2121                 kvm_change_c0_guest_cause(cop0, (0xff),
2122                                           (EXCCODE_RI << CAUSEB_EXCCODE));
2123
2124                 /* Set PC to the exception entry point */
2125                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2126
2127         } else {
2128                 kvm_err("Trying to deliver RI when EXL is already set\n");
2129                 er = EMULATE_FAIL;
2130         }
2131
2132         return er;
2133 }
2134
2135 enum emulation_result kvm_mips_emulate_bp_exc(u32 cause,
2136                                               u32 *opc,
2137                                               struct kvm_run *run,
2138                                               struct kvm_vcpu *vcpu)
2139 {
2140         struct mips_coproc *cop0 = vcpu->arch.cop0;
2141         struct kvm_vcpu_arch *arch = &vcpu->arch;
2142         enum emulation_result er = EMULATE_DONE;
2143
2144         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2145                 /* save old pc */
2146                 kvm_write_c0_guest_epc(cop0, arch->pc);
2147                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2148
2149                 if (cause & CAUSEF_BD)
2150                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2151                 else
2152                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2153
2154                 kvm_debug("Delivering BP @ pc %#lx\n", arch->pc);
2155
2156                 kvm_change_c0_guest_cause(cop0, (0xff),
2157                                           (EXCCODE_BP << CAUSEB_EXCCODE));
2158
2159                 /* Set PC to the exception entry point */
2160                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2161
2162         } else {
2163                 kvm_err("Trying to deliver BP when EXL is already set\n");
2164                 er = EMULATE_FAIL;
2165         }
2166
2167         return er;
2168 }
2169
2170 enum emulation_result kvm_mips_emulate_trap_exc(u32 cause,
2171                                                 u32 *opc,
2172                                                 struct kvm_run *run,
2173                                                 struct kvm_vcpu *vcpu)
2174 {
2175         struct mips_coproc *cop0 = vcpu->arch.cop0;
2176         struct kvm_vcpu_arch *arch = &vcpu->arch;
2177         enum emulation_result er = EMULATE_DONE;
2178
2179         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2180                 /* save old pc */
2181                 kvm_write_c0_guest_epc(cop0, arch->pc);
2182                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2183
2184                 if (cause & CAUSEF_BD)
2185                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2186                 else
2187                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2188
2189                 kvm_debug("Delivering TRAP @ pc %#lx\n", arch->pc);
2190
2191                 kvm_change_c0_guest_cause(cop0, (0xff),
2192                                           (EXCCODE_TR << CAUSEB_EXCCODE));
2193
2194                 /* Set PC to the exception entry point */
2195                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2196
2197         } else {
2198                 kvm_err("Trying to deliver TRAP when EXL is already set\n");
2199                 er = EMULATE_FAIL;
2200         }
2201
2202         return er;
2203 }
2204
2205 enum emulation_result kvm_mips_emulate_msafpe_exc(u32 cause,
2206                                                   u32 *opc,
2207                                                   struct kvm_run *run,
2208                                                   struct kvm_vcpu *vcpu)
2209 {
2210         struct mips_coproc *cop0 = vcpu->arch.cop0;
2211         struct kvm_vcpu_arch *arch = &vcpu->arch;
2212         enum emulation_result er = EMULATE_DONE;
2213
2214         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2215                 /* save old pc */
2216                 kvm_write_c0_guest_epc(cop0, arch->pc);
2217                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2218
2219                 if (cause & CAUSEF_BD)
2220                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2221                 else
2222                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2223
2224                 kvm_debug("Delivering MSAFPE @ pc %#lx\n", arch->pc);
2225
2226                 kvm_change_c0_guest_cause(cop0, (0xff),
2227                                           (EXCCODE_MSAFPE << CAUSEB_EXCCODE));
2228
2229                 /* Set PC to the exception entry point */
2230                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2231
2232         } else {
2233                 kvm_err("Trying to deliver MSAFPE when EXL is already set\n");
2234                 er = EMULATE_FAIL;
2235         }
2236
2237         return er;
2238 }
2239
2240 enum emulation_result kvm_mips_emulate_fpe_exc(u32 cause,
2241                                                u32 *opc,
2242                                                struct kvm_run *run,
2243                                                struct kvm_vcpu *vcpu)
2244 {
2245         struct mips_coproc *cop0 = vcpu->arch.cop0;
2246         struct kvm_vcpu_arch *arch = &vcpu->arch;
2247         enum emulation_result er = EMULATE_DONE;
2248
2249         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2250                 /* save old pc */
2251                 kvm_write_c0_guest_epc(cop0, arch->pc);
2252                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2253
2254                 if (cause & CAUSEF_BD)
2255                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2256                 else
2257                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2258
2259                 kvm_debug("Delivering FPE @ pc %#lx\n", arch->pc);
2260
2261                 kvm_change_c0_guest_cause(cop0, (0xff),
2262                                           (EXCCODE_FPE << CAUSEB_EXCCODE));
2263
2264                 /* Set PC to the exception entry point */
2265                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2266
2267         } else {
2268                 kvm_err("Trying to deliver FPE when EXL is already set\n");
2269                 er = EMULATE_FAIL;
2270         }
2271
2272         return er;
2273 }
2274
2275 enum emulation_result kvm_mips_emulate_msadis_exc(u32 cause,
2276                                                   u32 *opc,
2277                                                   struct kvm_run *run,
2278                                                   struct kvm_vcpu *vcpu)
2279 {
2280         struct mips_coproc *cop0 = vcpu->arch.cop0;
2281         struct kvm_vcpu_arch *arch = &vcpu->arch;
2282         enum emulation_result er = EMULATE_DONE;
2283
2284         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2285                 /* save old pc */
2286                 kvm_write_c0_guest_epc(cop0, arch->pc);
2287                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2288
2289                 if (cause & CAUSEF_BD)
2290                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2291                 else
2292                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2293
2294                 kvm_debug("Delivering MSADIS @ pc %#lx\n", arch->pc);
2295
2296                 kvm_change_c0_guest_cause(cop0, (0xff),
2297                                           (EXCCODE_MSADIS << CAUSEB_EXCCODE));
2298
2299                 /* Set PC to the exception entry point */
2300                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2301
2302         } else {
2303                 kvm_err("Trying to deliver MSADIS when EXL is already set\n");
2304                 er = EMULATE_FAIL;
2305         }
2306
2307         return er;
2308 }
2309
2310 enum emulation_result kvm_mips_handle_ri(u32 cause, u32 *opc,
2311                                          struct kvm_run *run,
2312                                          struct kvm_vcpu *vcpu)
2313 {
2314         struct mips_coproc *cop0 = vcpu->arch.cop0;
2315         struct kvm_vcpu_arch *arch = &vcpu->arch;
2316         enum emulation_result er = EMULATE_DONE;
2317         unsigned long curr_pc;
2318         union mips_instruction inst;
2319
2320         /*
2321          * Update PC and hold onto current PC in case there is
2322          * an error and we want to rollback the PC
2323          */
2324         curr_pc = vcpu->arch.pc;
2325         er = update_pc(vcpu, cause);
2326         if (er == EMULATE_FAIL)
2327                 return er;
2328
2329         /* Fetch the instruction. */
2330         if (cause & CAUSEF_BD)
2331                 opc += 1;
2332
2333         inst.word = kvm_get_inst(opc, vcpu);
2334
2335         if (inst.word == KVM_INVALID_INST) {
2336                 kvm_err("%s: Cannot get inst @ %p\n", __func__, opc);
2337                 return EMULATE_FAIL;
2338         }
2339
2340         if (inst.r_format.opcode == spec3_op &&
2341             inst.r_format.func == rdhwr_op) {
2342                 int usermode = !KVM_GUEST_KERNEL_MODE(vcpu);
2343                 int rd = inst.r_format.rd;
2344                 int rt = inst.r_format.rt;
2345                 int sel = inst.r_format.re & 0x7;
2346
2347                 /* If usermode, check RDHWR rd is allowed by guest HWREna */
2348                 if (usermode && !(kvm_read_c0_guest_hwrena(cop0) & BIT(rd))) {
2349                         kvm_debug("RDHWR %#x disallowed by HWREna @ %p\n",
2350                                   rd, opc);
2351                         goto emulate_ri;
2352                 }
2353                 switch (rd) {
2354                 case MIPS_HWR_CPUNUM:           /* CPU number */
2355                         arch->gprs[rt] = vcpu->vcpu_id;
2356                         break;
2357                 case MIPS_HWR_SYNCISTEP:        /* SYNCI length */
2358                         arch->gprs[rt] = min(current_cpu_data.dcache.linesz,
2359                                              current_cpu_data.icache.linesz);
2360                         break;
2361                 case MIPS_HWR_CC:               /* Read count register */
2362                         arch->gprs[rt] = kvm_mips_read_count(vcpu);
2363                         break;
2364                 case MIPS_HWR_CCRES:            /* Count register resolution */
2365                         switch (current_cpu_data.cputype) {
2366                         case CPU_20KC:
2367                         case CPU_25KF:
2368                                 arch->gprs[rt] = 1;
2369                                 break;
2370                         default:
2371                                 arch->gprs[rt] = 2;
2372                         }
2373                         break;
2374                 case MIPS_HWR_ULR:              /* Read UserLocal register */
2375                         arch->gprs[rt] = kvm_read_c0_guest_userlocal(cop0);
2376                         break;
2377
2378                 default:
2379                         kvm_debug("RDHWR %#x not supported @ %p\n", rd, opc);
2380                         goto emulate_ri;
2381                 }
2382
2383                 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, KVM_TRACE_HWR(rd, sel),
2384                               vcpu->arch.gprs[rt]);
2385         } else {
2386                 kvm_debug("Emulate RI not supported @ %p: %#x\n",
2387                           opc, inst.word);
2388                 goto emulate_ri;
2389         }
2390
2391         return EMULATE_DONE;
2392
2393 emulate_ri:
2394         /*
2395          * Rollback PC (if in branch delay slot then the PC already points to
2396          * branch target), and pass the RI exception to the guest OS.
2397          */
2398         vcpu->arch.pc = curr_pc;
2399         return kvm_mips_emulate_ri_exc(cause, opc, run, vcpu);
2400 }
2401
2402 enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu,
2403                                                   struct kvm_run *run)
2404 {
2405         unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
2406         enum emulation_result er = EMULATE_DONE;
2407
2408         if (run->mmio.len > sizeof(*gpr)) {
2409                 kvm_err("Bad MMIO length: %d", run->mmio.len);
2410                 er = EMULATE_FAIL;
2411                 goto done;
2412         }
2413
2414         er = update_pc(vcpu, vcpu->arch.pending_load_cause);
2415         if (er == EMULATE_FAIL)
2416                 return er;
2417
2418         switch (run->mmio.len) {
2419         case 4:
2420                 *gpr = *(s32 *) run->mmio.data;
2421                 break;
2422
2423         case 2:
2424                 if (vcpu->mmio_needed == 2)
2425                         *gpr = *(s16 *) run->mmio.data;
2426                 else
2427                         *gpr = *(u16 *)run->mmio.data;
2428
2429                 break;
2430         case 1:
2431                 if (vcpu->mmio_needed == 2)
2432                         *gpr = *(s8 *) run->mmio.data;
2433                 else
2434                         *gpr = *(u8 *) run->mmio.data;
2435                 break;
2436         }
2437
2438         if (vcpu->arch.pending_load_cause & CAUSEF_BD)
2439                 kvm_debug("[%#lx] Completing %d byte BD Load to gpr %d (0x%08lx) type %d\n",
2440                           vcpu->arch.pc, run->mmio.len, vcpu->arch.io_gpr, *gpr,
2441                           vcpu->mmio_needed);
2442
2443 done:
2444         return er;
2445 }
2446
2447 static enum emulation_result kvm_mips_emulate_exc(u32 cause,
2448                                                   u32 *opc,
2449                                                   struct kvm_run *run,
2450                                                   struct kvm_vcpu *vcpu)
2451 {
2452         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2453         struct mips_coproc *cop0 = vcpu->arch.cop0;
2454         struct kvm_vcpu_arch *arch = &vcpu->arch;
2455         enum emulation_result er = EMULATE_DONE;
2456
2457         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2458                 /* save old pc */
2459                 kvm_write_c0_guest_epc(cop0, arch->pc);
2460                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2461
2462                 if (cause & CAUSEF_BD)
2463                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2464                 else
2465                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2466
2467                 kvm_change_c0_guest_cause(cop0, (0xff),
2468                                           (exccode << CAUSEB_EXCCODE));
2469
2470                 /* Set PC to the exception entry point */
2471                 arch->pc = KVM_GUEST_KSEG0 + 0x180;
2472                 kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2473
2474                 kvm_debug("Delivering EXC %d @ pc %#lx, badVaddr: %#lx\n",
2475                           exccode, kvm_read_c0_guest_epc(cop0),
2476                           kvm_read_c0_guest_badvaddr(cop0));
2477         } else {
2478                 kvm_err("Trying to deliver EXC when EXL is already set\n");
2479                 er = EMULATE_FAIL;
2480         }
2481
2482         return er;
2483 }
2484
2485 enum emulation_result kvm_mips_check_privilege(u32 cause,
2486                                                u32 *opc,
2487                                                struct kvm_run *run,
2488                                                struct kvm_vcpu *vcpu)
2489 {
2490         enum emulation_result er = EMULATE_DONE;
2491         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2492         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
2493
2494         int usermode = !KVM_GUEST_KERNEL_MODE(vcpu);
2495
2496         if (usermode) {
2497                 switch (exccode) {
2498                 case EXCCODE_INT:
2499                 case EXCCODE_SYS:
2500                 case EXCCODE_BP:
2501                 case EXCCODE_RI:
2502                 case EXCCODE_TR:
2503                 case EXCCODE_MSAFPE:
2504                 case EXCCODE_FPE:
2505                 case EXCCODE_MSADIS:
2506                         break;
2507
2508                 case EXCCODE_CPU:
2509                         if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 0)
2510                                 er = EMULATE_PRIV_FAIL;
2511                         break;
2512
2513                 case EXCCODE_MOD:
2514                         break;
2515
2516                 case EXCCODE_TLBL:
2517                         /*
2518                          * We we are accessing Guest kernel space, then send an
2519                          * address error exception to the guest
2520                          */
2521                         if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) {
2522                                 kvm_debug("%s: LD MISS @ %#lx\n", __func__,
2523                                           badvaddr);
2524                                 cause &= ~0xff;
2525                                 cause |= (EXCCODE_ADEL << CAUSEB_EXCCODE);
2526                                 er = EMULATE_PRIV_FAIL;
2527                         }
2528                         break;
2529
2530                 case EXCCODE_TLBS:
2531                         /*
2532                          * We we are accessing Guest kernel space, then send an
2533                          * address error exception to the guest
2534                          */
2535                         if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) {
2536                                 kvm_debug("%s: ST MISS @ %#lx\n", __func__,
2537                                           badvaddr);
2538                                 cause &= ~0xff;
2539                                 cause |= (EXCCODE_ADES << CAUSEB_EXCCODE);
2540                                 er = EMULATE_PRIV_FAIL;
2541                         }
2542                         break;
2543
2544                 case EXCCODE_ADES:
2545                         kvm_debug("%s: address error ST @ %#lx\n", __func__,
2546                                   badvaddr);
2547                         if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) {
2548                                 cause &= ~0xff;
2549                                 cause |= (EXCCODE_TLBS << CAUSEB_EXCCODE);
2550                         }
2551                         er = EMULATE_PRIV_FAIL;
2552                         break;
2553                 case EXCCODE_ADEL:
2554                         kvm_debug("%s: address error LD @ %#lx\n", __func__,
2555                                   badvaddr);
2556                         if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) {
2557                                 cause &= ~0xff;
2558                                 cause |= (EXCCODE_TLBL << CAUSEB_EXCCODE);
2559                         }
2560                         er = EMULATE_PRIV_FAIL;
2561                         break;
2562                 default:
2563                         er = EMULATE_PRIV_FAIL;
2564                         break;
2565                 }
2566         }
2567
2568         if (er == EMULATE_PRIV_FAIL)
2569                 kvm_mips_emulate_exc(cause, opc, run, vcpu);
2570
2571         return er;
2572 }
2573
2574 /*
2575  * User Address (UA) fault, this could happen if
2576  * (1) TLB entry not present/valid in both Guest and shadow host TLBs, in this
2577  *     case we pass on the fault to the guest kernel and let it handle it.
2578  * (2) TLB entry is present in the Guest TLB but not in the shadow, in this
2579  *     case we inject the TLB from the Guest TLB into the shadow host TLB
2580  */
2581 enum emulation_result kvm_mips_handle_tlbmiss(u32 cause,
2582                                               u32 *opc,
2583                                               struct kvm_run *run,
2584                                               struct kvm_vcpu *vcpu)
2585 {
2586         enum emulation_result er = EMULATE_DONE;
2587         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2588         unsigned long va = vcpu->arch.host_cp0_badvaddr;
2589         int index;
2590
2591         kvm_debug("kvm_mips_handle_tlbmiss: badvaddr: %#lx\n",
2592                   vcpu->arch.host_cp0_badvaddr);
2593
2594         /*
2595          * KVM would not have got the exception if this entry was valid in the
2596          * shadow host TLB. Check the Guest TLB, if the entry is not there then
2597          * send the guest an exception. The guest exc handler should then inject
2598          * an entry into the guest TLB.
2599          */
2600         index = kvm_mips_guest_tlb_lookup(vcpu,
2601                       (va & VPN2_MASK) |
2602                       (kvm_read_c0_guest_entryhi(vcpu->arch.cop0) &
2603                        KVM_ENTRYHI_ASID));
2604         if (index < 0) {
2605                 if (exccode == EXCCODE_TLBL) {
2606                         er = kvm_mips_emulate_tlbmiss_ld(cause, opc, run, vcpu);
2607                 } else if (exccode == EXCCODE_TLBS) {
2608                         er = kvm_mips_emulate_tlbmiss_st(cause, opc, run, vcpu);
2609                 } else {
2610                         kvm_err("%s: invalid exc code: %d\n", __func__,
2611                                 exccode);
2612                         er = EMULATE_FAIL;
2613                 }
2614         } else {
2615                 struct kvm_mips_tlb *tlb = &vcpu->arch.guest_tlb[index];
2616
2617                 /*
2618                  * Check if the entry is valid, if not then setup a TLB invalid
2619                  * exception to the guest
2620                  */
2621                 if (!TLB_IS_VALID(*tlb, va)) {
2622                         if (exccode == EXCCODE_TLBL) {
2623                                 er = kvm_mips_emulate_tlbinv_ld(cause, opc, run,
2624                                                                 vcpu);
2625                         } else if (exccode == EXCCODE_TLBS) {
2626                                 er = kvm_mips_emulate_tlbinv_st(cause, opc, run,
2627                                                                 vcpu);
2628                         } else {
2629                                 kvm_err("%s: invalid exc code: %d\n", __func__,
2630                                         exccode);
2631                                 er = EMULATE_FAIL;
2632                         }
2633                 } else {
2634                         kvm_debug("Injecting hi: %#lx, lo0: %#lx, lo1: %#lx into shadow host TLB\n",
2635                                   tlb->tlb_hi, tlb->tlb_lo[0], tlb->tlb_lo[1]);
2636                         /*
2637                          * OK we have a Guest TLB entry, now inject it into the
2638                          * shadow host TLB
2639                          */
2640                         kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb);
2641                 }
2642         }
2643
2644         return er;
2645 }