arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  *               Christian Borntraeger <borntraeger@de.ibm.com>
  12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  14  *               Jason J. Herne <jjherne@us.ibm.com>
  15  */
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/mman.h>
  25 #include <linux/module.h>
  26 #include <linux/random.h>
  27 #include <linux/slab.h>
  28 #include <linux/timer.h>
  29 #include <linux/vmalloc.h>
  30 #include <linux/bitmap.h>
  31 #include <asm/asm-offsets.h>
  32 #include <asm/lowcore.h>
  33 #include <asm/stp.h>
  34 #include <asm/pgtable.h>
  35 #include <asm/gmap.h>
  36 #include <asm/nmi.h>
  37 #include <asm/switch_to.h>
  38 #include <asm/isc.h>
  39 #include <asm/sclp.h>
  40 #include <asm/cpacf.h>
  41 #include <asm/timex.h>
  42 #include "kvm-s390.h"
  43 #include "gaccess.h"
  44
  45 #define KMSG_COMPONENT "kvm-s390"
  46 #undef pr_fmt
  47 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  48
  49 #define CREATE_TRACE_POINTS
  50 #include "trace.h"
  51 #include "trace-s390.h"
  52
  53 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
  54 #define LOCAL_IRQS 32
  55 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  56                            (KVM_MAX_VCPUS + LOCAL_IRQS))
  57
  58 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  59
  60 struct kvm_stats_debugfs_item debugfs_entries[] = {
  61         { "userspace_handled", VCPU_STAT(exit_userspace) },
  62         { "exit_null", VCPU_STAT(exit_null) },
  63         { "exit_validity", VCPU_STAT(exit_validity) },
  64         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  65         { "exit_external_request", VCPU_STAT(exit_external_request) },
  66         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  67         { "exit_instruction", VCPU_STAT(exit_instruction) },
  68         { "exit_pei", VCPU_STAT(exit_pei) },
  69         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  70         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  71         { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
  72         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
  73         { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
  74         { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
  75         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  76         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  77         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  78         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  79         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  80         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  81         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  82         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  83         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  84         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  85         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  86         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  87         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  88         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  89         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  90         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  91         { "instruction_spx", VCPU_STAT(instruction_spx) },
  92         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  93         { "instruction_stap", VCPU_STAT(instruction_stap) },
  94         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  95         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
  96         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  97         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  98         { "instruction_essa", VCPU_STAT(instruction_essa) },
  99         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
 100         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
 101         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
 102         { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
 103         { "instruction_sie", VCPU_STAT(instruction_sie) },
 104         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
 105         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
 106         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
 107         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
 108         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
 109         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
 110         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
 111         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
 112         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
 113         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
 114         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 115         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 116         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 117         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
 118         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
 119         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
 120         { "diagnose_10", VCPU_STAT(diagnose_10) },
 121         { "diagnose_44", VCPU_STAT(diagnose_44) },
 122         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
 123         { "diagnose_258", VCPU_STAT(diagnose_258) },
 124         { "diagnose_308", VCPU_STAT(diagnose_308) },
 125         { "diagnose_500", VCPU_STAT(diagnose_500) },
 126         { NULL }
 127 };
 128
 129 /* allow nested virtualization in KVM (if enabled by user space) */
 130 static int nested;
 131 module_param(nested, int, S_IRUGO);
 132 MODULE_PARM_DESC(nested, "Nested virtualization support");
 133
 134 /* upper facilities limit for kvm */
 135 unsigned long kvm_s390_fac_list_mask[16] = {
 136         0xffe6000000000000UL,
 137         0x005e000000000000UL,
 138 };
 139
 140 unsigned long kvm_s390_fac_list_mask_size(void)
 141 {
 142         BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
 143         return ARRAY_SIZE(kvm_s390_fac_list_mask);
 144 }
 145
 146 /* available cpu features supported by kvm */
 147 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
 148 /* available subfunctions indicated via query / "test bit" */
 149 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
 150
 151 static struct gmap_notifier gmap_notifier;
 152 static struct gmap_notifier vsie_gmap_notifier;
 153 debug_info_t *kvm_s390_dbf;
 154
 155 /* Section: not file related */
 156 int kvm_arch_hardware_enable(void)
 157 {
 158         /* every s390 is virtualization enabled ;-) */
 159         return 0;
 160 }
 161
 162 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
 163                               unsigned long end);
 164
 165 /*
 166  * This callback is executed during stop_machine(). All CPUs are therefore
 167  * temporarily stopped. In order not to change guest behavior, we have to
 168  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
 169  * so a CPU won't be stopped while calculating with the epoch.
 170  */
 171 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
 172                           void *v)
 173 {
 174         struct kvm *kvm;
 175         struct kvm_vcpu *vcpu;
 176         int i;
 177         unsigned long long *delta = v;
 178
 179         list_for_each_entry(kvm, &vm_list, vm_list) {
 180                 kvm->arch.epoch -= *delta;
 181                 kvm_for_each_vcpu(i, vcpu, kvm) {
 182                         vcpu->arch.sie_block->epoch -= *delta;
 183                         if (vcpu->arch.cputm_enabled)
 184                                 vcpu->arch.cputm_start += *delta;
 185                         if (vcpu->arch.vsie_block)
 186                                 vcpu->arch.vsie_block->epoch -= *delta;
 187                 }
 188         }
 189         return NOTIFY_OK;
 190 }
 191
 192 static struct notifier_block kvm_clock_notifier = {
 193         .notifier_call = kvm_clock_sync,
 194 };
 195
 196 int kvm_arch_hardware_setup(void)
 197 {
 198         gmap_notifier.notifier_call = kvm_gmap_notifier;
 199         gmap_register_pte_notifier(&gmap_notifier);
 200         vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
 201         gmap_register_pte_notifier(&vsie_gmap_notifier);
 202         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
 203                                        &kvm_clock_notifier);
 204         return 0;
 205 }
 206
 207 void kvm_arch_hardware_unsetup(void)
 208 {
 209         gmap_unregister_pte_notifier(&gmap_notifier);
 210         gmap_unregister_pte_notifier(&vsie_gmap_notifier);
 211         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
 212                                          &kvm_clock_notifier);
 213 }
 214
 215 static void allow_cpu_feat(unsigned long nr)
 216 {
 217         set_bit_inv(nr, kvm_s390_available_cpu_feat);
 218 }
 219
 220 static inline int plo_test_bit(unsigned char nr)
 221 {
 222         register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
 223         int cc = 3; /* subfunction not available */
 224
 225         asm volatile(
 226                 /* Parameter registers are ignored for "test bit" */
 227                 "       plo     0,0,0,0(0)\n"
 228                 "       ipm     %0\n"
 229                 "       srl     %0,28\n"
 230                 : "=d" (cc)
 231                 : "d" (r0)
 232                 : "cc");
 233         return cc == 0;
 234 }
 235
 236 static void kvm_s390_cpu_feat_init(void)
 237 {
 238         int i;
 239
 240         for (i = 0; i < 256; ++i) {
 241                 if (plo_test_bit(i))
 242                         kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
 243         }
 244
 245         if (test_facility(28)) /* TOD-clock steering */
 246                 ptff(kvm_s390_available_subfunc.ptff,
 247                      sizeof(kvm_s390_available_subfunc.ptff),
 248                      PTFF_QAF);
 249
 250         if (test_facility(17)) { /* MSA */
 251                 __cpacf_query(CPACF_KMAC, kvm_s390_available_subfunc.kmac);
 252                 __cpacf_query(CPACF_KMC, kvm_s390_available_subfunc.kmc);
 253                 __cpacf_query(CPACF_KM, kvm_s390_available_subfunc.km);
 254                 __cpacf_query(CPACF_KIMD, kvm_s390_available_subfunc.kimd);
 255                 __cpacf_query(CPACF_KLMD, kvm_s390_available_subfunc.klmd);
 256         }
 257         if (test_facility(76)) /* MSA3 */
 258                 __cpacf_query(CPACF_PCKMO, kvm_s390_available_subfunc.pckmo);
 259         if (test_facility(77)) { /* MSA4 */
 260                 __cpacf_query(CPACF_KMCTR, kvm_s390_available_subfunc.kmctr);
 261                 __cpacf_query(CPACF_KMF, kvm_s390_available_subfunc.kmf);
 262                 __cpacf_query(CPACF_KMO, kvm_s390_available_subfunc.kmo);
 263                 __cpacf_query(CPACF_PCC, kvm_s390_available_subfunc.pcc);
 264         }
 265         if (test_facility(57)) /* MSA5 */
 266                 __cpacf_query(CPACF_PPNO, kvm_s390_available_subfunc.ppno);
 267
 268         if (MACHINE_HAS_ESOP)
 269                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
 270         /*
 271          * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
 272          * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
 273          */
 274         if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
 275             !test_facility(3) || !nested)
 276                 return;
 277         allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
 278         if (sclp.has_64bscao)
 279                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
 280         if (sclp.has_siif)
 281                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
 282         if (sclp.has_gpere)
 283                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
 284         if (sclp.has_gsls)
 285                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
 286         if (sclp.has_ib)
 287                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
 288         if (sclp.has_cei)
 289                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
 290         if (sclp.has_ibs)
 291                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
 292         /*
 293          * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
 294          * all skey handling functions read/set the skey from the PGSTE
 295          * instead of the real storage key.
 296          *
 297          * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
 298          * pages being detected as preserved although they are resident.
 299          *
 300          * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
 301          * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
 302          *
 303          * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
 304          * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
 305          * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
 306          *
 307          * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
 308          * cannot easily shadow the SCA because of the ipte lock.
 309          */
 310 }
 311
 312 int kvm_arch_init(void *opaque)
 313 {
 314         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
 315         if (!kvm_s390_dbf)
 316                 return -ENOMEM;
 317
 318         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
 319                 debug_unregister(kvm_s390_dbf);
 320                 return -ENOMEM;
 321         }
 322
 323         kvm_s390_cpu_feat_init();
 324
 325         /* Register floating interrupt controller interface. */
 326         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 327 }
 328
 329 void kvm_arch_exit(void)
 330 {
 331         debug_unregister(kvm_s390_dbf);
 332 }
 333
 334 /* Section: device related */
 335 long kvm_arch_dev_ioctl(struct file *filp,
 336                         unsigned int ioctl, unsigned long arg)
 337 {
 338         if (ioctl == KVM_S390_ENABLE_SIE)
 339                 return s390_enable_sie();
 340         return -EINVAL;
 341 }
 342
 343 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 344 {
 345         int r;
 346
 347         switch (ext) {
 348         case KVM_CAP_S390_PSW:
 349         case KVM_CAP_S390_GMAP:
 350         case KVM_CAP_SYNC_MMU:
 351 #ifdef CONFIG_KVM_S390_UCONTROL
 352         case KVM_CAP_S390_UCONTROL:
 353 #endif
 354         case KVM_CAP_ASYNC_PF:
 355         case KVM_CAP_SYNC_REGS:
 356         case KVM_CAP_ONE_REG:
 357         case KVM_CAP_ENABLE_CAP:
 358         case KVM_CAP_S390_CSS_SUPPORT:
 359         case KVM_CAP_IOEVENTFD:
 360         case KVM_CAP_DEVICE_CTRL:
 361         case KVM_CAP_ENABLE_CAP_VM:
 362         case KVM_CAP_S390_IRQCHIP:
 363         case KVM_CAP_VM_ATTRIBUTES:
 364         case KVM_CAP_MP_STATE:
 365         case KVM_CAP_S390_INJECT_IRQ:
 366         case KVM_CAP_S390_USER_SIGP:
 367         case KVM_CAP_S390_USER_STSI:
 368         case KVM_CAP_S390_SKEYS:
 369         case KVM_CAP_S390_IRQ_STATE:
 370         case KVM_CAP_S390_USER_INSTR0:
 371                 r = 1;
 372                 break;
 373         case KVM_CAP_S390_MEM_OP:
 374                 r = MEM_OP_MAX_SIZE;
 375                 break;
 376         case KVM_CAP_NR_VCPUS:
 377         case KVM_CAP_MAX_VCPUS:
 378                 r = KVM_S390_BSCA_CPU_SLOTS;
 379                 if (sclp.has_esca && sclp.has_64bscao)
 380                         r = KVM_S390_ESCA_CPU_SLOTS;
 381                 break;
 382         case KVM_CAP_NR_MEMSLOTS:
 383                 r = KVM_USER_MEM_SLOTS;
 384                 break;
 385         case KVM_CAP_S390_COW:
 386                 r = MACHINE_HAS_ESOP;
 387                 break;
 388         case KVM_CAP_S390_VECTOR_REGISTERS:
 389                 r = MACHINE_HAS_VX;
 390                 break;
 391         case KVM_CAP_S390_RI:
 392                 r = test_facility(64);
 393                 break;
 394         default:
 395                 r = 0;
 396         }
 397         return r;
 398 }
 399
 400 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 401                                         struct kvm_memory_slot *memslot)
 402 {
 403         gfn_t cur_gfn, last_gfn;
 404         unsigned long address;
 405         struct gmap *gmap = kvm->arch.gmap;
 406
 407         /* Loop over all guest pages */
 408         last_gfn = memslot->base_gfn + memslot->npages;
 409         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 410                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 411
 412                 if (test_and_clear_guest_dirty(gmap->mm, address))
 413                         mark_page_dirty(kvm, cur_gfn);
 414                 if (fatal_signal_pending(current))
 415                         return;
 416                 cond_resched();
 417         }
 418 }
 419
 420 /* Section: vm related */
 421 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
 422
 423 /*
 424  * Get (and clear) the dirty memory log for a memory slot.
 425  */
 426 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 427                                struct kvm_dirty_log *log)
 428 {
 429         int r;
 430         unsigned long n;
 431         struct kvm_memslots *slots;
 432         struct kvm_memory_slot *memslot;
 433         int is_dirty = 0;
 434
 435         mutex_lock(&kvm->slots_lock);
 436
 437         r = -EINVAL;
 438         if (log->slot >= KVM_USER_MEM_SLOTS)
 439                 goto out;
 440
 441         slots = kvm_memslots(kvm);
 442         memslot = id_to_memslot(slots, log->slot);
 443         r = -ENOENT;
 444         if (!memslot->dirty_bitmap)
 445                 goto out;
 446
 447         kvm_s390_sync_dirty_log(kvm, memslot);
 448         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 449         if (r)
 450                 goto out;
 451
 452         /* Clear the dirty log */
 453         if (is_dirty) {
 454                 n = kvm_dirty_bitmap_bytes(memslot);
 455                 memset(memslot->dirty_bitmap, 0, n);
 456         }
 457         r = 0;
 458 out:
 459         mutex_unlock(&kvm->slots_lock);
 460         return r;
 461 }
 462
 463 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
 464 {
 465         unsigned int i;
 466         struct kvm_vcpu *vcpu;
 467
 468         kvm_for_each_vcpu(i, vcpu, kvm) {
 469                 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
 470         }
 471 }
 472
 473 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 474 {
 475         int r;
 476
 477         if (cap->flags)
 478                 return -EINVAL;
 479
 480         switch (cap->cap) {
 481         case KVM_CAP_S390_IRQCHIP:
 482                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
 483                 kvm->arch.use_irqchip = 1;
 484                 r = 0;
 485                 break;
 486         case KVM_CAP_S390_USER_SIGP:
 487                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
 488                 kvm->arch.user_sigp = 1;
 489                 r = 0;
 490                 break;
 491         case KVM_CAP_S390_VECTOR_REGISTERS:
 492                 mutex_lock(&kvm->lock);
 493                 if (kvm->created_vcpus) {
 494                         r = -EBUSY;
 495                 } else if (MACHINE_HAS_VX) {
 496                         set_kvm_facility(kvm->arch.model.fac_mask, 129);
 497                         set_kvm_facility(kvm->arch.model.fac_list, 129);
 498                         r = 0;
 499                 } else
 500                         r = -EINVAL;
 501                 mutex_unlock(&kvm->lock);
 502                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
 503                          r ? "(not available)" : "(success)");
 504                 break;
 505         case KVM_CAP_S390_RI:
 506                 r = -EINVAL;
 507                 mutex_lock(&kvm->lock);
 508                 if (kvm->created_vcpus) {
 509                         r = -EBUSY;
 510                 } else if (test_facility(64)) {
 511                         set_kvm_facility(kvm->arch.model.fac_mask, 64);
 512                         set_kvm_facility(kvm->arch.model.fac_list, 64);
 513                         r = 0;
 514                 }
 515                 mutex_unlock(&kvm->lock);
 516                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
 517                          r ? "(not available)" : "(success)");
 518                 break;
 519         case KVM_CAP_S390_USER_STSI:
 520                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
 521                 kvm->arch.user_stsi = 1;
 522                 r = 0;
 523                 break;
 524         case KVM_CAP_S390_USER_INSTR0:
 525                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
 526                 kvm->arch.user_instr0 = 1;
 527                 icpt_operexc_on_all_vcpus(kvm);
 528                 r = 0;
 529                 break;
 530         default:
 531                 r = -EINVAL;
 532                 break;
 533         }
 534         return r;
 535 }
 536
 537 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 538 {
 539         int ret;
 540
 541         switch (attr->attr) {
 542         case KVM_S390_VM_MEM_LIMIT_SIZE:
 543                 ret = 0;
 544                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
 545                          kvm->arch.mem_limit);
 546                 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
 547                         ret = -EFAULT;
 548                 break;
 549         default:
 550                 ret = -ENXIO;
 551                 break;
 552         }
 553         return ret;
 554 }
 555
 556 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 557 {
 558         int ret;
 559         unsigned int idx;
 560         switch (attr->attr) {
 561         case KVM_S390_VM_MEM_ENABLE_CMMA:
 562                 ret = -ENXIO;
 563                 if (!sclp.has_cmma)
 564                         break;
 565
 566                 ret = -EBUSY;
 567                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
 568                 mutex_lock(&kvm->lock);
 569                 if (!kvm->created_vcpus) {
 570                         kvm->arch.use_cmma = 1;
 571                         ret = 0;
 572                 }
 573                 mutex_unlock(&kvm->lock);
 574                 break;
 575         case KVM_S390_VM_MEM_CLR_CMMA:
 576                 ret = -ENXIO;
 577                 if (!sclp.has_cmma)
 578                         break;
 579                 ret = -EINVAL;
 580                 if (!kvm->arch.use_cmma)
 581                         break;
 582
 583                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
 584                 mutex_lock(&kvm->lock);
 585                 idx = srcu_read_lock(&kvm->srcu);
 586                 s390_reset_cmma(kvm->arch.gmap->mm);
 587                 srcu_read_unlock(&kvm->srcu, idx);
 588                 mutex_unlock(&kvm->lock);
 589                 ret = 0;
 590                 break;
 591         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 592                 unsigned long new_limit;
 593
 594                 if (kvm_is_ucontrol(kvm))
 595                         return -EINVAL;
 596
 597                 if (get_user(new_limit, (u64 __user *)attr->addr))
 598                         return -EFAULT;
 599
 600                 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
 601                     new_limit > kvm->arch.mem_limit)
 602                         return -E2BIG;
 603
 604                 if (!new_limit)
 605                         return -EINVAL;
 606
 607                 /* gmap_create takes last usable address */
 608                 if (new_limit != KVM_S390_NO_MEM_LIMIT)
 609                         new_limit -= 1;
 610
 611                 ret = -EBUSY;
 612                 mutex_lock(&kvm->lock);
 613                 if (!kvm->created_vcpus) {
 614                         /* gmap_create will round the limit up */
 615                         struct gmap *new = gmap_create(current->mm, new_limit);
 616
 617                         if (!new) {
 618                                 ret = -ENOMEM;
 619                         } else {
 620                                 gmap_remove(kvm->arch.gmap);
 621                                 new->private = kvm;
 622                                 kvm->arch.gmap = new;
 623                                 ret = 0;
 624                         }
 625                 }
 626                 mutex_unlock(&kvm->lock);
 627                 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
 628                 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
 629                          (void *) kvm->arch.gmap->asce);
 630                 break;
 631         }
 632         default:
 633                 ret = -ENXIO;
 634                 break;
 635         }
 636         return ret;
 637 }
 638
 639 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 640
 641 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 642 {
 643         struct kvm_vcpu *vcpu;
 644         int i;
 645
 646         if (!test_kvm_facility(kvm, 76))
 647                 return -EINVAL;
 648
 649         mutex_lock(&kvm->lock);
 650         switch (attr->attr) {
 651         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 652                 get_random_bytes(
 653                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 654                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 655                 kvm->arch.crypto.aes_kw = 1;
 656                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
 657                 break;
 658         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 659                 get_random_bytes(
 660                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 661                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 662                 kvm->arch.crypto.dea_kw = 1;
 663                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
 664                 break;
 665         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 666                 kvm->arch.crypto.aes_kw = 0;
 667                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 668                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 669                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
 670                 break;
 671         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 672                 kvm->arch.crypto.dea_kw = 0;
 673                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 674                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 675                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
 676                 break;
 677         default:
 678                 mutex_unlock(&kvm->lock);
 679                 return -ENXIO;
 680         }
 681
 682         kvm_for_each_vcpu(i, vcpu, kvm) {
 683                 kvm_s390_vcpu_crypto_setup(vcpu);
 684                 exit_sie(vcpu);
 685         }
 686         mutex_unlock(&kvm->lock);
 687         return 0;
 688 }
 689
 690 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 691 {
 692         u8 gtod_high;
 693
 694         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
 695                                            sizeof(gtod_high)))
 696                 return -EFAULT;
 697
 698         if (gtod_high != 0)
 699                 return -EINVAL;
 700         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
 701
 702         return 0;
 703 }
 704
 705 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 706 {
 707         u64 gtod;
 708
 709         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
 710                 return -EFAULT;
 711
 712         kvm_s390_set_tod_clock(kvm, gtod);
 713         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
 714         return 0;
 715 }
 716
 717 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 718 {
 719         int ret;
 720
 721         if (attr->flags)
 722                 return -EINVAL;
 723
 724         switch (attr->attr) {
 725         case KVM_S390_VM_TOD_HIGH:
 726                 ret = kvm_s390_set_tod_high(kvm, attr);
 727                 break;
 728         case KVM_S390_VM_TOD_LOW:
 729                 ret = kvm_s390_set_tod_low(kvm, attr);
 730                 break;
 731         default:
 732                 ret = -ENXIO;
 733                 break;
 734         }
 735         return ret;
 736 }
 737
 738 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 739 {
 740         u8 gtod_high = 0;
 741
 742         if (copy_to_user((void __user *)attr->addr, &gtod_high,
 743                                          sizeof(gtod_high)))
 744                 return -EFAULT;
 745         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
 746
 747         return 0;
 748 }
 749
 750 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 751 {
 752         u64 gtod;
 753
 754         gtod = kvm_s390_get_tod_clock_fast(kvm);
 755         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
 756                 return -EFAULT;
 757         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
 758
 759         return 0;
 760 }
 761
 762 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 763 {
 764         int ret;
 765
 766         if (attr->flags)
 767                 return -EINVAL;
 768
 769         switch (attr->attr) {
 770         case KVM_S390_VM_TOD_HIGH:
 771                 ret = kvm_s390_get_tod_high(kvm, attr);
 772                 break;
 773         case KVM_S390_VM_TOD_LOW:
 774                 ret = kvm_s390_get_tod_low(kvm, attr);
 775                 break;
 776         default:
 777                 ret = -ENXIO;
 778                 break;
 779         }
 780         return ret;
 781 }
 782
 783 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 784 {
 785         struct kvm_s390_vm_cpu_processor *proc;
 786         u16 lowest_ibc, unblocked_ibc;
 787         int ret = 0;
 788
 789         mutex_lock(&kvm->lock);
 790         if (kvm->created_vcpus) {
 791                 ret = -EBUSY;
 792                 goto out;
 793         }
 794         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 795         if (!proc) {
 796                 ret = -ENOMEM;
 797                 goto out;
 798         }
 799         if (!copy_from_user(proc, (void __user *)attr->addr,
 800                             sizeof(*proc))) {
 801                 kvm->arch.model.cpuid = proc->cpuid;
 802                 lowest_ibc = sclp.ibc >> 16 & 0xfff;
 803                 unblocked_ibc = sclp.ibc & 0xfff;
 804                 if (lowest_ibc && proc->ibc) {
 805                         if (proc->ibc > unblocked_ibc)
 806                                 kvm->arch.model.ibc = unblocked_ibc;
 807                         else if (proc->ibc < lowest_ibc)
 808                                 kvm->arch.model.ibc = lowest_ibc;
 809                         else
 810                                 kvm->arch.model.ibc = proc->ibc;
 811                 }
 812                 memcpy(kvm->arch.model.fac_list, proc->fac_list,
 813                        S390_ARCH_FAC_LIST_SIZE_BYTE);
 814         } else
 815                 ret = -EFAULT;
 816         kfree(proc);
 817 out:
 818         mutex_unlock(&kvm->lock);
 819         return ret;
 820 }
 821
 822 static int kvm_s390_set_processor_feat(struct kvm *kvm,
 823                                        struct kvm_device_attr *attr)
 824 {
 825         struct kvm_s390_vm_cpu_feat data;
 826         int ret = -EBUSY;
 827
 828         if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
 829                 return -EFAULT;
 830         if (!bitmap_subset((unsigned long *) data.feat,
 831                            kvm_s390_available_cpu_feat,
 832                            KVM_S390_VM_CPU_FEAT_NR_BITS))
 833                 return -EINVAL;
 834
 835         mutex_lock(&kvm->lock);
 836         if (!atomic_read(&kvm->online_vcpus)) {
 837                 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
 838                             KVM_S390_VM_CPU_FEAT_NR_BITS);
 839                 ret = 0;
 840         }
 841         mutex_unlock(&kvm->lock);
 842         return ret;
 843 }
 844
 845 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
 846                                           struct kvm_device_attr *attr)
 847 {
 848         /*
 849          * Once supported by kernel + hw, we have to store the subfunctions
 850          * in kvm->arch and remember that user space configured them.
 851          */
 852         return -ENXIO;
 853 }
 854
 855 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 856 {
 857         int ret = -ENXIO;
 858
 859         switch (attr->attr) {
 860         case KVM_S390_VM_CPU_PROCESSOR:
 861                 ret = kvm_s390_set_processor(kvm, attr);
 862                 break;
 863         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
 864                 ret = kvm_s390_set_processor_feat(kvm, attr);
 865                 break;
 866         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
 867                 ret = kvm_s390_set_processor_subfunc(kvm, attr);
 868                 break;
 869         }
 870         return ret;
 871 }
 872
 873 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 874 {
 875         struct kvm_s390_vm_cpu_processor *proc;
 876         int ret = 0;
 877
 878         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 879         if (!proc) {
 880                 ret = -ENOMEM;
 881                 goto out;
 882         }
 883         proc->cpuid = kvm->arch.model.cpuid;
 884         proc->ibc = kvm->arch.model.ibc;
 885         memcpy(&proc->fac_list, kvm->arch.model.fac_list,
 886                S390_ARCH_FAC_LIST_SIZE_BYTE);
 887         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
 888                 ret = -EFAULT;
 889         kfree(proc);
 890 out:
 891         return ret;
 892 }
 893
 894 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
 895 {
 896         struct kvm_s390_vm_cpu_machine *mach;
 897         int ret = 0;
 898
 899         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
 900         if (!mach) {
 901                 ret = -ENOMEM;
 902                 goto out;
 903         }
 904         get_cpu_id((struct cpuid *) &mach->cpuid);
 905         mach->ibc = sclp.ibc;
 906         memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
 907                S390_ARCH_FAC_LIST_SIZE_BYTE);
 908         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
 909                S390_ARCH_FAC_LIST_SIZE_BYTE);
 910         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
 911                 ret = -EFAULT;
 912         kfree(mach);
 913 out:
 914         return ret;
 915 }
 916
 917 static int kvm_s390_get_processor_feat(struct kvm *kvm,
 918                                        struct kvm_device_attr *attr)
 919 {
 920         struct kvm_s390_vm_cpu_feat data;
 921
 922         bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
 923                     KVM_S390_VM_CPU_FEAT_NR_BITS);
 924         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
 925                 return -EFAULT;
 926         return 0;
 927 }
 928
 929 static int kvm_s390_get_machine_feat(struct kvm *kvm,
 930                                      struct kvm_device_attr *attr)
 931 {
 932         struct kvm_s390_vm_cpu_feat data;
 933
 934         bitmap_copy((unsigned long *) data.feat,
 935                     kvm_s390_available_cpu_feat,
 936                     KVM_S390_VM_CPU_FEAT_NR_BITS);
 937         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
 938                 return -EFAULT;
 939         return 0;
 940 }
 941
 942 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
 943                                           struct kvm_device_attr *attr)
 944 {
 945         /*
 946          * Once we can actually configure subfunctions (kernel + hw support),
 947          * we have to check if they were already set by user space, if so copy
 948          * them from kvm->arch.
 949          */
 950         return -ENXIO;
 951 }
 952
 953 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
 954                                         struct kvm_device_attr *attr)
 955 {
 956         if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
 957             sizeof(struct kvm_s390_vm_cpu_subfunc)))
 958                 return -EFAULT;
 959         return 0;
 960 }
 961 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 962 {
 963         int ret = -ENXIO;
 964
 965         switch (attr->attr) {
 966         case KVM_S390_VM_CPU_PROCESSOR:
 967                 ret = kvm_s390_get_processor(kvm, attr);
 968                 break;
 969         case KVM_S390_VM_CPU_MACHINE:
 970                 ret = kvm_s390_get_machine(kvm, attr);
 971                 break;
 972         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
 973                 ret = kvm_s390_get_processor_feat(kvm, attr);
 974                 break;
 975         case KVM_S390_VM_CPU_MACHINE_FEAT:
 976                 ret = kvm_s390_get_machine_feat(kvm, attr);
 977                 break;
 978         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
 979                 ret = kvm_s390_get_processor_subfunc(kvm, attr);
 980                 break;
 981         case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
 982                 ret = kvm_s390_get_machine_subfunc(kvm, attr);
 983                 break;
 984         }
 985         return ret;
 986 }
 987
 988 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 989 {
 990         int ret;
 991
 992         switch (attr->group) {
 993         case KVM_S390_VM_MEM_CTRL:
 994                 ret = kvm_s390_set_mem_control(kvm, attr);
 995                 break;
 996         case KVM_S390_VM_TOD:
 997                 ret = kvm_s390_set_tod(kvm, attr);
 998                 break;
 999         case KVM_S390_VM_CPU_MODEL:
1000                 ret = kvm_s390_set_cpu_model(kvm, attr);
1001                 break;
1002         case KVM_S390_VM_CRYPTO:
1003                 ret = kvm_s390_vm_set_crypto(kvm, attr);
1004                 break;
1005         default:
1006                 ret = -ENXIO;
1007                 break;
1008         }
1009
1010         return ret;
1011 }
1012
1013 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1014 {
1015         int ret;
1016
1017         switch (attr->group) {
1018         case KVM_S390_VM_MEM_CTRL:
1019                 ret = kvm_s390_get_mem_control(kvm, attr);
1020                 break;
1021         case KVM_S390_VM_TOD:
1022                 ret = kvm_s390_get_tod(kvm, attr);
1023                 break;
1024         case KVM_S390_VM_CPU_MODEL:
1025                 ret = kvm_s390_get_cpu_model(kvm, attr);
1026                 break;
1027         default:
1028                 ret = -ENXIO;
1029                 break;
1030         }
1031
1032         return ret;
1033 }
1034
1035 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1036 {
1037         int ret;
1038
1039         switch (attr->group) {
1040         case KVM_S390_VM_MEM_CTRL:
1041                 switch (attr->attr) {
1042                 case KVM_S390_VM_MEM_ENABLE_CMMA:
1043                 case KVM_S390_VM_MEM_CLR_CMMA:
1044                         ret = sclp.has_cmma ? 0 : -ENXIO;
1045                         break;
1046                 case KVM_S390_VM_MEM_LIMIT_SIZE:
1047                         ret = 0;
1048                         break;
1049                 default:
1050                         ret = -ENXIO;
1051                         break;
1052                 }
1053                 break;
1054         case KVM_S390_VM_TOD:
1055                 switch (attr->attr) {
1056                 case KVM_S390_VM_TOD_LOW:
1057                 case KVM_S390_VM_TOD_HIGH:
1058                         ret = 0;
1059                         break;
1060                 default:
1061                         ret = -ENXIO;
1062                         break;
1063                 }
1064                 break;
1065         case KVM_S390_VM_CPU_MODEL:
1066                 switch (attr->attr) {
1067                 case KVM_S390_VM_CPU_PROCESSOR:
1068                 case KVM_S390_VM_CPU_MACHINE:
1069                 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1070                 case KVM_S390_VM_CPU_MACHINE_FEAT:
1071                 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1072                         ret = 0;
1073                         break;
1074                 /* configuring subfunctions is not supported yet */
1075                 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1076                 default:
1077                         ret = -ENXIO;
1078                         break;
1079                 }
1080                 break;
1081         case KVM_S390_VM_CRYPTO:
1082                 switch (attr->attr) {
1083                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1084                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1085                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1086                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1087                         ret = 0;
1088                         break;
1089                 default:
1090                         ret = -ENXIO;
1091                         break;
1092                 }
1093                 break;
1094         default:
1095                 ret = -ENXIO;
1096                 break;
1097         }
1098
1099         return ret;
1100 }
1101
1102 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1103 {
1104         uint8_t *keys;
1105         uint64_t hva;
1106         int i, r = 0;
1107
1108         if (args->flags != 0)
1109                 return -EINVAL;
1110
1111         /* Is this guest using storage keys? */
1112         if (!mm_use_skey(current->mm))
1113                 return KVM_S390_GET_SKEYS_NONE;
1114
1115         /* Enforce sane limit on memory allocation */
1116         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1117                 return -EINVAL;
1118
1119         keys = kmalloc_array(args->count, sizeof(uint8_t),
1120                              GFP_KERNEL | __GFP_NOWARN);
1121         if (!keys)
1122                 keys = vmalloc(sizeof(uint8_t) * args->count);
1123         if (!keys)
1124                 return -ENOMEM;
1125
1126         down_read(&current->mm->mmap_sem);
1127         for (i = 0; i < args->count; i++) {
1128                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1129                 if (kvm_is_error_hva(hva)) {
1130                         r = -EFAULT;
1131                         break;
1132                 }
1133
1134                 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1135                 if (r)
1136                         break;
1137         }
1138         up_read(&current->mm->mmap_sem);
1139
1140         if (!r) {
1141                 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1142                                  sizeof(uint8_t) * args->count);
1143                 if (r)
1144                         r = -EFAULT;
1145         }
1146
1147         kvfree(keys);
1148         return r;
1149 }
1150
1151 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1152 {
1153         uint8_t *keys;
1154         uint64_t hva;
1155         int i, r = 0;
1156
1157         if (args->flags != 0)
1158                 return -EINVAL;
1159
1160         /* Enforce sane limit on memory allocation */
1161         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1162                 return -EINVAL;
1163
1164         keys = kmalloc_array(args->count, sizeof(uint8_t),
1165                              GFP_KERNEL | __GFP_NOWARN);
1166         if (!keys)
1167                 keys = vmalloc(sizeof(uint8_t) * args->count);
1168         if (!keys)
1169                 return -ENOMEM;
1170
1171         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1172                            sizeof(uint8_t) * args->count);
1173         if (r) {
1174                 r = -EFAULT;
1175                 goto out;
1176         }
1177
1178         /* Enable storage key handling for the guest */
1179         r = s390_enable_skey();
1180         if (r)
1181                 goto out;
1182
1183         down_read(&current->mm->mmap_sem);
1184         for (i = 0; i < args->count; i++) {
1185                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1186                 if (kvm_is_error_hva(hva)) {
1187                         r = -EFAULT;
1188                         break;
1189                 }
1190
1191                 /* Lowest order bit is reserved */
1192                 if (keys[i] & 0x01) {
1193                         r = -EINVAL;
1194                         break;
1195                 }
1196
1197                 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1198                 if (r)
1199                         break;
1200         }
1201         up_read(&current->mm->mmap_sem);
1202 out:
1203         kvfree(keys);
1204         return r;
1205 }
1206
1207 long kvm_arch_vm_ioctl(struct file *filp,
1208                        unsigned int ioctl, unsigned long arg)
1209 {
1210         struct kvm *kvm = filp->private_data;
1211         void __user *argp = (void __user *)arg;
1212         struct kvm_device_attr attr;
1213         int r;
1214
1215         switch (ioctl) {
1216         case KVM_S390_INTERRUPT: {
1217                 struct kvm_s390_interrupt s390int;
1218
1219                 r = -EFAULT;
1220                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1221                         break;
1222                 r = kvm_s390_inject_vm(kvm, &s390int);
1223                 break;
1224         }
1225         case KVM_ENABLE_CAP: {
1226                 struct kvm_enable_cap cap;
1227                 r = -EFAULT;
1228                 if (copy_from_user(&cap, argp, sizeof(cap)))
1229                         break;
1230                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1231                 break;
1232         }
1233         case KVM_CREATE_IRQCHIP: {
1234                 struct kvm_irq_routing_entry routing;
1235
1236                 r = -EINVAL;
1237                 if (kvm->arch.use_irqchip) {
1238                         /* Set up dummy routing. */
1239                         memset(&routing, 0, sizeof(routing));
1240                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
1241                 }
1242                 break;
1243         }
1244         case KVM_SET_DEVICE_ATTR: {
1245                 r = -EFAULT;
1246                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1247                         break;
1248                 r = kvm_s390_vm_set_attr(kvm, &attr);
1249                 break;
1250         }
1251         case KVM_GET_DEVICE_ATTR: {
1252                 r = -EFAULT;
1253                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1254                         break;
1255                 r = kvm_s390_vm_get_attr(kvm, &attr);
1256                 break;
1257         }
1258         case KVM_HAS_DEVICE_ATTR: {
1259                 r = -EFAULT;
1260                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1261                         break;
1262                 r = kvm_s390_vm_has_attr(kvm, &attr);
1263                 break;
1264         }
1265         case KVM_S390_GET_SKEYS: {
1266                 struct kvm_s390_skeys args;
1267
1268                 r = -EFAULT;
1269                 if (copy_from_user(&args, argp,
1270                                    sizeof(struct kvm_s390_skeys)))
1271                         break;
1272                 r = kvm_s390_get_skeys(kvm, &args);
1273                 break;
1274         }
1275         case KVM_S390_SET_SKEYS: {
1276                 struct kvm_s390_skeys args;
1277
1278                 r = -EFAULT;
1279                 if (copy_from_user(&args, argp,
1280                                    sizeof(struct kvm_s390_skeys)))
1281                         break;
1282                 r = kvm_s390_set_skeys(kvm, &args);
1283                 break;
1284         }
1285         default:
1286                 r = -ENOTTY;
1287         }
1288
1289         return r;
1290 }
1291
1292 static int kvm_s390_query_ap_config(u8 *config)
1293 {
1294         u32 fcn_code = 0x04000000UL;
1295         u32 cc = 0;
1296
1297         memset(config, 0, 128);
1298         asm volatile(
1299                 "lgr 0,%1\n"
1300                 "lgr 2,%2\n"
1301                 ".long 0xb2af0000\n"            /* PQAP(QCI) */
1302                 "0: ipm %0\n"
1303                 "srl %0,28\n"
1304                 "1:\n"
1305                 EX_TABLE(0b, 1b)
1306                 : "+r" (cc)
1307                 : "r" (fcn_code), "r" (config)
1308                 : "cc", "0", "2", "memory"
1309         );
1310
1311         return cc;
1312 }
1313
1314 static int kvm_s390_apxa_installed(void)
1315 {
1316         u8 config[128];
1317         int cc;
1318
1319         if (test_facility(12)) {
1320                 cc = kvm_s390_query_ap_config(config);
1321
1322                 if (cc)
1323                         pr_err("PQAP(QCI) failed with cc=%d", cc);
1324                 else
1325                         return config[0] & 0x40;
1326         }
1327
1328         return 0;
1329 }
1330
1331 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1332 {
1333         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1334
1335         if (kvm_s390_apxa_installed())
1336                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1337         else
1338                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1339 }
1340
1341 static u64 kvm_s390_get_initial_cpuid(void)
1342 {
1343         struct cpuid cpuid;
1344
1345         get_cpu_id(&cpuid);
1346         cpuid.version = 0xff;
1347         return *((u64 *) &cpuid);
1348 }
1349
1350 static void kvm_s390_crypto_init(struct kvm *kvm)
1351 {
1352         if (!test_kvm_facility(kvm, 76))
1353                 return;
1354
1355         kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
1356         kvm_s390_set_crycb_format(kvm);
1357
1358         /* Enable AES/DEA protected key functions by default */
1359         kvm->arch.crypto.aes_kw = 1;
1360         kvm->arch.crypto.dea_kw = 1;
1361         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1362                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1363         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1364                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1365 }
1366
1367 static void sca_dispose(struct kvm *kvm)
1368 {
1369         if (kvm->arch.use_esca)
1370                 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1371         else
1372                 free_page((unsigned long)(kvm->arch.sca));
1373         kvm->arch.sca = NULL;
1374 }
1375
1376 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1377 {
1378         gfp_t alloc_flags = GFP_KERNEL;
1379         int i, rc;
1380         char debug_name[16];
1381         static unsigned long sca_offset;
1382
1383         rc = -EINVAL;
1384 #ifdef CONFIG_KVM_S390_UCONTROL
1385         if (type & ~KVM_VM_S390_UCONTROL)
1386                 goto out_err;
1387         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1388                 goto out_err;
1389 #else
1390         if (type)
1391                 goto out_err;
1392 #endif
1393
1394         rc = s390_enable_sie();
1395         if (rc)
1396                 goto out_err;
1397
1398         rc = -ENOMEM;
1399
1400         ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1401
1402         kvm->arch.use_esca = 0; /* start with basic SCA */
1403         if (!sclp.has_64bscao)
1404                 alloc_flags |= GFP_DMA;
1405         rwlock_init(&kvm->arch.sca_lock);
1406         kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
1407         if (!kvm->arch.sca)
1408                 goto out_err;
1409         spin_lock(&kvm_lock);
1410         sca_offset += 16;
1411         if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1412                 sca_offset = 0;
1413         kvm->arch.sca = (struct bsca_block *)
1414                         ((char *) kvm->arch.sca + sca_offset);
1415         spin_unlock(&kvm_lock);
1416
1417         sprintf(debug_name, "kvm-%u", current->pid);
1418
1419         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1420         if (!kvm->arch.dbf)
1421                 goto out_err;
1422
1423         kvm->arch.sie_page2 =
1424              (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1425         if (!kvm->arch.sie_page2)
1426                 goto out_err;
1427
1428         /* Populate the facility mask initially. */
1429         memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
1430                S390_ARCH_FAC_LIST_SIZE_BYTE);
1431         for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1432                 if (i < kvm_s390_fac_list_mask_size())
1433                         kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
1434                 else
1435                         kvm->arch.model.fac_mask[i] = 0UL;
1436         }
1437
1438         /* Populate the facility list initially. */
1439         kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1440         memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
1441                S390_ARCH_FAC_LIST_SIZE_BYTE);
1442
1443         set_kvm_facility(kvm->arch.model.fac_mask, 74);
1444         set_kvm_facility(kvm->arch.model.fac_list, 74);
1445
1446         kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
1447         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1448
1449         kvm_s390_crypto_init(kvm);
1450
1451         spin_lock_init(&kvm->arch.float_int.lock);
1452         for (i = 0; i < FIRQ_LIST_COUNT; i++)
1453                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1454         init_waitqueue_head(&kvm->arch.ipte_wq);
1455         mutex_init(&kvm->arch.ipte_mutex);
1456
1457         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1458         VM_EVENT(kvm, 3, "vm created with type %lu", type);
1459
1460         if (type & KVM_VM_S390_UCONTROL) {
1461                 kvm->arch.gmap = NULL;
1462                 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1463         } else {
1464                 if (sclp.hamax == U64_MAX)
1465                         kvm->arch.mem_limit = TASK_MAX_SIZE;
1466                 else
1467                         kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1468                                                     sclp.hamax + 1);
1469                 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
1470                 if (!kvm->arch.gmap)
1471                         goto out_err;
1472                 kvm->arch.gmap->private = kvm;
1473                 kvm->arch.gmap->pfault_enabled = 0;
1474         }
1475
1476         kvm->arch.css_support = 0;
1477         kvm->arch.use_irqchip = 0;
1478         kvm->arch.epoch = 0;
1479
1480         spin_lock_init(&kvm->arch.start_stop_lock);
1481         kvm_s390_vsie_init(kvm);
1482         KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1483
1484         return 0;
1485 out_err:
1486         free_page((unsigned long)kvm->arch.sie_page2);
1487         debug_unregister(kvm->arch.dbf);
1488         sca_dispose(kvm);
1489         KVM_EVENT(3, "creation of vm failed: %d", rc);
1490         return rc;
1491 }
1492
1493 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1494 {
1495         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1496         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1497         kvm_s390_clear_local_irqs(vcpu);
1498         kvm_clear_async_pf_completion_queue(vcpu);
1499         if (!kvm_is_ucontrol(vcpu->kvm))
1500                 sca_del_vcpu(vcpu);
1501
1502         if (kvm_is_ucontrol(vcpu->kvm))
1503                 gmap_remove(vcpu->arch.gmap);
1504
1505         if (vcpu->kvm->arch.use_cmma)
1506                 kvm_s390_vcpu_unsetup_cmma(vcpu);
1507         free_page((unsigned long)(vcpu->arch.sie_block));
1508
1509         kvm_vcpu_uninit(vcpu);
1510         kmem_cache_free(kvm_vcpu_cache, vcpu);
1511 }
1512
1513 static void kvm_free_vcpus(struct kvm *kvm)
1514 {
1515         unsigned int i;
1516         struct kvm_vcpu *vcpu;
1517
1518         kvm_for_each_vcpu(i, vcpu, kvm)
1519                 kvm_arch_vcpu_destroy(vcpu);
1520
1521         mutex_lock(&kvm->lock);
1522         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1523                 kvm->vcpus[i] = NULL;
1524
1525         atomic_set(&kvm->online_vcpus, 0);
1526         mutex_unlock(&kvm->lock);
1527 }
1528
1529 void kvm_arch_destroy_vm(struct kvm *kvm)
1530 {
1531         kvm_free_vcpus(kvm);
1532         sca_dispose(kvm);
1533         debug_unregister(kvm->arch.dbf);
1534         free_page((unsigned long)kvm->arch.sie_page2);
1535         if (!kvm_is_ucontrol(kvm))
1536                 gmap_remove(kvm->arch.gmap);
1537         kvm_s390_destroy_adapters(kvm);
1538         kvm_s390_clear_float_irqs(kvm);
1539         kvm_s390_vsie_destroy(kvm);
1540         KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
1541 }
1542
1543 /* Section: vcpu related */
1544 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1545 {
1546         vcpu->arch.gmap = gmap_create(current->mm, -1UL);
1547         if (!vcpu->arch.gmap)
1548                 return -ENOMEM;
1549         vcpu->arch.gmap->private = vcpu->kvm;
1550
1551         return 0;
1552 }
1553
1554 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1555 {
1556         read_lock(&vcpu->kvm->arch.sca_lock);
1557         if (vcpu->kvm->arch.use_esca) {
1558                 struct esca_block *sca = vcpu->kvm->arch.sca;
1559
1560                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1561                 sca->cpu[vcpu->vcpu_id].sda = 0;
1562         } else {
1563                 struct bsca_block *sca = vcpu->kvm->arch.sca;
1564
1565                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1566                 sca->cpu[vcpu->vcpu_id].sda = 0;
1567         }
1568         read_unlock(&vcpu->kvm->arch.sca_lock);
1569 }
1570
1571 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
1572 {
1573         read_lock(&vcpu->kvm->arch.sca_lock);
1574         if (vcpu->kvm->arch.use_esca) {
1575                 struct esca_block *sca = vcpu->kvm->arch.sca;
1576
1577                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1578                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1579                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
1580                 vcpu->arch.sie_block->ecb2 |= 0x04U;
1581                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1582         } else {
1583                 struct bsca_block *sca = vcpu->kvm->arch.sca;
1584
1585                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1586                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1587                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1588                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1589         }
1590         read_unlock(&vcpu->kvm->arch.sca_lock);
1591 }
1592
1593 /* Basic SCA to Extended SCA data copy routines */
1594 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1595 {
1596         d->sda = s->sda;
1597         d->sigp_ctrl.c = s->sigp_ctrl.c;
1598         d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1599 }
1600
1601 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1602 {
1603         int i;
1604
1605         d->ipte_control = s->ipte_control;
1606         d->mcn[0] = s->mcn;
1607         for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1608                 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1609 }
1610
1611 static int sca_switch_to_extended(struct kvm *kvm)
1612 {
1613         struct bsca_block *old_sca = kvm->arch.sca;
1614         struct esca_block *new_sca;
1615         struct kvm_vcpu *vcpu;
1616         unsigned int vcpu_idx;
1617         u32 scaol, scaoh;
1618
1619         new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1620         if (!new_sca)
1621                 return -ENOMEM;
1622
1623         scaoh = (u32)((u64)(new_sca) >> 32);
1624         scaol = (u32)(u64)(new_sca) & ~0x3fU;
1625
1626         kvm_s390_vcpu_block_all(kvm);
1627         write_lock(&kvm->arch.sca_lock);
1628
1629         sca_copy_b_to_e(new_sca, old_sca);
1630
1631         kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1632                 vcpu->arch.sie_block->scaoh = scaoh;
1633                 vcpu->arch.sie_block->scaol = scaol;
1634                 vcpu->arch.sie_block->ecb2 |= 0x04U;
1635         }
1636         kvm->arch.sca = new_sca;
1637         kvm->arch.use_esca = 1;
1638
1639         write_unlock(&kvm->arch.sca_lock);
1640         kvm_s390_vcpu_unblock_all(kvm);
1641
1642         free_page((unsigned long)old_sca);
1643
1644         VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1645                  old_sca, kvm->arch.sca);
1646         return 0;
1647 }
1648
1649 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1650 {
1651         int rc;
1652
1653         if (id < KVM_S390_BSCA_CPU_SLOTS)
1654                 return true;
1655         if (!sclp.has_esca || !sclp.has_64bscao)
1656                 return false;
1657
1658         mutex_lock(&kvm->lock);
1659         rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1660         mutex_unlock(&kvm->lock);
1661
1662         return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
1663 }
1664
1665 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1666 {
1667         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1668         kvm_clear_async_pf_completion_queue(vcpu);
1669         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1670                                     KVM_SYNC_GPRS |
1671                                     KVM_SYNC_ACRS |
1672                                     KVM_SYNC_CRS |
1673                                     KVM_SYNC_ARCH0 |
1674                                     KVM_SYNC_PFAULT;
1675         kvm_s390_set_prefix(vcpu, 0);
1676         if (test_kvm_facility(vcpu->kvm, 64))
1677                 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
1678         /* fprs can be synchronized via vrs, even if the guest has no vx. With
1679          * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1680          */
1681         if (MACHINE_HAS_VX)
1682                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1683         else
1684                 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
1685
1686         if (kvm_is_ucontrol(vcpu->kvm))
1687                 return __kvm_ucontrol_vcpu_init(vcpu);
1688
1689         return 0;
1690 }
1691
1692 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1693 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1694 {
1695         WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
1696         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1697         vcpu->arch.cputm_start = get_tod_clock_fast();
1698         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1699 }
1700
1701 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1702 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1703 {
1704         WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
1705         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1706         vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1707         vcpu->arch.cputm_start = 0;
1708         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1709 }
1710
1711 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1712 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1713 {
1714         WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1715         vcpu->arch.cputm_enabled = true;
1716         __start_cpu_timer_accounting(vcpu);
1717 }
1718
1719 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1720 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1721 {
1722         WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1723         __stop_cpu_timer_accounting(vcpu);
1724         vcpu->arch.cputm_enabled = false;
1725 }
1726
1727 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1728 {
1729         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1730         __enable_cpu_timer_accounting(vcpu);
1731         preempt_enable();
1732 }
1733
1734 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1735 {
1736         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1737         __disable_cpu_timer_accounting(vcpu);
1738         preempt_enable();
1739 }
1740
1741 /* set the cpu timer - may only be called from the VCPU thread itself */
1742 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1743 {
1744         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1745         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1746         if (vcpu->arch.cputm_enabled)
1747                 vcpu->arch.cputm_start = get_tod_clock_fast();
1748         vcpu->arch.sie_block->cputm = cputm;
1749         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1750         preempt_enable();
1751 }
1752
1753 /* update and get the cpu timer - can also be called from other VCPU threads */
1754 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1755 {
1756         unsigned int seq;
1757         __u64 value;
1758
1759         if (unlikely(!vcpu->arch.cputm_enabled))
1760                 return vcpu->arch.sie_block->cputm;
1761
1762         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1763         do {
1764                 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1765                 /*
1766                  * If the writer would ever execute a read in the critical
1767                  * section, e.g. in irq context, we have a deadlock.
1768                  */
1769                 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1770                 value = vcpu->arch.sie_block->cputm;
1771                 /* if cputm_start is 0, accounting is being started/stopped */
1772                 if (likely(vcpu->arch.cputm_start))
1773                         value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1774         } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1775         preempt_enable();
1776         return value;
1777 }
1778
1779 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1780 {
1781         /* Save host register state */
1782         save_fpu_regs();
1783         vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1784         vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
1785
1786         if (MACHINE_HAS_VX)
1787                 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1788         else
1789                 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
1790         current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1791         if (test_fp_ctl(current->thread.fpu.fpc))
1792                 /* User space provided an invalid FPC, let's clear it */
1793                 current->thread.fpu.fpc = 0;
1794
1795         save_access_regs(vcpu->arch.host_acrs);
1796         restore_access_regs(vcpu->run->s.regs.acrs);
1797         gmap_enable(vcpu->arch.enabled_gmap);
1798         atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1799         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1800                 __start_cpu_timer_accounting(vcpu);
1801         vcpu->cpu = cpu;
1802 }
1803
1804 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1805 {
1806         vcpu->cpu = -1;
1807         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1808                 __stop_cpu_timer_accounting(vcpu);
1809         atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1810         vcpu->arch.enabled_gmap = gmap_get_enabled();
1811         gmap_disable(vcpu->arch.enabled_gmap);
1812
1813         /* Save guest register state */
1814         save_fpu_regs();
1815         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1816
1817         /* Restore host register state */
1818         current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1819         current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
1820
1821         save_access_regs(vcpu->run->s.regs.acrs);
1822         restore_access_regs(vcpu->arch.host_acrs);
1823 }
1824
1825 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1826 {
1827         /* this equals initial cpu reset in pop, but we don't switch to ESA */
1828         vcpu->arch.sie_block->gpsw.mask = 0UL;
1829         vcpu->arch.sie_block->gpsw.addr = 0UL;
1830         kvm_s390_set_prefix(vcpu, 0);
1831         kvm_s390_set_cpu_timer(vcpu, 0);
1832         vcpu->arch.sie_block->ckc       = 0UL;
1833         vcpu->arch.sie_block->todpr     = 0;
1834         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1835         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
1836         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1837         /* make sure the new fpc will be lazily loaded */
1838         save_fpu_regs();
1839         current->thread.fpu.fpc = 0;
1840         vcpu->arch.sie_block->gbea = 1;
1841         vcpu->arch.sie_block->pp = 0;
1842         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1843         kvm_clear_async_pf_completion_queue(vcpu);
1844         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1845                 kvm_s390_vcpu_stop(vcpu);
1846         kvm_s390_clear_local_irqs(vcpu);
1847 }
1848
1849 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1850 {
1851         mutex_lock(&vcpu->kvm->lock);
1852         preempt_disable();
1853         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1854         preempt_enable();
1855         mutex_unlock(&vcpu->kvm->lock);
1856         if (!kvm_is_ucontrol(vcpu->kvm)) {
1857                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1858                 sca_add_vcpu(vcpu);
1859         }
1860         if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
1861                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
1862         /* make vcpu_load load the right gmap on the first trigger */
1863         vcpu->arch.enabled_gmap = vcpu->arch.gmap;
1864 }
1865
1866 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1867 {
1868         if (!test_kvm_facility(vcpu->kvm, 76))
1869                 return;
1870
1871         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1872
1873         if (vcpu->kvm->arch.crypto.aes_kw)
1874                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1875         if (vcpu->kvm->arch.crypto.dea_kw)
1876                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1877
1878         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1879 }
1880
1881 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1882 {
1883         free_page(vcpu->arch.sie_block->cbrlo);
1884         vcpu->arch.sie_block->cbrlo = 0;
1885 }
1886
1887 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1888 {
1889         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1890         if (!vcpu->arch.sie_block->cbrlo)
1891                 return -ENOMEM;
1892
1893         vcpu->arch.sie_block->ecb2 |= 0x80;
1894         vcpu->arch.sie_block->ecb2 &= ~0x08;
1895         return 0;
1896 }
1897
1898 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1899 {
1900         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1901
1902         vcpu->arch.sie_block->ibc = model->ibc;
1903         if (test_kvm_facility(vcpu->kvm, 7))
1904                 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
1905 }
1906
1907 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1908 {
1909         int rc = 0;
1910
1911         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1912                                                     CPUSTAT_SM |
1913                                                     CPUSTAT_STOPPED);
1914
1915         if (test_kvm_facility(vcpu->kvm, 78))
1916                 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1917         else if (test_kvm_facility(vcpu->kvm, 8))
1918                 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1919
1920         kvm_s390_vcpu_setup_model(vcpu);
1921
1922         /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
1923         if (MACHINE_HAS_ESOP)
1924                 vcpu->arch.sie_block->ecb |= 0x02;
1925         if (test_kvm_facility(vcpu->kvm, 9))
1926                 vcpu->arch.sie_block->ecb |= 0x04;
1927         if (test_kvm_facility(vcpu->kvm, 73))
1928                 vcpu->arch.sie_block->ecb |= 0x10;
1929
1930         if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
1931                 vcpu->arch.sie_block->ecb2 |= 0x08;
1932         vcpu->arch.sie_block->eca = 0x1002000U;
1933         if (sclp.has_cei)
1934                 vcpu->arch.sie_block->eca |= 0x80000000U;
1935         if (sclp.has_ib)
1936                 vcpu->arch.sie_block->eca |= 0x40000000U;
1937         if (sclp.has_siif)
1938                 vcpu->arch.sie_block->eca |= 1;
1939         if (sclp.has_sigpif)
1940                 vcpu->arch.sie_block->eca |= 0x10000000U;
1941         if (test_kvm_facility(vcpu->kvm, 64))
1942                 vcpu->arch.sie_block->ecb3 |= 0x01;
1943         if (test_kvm_facility(vcpu->kvm, 129)) {
1944                 vcpu->arch.sie_block->eca |= 0x00020000;
1945                 vcpu->arch.sie_block->ecd |= 0x20000000;
1946         }
1947         vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
1948         vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1949
1950         if (vcpu->kvm->arch.use_cmma) {
1951                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1952                 if (rc)
1953                         return rc;
1954         }
1955         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1956         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1957
1958         kvm_s390_vcpu_crypto_setup(vcpu);
1959
1960         return rc;
1961 }
1962
1963 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1964                                       unsigned int id)
1965 {
1966         struct kvm_vcpu *vcpu;
1967         struct sie_page *sie_page;
1968         int rc = -EINVAL;
1969
1970         if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
1971                 goto out;
1972
1973         rc = -ENOMEM;
1974
1975         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1976         if (!vcpu)
1977                 goto out;
1978
1979         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1980         if (!sie_page)
1981                 goto out_free_cpu;
1982
1983         vcpu->arch.sie_block = &sie_page->sie_block;
1984         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1985
1986         /* the real guest size will always be smaller than msl */
1987         vcpu->arch.sie_block->mso = 0;
1988         vcpu->arch.sie_block->msl = sclp.hamax;
1989
1990         vcpu->arch.sie_block->icpua = id;
1991         spin_lock_init(&vcpu->arch.local_int.lock);
1992         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1993         vcpu->arch.local_int.wq = &vcpu->wq;
1994         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1995         seqcount_init(&vcpu->arch.cputm_seqcount);
1996
1997         rc = kvm_vcpu_init(vcpu, kvm, id);
1998         if (rc)
1999                 goto out_free_sie_block;
2000         VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
2001                  vcpu->arch.sie_block);
2002         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
2003
2004         return vcpu;
2005 out_free_sie_block:
2006         free_page((unsigned long)(vcpu->arch.sie_block));
2007 out_free_cpu:
2008         kmem_cache_free(kvm_vcpu_cache, vcpu);
2009 out:
2010         return ERR_PTR(rc);
2011 }
2012
2013 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2014 {
2015         return kvm_s390_vcpu_has_irq(vcpu, 0);
2016 }
2017
2018 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
2019 {
2020         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2021         exit_sie(vcpu);
2022 }
2023
2024 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
2025 {
2026         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2027 }
2028
2029 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2030 {
2031         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2032         exit_sie(vcpu);
2033 }
2034
2035 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2036 {
2037         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2038 }
2039
2040 /*
2041  * Kick a guest cpu out of SIE and wait until SIE is not running.
2042  * If the CPU is not running (e.g. waiting as idle) the function will
2043  * return immediately. */
2044 void exit_sie(struct kvm_vcpu *vcpu)
2045 {
2046         atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
2047         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2048                 cpu_relax();
2049 }
2050
2051 /* Kick a guest cpu out of SIE to process a request synchronously */
2052 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
2053 {
2054         kvm_make_request(req, vcpu);
2055         kvm_s390_vcpu_request(vcpu);
2056 }
2057
2058 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2059                               unsigned long end)
2060 {
2061         struct kvm *kvm = gmap->private;
2062         struct kvm_vcpu *vcpu;
2063         unsigned long prefix;
2064         int i;
2065
2066         if (gmap_is_shadow(gmap))
2067                 return;
2068         if (start >= 1UL << 31)
2069                 /* We are only interested in prefix pages */
2070                 return;
2071         kvm_for_each_vcpu(i, vcpu, kvm) {
2072                 /* match against both prefix pages */
2073                 prefix = kvm_s390_get_prefix(vcpu);
2074                 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2075                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2076                                    start, end);
2077                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2078                 }
2079         }
2080 }
2081
2082 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2083 {
2084         /* kvm common code refers to this, but never calls it */
2085         BUG();
2086         return 0;
2087 }
2088
2089 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2090                                            struct kvm_one_reg *reg)
2091 {
2092         int r = -EINVAL;
2093
2094         switch (reg->id) {
2095         case KVM_REG_S390_TODPR:
2096                 r = put_user(vcpu->arch.sie_block->todpr,
2097                              (u32 __user *)reg->addr);
2098                 break;
2099         case KVM_REG_S390_EPOCHDIFF:
2100                 r = put_user(vcpu->arch.sie_block->epoch,
2101                              (u64 __user *)reg->addr);
2102                 break;
2103         case KVM_REG_S390_CPU_TIMER:
2104                 r = put_user(kvm_s390_get_cpu_timer(vcpu),
2105                              (u64 __user *)reg->addr);
2106                 break;
2107         case KVM_REG_S390_CLOCK_COMP:
2108                 r = put_user(vcpu->arch.sie_block->ckc,
2109                              (u64 __user *)reg->addr);
2110                 break;
2111         case KVM_REG_S390_PFTOKEN:
2112                 r = put_user(vcpu->arch.pfault_token,
2113                              (u64 __user *)reg->addr);
2114                 break;
2115         case KVM_REG_S390_PFCOMPARE:
2116                 r = put_user(vcpu->arch.pfault_compare,
2117                              (u64 __user *)reg->addr);
2118                 break;
2119         case KVM_REG_S390_PFSELECT:
2120                 r = put_user(vcpu->arch.pfault_select,
2121                              (u64 __user *)reg->addr);
2122                 break;
2123         case KVM_REG_S390_PP:
2124                 r = put_user(vcpu->arch.sie_block->pp,
2125                              (u64 __user *)reg->addr);
2126                 break;
2127         case KVM_REG_S390_GBEA:
2128                 r = put_user(vcpu->arch.sie_block->gbea,
2129                              (u64 __user *)reg->addr);
2130                 break;
2131         default:
2132                 break;
2133         }
2134
2135         return r;
2136 }
2137
2138 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2139                                            struct kvm_one_reg *reg)
2140 {
2141         int r = -EINVAL;
2142         __u64 val;
2143
2144         switch (reg->id) {
2145         case KVM_REG_S390_TODPR:
2146                 r = get_user(vcpu->arch.sie_block->todpr,
2147                              (u32 __user *)reg->addr);
2148                 break;
2149         case KVM_REG_S390_EPOCHDIFF:
2150                 r = get_user(vcpu->arch.sie_block->epoch,
2151                              (u64 __user *)reg->addr);
2152                 break;
2153         case KVM_REG_S390_CPU_TIMER:
2154                 r = get_user(val, (u64 __user *)reg->addr);
2155                 if (!r)
2156                         kvm_s390_set_cpu_timer(vcpu, val);
2157                 break;
2158         case KVM_REG_S390_CLOCK_COMP:
2159                 r = get_user(vcpu->arch.sie_block->ckc,
2160                              (u64 __user *)reg->addr);
2161                 break;
2162         case KVM_REG_S390_PFTOKEN:
2163                 r = get_user(vcpu->arch.pfault_token,
2164                              (u64 __user *)reg->addr);
2165                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2166                         kvm_clear_async_pf_completion_queue(vcpu);
2167                 break;
2168         case KVM_REG_S390_PFCOMPARE:
2169                 r = get_user(vcpu->arch.pfault_compare,
2170                              (u64 __user *)reg->addr);
2171                 break;
2172         case KVM_REG_S390_PFSELECT:
2173                 r = get_user(vcpu->arch.pfault_select,
2174                              (u64 __user *)reg->addr);
2175                 break;
2176         case KVM_REG_S390_PP:
2177                 r = get_user(vcpu->arch.sie_block->pp,
2178                              (u64 __user *)reg->addr);
2179                 break;
2180         case KVM_REG_S390_GBEA:
2181                 r = get_user(vcpu->arch.sie_block->gbea,
2182                              (u64 __user *)reg->addr);
2183                 break;
2184         default:
2185                 break;
2186         }
2187
2188         return r;
2189 }
2190
2191 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2192 {
2193         kvm_s390_vcpu_initial_reset(vcpu);
2194         return 0;
2195 }
2196
2197 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2198 {
2199         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
2200         return 0;
2201 }
2202
2203 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2204 {
2205         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
2206         return 0;
2207 }
2208
2209 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2210                                   struct kvm_sregs *sregs)
2211 {
2212         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
2213         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
2214         restore_access_regs(vcpu->run->s.regs.acrs);
2215         return 0;
2216 }
2217
2218 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2219                                   struct kvm_sregs *sregs)
2220 {
2221         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
2222         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
2223         return 0;
2224 }
2225
2226 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2227 {
2228         /* make sure the new values will be lazily loaded */
2229         save_fpu_regs();
2230         if (test_fp_ctl(fpu->fpc))
2231                 return -EINVAL;
2232         current->thread.fpu.fpc = fpu->fpc;
2233         if (MACHINE_HAS_VX)
2234                 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
2235         else
2236                 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
2237         return 0;
2238 }
2239
2240 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2241 {
2242         /* make sure we have the latest values */
2243         save_fpu_regs();
2244         if (MACHINE_HAS_VX)
2245                 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
2246         else
2247                 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
2248         fpu->fpc = current->thread.fpu.fpc;
2249         return 0;
2250 }
2251
2252 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2253 {
2254         int rc = 0;
2255
2256         if (!is_vcpu_stopped(vcpu))
2257                 rc = -EBUSY;
2258         else {
2259                 vcpu->run->psw_mask = psw.mask;
2260                 vcpu->run->psw_addr = psw.addr;
2261         }
2262         return rc;
2263 }
2264
2265 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2266                                   struct kvm_translation *tr)
2267 {
2268         return -EINVAL; /* not implemented yet */
2269 }
2270
2271 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2272                               KVM_GUESTDBG_USE_HW_BP | \
2273                               KVM_GUESTDBG_ENABLE)
2274
2275 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2276                                         struct kvm_guest_debug *dbg)
2277 {
2278         int rc = 0;
2279
2280         vcpu->guest_debug = 0;
2281         kvm_s390_clear_bp_data(vcpu);
2282
2283         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
2284                 return -EINVAL;
2285         if (!sclp.has_gpere)
2286                 return -EINVAL;
2287
2288         if (dbg->control & KVM_GUESTDBG_ENABLE) {
2289                 vcpu->guest_debug = dbg->control;
2290                 /* enforce guest PER */
2291                 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2292
2293                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2294                         rc = kvm_s390_import_bp_data(vcpu, dbg);
2295         } else {
2296                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2297                 vcpu->arch.guestdbg.last_bp = 0;
2298         }
2299
2300         if (rc) {
2301                 vcpu->guest_debug = 0;
2302                 kvm_s390_clear_bp_data(vcpu);
2303                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2304         }
2305
2306         return rc;
2307 }
2308
2309 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2310                                     struct kvm_mp_state *mp_state)
2311 {
2312         /* CHECK_STOP and LOAD are not supported yet */
2313         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2314                                        KVM_MP_STATE_OPERATING;
2315 }
2316
2317 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2318                                     struct kvm_mp_state *mp_state)
2319 {
2320         int rc = 0;
2321
2322         /* user space knows about this interface - let it control the state */
2323         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2324
2325         switch (mp_state->mp_state) {
2326         case KVM_MP_STATE_STOPPED:
2327                 kvm_s390_vcpu_stop(vcpu);
2328                 break;
2329         case KVM_MP_STATE_OPERATING:
2330                 kvm_s390_vcpu_start(vcpu);
2331                 break;
2332         case KVM_MP_STATE_LOAD:
2333         case KVM_MP_STATE_CHECK_STOP:
2334                 /* fall through - CHECK_STOP and LOAD are not supported yet */
2335         default:
2336                 rc = -ENXIO;
2337         }
2338
2339         return rc;
2340 }
2341
2342 static bool ibs_enabled(struct kvm_vcpu *vcpu)
2343 {
2344         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2345 }
2346
2347 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2348 {
2349 retry:
2350         kvm_s390_vcpu_request_handled(vcpu);
2351         if (!vcpu->requests)
2352                 return 0;
2353         /*
2354          * We use MMU_RELOAD just to re-arm the ipte notifier for the
2355          * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2356          * This ensures that the ipte instruction for this request has
2357          * already finished. We might race against a second unmapper that
2358          * wants to set the blocking bit. Lets just retry the request loop.
2359          */
2360         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2361                 int rc;
2362                 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2363                                           kvm_s390_get_prefix(vcpu),
2364                                           PAGE_SIZE * 2, PROT_WRITE);
2365                 if (rc) {
2366                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2367                         return rc;
2368                 }
2369                 goto retry;
2370         }
2371
2372         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2373                 vcpu->arch.sie_block->ihcpu = 0xffff;
2374                 goto retry;
2375         }
2376
2377         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2378                 if (!ibs_enabled(vcpu)) {
2379                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
2380                         atomic_or(CPUSTAT_IBS,
2381                                         &vcpu->arch.sie_block->cpuflags);
2382                 }
2383                 goto retry;
2384         }
2385
2386         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2387                 if (ibs_enabled(vcpu)) {
2388                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2389                         atomic_andnot(CPUSTAT_IBS,
2390                                           &vcpu->arch.sie_block->cpuflags);
2391                 }
2392                 goto retry;
2393         }
2394
2395         if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2396                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2397                 goto retry;
2398         }
2399
2400         /* nothing to do, just clear the request */
2401         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2402
2403         return 0;
2404 }
2405
2406 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2407 {
2408         struct kvm_vcpu *vcpu;
2409         int i;
2410
2411         mutex_lock(&kvm->lock);
2412         preempt_disable();
2413         kvm->arch.epoch = tod - get_tod_clock();
2414         kvm_s390_vcpu_block_all(kvm);
2415         kvm_for_each_vcpu(i, vcpu, kvm)
2416                 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2417         kvm_s390_vcpu_unblock_all(kvm);
2418         preempt_enable();
2419         mutex_unlock(&kvm->lock);
2420 }
2421
2422 /**
2423  * kvm_arch_fault_in_page - fault-in guest page if necessary
2424  * @vcpu: The corresponding virtual cpu
2425  * @gpa: Guest physical address
2426  * @writable: Whether the page should be writable or not
2427  *
2428  * Make sure that a guest page has been faulted-in on the host.
2429  *
2430  * Return: Zero on success, negative error code otherwise.
2431  */
2432 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2433 {
2434         return gmap_fault(vcpu->arch.gmap, gpa,
2435                           writable ? FAULT_FLAG_WRITE : 0);
2436 }
2437
2438 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2439                                       unsigned long token)
2440 {
2441         struct kvm_s390_interrupt inti;
2442         struct kvm_s390_irq irq;
2443
2444         if (start_token) {
2445                 irq.u.ext.ext_params2 = token;
2446                 irq.type = KVM_S390_INT_PFAULT_INIT;
2447                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
2448         } else {
2449                 inti.type = KVM_S390_INT_PFAULT_DONE;
2450                 inti.parm64 = token;
2451                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2452         }
2453 }
2454
2455 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2456                                      struct kvm_async_pf *work)
2457 {
2458         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2459         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2460 }
2461
2462 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2463                                  struct kvm_async_pf *work)
2464 {
2465         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2466         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2467 }
2468
2469 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2470                                struct kvm_async_pf *work)
2471 {
2472         /* s390 will always inject the page directly */
2473 }
2474
2475 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2476 {
2477         /*
2478          * s390 will always inject the page directly,
2479          * but we still want check_async_completion to cleanup
2480          */
2481         return true;
2482 }
2483
2484 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2485 {
2486         hva_t hva;
2487         struct kvm_arch_async_pf arch;
2488         int rc;
2489
2490         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2491                 return 0;
2492         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2493             vcpu->arch.pfault_compare)
2494                 return 0;
2495         if (psw_extint_disabled(vcpu))
2496                 return 0;
2497         if (kvm_s390_vcpu_has_irq(vcpu, 0))
2498                 return 0;
2499         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2500                 return 0;
2501         if (!vcpu->arch.gmap->pfault_enabled)
2502                 return 0;
2503
2504         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2505         hva += current->thread.gmap_addr & ~PAGE_MASK;
2506         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
2507                 return 0;
2508
2509         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2510         return rc;
2511 }
2512
2513 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2514 {
2515         int rc, cpuflags;
2516
2517         /*
2518          * On s390 notifications for arriving pages will be delivered directly
2519          * to the guest but the house keeping for completed pfaults is
2520          * handled outside the worker.
2521          */
2522         kvm_check_async_pf_completion(vcpu);
2523
2524         vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2525         vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
2526
2527         if (need_resched())
2528                 schedule();
2529
2530         if (test_cpu_flag(CIF_MCCK_PENDING))
2531                 s390_handle_mcck();
2532
2533         if (!kvm_is_ucontrol(vcpu->kvm)) {
2534                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2535                 if (rc)
2536                         return rc;
2537         }
2538
2539         rc = kvm_s390_handle_requests(vcpu);
2540         if (rc)
2541                 return rc;
2542
2543         if (guestdbg_enabled(vcpu)) {
2544                 kvm_s390_backup_guest_per_regs(vcpu);
2545                 kvm_s390_patch_guest_per_regs(vcpu);
2546         }
2547
2548         vcpu->arch.sie_block->icptcode = 0;
2549         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2550         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2551         trace_kvm_s390_sie_enter(vcpu, cpuflags);
2552
2553         return 0;
2554 }
2555
2556 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2557 {
2558         struct kvm_s390_pgm_info pgm_info = {
2559                 .code = PGM_ADDRESSING,
2560         };
2561         u8 opcode, ilen;
2562         int rc;
2563
2564         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2565         trace_kvm_s390_sie_fault(vcpu);
2566
2567         /*
2568          * We want to inject an addressing exception, which is defined as a
2569          * suppressing or terminating exception. However, since we came here
2570          * by a DAT access exception, the PSW still points to the faulting
2571          * instruction since DAT exceptions are nullifying. So we've got
2572          * to look up the current opcode to get the length of the instruction
2573          * to be able to forward the PSW.
2574          */
2575         rc = read_guest_instr(vcpu, &opcode, 1);
2576         ilen = insn_length(opcode);
2577         if (rc < 0) {
2578                 return rc;
2579         } else if (rc) {
2580                 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2581                  * Forward by arbitrary ilc, injection will take care of
2582                  * nullification if necessary.
2583                  */
2584                 pgm_info = vcpu->arch.pgm;
2585                 ilen = 4;
2586         }
2587         pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2588         kvm_s390_forward_psw(vcpu, ilen);
2589         return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
2590 }
2591
2592 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2593 {
2594         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2595                    vcpu->arch.sie_block->icptcode);
2596         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2597
2598         if (guestdbg_enabled(vcpu))
2599                 kvm_s390_restore_guest_per_regs(vcpu);
2600
2601         vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2602         vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
2603
2604         if (vcpu->arch.sie_block->icptcode > 0) {
2605                 int rc = kvm_handle_sie_intercept(vcpu);
2606
2607                 if (rc != -EOPNOTSUPP)
2608                         return rc;
2609                 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2610                 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2611                 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2612                 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2613                 return -EREMOTE;
2614         } else if (exit_reason != -EFAULT) {
2615                 vcpu->stat.exit_null++;
2616                 return 0;
2617         } else if (kvm_is_ucontrol(vcpu->kvm)) {
2618                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2619                 vcpu->run->s390_ucontrol.trans_exc_code =
2620                                                 current->thread.gmap_addr;
2621                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2622                 return -EREMOTE;
2623         } else if (current->thread.gmap_pfault) {
2624                 trace_kvm_s390_major_guest_pfault(vcpu);
2625                 current->thread.gmap_pfault = 0;
2626                 if (kvm_arch_setup_async_pf(vcpu))
2627                         return 0;
2628                 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
2629         }
2630         return vcpu_post_run_fault_in_sie(vcpu);
2631 }
2632
2633 static int __vcpu_run(struct kvm_vcpu *vcpu)
2634 {
2635         int rc, exit_reason;
2636
2637         /*
2638          * We try to hold kvm->srcu during most of vcpu_run (except when run-
2639          * ning the guest), so that memslots (and other stuff) are protected
2640          */
2641         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2642
2643         do {
2644                 rc = vcpu_pre_run(vcpu);
2645                 if (rc)
2646                         break;
2647
2648                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2649                 /*
2650                  * As PF_VCPU will be used in fault handler, between
2651                  * guest_enter and guest_exit should be no uaccess.
2652                  */
2653                 local_irq_disable();
2654                 guest_enter_irqoff();
2655                 __disable_cpu_timer_accounting(vcpu);
2656                 local_irq_enable();
2657                 exit_reason = sie64a(vcpu->arch.sie_block,
2658                                      vcpu->run->s.regs.gprs);
2659                 local_irq_disable();
2660                 __enable_cpu_timer_accounting(vcpu);
2661                 guest_exit_irqoff();
2662                 local_irq_enable();
2663                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2664
2665                 rc = vcpu_post_run(vcpu, exit_reason);
2666         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2667
2668         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2669         return rc;
2670 }
2671
2672 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2673 {
2674         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2675         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2676         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2677                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2678         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2679                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2680                 /* some control register changes require a tlb flush */
2681                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2682         }
2683         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2684                 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
2685                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2686                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2687                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2688                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2689         }
2690         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2691                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2692                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2693                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2694                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2695                         kvm_clear_async_pf_completion_queue(vcpu);
2696         }
2697         kvm_run->kvm_dirty_regs = 0;
2698 }
2699
2700 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2701 {
2702         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2703         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2704         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2705         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2706         kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
2707         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2708         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2709         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2710         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2711         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2712         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2713         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2714 }
2715
2716 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2717 {
2718         int rc;
2719         sigset_t sigsaved;
2720
2721         if (guestdbg_exit_pending(vcpu)) {
2722                 kvm_s390_prepare_debug_exit(vcpu);
2723                 return 0;
2724         }
2725
2726         if (vcpu->sigset_active)
2727                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2728
2729         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2730                 kvm_s390_vcpu_start(vcpu);
2731         } else if (is_vcpu_stopped(vcpu)) {
2732                 pr_err_ratelimited("can't run stopped vcpu %d\n",
2733                                    vcpu->vcpu_id);
2734                 return -EINVAL;
2735         }
2736
2737         sync_regs(vcpu, kvm_run);
2738         enable_cpu_timer_accounting(vcpu);
2739
2740         might_fault();
2741         rc = __vcpu_run(vcpu);
2742
2743         if (signal_pending(current) && !rc) {
2744                 kvm_run->exit_reason = KVM_EXIT_INTR;
2745                 rc = -EINTR;
2746         }
2747
2748         if (guestdbg_exit_pending(vcpu) && !rc)  {
2749                 kvm_s390_prepare_debug_exit(vcpu);
2750                 rc = 0;
2751         }
2752
2753         if (rc == -EREMOTE) {
2754                 /* userspace support is needed, kvm_run has been prepared */
2755                 rc = 0;
2756         }
2757
2758         disable_cpu_timer_accounting(vcpu);
2759         store_regs(vcpu, kvm_run);
2760
2761         if (vcpu->sigset_active)
2762                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2763
2764         vcpu->stat.exit_userspace++;
2765         return rc;
2766 }
2767
2768 /*
2769  * store status at address
2770  * we use have two special cases:
2771  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2772  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2773  */
2774 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2775 {
2776         unsigned char archmode = 1;
2777         freg_t fprs[NUM_FPRS];
2778         unsigned int px;
2779         u64 clkcomp, cputm;
2780         int rc;
2781
2782         px = kvm_s390_get_prefix(vcpu);
2783         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2784                 if (write_guest_abs(vcpu, 163, &archmode, 1))
2785                         return -EFAULT;
2786                 gpa = 0;
2787         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2788                 if (write_guest_real(vcpu, 163, &archmode, 1))
2789                         return -EFAULT;
2790                 gpa = px;
2791         } else
2792                 gpa -= __LC_FPREGS_SAVE_AREA;
2793
2794         /* manually convert vector registers if necessary */
2795         if (MACHINE_HAS_VX) {
2796                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
2797                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2798                                      fprs, 128);
2799         } else {
2800                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2801                                      vcpu->run->s.regs.fprs, 128);
2802         }
2803         rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
2804                               vcpu->run->s.regs.gprs, 128);
2805         rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
2806                               &vcpu->arch.sie_block->gpsw, 16);
2807         rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
2808                               &px, 4);
2809         rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
2810                               &vcpu->run->s.regs.fpc, 4);
2811         rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
2812                               &vcpu->arch.sie_block->todpr, 4);
2813         cputm = kvm_s390_get_cpu_timer(vcpu);
2814         rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
2815                               &cputm, 8);
2816         clkcomp = vcpu->arch.sie_block->ckc >> 8;
2817         rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
2818                               &clkcomp, 8);
2819         rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
2820                               &vcpu->run->s.regs.acrs, 64);
2821         rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
2822                               &vcpu->arch.sie_block->gcr, 128);
2823         return rc ? -EFAULT : 0;
2824 }
2825
2826 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2827 {
2828         /*
2829          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2830          * copying in vcpu load/put. Lets update our copies before we save
2831          * it into the save area
2832          */
2833         save_fpu_regs();
2834         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2835         save_access_regs(vcpu->run->s.regs.acrs);
2836
2837         return kvm_s390_store_status_unloaded(vcpu, addr);
2838 }
2839
2840 /*
2841  * store additional status at address
2842  */
2843 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2844                                         unsigned long gpa)
2845 {
2846         /* Only bits 0-53 are used for address formation */
2847         if (!(gpa & ~0x3ff))
2848                 return 0;
2849
2850         return write_guest_abs(vcpu, gpa & ~0x3ff,
2851                                (void *)&vcpu->run->s.regs.vrs, 512);
2852 }
2853
2854 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2855 {
2856         if (!test_kvm_facility(vcpu->kvm, 129))
2857                 return 0;
2858
2859         /*
2860          * The guest VXRS are in the host VXRs due to the lazy
2861          * copying in vcpu load/put. We can simply call save_fpu_regs()
2862          * to save the current register state because we are in the
2863          * middle of a load/put cycle.
2864          *
2865          * Let's update our copies before we save it into the save area.
2866          */
2867         save_fpu_regs();
2868
2869         return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2870 }
2871
2872 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2873 {
2874         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2875         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2876 }
2877
2878 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2879 {
2880         unsigned int i;
2881         struct kvm_vcpu *vcpu;
2882
2883         kvm_for_each_vcpu(i, vcpu, kvm) {
2884                 __disable_ibs_on_vcpu(vcpu);
2885         }
2886 }
2887
2888 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2889 {
2890         if (!sclp.has_ibs)
2891                 return;
2892         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2893         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2894 }
2895
2896 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2897 {
2898         int i, online_vcpus, started_vcpus = 0;
2899
2900         if (!is_vcpu_stopped(vcpu))
2901                 return;
2902
2903         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2904         /* Only one cpu at a time may enter/leave the STOPPED state. */
2905         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2906         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2907
2908         for (i = 0; i < online_vcpus; i++) {
2909                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2910                         started_vcpus++;
2911         }
2912
2913         if (started_vcpus == 0) {
2914                 /* we're the only active VCPU -> speed it up */
2915                 __enable_ibs_on_vcpu(vcpu);
2916         } else if (started_vcpus == 1) {
2917                 /*
2918                  * As we are starting a second VCPU, we have to disable
2919                  * the IBS facility on all VCPUs to remove potentially
2920                  * oustanding ENABLE requests.
2921                  */
2922                 __disable_ibs_on_all_vcpus(vcpu->kvm);
2923         }
2924
2925         atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2926         /*
2927          * Another VCPU might have used IBS while we were offline.
2928          * Let's play safe and flush the VCPU at startup.
2929          */
2930         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2931         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2932         return;
2933 }
2934
2935 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2936 {
2937         int i, online_vcpus, started_vcpus = 0;
2938         struct kvm_vcpu *started_vcpu = NULL;
2939
2940         if (is_vcpu_stopped(vcpu))
2941                 return;
2942
2943         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2944         /* Only one cpu at a time may enter/leave the STOPPED state. */
2945         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2946         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2947
2948         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2949         kvm_s390_clear_stop_irq(vcpu);
2950
2951         atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2952         __disable_ibs_on_vcpu(vcpu);
2953
2954         for (i = 0; i < online_vcpus; i++) {
2955                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2956                         started_vcpus++;
2957                         started_vcpu = vcpu->kvm->vcpus[i];
2958                 }
2959         }
2960
2961         if (started_vcpus == 1) {
2962                 /*
2963                  * As we only have one VCPU left, we want to enable the
2964                  * IBS facility for that VCPU to speed it up.
2965                  */
2966                 __enable_ibs_on_vcpu(started_vcpu);
2967         }
2968
2969         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2970         return;
2971 }
2972
2973 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2974                                      struct kvm_enable_cap *cap)
2975 {
2976         int r;
2977
2978         if (cap->flags)
2979                 return -EINVAL;
2980
2981         switch (cap->cap) {
2982         case KVM_CAP_S390_CSS_SUPPORT:
2983                 if (!vcpu->kvm->arch.css_support) {
2984                         vcpu->kvm->arch.css_support = 1;
2985                         VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2986                         trace_kvm_s390_enable_css(vcpu->kvm);
2987                 }
2988                 r = 0;
2989                 break;
2990         default:
2991                 r = -EINVAL;
2992                 break;
2993         }
2994         return r;
2995 }
2996
2997 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2998                                   struct kvm_s390_mem_op *mop)
2999 {
3000         void __user *uaddr = (void __user *)mop->buf;
3001         void *tmpbuf = NULL;
3002         int r, srcu_idx;
3003         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3004                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
3005
3006         if (mop->flags & ~supported_flags)
3007                 return -EINVAL;
3008
3009         if (mop->size > MEM_OP_MAX_SIZE)
3010                 return -E2BIG;
3011
3012         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3013                 tmpbuf = vmalloc(mop->size);
3014                 if (!tmpbuf)
3015                         return -ENOMEM;
3016         }
3017
3018         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3019
3020         switch (mop->op) {
3021         case KVM_S390_MEMOP_LOGICAL_READ:
3022                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3023                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3024                                             mop->size, GACC_FETCH);
3025                         break;
3026                 }
3027                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3028                 if (r == 0) {
3029                         if (copy_to_user(uaddr, tmpbuf, mop->size))
3030                                 r = -EFAULT;
3031                 }
3032                 break;
3033         case KVM_S390_MEMOP_LOGICAL_WRITE:
3034                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3035                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3036                                             mop->size, GACC_STORE);
3037                         break;
3038                 }
3039                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3040                         r = -EFAULT;
3041                         break;
3042                 }
3043                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3044                 break;
3045         default:
3046                 r = -EINVAL;
3047         }
3048
3049         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3050
3051         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3052                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3053
3054         vfree(tmpbuf);
3055         return r;
3056 }
3057
3058 long kvm_arch_vcpu_ioctl(struct file *filp,
3059                          unsigned int ioctl, unsigned long arg)
3060 {
3061         struct kvm_vcpu *vcpu = filp->private_data;
3062         void __user *argp = (void __user *)arg;
3063         int idx;
3064         long r;
3065
3066         switch (ioctl) {
3067         case KVM_S390_IRQ: {
3068                 struct kvm_s390_irq s390irq;
3069
3070                 r = -EFAULT;
3071                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3072                         break;
3073                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3074                 break;
3075         }
3076         case KVM_S390_INTERRUPT: {
3077                 struct kvm_s390_interrupt s390int;
3078                 struct kvm_s390_irq s390irq;
3079
3080                 r = -EFAULT;
3081                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
3082                         break;
3083                 if (s390int_to_s390irq(&s390int, &s390irq))
3084                         return -EINVAL;
3085                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3086                 break;
3087         }
3088         case KVM_S390_STORE_STATUS:
3089                 idx = srcu_read_lock(&vcpu->kvm->srcu);
3090                 r = kvm_s390_vcpu_store_status(vcpu, arg);
3091                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
3092                 break;
3093         case KVM_S390_SET_INITIAL_PSW: {
3094                 psw_t psw;
3095
3096                 r = -EFAULT;
3097                 if (copy_from_user(&psw, argp, sizeof(psw)))
3098                         break;
3099                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3100                 break;
3101         }
3102         case KVM_S390_INITIAL_RESET:
3103                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3104                 break;
3105         case KVM_SET_ONE_REG:
3106         case KVM_GET_ONE_REG: {
3107                 struct kvm_one_reg reg;
3108                 r = -EFAULT;
3109                 if (copy_from_user(&reg, argp, sizeof(reg)))
3110                         break;
3111                 if (ioctl == KVM_SET_ONE_REG)
3112                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3113                 else
3114                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3115                 break;
3116         }
3117 #ifdef CONFIG_KVM_S390_UCONTROL
3118         case KVM_S390_UCAS_MAP: {
3119                 struct kvm_s390_ucas_mapping ucasmap;
3120
3121                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3122                         r = -EFAULT;
3123                         break;
3124                 }
3125
3126                 if (!kvm_is_ucontrol(vcpu->kvm)) {
3127                         r = -EINVAL;
3128                         break;
3129                 }
3130
3131                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3132                                      ucasmap.vcpu_addr, ucasmap.length);
3133                 break;
3134         }
3135         case KVM_S390_UCAS_UNMAP: {
3136                 struct kvm_s390_ucas_mapping ucasmap;
3137
3138                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3139                         r = -EFAULT;
3140                         break;
3141                 }
3142
3143                 if (!kvm_is_ucontrol(vcpu->kvm)) {
3144                         r = -EINVAL;
3145                         break;
3146                 }
3147
3148                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3149                         ucasmap.length);
3150                 break;
3151         }
3152 #endif
3153         case KVM_S390_VCPU_FAULT: {
3154                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
3155                 break;
3156         }
3157         case KVM_ENABLE_CAP:
3158         {
3159                 struct kvm_enable_cap cap;
3160                 r = -EFAULT;
3161                 if (copy_from_user(&cap, argp, sizeof(cap)))
3162                         break;
3163                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3164                 break;
3165         }
3166         case KVM_S390_MEM_OP: {
3167                 struct kvm_s390_mem_op mem_op;
3168
3169                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3170                         r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3171                 else
3172                         r = -EFAULT;
3173                 break;
3174         }
3175         case KVM_S390_SET_IRQ_STATE: {
3176                 struct kvm_s390_irq_state irq_state;
3177
3178                 r = -EFAULT;
3179                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3180                         break;
3181                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3182                     irq_state.len == 0 ||
3183                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3184                         r = -EINVAL;
3185                         break;
3186                 }
3187                 r = kvm_s390_set_irq_state(vcpu,
3188                                            (void __user *) irq_state.buf,
3189                                            irq_state.len);
3190                 break;
3191         }
3192         case KVM_S390_GET_IRQ_STATE: {
3193                 struct kvm_s390_irq_state irq_state;
3194
3195                 r = -EFAULT;
3196                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3197                         break;
3198                 if (irq_state.len == 0) {
3199                         r = -EINVAL;
3200                         break;
3201                 }
3202                 r = kvm_s390_get_irq_state(vcpu,
3203                                            (__u8 __user *)  irq_state.buf,
3204                                            irq_state.len);
3205                 break;
3206         }
3207         default:
3208                 r = -ENOTTY;
3209         }
3210         return r;
3211 }
3212
3213 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3214 {
3215 #ifdef CONFIG_KVM_S390_UCONTROL
3216         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3217                  && (kvm_is_ucontrol(vcpu->kvm))) {
3218                 vmf->page = virt_to_page(vcpu->arch.sie_block);
3219                 get_page(vmf->page);
3220                 return 0;
3221         }
3222 #endif
3223         return VM_FAULT_SIGBUS;
3224 }
3225
3226 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3227                             unsigned long npages)
3228 {
3229         return 0;
3230 }
3231
3232 /* Section: memory related */
3233 int kvm_arch_prepare_memory_region(struct kvm *kvm,
3234                                    struct kvm_memory_slot *memslot,
3235                                    const struct kvm_userspace_memory_region *mem,
3236                                    enum kvm_mr_change change)
3237 {
3238         /* A few sanity checks. We can have memory slots which have to be
3239            located/ended at a segment boundary (1MB). The memory in userland is
3240            ok to be fragmented into various different vmas. It is okay to mmap()
3241            and munmap() stuff in this slot after doing this call at any time */
3242
3243         if (mem->userspace_addr & 0xffffful)
3244                 return -EINVAL;
3245
3246         if (mem->memory_size & 0xffffful)
3247                 return -EINVAL;
3248
3249         if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3250                 return -EINVAL;
3251
3252         return 0;
3253 }
3254
3255 void kvm_arch_commit_memory_region(struct kvm *kvm,
3256                                 const struct kvm_userspace_memory_region *mem,
3257                                 const struct kvm_memory_slot *old,
3258                                 const struct kvm_memory_slot *new,
3259                                 enum kvm_mr_change change)
3260 {
3261         int rc;
3262
3263         /* If the basics of the memslot do not change, we do not want
3264          * to update the gmap. Every update causes several unnecessary
3265          * segment translation exceptions. This is usually handled just
3266          * fine by the normal fault handler + gmap, but it will also
3267          * cause faults on the prefix page of running guest CPUs.
3268          */
3269         if (old->userspace_addr == mem->userspace_addr &&
3270             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3271             old->npages * PAGE_SIZE == mem->memory_size)
3272                 return;
3273
3274         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3275                 mem->guest_phys_addr, mem->memory_size);
3276         if (rc)
3277                 pr_warn("failed to commit memory region\n");
3278         return;
3279 }
3280
3281 static inline unsigned long nonhyp_mask(int i)
3282 {
3283         unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3284
3285         return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3286 }
3287
3288 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3289 {
3290         vcpu->valid_wakeup = false;
3291 }
3292
3293 static int __init kvm_s390_init(void)
3294 {
3295         int i;
3296
3297         if (!sclp.has_sief2) {
3298                 pr_info("SIE not available\n");
3299                 return -ENODEV;
3300         }
3301
3302         for (i = 0; i < 16; i++)
3303                 kvm_s390_fac_list_mask[i] |=
3304                         S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3305
3306         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
3307 }
3308
3309 static void __exit kvm_s390_exit(void)
3310 {
3311         kvm_exit();
3312 }
3313
3314 module_init(kvm_s390_init);
3315 module_exit(kvm_s390_exit);
3316
3317 /*
3318  * Enable autoloading of the kvm module.
3319  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3320  * since x86 takes a different approach.
3321  */
3322 #include <linux/miscdevice.h>
3323 MODULE_ALIAS_MISCDEV(KVM_MINOR);
3324 MODULE_ALIAS("devname:kvm");