2 * hosting zSeries kernel virtual machines
4 * Copyright IBM Corp. 2008, 2009
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
17 #include <linux/compiler.h>
18 #include <linux/err.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/module.h>
25 #include <linux/random.h>
26 #include <linux/slab.h>
27 #include <linux/timer.h>
28 #include <linux/vmalloc.h>
29 #include <linux/bitmap.h>
30 #include <asm/asm-offsets.h>
31 #include <asm/lowcore.h>
33 #include <asm/pgtable.h>
36 #include <asm/switch_to.h>
39 #include <asm/cpacf.h>
44 #define KMSG_COMPONENT "kvm-s390"
46 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
48 #define CREATE_TRACE_POINTS
50 #include "trace-s390.h"
52 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
54 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
55 (KVM_MAX_VCPUS + LOCAL_IRQS))
57 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
59 struct kvm_stats_debugfs_item debugfs_entries[] = {
60 { "userspace_handled", VCPU_STAT(exit_userspace) },
61 { "exit_null", VCPU_STAT(exit_null) },
62 { "exit_validity", VCPU_STAT(exit_validity) },
63 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
64 { "exit_external_request", VCPU_STAT(exit_external_request) },
65 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
66 { "exit_instruction", VCPU_STAT(exit_instruction) },
67 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
68 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
69 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
70 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
71 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
72 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
73 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
74 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
75 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
76 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
77 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
78 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
79 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
80 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
81 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
82 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
83 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
84 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
85 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
86 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
87 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
88 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
89 { "instruction_spx", VCPU_STAT(instruction_spx) },
90 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
91 { "instruction_stap", VCPU_STAT(instruction_stap) },
92 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
93 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
94 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
95 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
96 { "instruction_essa", VCPU_STAT(instruction_essa) },
97 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
98 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
99 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
100 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
101 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
102 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
103 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
104 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
105 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
106 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
107 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
108 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
109 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
110 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
111 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
112 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
113 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
114 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
115 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
116 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
117 { "diagnose_10", VCPU_STAT(diagnose_10) },
118 { "diagnose_44", VCPU_STAT(diagnose_44) },
119 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
120 { "diagnose_258", VCPU_STAT(diagnose_258) },
121 { "diagnose_308", VCPU_STAT(diagnose_308) },
122 { "diagnose_500", VCPU_STAT(diagnose_500) },
126 /* upper facilities limit for kvm */
127 unsigned long kvm_s390_fac_list_mask[16] = {
128 0xffe6000000000000UL,
129 0x005e000000000000UL,
132 unsigned long kvm_s390_fac_list_mask_size(void)
134 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
135 return ARRAY_SIZE(kvm_s390_fac_list_mask);
138 /* available cpu features supported by kvm */
139 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
140 /* available subfunctions indicated via query / "test bit" */
141 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
143 static struct gmap_notifier gmap_notifier;
144 debug_info_t *kvm_s390_dbf;
146 /* Section: not file related */
147 int kvm_arch_hardware_enable(void)
149 /* every s390 is virtualization enabled ;-) */
153 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
156 * This callback is executed during stop_machine(). All CPUs are therefore
157 * temporarily stopped. In order not to change guest behavior, we have to
158 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
159 * so a CPU won't be stopped while calculating with the epoch.
161 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
165 struct kvm_vcpu *vcpu;
167 unsigned long long *delta = v;
169 list_for_each_entry(kvm, &vm_list, vm_list) {
170 kvm->arch.epoch -= *delta;
171 kvm_for_each_vcpu(i, vcpu, kvm) {
172 vcpu->arch.sie_block->epoch -= *delta;
173 if (vcpu->arch.cputm_enabled)
174 vcpu->arch.cputm_start += *delta;
180 static struct notifier_block kvm_clock_notifier = {
181 .notifier_call = kvm_clock_sync,
184 int kvm_arch_hardware_setup(void)
186 gmap_notifier.notifier_call = kvm_gmap_notifier;
187 gmap_register_ipte_notifier(&gmap_notifier);
188 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
189 &kvm_clock_notifier);
193 void kvm_arch_hardware_unsetup(void)
195 gmap_unregister_ipte_notifier(&gmap_notifier);
196 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
197 &kvm_clock_notifier);
200 static void allow_cpu_feat(unsigned long nr)
202 set_bit_inv(nr, kvm_s390_available_cpu_feat);
205 static inline int plo_test_bit(unsigned char nr)
207 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
208 int cc = 3; /* subfunction not available */
211 /* Parameter registers are ignored for "test bit" */
221 static void kvm_s390_cpu_feat_init(void)
225 for (i = 0; i < 256; ++i) {
227 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
230 if (test_facility(28)) /* TOD-clock steering */
231 etr_ptff(kvm_s390_available_subfunc.ptff, ETR_PTFF_QAF);
233 if (test_facility(17)) { /* MSA */
234 __cpacf_query(CPACF_KMAC, kvm_s390_available_subfunc.kmac);
235 __cpacf_query(CPACF_KMC, kvm_s390_available_subfunc.kmc);
236 __cpacf_query(CPACF_KM, kvm_s390_available_subfunc.km);
237 __cpacf_query(CPACF_KIMD, kvm_s390_available_subfunc.kimd);
238 __cpacf_query(CPACF_KLMD, kvm_s390_available_subfunc.klmd);
240 if (test_facility(76)) /* MSA3 */
241 __cpacf_query(CPACF_PCKMO, kvm_s390_available_subfunc.pckmo);
242 if (test_facility(77)) { /* MSA4 */
243 __cpacf_query(CPACF_KMCTR, kvm_s390_available_subfunc.kmctr);
244 __cpacf_query(CPACF_KMF, kvm_s390_available_subfunc.kmf);
245 __cpacf_query(CPACF_KMO, kvm_s390_available_subfunc.kmo);
246 __cpacf_query(CPACF_PCC, kvm_s390_available_subfunc.pcc);
248 if (test_facility(57)) /* MSA5 */
249 __cpacf_query(CPACF_PPNO, kvm_s390_available_subfunc.ppno);
251 if (MACHINE_HAS_ESOP)
252 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
255 int kvm_arch_init(void *opaque)
257 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
261 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
262 debug_unregister(kvm_s390_dbf);
266 kvm_s390_cpu_feat_init();
268 /* Register floating interrupt controller interface. */
269 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
272 void kvm_arch_exit(void)
274 debug_unregister(kvm_s390_dbf);
277 /* Section: device related */
278 long kvm_arch_dev_ioctl(struct file *filp,
279 unsigned int ioctl, unsigned long arg)
281 if (ioctl == KVM_S390_ENABLE_SIE)
282 return s390_enable_sie();
286 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
291 case KVM_CAP_S390_PSW:
292 case KVM_CAP_S390_GMAP:
293 case KVM_CAP_SYNC_MMU:
294 #ifdef CONFIG_KVM_S390_UCONTROL
295 case KVM_CAP_S390_UCONTROL:
297 case KVM_CAP_ASYNC_PF:
298 case KVM_CAP_SYNC_REGS:
299 case KVM_CAP_ONE_REG:
300 case KVM_CAP_ENABLE_CAP:
301 case KVM_CAP_S390_CSS_SUPPORT:
302 case KVM_CAP_IOEVENTFD:
303 case KVM_CAP_DEVICE_CTRL:
304 case KVM_CAP_ENABLE_CAP_VM:
305 case KVM_CAP_S390_IRQCHIP:
306 case KVM_CAP_VM_ATTRIBUTES:
307 case KVM_CAP_MP_STATE:
308 case KVM_CAP_S390_INJECT_IRQ:
309 case KVM_CAP_S390_USER_SIGP:
310 case KVM_CAP_S390_USER_STSI:
311 case KVM_CAP_S390_SKEYS:
312 case KVM_CAP_S390_IRQ_STATE:
315 case KVM_CAP_S390_MEM_OP:
318 case KVM_CAP_NR_VCPUS:
319 case KVM_CAP_MAX_VCPUS:
320 r = KVM_S390_BSCA_CPU_SLOTS;
321 if (sclp.has_esca && sclp.has_64bscao)
322 r = KVM_S390_ESCA_CPU_SLOTS;
324 case KVM_CAP_NR_MEMSLOTS:
325 r = KVM_USER_MEM_SLOTS;
327 case KVM_CAP_S390_COW:
328 r = MACHINE_HAS_ESOP;
330 case KVM_CAP_S390_VECTOR_REGISTERS:
333 case KVM_CAP_S390_RI:
334 r = test_facility(64);
342 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
343 struct kvm_memory_slot *memslot)
345 gfn_t cur_gfn, last_gfn;
346 unsigned long address;
347 struct gmap *gmap = kvm->arch.gmap;
349 /* Loop over all guest pages */
350 last_gfn = memslot->base_gfn + memslot->npages;
351 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
352 address = gfn_to_hva_memslot(memslot, cur_gfn);
354 if (test_and_clear_guest_dirty(gmap->mm, address))
355 mark_page_dirty(kvm, cur_gfn);
356 if (fatal_signal_pending(current))
362 /* Section: vm related */
363 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
366 * Get (and clear) the dirty memory log for a memory slot.
368 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
369 struct kvm_dirty_log *log)
373 struct kvm_memslots *slots;
374 struct kvm_memory_slot *memslot;
377 mutex_lock(&kvm->slots_lock);
380 if (log->slot >= KVM_USER_MEM_SLOTS)
383 slots = kvm_memslots(kvm);
384 memslot = id_to_memslot(slots, log->slot);
386 if (!memslot->dirty_bitmap)
389 kvm_s390_sync_dirty_log(kvm, memslot);
390 r = kvm_get_dirty_log(kvm, log, &is_dirty);
394 /* Clear the dirty log */
396 n = kvm_dirty_bitmap_bytes(memslot);
397 memset(memslot->dirty_bitmap, 0, n);
401 mutex_unlock(&kvm->slots_lock);
405 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
413 case KVM_CAP_S390_IRQCHIP:
414 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
415 kvm->arch.use_irqchip = 1;
418 case KVM_CAP_S390_USER_SIGP:
419 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
420 kvm->arch.user_sigp = 1;
423 case KVM_CAP_S390_VECTOR_REGISTERS:
424 mutex_lock(&kvm->lock);
425 if (atomic_read(&kvm->online_vcpus)) {
427 } else if (MACHINE_HAS_VX) {
428 set_kvm_facility(kvm->arch.model.fac_mask, 129);
429 set_kvm_facility(kvm->arch.model.fac_list, 129);
433 mutex_unlock(&kvm->lock);
434 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
435 r ? "(not available)" : "(success)");
437 case KVM_CAP_S390_RI:
439 mutex_lock(&kvm->lock);
440 if (atomic_read(&kvm->online_vcpus)) {
442 } else if (test_facility(64)) {
443 set_kvm_facility(kvm->arch.model.fac_mask, 64);
444 set_kvm_facility(kvm->arch.model.fac_list, 64);
447 mutex_unlock(&kvm->lock);
448 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
449 r ? "(not available)" : "(success)");
451 case KVM_CAP_S390_USER_STSI:
452 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
453 kvm->arch.user_stsi = 1;
463 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
467 switch (attr->attr) {
468 case KVM_S390_VM_MEM_LIMIT_SIZE:
470 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
471 kvm->arch.mem_limit);
472 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
482 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
486 switch (attr->attr) {
487 case KVM_S390_VM_MEM_ENABLE_CMMA:
493 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
494 mutex_lock(&kvm->lock);
495 if (atomic_read(&kvm->online_vcpus) == 0) {
496 kvm->arch.use_cmma = 1;
499 mutex_unlock(&kvm->lock);
501 case KVM_S390_VM_MEM_CLR_CMMA:
506 if (!kvm->arch.use_cmma)
509 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
510 mutex_lock(&kvm->lock);
511 idx = srcu_read_lock(&kvm->srcu);
512 s390_reset_cmma(kvm->arch.gmap->mm);
513 srcu_read_unlock(&kvm->srcu, idx);
514 mutex_unlock(&kvm->lock);
517 case KVM_S390_VM_MEM_LIMIT_SIZE: {
518 unsigned long new_limit;
520 if (kvm_is_ucontrol(kvm))
523 if (get_user(new_limit, (u64 __user *)attr->addr))
526 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
527 new_limit > kvm->arch.mem_limit)
533 /* gmap_alloc takes last usable address */
534 if (new_limit != KVM_S390_NO_MEM_LIMIT)
538 mutex_lock(&kvm->lock);
539 if (atomic_read(&kvm->online_vcpus) == 0) {
540 /* gmap_alloc will round the limit up */
541 struct gmap *new = gmap_alloc(current->mm, new_limit);
546 gmap_free(kvm->arch.gmap);
548 kvm->arch.gmap = new;
552 mutex_unlock(&kvm->lock);
553 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
554 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
555 (void *) kvm->arch.gmap->asce);
565 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
567 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
569 struct kvm_vcpu *vcpu;
572 if (!test_kvm_facility(kvm, 76))
575 mutex_lock(&kvm->lock);
576 switch (attr->attr) {
577 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
579 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
580 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
581 kvm->arch.crypto.aes_kw = 1;
582 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
584 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
586 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
587 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
588 kvm->arch.crypto.dea_kw = 1;
589 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
591 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
592 kvm->arch.crypto.aes_kw = 0;
593 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
594 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
595 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
597 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
598 kvm->arch.crypto.dea_kw = 0;
599 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
600 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
601 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
604 mutex_unlock(&kvm->lock);
608 kvm_for_each_vcpu(i, vcpu, kvm) {
609 kvm_s390_vcpu_crypto_setup(vcpu);
612 mutex_unlock(&kvm->lock);
616 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
620 if (copy_from_user(>od_high, (void __user *)attr->addr,
626 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
631 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
635 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod)))
638 kvm_s390_set_tod_clock(kvm, gtod);
639 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
643 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
650 switch (attr->attr) {
651 case KVM_S390_VM_TOD_HIGH:
652 ret = kvm_s390_set_tod_high(kvm, attr);
654 case KVM_S390_VM_TOD_LOW:
655 ret = kvm_s390_set_tod_low(kvm, attr);
664 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
668 if (copy_to_user((void __user *)attr->addr, >od_high,
671 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
676 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
680 gtod = kvm_s390_get_tod_clock_fast(kvm);
681 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
683 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
688 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
695 switch (attr->attr) {
696 case KVM_S390_VM_TOD_HIGH:
697 ret = kvm_s390_get_tod_high(kvm, attr);
699 case KVM_S390_VM_TOD_LOW:
700 ret = kvm_s390_get_tod_low(kvm, attr);
709 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
711 struct kvm_s390_vm_cpu_processor *proc;
712 u16 lowest_ibc, unblocked_ibc;
715 mutex_lock(&kvm->lock);
716 if (atomic_read(&kvm->online_vcpus)) {
720 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
725 if (!copy_from_user(proc, (void __user *)attr->addr,
727 kvm->arch.model.cpuid = proc->cpuid;
728 lowest_ibc = sclp.ibc >> 16 & 0xfff;
729 unblocked_ibc = sclp.ibc & 0xfff;
731 if (proc->ibc > unblocked_ibc)
732 kvm->arch.model.ibc = unblocked_ibc;
733 else if (proc->ibc < lowest_ibc)
734 kvm->arch.model.ibc = lowest_ibc;
736 kvm->arch.model.ibc = proc->ibc;
738 memcpy(kvm->arch.model.fac_list, proc->fac_list,
739 S390_ARCH_FAC_LIST_SIZE_BYTE);
744 mutex_unlock(&kvm->lock);
748 static int kvm_s390_set_processor_feat(struct kvm *kvm,
749 struct kvm_device_attr *attr)
751 struct kvm_s390_vm_cpu_feat data;
754 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
756 if (!bitmap_subset((unsigned long *) data.feat,
757 kvm_s390_available_cpu_feat,
758 KVM_S390_VM_CPU_FEAT_NR_BITS))
761 mutex_lock(&kvm->lock);
762 if (!atomic_read(&kvm->online_vcpus)) {
763 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
764 KVM_S390_VM_CPU_FEAT_NR_BITS);
767 mutex_unlock(&kvm->lock);
771 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
772 struct kvm_device_attr *attr)
775 * Once supported by kernel + hw, we have to store the subfunctions
776 * in kvm->arch and remember that user space configured them.
781 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
785 switch (attr->attr) {
786 case KVM_S390_VM_CPU_PROCESSOR:
787 ret = kvm_s390_set_processor(kvm, attr);
789 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
790 ret = kvm_s390_set_processor_feat(kvm, attr);
792 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
793 ret = kvm_s390_set_processor_subfunc(kvm, attr);
799 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
801 struct kvm_s390_vm_cpu_processor *proc;
804 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
809 proc->cpuid = kvm->arch.model.cpuid;
810 proc->ibc = kvm->arch.model.ibc;
811 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
812 S390_ARCH_FAC_LIST_SIZE_BYTE);
813 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
820 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
822 struct kvm_s390_vm_cpu_machine *mach;
825 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
830 get_cpu_id((struct cpuid *) &mach->cpuid);
831 mach->ibc = sclp.ibc;
832 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
833 S390_ARCH_FAC_LIST_SIZE_BYTE);
834 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
835 S390_ARCH_FAC_LIST_SIZE_BYTE);
836 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
843 static int kvm_s390_get_processor_feat(struct kvm *kvm,
844 struct kvm_device_attr *attr)
846 struct kvm_s390_vm_cpu_feat data;
848 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
849 KVM_S390_VM_CPU_FEAT_NR_BITS);
850 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
855 static int kvm_s390_get_machine_feat(struct kvm *kvm,
856 struct kvm_device_attr *attr)
858 struct kvm_s390_vm_cpu_feat data;
860 bitmap_copy((unsigned long *) data.feat,
861 kvm_s390_available_cpu_feat,
862 KVM_S390_VM_CPU_FEAT_NR_BITS);
863 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
868 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
869 struct kvm_device_attr *attr)
872 * Once we can actually configure subfunctions (kernel + hw support),
873 * we have to check if they were already set by user space, if so copy
874 * them from kvm->arch.
879 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
880 struct kvm_device_attr *attr)
882 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
883 sizeof(struct kvm_s390_vm_cpu_subfunc)))
887 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
891 switch (attr->attr) {
892 case KVM_S390_VM_CPU_PROCESSOR:
893 ret = kvm_s390_get_processor(kvm, attr);
895 case KVM_S390_VM_CPU_MACHINE:
896 ret = kvm_s390_get_machine(kvm, attr);
898 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
899 ret = kvm_s390_get_processor_feat(kvm, attr);
901 case KVM_S390_VM_CPU_MACHINE_FEAT:
902 ret = kvm_s390_get_machine_feat(kvm, attr);
904 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
905 ret = kvm_s390_get_processor_subfunc(kvm, attr);
907 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
908 ret = kvm_s390_get_machine_subfunc(kvm, attr);
914 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
918 switch (attr->group) {
919 case KVM_S390_VM_MEM_CTRL:
920 ret = kvm_s390_set_mem_control(kvm, attr);
922 case KVM_S390_VM_TOD:
923 ret = kvm_s390_set_tod(kvm, attr);
925 case KVM_S390_VM_CPU_MODEL:
926 ret = kvm_s390_set_cpu_model(kvm, attr);
928 case KVM_S390_VM_CRYPTO:
929 ret = kvm_s390_vm_set_crypto(kvm, attr);
939 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
943 switch (attr->group) {
944 case KVM_S390_VM_MEM_CTRL:
945 ret = kvm_s390_get_mem_control(kvm, attr);
947 case KVM_S390_VM_TOD:
948 ret = kvm_s390_get_tod(kvm, attr);
950 case KVM_S390_VM_CPU_MODEL:
951 ret = kvm_s390_get_cpu_model(kvm, attr);
961 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
965 switch (attr->group) {
966 case KVM_S390_VM_MEM_CTRL:
967 switch (attr->attr) {
968 case KVM_S390_VM_MEM_ENABLE_CMMA:
969 case KVM_S390_VM_MEM_CLR_CMMA:
970 ret = sclp.has_cmma ? 0 : -ENXIO;
972 case KVM_S390_VM_MEM_LIMIT_SIZE:
980 case KVM_S390_VM_TOD:
981 switch (attr->attr) {
982 case KVM_S390_VM_TOD_LOW:
983 case KVM_S390_VM_TOD_HIGH:
991 case KVM_S390_VM_CPU_MODEL:
992 switch (attr->attr) {
993 case KVM_S390_VM_CPU_PROCESSOR:
994 case KVM_S390_VM_CPU_MACHINE:
995 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
996 case KVM_S390_VM_CPU_MACHINE_FEAT:
997 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1000 /* configuring subfunctions is not supported yet */
1001 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1007 case KVM_S390_VM_CRYPTO:
1008 switch (attr->attr) {
1009 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1010 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1011 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1012 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1028 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1032 unsigned long curkey;
1035 if (args->flags != 0)
1038 /* Is this guest using storage keys? */
1039 if (!mm_use_skey(current->mm))
1040 return KVM_S390_GET_SKEYS_NONE;
1042 /* Enforce sane limit on memory allocation */
1043 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1046 keys = kmalloc_array(args->count, sizeof(uint8_t),
1047 GFP_KERNEL | __GFP_NOWARN);
1049 keys = vmalloc(sizeof(uint8_t) * args->count);
1053 for (i = 0; i < args->count; i++) {
1054 hva = gfn_to_hva(kvm, args->start_gfn + i);
1055 if (kvm_is_error_hva(hva)) {
1060 curkey = get_guest_storage_key(current->mm, hva);
1061 if (IS_ERR_VALUE(curkey)) {
1068 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1069 sizeof(uint8_t) * args->count);
1077 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1083 if (args->flags != 0)
1086 /* Enforce sane limit on memory allocation */
1087 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1090 keys = kmalloc_array(args->count, sizeof(uint8_t),
1091 GFP_KERNEL | __GFP_NOWARN);
1093 keys = vmalloc(sizeof(uint8_t) * args->count);
1097 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1098 sizeof(uint8_t) * args->count);
1104 /* Enable storage key handling for the guest */
1105 r = s390_enable_skey();
1109 for (i = 0; i < args->count; i++) {
1110 hva = gfn_to_hva(kvm, args->start_gfn + i);
1111 if (kvm_is_error_hva(hva)) {
1116 /* Lowest order bit is reserved */
1117 if (keys[i] & 0x01) {
1122 r = set_guest_storage_key(current->mm, hva,
1123 (unsigned long)keys[i], 0);
1132 long kvm_arch_vm_ioctl(struct file *filp,
1133 unsigned int ioctl, unsigned long arg)
1135 struct kvm *kvm = filp->private_data;
1136 void __user *argp = (void __user *)arg;
1137 struct kvm_device_attr attr;
1141 case KVM_S390_INTERRUPT: {
1142 struct kvm_s390_interrupt s390int;
1145 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1147 r = kvm_s390_inject_vm(kvm, &s390int);
1150 case KVM_ENABLE_CAP: {
1151 struct kvm_enable_cap cap;
1153 if (copy_from_user(&cap, argp, sizeof(cap)))
1155 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1158 case KVM_CREATE_IRQCHIP: {
1159 struct kvm_irq_routing_entry routing;
1162 if (kvm->arch.use_irqchip) {
1163 /* Set up dummy routing. */
1164 memset(&routing, 0, sizeof(routing));
1165 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
1169 case KVM_SET_DEVICE_ATTR: {
1171 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1173 r = kvm_s390_vm_set_attr(kvm, &attr);
1176 case KVM_GET_DEVICE_ATTR: {
1178 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1180 r = kvm_s390_vm_get_attr(kvm, &attr);
1183 case KVM_HAS_DEVICE_ATTR: {
1185 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1187 r = kvm_s390_vm_has_attr(kvm, &attr);
1190 case KVM_S390_GET_SKEYS: {
1191 struct kvm_s390_skeys args;
1194 if (copy_from_user(&args, argp,
1195 sizeof(struct kvm_s390_skeys)))
1197 r = kvm_s390_get_skeys(kvm, &args);
1200 case KVM_S390_SET_SKEYS: {
1201 struct kvm_s390_skeys args;
1204 if (copy_from_user(&args, argp,
1205 sizeof(struct kvm_s390_skeys)))
1207 r = kvm_s390_set_skeys(kvm, &args);
1217 static int kvm_s390_query_ap_config(u8 *config)
1219 u32 fcn_code = 0x04000000UL;
1222 memset(config, 0, 128);
1226 ".long 0xb2af0000\n" /* PQAP(QCI) */
1232 : "r" (fcn_code), "r" (config)
1233 : "cc", "0", "2", "memory"
1239 static int kvm_s390_apxa_installed(void)
1244 if (test_facility(12)) {
1245 cc = kvm_s390_query_ap_config(config);
1248 pr_err("PQAP(QCI) failed with cc=%d", cc);
1250 return config[0] & 0x40;
1256 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1258 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1260 if (kvm_s390_apxa_installed())
1261 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1263 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1266 static u64 kvm_s390_get_initial_cpuid(void)
1271 cpuid.version = 0xff;
1272 return *((u64 *) &cpuid);
1275 static void kvm_s390_crypto_init(struct kvm *kvm)
1277 if (!test_kvm_facility(kvm, 76))
1280 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
1281 kvm_s390_set_crycb_format(kvm);
1283 /* Enable AES/DEA protected key functions by default */
1284 kvm->arch.crypto.aes_kw = 1;
1285 kvm->arch.crypto.dea_kw = 1;
1286 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1287 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1288 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1289 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1292 static void sca_dispose(struct kvm *kvm)
1294 if (kvm->arch.use_esca)
1295 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1297 free_page((unsigned long)(kvm->arch.sca));
1298 kvm->arch.sca = NULL;
1301 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1303 gfp_t alloc_flags = GFP_KERNEL;
1305 char debug_name[16];
1306 static unsigned long sca_offset;
1309 #ifdef CONFIG_KVM_S390_UCONTROL
1310 if (type & ~KVM_VM_S390_UCONTROL)
1312 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1319 rc = s390_enable_sie();
1325 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1327 kvm->arch.use_esca = 0; /* start with basic SCA */
1328 if (!sclp.has_64bscao)
1329 alloc_flags |= GFP_DMA;
1330 rwlock_init(&kvm->arch.sca_lock);
1331 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
1334 spin_lock(&kvm_lock);
1336 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1338 kvm->arch.sca = (struct bsca_block *)
1339 ((char *) kvm->arch.sca + sca_offset);
1340 spin_unlock(&kvm_lock);
1342 sprintf(debug_name, "kvm-%u", current->pid);
1344 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1348 kvm->arch.sie_page2 =
1349 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1350 if (!kvm->arch.sie_page2)
1353 /* Populate the facility mask initially. */
1354 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
1355 S390_ARCH_FAC_LIST_SIZE_BYTE);
1356 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1357 if (i < kvm_s390_fac_list_mask_size())
1358 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
1360 kvm->arch.model.fac_mask[i] = 0UL;
1363 /* Populate the facility list initially. */
1364 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1365 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
1366 S390_ARCH_FAC_LIST_SIZE_BYTE);
1368 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1369 set_kvm_facility(kvm->arch.model.fac_list, 74);
1371 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
1372 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1374 kvm_s390_crypto_init(kvm);
1376 spin_lock_init(&kvm->arch.float_int.lock);
1377 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1378 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1379 init_waitqueue_head(&kvm->arch.ipte_wq);
1380 mutex_init(&kvm->arch.ipte_mutex);
1382 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1383 VM_EVENT(kvm, 3, "vm created with type %lu", type);
1385 if (type & KVM_VM_S390_UCONTROL) {
1386 kvm->arch.gmap = NULL;
1387 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1389 if (sclp.hamax == U64_MAX)
1390 kvm->arch.mem_limit = TASK_MAX_SIZE;
1392 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1394 kvm->arch.gmap = gmap_alloc(current->mm, kvm->arch.mem_limit - 1);
1395 if (!kvm->arch.gmap)
1397 kvm->arch.gmap->private = kvm;
1398 kvm->arch.gmap->pfault_enabled = 0;
1401 kvm->arch.css_support = 0;
1402 kvm->arch.use_irqchip = 0;
1403 kvm->arch.epoch = 0;
1405 spin_lock_init(&kvm->arch.start_stop_lock);
1406 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1410 free_page((unsigned long)kvm->arch.sie_page2);
1411 debug_unregister(kvm->arch.dbf);
1413 KVM_EVENT(3, "creation of vm failed: %d", rc);
1417 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1419 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1420 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1421 kvm_s390_clear_local_irqs(vcpu);
1422 kvm_clear_async_pf_completion_queue(vcpu);
1423 if (!kvm_is_ucontrol(vcpu->kvm))
1426 if (kvm_is_ucontrol(vcpu->kvm))
1427 gmap_free(vcpu->arch.gmap);
1429 if (vcpu->kvm->arch.use_cmma)
1430 kvm_s390_vcpu_unsetup_cmma(vcpu);
1431 free_page((unsigned long)(vcpu->arch.sie_block));
1433 kvm_vcpu_uninit(vcpu);
1434 kmem_cache_free(kvm_vcpu_cache, vcpu);
1437 static void kvm_free_vcpus(struct kvm *kvm)
1440 struct kvm_vcpu *vcpu;
1442 kvm_for_each_vcpu(i, vcpu, kvm)
1443 kvm_arch_vcpu_destroy(vcpu);
1445 mutex_lock(&kvm->lock);
1446 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1447 kvm->vcpus[i] = NULL;
1449 atomic_set(&kvm->online_vcpus, 0);
1450 mutex_unlock(&kvm->lock);
1453 void kvm_arch_destroy_vm(struct kvm *kvm)
1455 kvm_free_vcpus(kvm);
1457 debug_unregister(kvm->arch.dbf);
1458 free_page((unsigned long)kvm->arch.sie_page2);
1459 if (!kvm_is_ucontrol(kvm))
1460 gmap_free(kvm->arch.gmap);
1461 kvm_s390_destroy_adapters(kvm);
1462 kvm_s390_clear_float_irqs(kvm);
1463 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
1466 /* Section: vcpu related */
1467 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1469 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1470 if (!vcpu->arch.gmap)
1472 vcpu->arch.gmap->private = vcpu->kvm;
1477 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1479 read_lock(&vcpu->kvm->arch.sca_lock);
1480 if (vcpu->kvm->arch.use_esca) {
1481 struct esca_block *sca = vcpu->kvm->arch.sca;
1483 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1484 sca->cpu[vcpu->vcpu_id].sda = 0;
1486 struct bsca_block *sca = vcpu->kvm->arch.sca;
1488 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1489 sca->cpu[vcpu->vcpu_id].sda = 0;
1491 read_unlock(&vcpu->kvm->arch.sca_lock);
1494 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
1496 read_lock(&vcpu->kvm->arch.sca_lock);
1497 if (vcpu->kvm->arch.use_esca) {
1498 struct esca_block *sca = vcpu->kvm->arch.sca;
1500 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1501 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1502 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
1503 vcpu->arch.sie_block->ecb2 |= 0x04U;
1504 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1506 struct bsca_block *sca = vcpu->kvm->arch.sca;
1508 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1509 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1510 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1511 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1513 read_unlock(&vcpu->kvm->arch.sca_lock);
1516 /* Basic SCA to Extended SCA data copy routines */
1517 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1520 d->sigp_ctrl.c = s->sigp_ctrl.c;
1521 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1524 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1528 d->ipte_control = s->ipte_control;
1530 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1531 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1534 static int sca_switch_to_extended(struct kvm *kvm)
1536 struct bsca_block *old_sca = kvm->arch.sca;
1537 struct esca_block *new_sca;
1538 struct kvm_vcpu *vcpu;
1539 unsigned int vcpu_idx;
1542 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1546 scaoh = (u32)((u64)(new_sca) >> 32);
1547 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1549 kvm_s390_vcpu_block_all(kvm);
1550 write_lock(&kvm->arch.sca_lock);
1552 sca_copy_b_to_e(new_sca, old_sca);
1554 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1555 vcpu->arch.sie_block->scaoh = scaoh;
1556 vcpu->arch.sie_block->scaol = scaol;
1557 vcpu->arch.sie_block->ecb2 |= 0x04U;
1559 kvm->arch.sca = new_sca;
1560 kvm->arch.use_esca = 1;
1562 write_unlock(&kvm->arch.sca_lock);
1563 kvm_s390_vcpu_unblock_all(kvm);
1565 free_page((unsigned long)old_sca);
1567 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1568 old_sca, kvm->arch.sca);
1572 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1576 if (id < KVM_S390_BSCA_CPU_SLOTS)
1578 if (!sclp.has_esca || !sclp.has_64bscao)
1581 mutex_lock(&kvm->lock);
1582 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1583 mutex_unlock(&kvm->lock);
1585 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
1588 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1590 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1591 kvm_clear_async_pf_completion_queue(vcpu);
1592 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1598 if (test_kvm_facility(vcpu->kvm, 64))
1599 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
1600 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1601 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1604 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1606 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
1608 if (kvm_is_ucontrol(vcpu->kvm))
1609 return __kvm_ucontrol_vcpu_init(vcpu);
1614 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1615 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1617 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
1618 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1619 vcpu->arch.cputm_start = get_tod_clock_fast();
1620 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1623 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1624 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1626 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
1627 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1628 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1629 vcpu->arch.cputm_start = 0;
1630 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1633 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1634 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1636 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1637 vcpu->arch.cputm_enabled = true;
1638 __start_cpu_timer_accounting(vcpu);
1641 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1642 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1644 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1645 __stop_cpu_timer_accounting(vcpu);
1646 vcpu->arch.cputm_enabled = false;
1649 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1651 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1652 __enable_cpu_timer_accounting(vcpu);
1656 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1658 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1659 __disable_cpu_timer_accounting(vcpu);
1663 /* set the cpu timer - may only be called from the VCPU thread itself */
1664 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1666 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1667 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1668 if (vcpu->arch.cputm_enabled)
1669 vcpu->arch.cputm_start = get_tod_clock_fast();
1670 vcpu->arch.sie_block->cputm = cputm;
1671 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1675 /* update and get the cpu timer - can also be called from other VCPU threads */
1676 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1681 if (unlikely(!vcpu->arch.cputm_enabled))
1682 return vcpu->arch.sie_block->cputm;
1684 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1686 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1688 * If the writer would ever execute a read in the critical
1689 * section, e.g. in irq context, we have a deadlock.
1691 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1692 value = vcpu->arch.sie_block->cputm;
1693 /* if cputm_start is 0, accounting is being started/stopped */
1694 if (likely(vcpu->arch.cputm_start))
1695 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1696 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1701 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1703 /* Save host register state */
1705 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1706 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
1709 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1711 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
1712 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1713 if (test_fp_ctl(current->thread.fpu.fpc))
1714 /* User space provided an invalid FPC, let's clear it */
1715 current->thread.fpu.fpc = 0;
1717 save_access_regs(vcpu->arch.host_acrs);
1718 restore_access_regs(vcpu->run->s.regs.acrs);
1719 gmap_enable(vcpu->arch.gmap);
1720 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1721 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1722 __start_cpu_timer_accounting(vcpu);
1726 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1729 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1730 __stop_cpu_timer_accounting(vcpu);
1731 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1732 gmap_disable(vcpu->arch.gmap);
1734 /* Save guest register state */
1736 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1738 /* Restore host register state */
1739 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1740 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
1742 save_access_regs(vcpu->run->s.regs.acrs);
1743 restore_access_regs(vcpu->arch.host_acrs);
1746 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1748 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1749 vcpu->arch.sie_block->gpsw.mask = 0UL;
1750 vcpu->arch.sie_block->gpsw.addr = 0UL;
1751 kvm_s390_set_prefix(vcpu, 0);
1752 kvm_s390_set_cpu_timer(vcpu, 0);
1753 vcpu->arch.sie_block->ckc = 0UL;
1754 vcpu->arch.sie_block->todpr = 0;
1755 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1756 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1757 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1758 /* make sure the new fpc will be lazily loaded */
1760 current->thread.fpu.fpc = 0;
1761 vcpu->arch.sie_block->gbea = 1;
1762 vcpu->arch.sie_block->pp = 0;
1763 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1764 kvm_clear_async_pf_completion_queue(vcpu);
1765 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1766 kvm_s390_vcpu_stop(vcpu);
1767 kvm_s390_clear_local_irqs(vcpu);
1770 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1772 mutex_lock(&vcpu->kvm->lock);
1774 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1776 mutex_unlock(&vcpu->kvm->lock);
1777 if (!kvm_is_ucontrol(vcpu->kvm)) {
1778 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1784 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1786 if (!test_kvm_facility(vcpu->kvm, 76))
1789 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1791 if (vcpu->kvm->arch.crypto.aes_kw)
1792 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1793 if (vcpu->kvm->arch.crypto.dea_kw)
1794 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1796 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1799 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1801 free_page(vcpu->arch.sie_block->cbrlo);
1802 vcpu->arch.sie_block->cbrlo = 0;
1805 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1807 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1808 if (!vcpu->arch.sie_block->cbrlo)
1811 vcpu->arch.sie_block->ecb2 |= 0x80;
1812 vcpu->arch.sie_block->ecb2 &= ~0x08;
1816 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1818 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1820 vcpu->arch.sie_block->ibc = model->ibc;
1821 if (test_kvm_facility(vcpu->kvm, 7))
1822 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
1825 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1829 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1833 if (test_kvm_facility(vcpu->kvm, 78))
1834 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1835 else if (test_kvm_facility(vcpu->kvm, 8))
1836 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1838 kvm_s390_vcpu_setup_model(vcpu);
1840 vcpu->arch.sie_block->ecb = 0x02;
1841 if (test_kvm_facility(vcpu->kvm, 9))
1842 vcpu->arch.sie_block->ecb |= 0x04;
1843 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1844 vcpu->arch.sie_block->ecb |= 0x10;
1846 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
1847 vcpu->arch.sie_block->ecb2 |= 0x08;
1848 vcpu->arch.sie_block->eca = 0x1002000U;
1850 vcpu->arch.sie_block->eca |= 0x80000000U;
1852 vcpu->arch.sie_block->eca |= 0x40000000U;
1854 vcpu->arch.sie_block->eca |= 1;
1855 if (sclp.has_sigpif)
1856 vcpu->arch.sie_block->eca |= 0x10000000U;
1857 if (test_kvm_facility(vcpu->kvm, 64))
1858 vcpu->arch.sie_block->ecb3 |= 0x01;
1859 if (test_kvm_facility(vcpu->kvm, 129)) {
1860 vcpu->arch.sie_block->eca |= 0x00020000;
1861 vcpu->arch.sie_block->ecd |= 0x20000000;
1863 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
1864 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1865 if (test_kvm_facility(vcpu->kvm, 74))
1866 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
1868 if (vcpu->kvm->arch.use_cmma) {
1869 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1873 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1874 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1876 kvm_s390_vcpu_crypto_setup(vcpu);
1881 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1884 struct kvm_vcpu *vcpu;
1885 struct sie_page *sie_page;
1888 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
1893 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1897 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1901 vcpu->arch.sie_block = &sie_page->sie_block;
1902 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1904 /* the real guest size will always be smaller than msl */
1905 vcpu->arch.sie_block->mso = 0;
1906 vcpu->arch.sie_block->msl = sclp.hamax;
1908 vcpu->arch.sie_block->icpua = id;
1909 spin_lock_init(&vcpu->arch.local_int.lock);
1910 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1911 vcpu->arch.local_int.wq = &vcpu->wq;
1912 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1913 seqcount_init(&vcpu->arch.cputm_seqcount);
1915 rc = kvm_vcpu_init(vcpu, kvm, id);
1917 goto out_free_sie_block;
1918 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
1919 vcpu->arch.sie_block);
1920 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1924 free_page((unsigned long)(vcpu->arch.sie_block));
1926 kmem_cache_free(kvm_vcpu_cache, vcpu);
1931 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1933 return kvm_s390_vcpu_has_irq(vcpu, 0);
1936 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1938 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1942 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1944 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1947 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1949 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1953 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1955 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1959 * Kick a guest cpu out of SIE and wait until SIE is not running.
1960 * If the CPU is not running (e.g. waiting as idle) the function will
1961 * return immediately. */
1962 void exit_sie(struct kvm_vcpu *vcpu)
1964 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1965 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1969 /* Kick a guest cpu out of SIE to process a request synchronously */
1970 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1972 kvm_make_request(req, vcpu);
1973 kvm_s390_vcpu_request(vcpu);
1976 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1979 struct kvm *kvm = gmap->private;
1980 struct kvm_vcpu *vcpu;
1982 kvm_for_each_vcpu(i, vcpu, kvm) {
1983 /* match against both prefix pages */
1984 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1985 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1986 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1991 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1993 /* kvm common code refers to this, but never calls it */
1998 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1999 struct kvm_one_reg *reg)
2004 case KVM_REG_S390_TODPR:
2005 r = put_user(vcpu->arch.sie_block->todpr,
2006 (u32 __user *)reg->addr);
2008 case KVM_REG_S390_EPOCHDIFF:
2009 r = put_user(vcpu->arch.sie_block->epoch,
2010 (u64 __user *)reg->addr);
2012 case KVM_REG_S390_CPU_TIMER:
2013 r = put_user(kvm_s390_get_cpu_timer(vcpu),
2014 (u64 __user *)reg->addr);
2016 case KVM_REG_S390_CLOCK_COMP:
2017 r = put_user(vcpu->arch.sie_block->ckc,
2018 (u64 __user *)reg->addr);
2020 case KVM_REG_S390_PFTOKEN:
2021 r = put_user(vcpu->arch.pfault_token,
2022 (u64 __user *)reg->addr);
2024 case KVM_REG_S390_PFCOMPARE:
2025 r = put_user(vcpu->arch.pfault_compare,
2026 (u64 __user *)reg->addr);
2028 case KVM_REG_S390_PFSELECT:
2029 r = put_user(vcpu->arch.pfault_select,
2030 (u64 __user *)reg->addr);
2032 case KVM_REG_S390_PP:
2033 r = put_user(vcpu->arch.sie_block->pp,
2034 (u64 __user *)reg->addr);
2036 case KVM_REG_S390_GBEA:
2037 r = put_user(vcpu->arch.sie_block->gbea,
2038 (u64 __user *)reg->addr);
2047 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2048 struct kvm_one_reg *reg)
2054 case KVM_REG_S390_TODPR:
2055 r = get_user(vcpu->arch.sie_block->todpr,
2056 (u32 __user *)reg->addr);
2058 case KVM_REG_S390_EPOCHDIFF:
2059 r = get_user(vcpu->arch.sie_block->epoch,
2060 (u64 __user *)reg->addr);
2062 case KVM_REG_S390_CPU_TIMER:
2063 r = get_user(val, (u64 __user *)reg->addr);
2065 kvm_s390_set_cpu_timer(vcpu, val);
2067 case KVM_REG_S390_CLOCK_COMP:
2068 r = get_user(vcpu->arch.sie_block->ckc,
2069 (u64 __user *)reg->addr);
2071 case KVM_REG_S390_PFTOKEN:
2072 r = get_user(vcpu->arch.pfault_token,
2073 (u64 __user *)reg->addr);
2074 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2075 kvm_clear_async_pf_completion_queue(vcpu);
2077 case KVM_REG_S390_PFCOMPARE:
2078 r = get_user(vcpu->arch.pfault_compare,
2079 (u64 __user *)reg->addr);
2081 case KVM_REG_S390_PFSELECT:
2082 r = get_user(vcpu->arch.pfault_select,
2083 (u64 __user *)reg->addr);
2085 case KVM_REG_S390_PP:
2086 r = get_user(vcpu->arch.sie_block->pp,
2087 (u64 __user *)reg->addr);
2089 case KVM_REG_S390_GBEA:
2090 r = get_user(vcpu->arch.sie_block->gbea,
2091 (u64 __user *)reg->addr);
2100 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2102 kvm_s390_vcpu_initial_reset(vcpu);
2106 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2108 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
2112 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2114 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
2118 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2119 struct kvm_sregs *sregs)
2121 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
2122 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
2123 restore_access_regs(vcpu->run->s.regs.acrs);
2127 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2128 struct kvm_sregs *sregs)
2130 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
2131 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
2135 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2137 /* make sure the new values will be lazily loaded */
2139 if (test_fp_ctl(fpu->fpc))
2141 current->thread.fpu.fpc = fpu->fpc;
2143 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
2145 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
2149 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2151 /* make sure we have the latest values */
2154 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
2156 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
2157 fpu->fpc = current->thread.fpu.fpc;
2161 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2165 if (!is_vcpu_stopped(vcpu))
2168 vcpu->run->psw_mask = psw.mask;
2169 vcpu->run->psw_addr = psw.addr;
2174 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2175 struct kvm_translation *tr)
2177 return -EINVAL; /* not implemented yet */
2180 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2181 KVM_GUESTDBG_USE_HW_BP | \
2182 KVM_GUESTDBG_ENABLE)
2184 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2185 struct kvm_guest_debug *dbg)
2189 vcpu->guest_debug = 0;
2190 kvm_s390_clear_bp_data(vcpu);
2192 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
2194 if (!sclp.has_gpere)
2197 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2198 vcpu->guest_debug = dbg->control;
2199 /* enforce guest PER */
2200 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2202 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2203 rc = kvm_s390_import_bp_data(vcpu, dbg);
2205 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2206 vcpu->arch.guestdbg.last_bp = 0;
2210 vcpu->guest_debug = 0;
2211 kvm_s390_clear_bp_data(vcpu);
2212 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2218 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2219 struct kvm_mp_state *mp_state)
2221 /* CHECK_STOP and LOAD are not supported yet */
2222 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2223 KVM_MP_STATE_OPERATING;
2226 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2227 struct kvm_mp_state *mp_state)
2231 /* user space knows about this interface - let it control the state */
2232 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2234 switch (mp_state->mp_state) {
2235 case KVM_MP_STATE_STOPPED:
2236 kvm_s390_vcpu_stop(vcpu);
2238 case KVM_MP_STATE_OPERATING:
2239 kvm_s390_vcpu_start(vcpu);
2241 case KVM_MP_STATE_LOAD:
2242 case KVM_MP_STATE_CHECK_STOP:
2243 /* fall through - CHECK_STOP and LOAD are not supported yet */
2251 static bool ibs_enabled(struct kvm_vcpu *vcpu)
2253 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2256 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2259 kvm_s390_vcpu_request_handled(vcpu);
2260 if (!vcpu->requests)
2263 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2264 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
2265 * This ensures that the ipte instruction for this request has
2266 * already finished. We might race against a second unmapper that
2267 * wants to set the blocking bit. Lets just retry the request loop.
2269 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2271 rc = gmap_ipte_notify(vcpu->arch.gmap,
2272 kvm_s390_get_prefix(vcpu),
2279 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2280 vcpu->arch.sie_block->ihcpu = 0xffff;
2284 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2285 if (!ibs_enabled(vcpu)) {
2286 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
2287 atomic_or(CPUSTAT_IBS,
2288 &vcpu->arch.sie_block->cpuflags);
2293 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2294 if (ibs_enabled(vcpu)) {
2295 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2296 atomic_andnot(CPUSTAT_IBS,
2297 &vcpu->arch.sie_block->cpuflags);
2302 /* nothing to do, just clear the request */
2303 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2308 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2310 struct kvm_vcpu *vcpu;
2313 mutex_lock(&kvm->lock);
2315 kvm->arch.epoch = tod - get_tod_clock();
2316 kvm_s390_vcpu_block_all(kvm);
2317 kvm_for_each_vcpu(i, vcpu, kvm)
2318 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2319 kvm_s390_vcpu_unblock_all(kvm);
2321 mutex_unlock(&kvm->lock);
2325 * kvm_arch_fault_in_page - fault-in guest page if necessary
2326 * @vcpu: The corresponding virtual cpu
2327 * @gpa: Guest physical address
2328 * @writable: Whether the page should be writable or not
2330 * Make sure that a guest page has been faulted-in on the host.
2332 * Return: Zero on success, negative error code otherwise.
2334 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2336 return gmap_fault(vcpu->arch.gmap, gpa,
2337 writable ? FAULT_FLAG_WRITE : 0);
2340 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2341 unsigned long token)
2343 struct kvm_s390_interrupt inti;
2344 struct kvm_s390_irq irq;
2347 irq.u.ext.ext_params2 = token;
2348 irq.type = KVM_S390_INT_PFAULT_INIT;
2349 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
2351 inti.type = KVM_S390_INT_PFAULT_DONE;
2352 inti.parm64 = token;
2353 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2357 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2358 struct kvm_async_pf *work)
2360 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2361 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2364 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2365 struct kvm_async_pf *work)
2367 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2368 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2371 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2372 struct kvm_async_pf *work)
2374 /* s390 will always inject the page directly */
2377 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2380 * s390 will always inject the page directly,
2381 * but we still want check_async_completion to cleanup
2386 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2389 struct kvm_arch_async_pf arch;
2392 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2394 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2395 vcpu->arch.pfault_compare)
2397 if (psw_extint_disabled(vcpu))
2399 if (kvm_s390_vcpu_has_irq(vcpu, 0))
2401 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2403 if (!vcpu->arch.gmap->pfault_enabled)
2406 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2407 hva += current->thread.gmap_addr & ~PAGE_MASK;
2408 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
2411 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2415 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2420 * On s390 notifications for arriving pages will be delivered directly
2421 * to the guest but the house keeping for completed pfaults is
2422 * handled outside the worker.
2424 kvm_check_async_pf_completion(vcpu);
2426 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2427 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
2432 if (test_cpu_flag(CIF_MCCK_PENDING))
2435 if (!kvm_is_ucontrol(vcpu->kvm)) {
2436 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2441 rc = kvm_s390_handle_requests(vcpu);
2445 if (guestdbg_enabled(vcpu)) {
2446 kvm_s390_backup_guest_per_regs(vcpu);
2447 kvm_s390_patch_guest_per_regs(vcpu);
2450 vcpu->arch.sie_block->icptcode = 0;
2451 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2452 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2453 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2458 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2460 struct kvm_s390_pgm_info pgm_info = {
2461 .code = PGM_ADDRESSING,
2466 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2467 trace_kvm_s390_sie_fault(vcpu);
2470 * We want to inject an addressing exception, which is defined as a
2471 * suppressing or terminating exception. However, since we came here
2472 * by a DAT access exception, the PSW still points to the faulting
2473 * instruction since DAT exceptions are nullifying. So we've got
2474 * to look up the current opcode to get the length of the instruction
2475 * to be able to forward the PSW.
2477 rc = read_guest_instr(vcpu, &opcode, 1);
2478 ilen = insn_length(opcode);
2482 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2483 * Forward by arbitrary ilc, injection will take care of
2484 * nullification if necessary.
2486 pgm_info = vcpu->arch.pgm;
2489 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2490 kvm_s390_forward_psw(vcpu, ilen);
2491 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
2494 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2496 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2497 vcpu->arch.sie_block->icptcode);
2498 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2500 if (guestdbg_enabled(vcpu))
2501 kvm_s390_restore_guest_per_regs(vcpu);
2503 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2504 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
2506 if (vcpu->arch.sie_block->icptcode > 0) {
2507 int rc = kvm_handle_sie_intercept(vcpu);
2509 if (rc != -EOPNOTSUPP)
2511 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2512 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2513 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2514 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2516 } else if (exit_reason != -EFAULT) {
2517 vcpu->stat.exit_null++;
2519 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2520 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2521 vcpu->run->s390_ucontrol.trans_exc_code =
2522 current->thread.gmap_addr;
2523 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2525 } else if (current->thread.gmap_pfault) {
2526 trace_kvm_s390_major_guest_pfault(vcpu);
2527 current->thread.gmap_pfault = 0;
2528 if (kvm_arch_setup_async_pf(vcpu))
2530 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
2532 return vcpu_post_run_fault_in_sie(vcpu);
2535 static int __vcpu_run(struct kvm_vcpu *vcpu)
2537 int rc, exit_reason;
2540 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2541 * ning the guest), so that memslots (and other stuff) are protected
2543 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2546 rc = vcpu_pre_run(vcpu);
2550 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2552 * As PF_VCPU will be used in fault handler, between
2553 * guest_enter and guest_exit should be no uaccess.
2555 local_irq_disable();
2556 __kvm_guest_enter();
2557 __disable_cpu_timer_accounting(vcpu);
2559 exit_reason = sie64a(vcpu->arch.sie_block,
2560 vcpu->run->s.regs.gprs);
2561 local_irq_disable();
2562 __enable_cpu_timer_accounting(vcpu);
2565 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2567 rc = vcpu_post_run(vcpu, exit_reason);
2568 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2570 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2574 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2576 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2577 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2578 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2579 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2580 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2581 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2582 /* some control register changes require a tlb flush */
2583 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2585 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2586 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
2587 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2588 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2589 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2590 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2592 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2593 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2594 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2595 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2596 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2597 kvm_clear_async_pf_completion_queue(vcpu);
2599 kvm_run->kvm_dirty_regs = 0;
2602 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2604 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2605 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2606 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2607 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2608 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
2609 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2610 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2611 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2612 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2613 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2614 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2615 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2618 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2623 if (guestdbg_exit_pending(vcpu)) {
2624 kvm_s390_prepare_debug_exit(vcpu);
2628 if (vcpu->sigset_active)
2629 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2631 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2632 kvm_s390_vcpu_start(vcpu);
2633 } else if (is_vcpu_stopped(vcpu)) {
2634 pr_err_ratelimited("can't run stopped vcpu %d\n",
2639 sync_regs(vcpu, kvm_run);
2640 enable_cpu_timer_accounting(vcpu);
2643 rc = __vcpu_run(vcpu);
2645 if (signal_pending(current) && !rc) {
2646 kvm_run->exit_reason = KVM_EXIT_INTR;
2650 if (guestdbg_exit_pending(vcpu) && !rc) {
2651 kvm_s390_prepare_debug_exit(vcpu);
2655 if (rc == -EREMOTE) {
2656 /* userspace support is needed, kvm_run has been prepared */
2660 disable_cpu_timer_accounting(vcpu);
2661 store_regs(vcpu, kvm_run);
2663 if (vcpu->sigset_active)
2664 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2666 vcpu->stat.exit_userspace++;
2671 * store status at address
2672 * we use have two special cases:
2673 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2674 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2676 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2678 unsigned char archmode = 1;
2679 freg_t fprs[NUM_FPRS];
2684 px = kvm_s390_get_prefix(vcpu);
2685 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2686 if (write_guest_abs(vcpu, 163, &archmode, 1))
2689 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2690 if (write_guest_real(vcpu, 163, &archmode, 1))
2694 gpa -= __LC_FPREGS_SAVE_AREA;
2696 /* manually convert vector registers if necessary */
2697 if (MACHINE_HAS_VX) {
2698 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
2699 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2702 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2703 vcpu->run->s.regs.fprs, 128);
2705 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
2706 vcpu->run->s.regs.gprs, 128);
2707 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
2708 &vcpu->arch.sie_block->gpsw, 16);
2709 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
2711 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
2712 &vcpu->run->s.regs.fpc, 4);
2713 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
2714 &vcpu->arch.sie_block->todpr, 4);
2715 cputm = kvm_s390_get_cpu_timer(vcpu);
2716 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
2718 clkcomp = vcpu->arch.sie_block->ckc >> 8;
2719 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
2721 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
2722 &vcpu->run->s.regs.acrs, 64);
2723 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
2724 &vcpu->arch.sie_block->gcr, 128);
2725 return rc ? -EFAULT : 0;
2728 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2731 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2732 * copying in vcpu load/put. Lets update our copies before we save
2733 * it into the save area
2736 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2737 save_access_regs(vcpu->run->s.regs.acrs);
2739 return kvm_s390_store_status_unloaded(vcpu, addr);
2743 * store additional status at address
2745 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2748 /* Only bits 0-53 are used for address formation */
2749 if (!(gpa & ~0x3ff))
2752 return write_guest_abs(vcpu, gpa & ~0x3ff,
2753 (void *)&vcpu->run->s.regs.vrs, 512);
2756 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2758 if (!test_kvm_facility(vcpu->kvm, 129))
2762 * The guest VXRS are in the host VXRs due to the lazy
2763 * copying in vcpu load/put. We can simply call save_fpu_regs()
2764 * to save the current register state because we are in the
2765 * middle of a load/put cycle.
2767 * Let's update our copies before we save it into the save area.
2771 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2774 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2776 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2777 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2780 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2783 struct kvm_vcpu *vcpu;
2785 kvm_for_each_vcpu(i, vcpu, kvm) {
2786 __disable_ibs_on_vcpu(vcpu);
2790 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2792 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2793 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2796 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2798 int i, online_vcpus, started_vcpus = 0;
2800 if (!is_vcpu_stopped(vcpu))
2803 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2804 /* Only one cpu at a time may enter/leave the STOPPED state. */
2805 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2806 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2808 for (i = 0; i < online_vcpus; i++) {
2809 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2813 if (started_vcpus == 0) {
2814 /* we're the only active VCPU -> speed it up */
2815 __enable_ibs_on_vcpu(vcpu);
2816 } else if (started_vcpus == 1) {
2818 * As we are starting a second VCPU, we have to disable
2819 * the IBS facility on all VCPUs to remove potentially
2820 * oustanding ENABLE requests.
2822 __disable_ibs_on_all_vcpus(vcpu->kvm);
2825 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2827 * Another VCPU might have used IBS while we were offline.
2828 * Let's play safe and flush the VCPU at startup.
2830 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2831 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2835 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2837 int i, online_vcpus, started_vcpus = 0;
2838 struct kvm_vcpu *started_vcpu = NULL;
2840 if (is_vcpu_stopped(vcpu))
2843 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2844 /* Only one cpu at a time may enter/leave the STOPPED state. */
2845 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2846 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2848 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2849 kvm_s390_clear_stop_irq(vcpu);
2851 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2852 __disable_ibs_on_vcpu(vcpu);
2854 for (i = 0; i < online_vcpus; i++) {
2855 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2857 started_vcpu = vcpu->kvm->vcpus[i];
2861 if (started_vcpus == 1) {
2863 * As we only have one VCPU left, we want to enable the
2864 * IBS facility for that VCPU to speed it up.
2866 __enable_ibs_on_vcpu(started_vcpu);
2869 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2873 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2874 struct kvm_enable_cap *cap)
2882 case KVM_CAP_S390_CSS_SUPPORT:
2883 if (!vcpu->kvm->arch.css_support) {
2884 vcpu->kvm->arch.css_support = 1;
2885 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2886 trace_kvm_s390_enable_css(vcpu->kvm);
2897 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2898 struct kvm_s390_mem_op *mop)
2900 void __user *uaddr = (void __user *)mop->buf;
2901 void *tmpbuf = NULL;
2903 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2904 | KVM_S390_MEMOP_F_CHECK_ONLY;
2906 if (mop->flags & ~supported_flags)
2909 if (mop->size > MEM_OP_MAX_SIZE)
2912 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2913 tmpbuf = vmalloc(mop->size);
2918 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2921 case KVM_S390_MEMOP_LOGICAL_READ:
2922 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2923 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2924 mop->size, GACC_FETCH);
2927 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2929 if (copy_to_user(uaddr, tmpbuf, mop->size))
2933 case KVM_S390_MEMOP_LOGICAL_WRITE:
2934 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2935 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2936 mop->size, GACC_STORE);
2939 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2943 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2949 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2951 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2952 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2958 long kvm_arch_vcpu_ioctl(struct file *filp,
2959 unsigned int ioctl, unsigned long arg)
2961 struct kvm_vcpu *vcpu = filp->private_data;
2962 void __user *argp = (void __user *)arg;
2967 case KVM_S390_IRQ: {
2968 struct kvm_s390_irq s390irq;
2971 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2973 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2976 case KVM_S390_INTERRUPT: {
2977 struct kvm_s390_interrupt s390int;
2978 struct kvm_s390_irq s390irq;
2981 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2983 if (s390int_to_s390irq(&s390int, &s390irq))
2985 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2988 case KVM_S390_STORE_STATUS:
2989 idx = srcu_read_lock(&vcpu->kvm->srcu);
2990 r = kvm_s390_vcpu_store_status(vcpu, arg);
2991 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2993 case KVM_S390_SET_INITIAL_PSW: {
2997 if (copy_from_user(&psw, argp, sizeof(psw)))
2999 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3002 case KVM_S390_INITIAL_RESET:
3003 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3005 case KVM_SET_ONE_REG:
3006 case KVM_GET_ONE_REG: {
3007 struct kvm_one_reg reg;
3009 if (copy_from_user(®, argp, sizeof(reg)))
3011 if (ioctl == KVM_SET_ONE_REG)
3012 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
3014 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
3017 #ifdef CONFIG_KVM_S390_UCONTROL
3018 case KVM_S390_UCAS_MAP: {
3019 struct kvm_s390_ucas_mapping ucasmap;
3021 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3026 if (!kvm_is_ucontrol(vcpu->kvm)) {
3031 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3032 ucasmap.vcpu_addr, ucasmap.length);
3035 case KVM_S390_UCAS_UNMAP: {
3036 struct kvm_s390_ucas_mapping ucasmap;
3038 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3043 if (!kvm_is_ucontrol(vcpu->kvm)) {
3048 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3053 case KVM_S390_VCPU_FAULT: {
3054 r = gmap_fault(vcpu->arch.gmap, arg, 0);
3057 case KVM_ENABLE_CAP:
3059 struct kvm_enable_cap cap;
3061 if (copy_from_user(&cap, argp, sizeof(cap)))
3063 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3066 case KVM_S390_MEM_OP: {
3067 struct kvm_s390_mem_op mem_op;
3069 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3070 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3075 case KVM_S390_SET_IRQ_STATE: {
3076 struct kvm_s390_irq_state irq_state;
3079 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3081 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3082 irq_state.len == 0 ||
3083 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3087 r = kvm_s390_set_irq_state(vcpu,
3088 (void __user *) irq_state.buf,
3092 case KVM_S390_GET_IRQ_STATE: {
3093 struct kvm_s390_irq_state irq_state;
3096 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3098 if (irq_state.len == 0) {
3102 r = kvm_s390_get_irq_state(vcpu,
3103 (__u8 __user *) irq_state.buf,
3113 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3115 #ifdef CONFIG_KVM_S390_UCONTROL
3116 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3117 && (kvm_is_ucontrol(vcpu->kvm))) {
3118 vmf->page = virt_to_page(vcpu->arch.sie_block);
3119 get_page(vmf->page);
3123 return VM_FAULT_SIGBUS;
3126 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3127 unsigned long npages)
3132 /* Section: memory related */
3133 int kvm_arch_prepare_memory_region(struct kvm *kvm,
3134 struct kvm_memory_slot *memslot,
3135 const struct kvm_userspace_memory_region *mem,
3136 enum kvm_mr_change change)
3138 /* A few sanity checks. We can have memory slots which have to be
3139 located/ended at a segment boundary (1MB). The memory in userland is
3140 ok to be fragmented into various different vmas. It is okay to mmap()
3141 and munmap() stuff in this slot after doing this call at any time */
3143 if (mem->userspace_addr & 0xffffful)
3146 if (mem->memory_size & 0xffffful)
3149 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3155 void kvm_arch_commit_memory_region(struct kvm *kvm,
3156 const struct kvm_userspace_memory_region *mem,
3157 const struct kvm_memory_slot *old,
3158 const struct kvm_memory_slot *new,
3159 enum kvm_mr_change change)
3163 /* If the basics of the memslot do not change, we do not want
3164 * to update the gmap. Every update causes several unnecessary
3165 * segment translation exceptions. This is usually handled just
3166 * fine by the normal fault handler + gmap, but it will also
3167 * cause faults on the prefix page of running guest CPUs.
3169 if (old->userspace_addr == mem->userspace_addr &&
3170 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3171 old->npages * PAGE_SIZE == mem->memory_size)
3174 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3175 mem->guest_phys_addr, mem->memory_size);
3177 pr_warn("failed to commit memory region\n");
3181 static inline unsigned long nonhyp_mask(int i)
3183 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3185 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3188 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3190 vcpu->valid_wakeup = false;
3193 static int __init kvm_s390_init(void)
3197 if (!sclp.has_sief2) {
3198 pr_info("SIE not available\n");
3202 for (i = 0; i < 16; i++)
3203 kvm_s390_fac_list_mask[i] |=
3204 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3206 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
3209 static void __exit kvm_s390_exit(void)
3214 module_init(kvm_s390_init);
3215 module_exit(kvm_s390_exit);
3218 * Enable autoloading of the kvm module.
3219 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3220 * since x86 takes a different approach.
3222 #include <linux/miscdevice.h>
3223 MODULE_ALIAS_MISCDEV(KVM_MINOR);
3224 MODULE_ALIAS("devname:kvm");