Merge branch 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[cascardo/linux.git] / arch / s390 / kvm / kvm-s390.c
index f32f843..c91eb94 100644 (file)
@@ -28,6 +28,7 @@
 #include <linux/vmalloc.h>
 #include <asm/asm-offsets.h>
 #include <asm/lowcore.h>
+#include <asm/etr.h>
 #include <asm/pgtable.h>
 #include <asm/nmi.h>
 #include <asm/switch_to.h>
@@ -108,6 +109,9 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
        { "diagnose_10", VCPU_STAT(diagnose_10) },
        { "diagnose_44", VCPU_STAT(diagnose_44) },
        { "diagnose_9c", VCPU_STAT(diagnose_9c) },
+       { "diagnose_258", VCPU_STAT(diagnose_258) },
+       { "diagnose_308", VCPU_STAT(diagnose_308) },
+       { "diagnose_500", VCPU_STAT(diagnose_500) },
        { NULL }
 };
 
@@ -124,6 +128,7 @@ unsigned long kvm_s390_fac_list_mask_size(void)
 }
 
 static struct gmap_notifier gmap_notifier;
+debug_info_t *kvm_s390_dbf;
 
 /* Section: not file related */
 int kvm_arch_hardware_enable(void)
@@ -134,24 +139,69 @@ int kvm_arch_hardware_enable(void)
 
 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
 
+/*
+ * This callback is executed during stop_machine(). All CPUs are therefore
+ * temporarily stopped. In order not to change guest behavior, we have to
+ * disable preemption whenever we touch the epoch of kvm and the VCPUs,
+ * so a CPU won't be stopped while calculating with the epoch.
+ */
+static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
+                         void *v)
+{
+       struct kvm *kvm;
+       struct kvm_vcpu *vcpu;
+       int i;
+       unsigned long long *delta = v;
+
+       list_for_each_entry(kvm, &vm_list, vm_list) {
+               kvm->arch.epoch -= *delta;
+               kvm_for_each_vcpu(i, vcpu, kvm) {
+                       vcpu->arch.sie_block->epoch -= *delta;
+               }
+       }
+       return NOTIFY_OK;
+}
+
+static struct notifier_block kvm_clock_notifier = {
+       .notifier_call = kvm_clock_sync,
+};
+
 int kvm_arch_hardware_setup(void)
 {
        gmap_notifier.notifier_call = kvm_gmap_notifier;
        gmap_register_ipte_notifier(&gmap_notifier);
+       atomic_notifier_chain_register(&s390_epoch_delta_notifier,
+                                      &kvm_clock_notifier);
        return 0;
 }
 
 void kvm_arch_hardware_unsetup(void)
 {
        gmap_unregister_ipte_notifier(&gmap_notifier);
+       atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
+                                        &kvm_clock_notifier);
 }
 
 int kvm_arch_init(void *opaque)
 {
+       kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
+       if (!kvm_s390_dbf)
+               return -ENOMEM;
+
+       if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
+               debug_unregister(kvm_s390_dbf);
+               return -ENOMEM;
+       }
+
        /* Register floating interrupt controller interface. */
        return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 }
 
+void kvm_arch_exit(void)
+{
+       debug_unregister(kvm_s390_dbf);
+}
+
 /* Section: device related */
 long kvm_arch_dev_ioctl(struct file *filp,
                        unsigned int ioctl, unsigned long arg)
@@ -281,10 +331,12 @@ static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 
        switch (cap->cap) {
        case KVM_CAP_S390_IRQCHIP:
+               VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
                kvm->arch.use_irqchip = 1;
                r = 0;
                break;
        case KVM_CAP_S390_USER_SIGP:
+               VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
                kvm->arch.user_sigp = 1;
                r = 0;
                break;
@@ -295,8 +347,11 @@ static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
                        r = 0;
                } else
                        r = -EINVAL;
+               VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
+                        r ? "(not available)" : "(success)");
                break;
        case KVM_CAP_S390_USER_STSI:
+               VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
                kvm->arch.user_stsi = 1;
                r = 0;
                break;
@@ -314,6 +369,8 @@ static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *att
        switch (attr->attr) {
        case KVM_S390_VM_MEM_LIMIT_SIZE:
                ret = 0;
+               VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
+                        kvm->arch.gmap->asce_end);
                if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
                        ret = -EFAULT;
                break;
@@ -330,7 +387,13 @@ static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *att
        unsigned int idx;
        switch (attr->attr) {
        case KVM_S390_VM_MEM_ENABLE_CMMA:
+               /* enable CMMA only for z10 and later (EDAT_1) */
+               ret = -EINVAL;
+               if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
+                       break;
+
                ret = -EBUSY;
+               VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
                mutex_lock(&kvm->lock);
                if (atomic_read(&kvm->online_vcpus) == 0) {
                        kvm->arch.use_cmma = 1;
@@ -339,6 +402,11 @@ static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *att
                mutex_unlock(&kvm->lock);
                break;
        case KVM_S390_VM_MEM_CLR_CMMA:
+               ret = -EINVAL;
+               if (!kvm->arch.use_cmma)
+                       break;
+
+               VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
                mutex_lock(&kvm->lock);
                idx = srcu_read_lock(&kvm->srcu);
                s390_reset_cmma(kvm->arch.gmap->mm);
@@ -374,6 +442,7 @@ static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *att
                        }
                }
                mutex_unlock(&kvm->lock);
+               VM_EVENT(kvm, 3, "SET: max guest memory: %lu bytes", new_limit);
                break;
        }
        default:
@@ -400,22 +469,26 @@ static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
                        kvm->arch.crypto.crycb->aes_wrapping_key_mask,
                        sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
                kvm->arch.crypto.aes_kw = 1;
+               VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
                break;
        case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
                get_random_bytes(
                        kvm->arch.crypto.crycb->dea_wrapping_key_mask,
                        sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
                kvm->arch.crypto.dea_kw = 1;
+               VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
                break;
        case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
                kvm->arch.crypto.aes_kw = 0;
                memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
                        sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
+               VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
                break;
        case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
                kvm->arch.crypto.dea_kw = 0;
                memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
                        sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
+               VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
                break;
        default:
                mutex_unlock(&kvm->lock);
@@ -440,6 +513,7 @@ static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 
        if (gtod_high != 0)
                return -EINVAL;
+       VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x\n", gtod_high);
 
        return 0;
 }
@@ -459,12 +533,15 @@ static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
                return r;
 
        mutex_lock(&kvm->lock);
+       preempt_disable();
        kvm->arch.epoch = gtod - host_tod;
        kvm_s390_vcpu_block_all(kvm);
        kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm)
                cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
        kvm_s390_vcpu_unblock_all(kvm);
+       preempt_enable();
        mutex_unlock(&kvm->lock);
+       VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx\n", gtod);
        return 0;
 }
 
@@ -496,6 +573,7 @@ static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
        if (copy_to_user((void __user *)attr->addr, &gtod_high,
                                         sizeof(gtod_high)))
                return -EFAULT;
+       VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x\n", gtod_high);
 
        return 0;
 }
@@ -509,9 +587,12 @@ static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
        if (r)
                return r;
 
+       preempt_disable();
        gtod = host_tod + kvm->arch.epoch;
+       preempt_enable();
        if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
                return -EFAULT;
+       VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx\n", gtod);
 
        return 0;
 }
@@ -821,7 +902,9 @@ static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
        }
 
        /* Enable storage key handling for the guest */
-       s390_enable_skey();
+       r = s390_enable_skey();
+       if (r)
+               goto out;
 
        for (i = 0; i < args->count; i++) {
                hva = gfn_to_hva(kvm, args->start_gfn + i);
@@ -879,8 +962,7 @@ long kvm_arch_vm_ioctl(struct file *filp,
                if (kvm->arch.use_irqchip) {
                        /* Set up dummy routing. */
                        memset(&routing, 0, sizeof(routing));
-                       kvm_set_irq_routing(kvm, &routing, 0, 0);
-                       r = 0;
+                       r = kvm_set_irq_routing(kvm, &routing, 0, 0);
                }
                break;
        }
@@ -1043,7 +1125,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 
        sprintf(debug_name, "kvm-%u", current->pid);
 
-       kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
+       kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
        if (!kvm->arch.dbf)
                goto out_err;
 
@@ -1086,7 +1168,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
        mutex_init(&kvm->arch.ipte_mutex);
 
        debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
-       VM_EVENT(kvm, 3, "%s", "vm created");
+       VM_EVENT(kvm, 3, "vm created with type %lu", type);
 
        if (type & KVM_VM_S390_UCONTROL) {
                kvm->arch.gmap = NULL;
@@ -1103,6 +1185,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
        kvm->arch.epoch = 0;
 
        spin_lock_init(&kvm->arch.start_stop_lock);
+       KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);
 
        return 0;
 out_err:
@@ -1110,6 +1193,7 @@ out_err:
        free_page((unsigned long)kvm->arch.model.fac);
        debug_unregister(kvm->arch.dbf);
        free_page((unsigned long)(kvm->arch.sca));
+       KVM_EVENT(3, "creation of vm failed: %d", rc);
        return rc;
 }
 
@@ -1131,7 +1215,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
        if (kvm_is_ucontrol(vcpu->kvm))
                gmap_free(vcpu->arch.gmap);
 
-       if (kvm_s390_cmma_enabled(vcpu->kvm))
+       if (vcpu->kvm->arch.use_cmma)
                kvm_s390_vcpu_unsetup_cmma(vcpu);
        free_page((unsigned long)(vcpu->arch.sie_block));
 
@@ -1166,6 +1250,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
                gmap_free(kvm->arch.gmap);
        kvm_s390_destroy_adapters(kvm);
        kvm_s390_clear_float_irqs(kvm);
+       KVM_EVENT(3, "vm 0x%p destroyed", kvm);
 }
 
 /* Section: vcpu related */
@@ -1198,43 +1283,79 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
        return 0;
 }
 
+/*
+ * Backs up the current FP/VX register save area on a particular
+ * destination.  Used to switch between different register save
+ * areas.
+ */
+static inline void save_fpu_to(struct fpu *dst)
+{
+       dst->fpc = current->thread.fpu.fpc;
+       dst->flags = current->thread.fpu.flags;
+       dst->regs = current->thread.fpu.regs;
+}
+
+/*
+ * Switches the FP/VX register save area from which to lazy
+ * restore register contents.
+ */
+static inline void load_fpu_from(struct fpu *from)
+{
+       current->thread.fpu.fpc = from->fpc;
+       current->thread.fpu.flags = from->flags;
+       current->thread.fpu.regs = from->regs;
+}
+
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
-       save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
-       if (test_kvm_facility(vcpu->kvm, 129))
-               save_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
-       else
-               save_fp_regs(vcpu->arch.host_fpregs.fprs);
-       save_access_regs(vcpu->arch.host_acrs);
+       /* Save host register state */
+       save_fpu_regs();
+       save_fpu_to(&vcpu->arch.host_fpregs);
+
        if (test_kvm_facility(vcpu->kvm, 129)) {
-               restore_fp_ctl(&vcpu->run->s.regs.fpc);
-               restore_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
-       } else {
-               restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
-               restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
-       }
+               current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
+               current->thread.fpu.flags = FPU_USE_VX;
+               /*
+                * Use the register save area in the SIE-control block
+                * for register restore and save in kvm_arch_vcpu_put()
+                */
+               current->thread.fpu.vxrs =
+                       (__vector128 *)&vcpu->run->s.regs.vrs;
+               /* Always enable the vector extension for KVM */
+               __ctl_set_vx();
+       } else
+               load_fpu_from(&vcpu->arch.guest_fpregs);
+
+       if (test_fp_ctl(current->thread.fpu.fpc))
+               /* User space provided an invalid FPC, let's clear it */
+               current->thread.fpu.fpc = 0;
+
+       save_access_regs(vcpu->arch.host_acrs);
        restore_access_regs(vcpu->run->s.regs.acrs);
        gmap_enable(vcpu->arch.gmap);
-       atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
+       atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
 }
 
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 {
-       atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
+       atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
        gmap_disable(vcpu->arch.gmap);
-       if (test_kvm_facility(vcpu->kvm, 129)) {
-               save_fp_ctl(&vcpu->run->s.regs.fpc);
-               save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
-       } else {
-               save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
-               save_fp_regs(vcpu->arch.guest_fpregs.fprs);
-       }
-       save_access_regs(vcpu->run->s.regs.acrs);
-       restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
+
+       save_fpu_regs();
+
        if (test_kvm_facility(vcpu->kvm, 129))
-               restore_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
+               /*
+                * kvm_arch_vcpu_load() set up the register save area to
+                * the &vcpu->run->s.regs.vrs and, thus, the vector registers
+                * are already saved.  Only the floating-point control must be
+                * copied.
+                */
+               vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
        else
-               restore_fp_regs(vcpu->arch.host_fpregs.fprs);
+               save_fpu_to(&vcpu->arch.guest_fpregs);
+       load_fpu_from(&vcpu->arch.host_fpregs);
+
+       save_access_regs(vcpu->run->s.regs.acrs);
        restore_access_regs(vcpu->arch.host_acrs);
 }
 
@@ -1264,7 +1385,9 @@ static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 {
        mutex_lock(&vcpu->kvm->lock);
+       preempt_disable();
        vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
+       preempt_enable();
        mutex_unlock(&vcpu->kvm->lock);
        if (!kvm_is_ucontrol(vcpu->kvm))
                vcpu->arch.gmap = vcpu->kvm->arch.gmap;
@@ -1320,9 +1443,9 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
                                                    CPUSTAT_STOPPED);
 
        if (test_kvm_facility(vcpu->kvm, 78))
-               atomic_set_mask(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
+               atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
        else if (test_kvm_facility(vcpu->kvm, 8))
-               atomic_set_mask(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
+               atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
 
        kvm_s390_vcpu_setup_model(vcpu);
 
@@ -1342,7 +1465,7 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
        }
        vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
 
-       if (kvm_s390_cmma_enabled(vcpu->kvm)) {
+       if (vcpu->kvm->arch.use_cmma) {
                rc = kvm_s390_vcpu_setup_cmma(vcpu);
                if (rc)
                        return rc;
@@ -1377,7 +1500,6 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 
        vcpu->arch.sie_block = &sie_page->sie_block;
        vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
-       vcpu->arch.host_vregs = &sie_page->vregs;
 
        vcpu->arch.sie_block->icpua = id;
        if (!kvm_is_ucontrol(kvm)) {
@@ -1399,6 +1521,19 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
        vcpu->arch.local_int.wq = &vcpu->wq;
        vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
 
+       /*
+        * Allocate a save area for floating-point registers.  If the vector
+        * extension is available, register contents are saved in the SIE
+        * control block.  The allocated save area is still required in
+        * particular places, for example, in kvm_s390_vcpu_store_status().
+        */
+       vcpu->arch.guest_fpregs.fprs = kzalloc(sizeof(freg_t) * __NUM_FPRS,
+                                              GFP_KERNEL);
+       if (!vcpu->arch.guest_fpregs.fprs) {
+               rc = -ENOMEM;
+               goto out_free_sie_block;
+       }
+
        rc = kvm_vcpu_init(vcpu, kvm, id);
        if (rc)
                goto out_free_sie_block;
@@ -1422,24 +1557,24 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 
 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
 {
-       atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
+       atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
        exit_sie(vcpu);
 }
 
 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
 {
-       atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
+       atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
 }
 
 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
 {
-       atomic_set_mask(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
+       atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
        exit_sie(vcpu);
 }
 
 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
 {
-       atomic_clear_mask(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
+       atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
 }
 
 /*
@@ -1448,7 +1583,7 @@ static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
  * return immediately. */
 void exit_sie(struct kvm_vcpu *vcpu)
 {
-       atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
+       atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
        while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
                cpu_relax();
 }
@@ -1621,16 +1756,16 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
        if (test_fp_ctl(fpu->fpc))
                return -EINVAL;
-       memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
+       memcpy(vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
        vcpu->arch.guest_fpregs.fpc = fpu->fpc;
-       restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
-       restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
+       save_fpu_regs();
+       load_fpu_from(&vcpu->arch.guest_fpregs);
        return 0;
 }
 
 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
-       memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
+       memcpy(&fpu->fprs, vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
        fpu->fpc = vcpu->arch.guest_fpregs.fpc;
        return 0;
 }
@@ -1672,19 +1807,19 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
        if (dbg->control & KVM_GUESTDBG_ENABLE) {
                vcpu->guest_debug = dbg->control;
                /* enforce guest PER */
-               atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
+               atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
 
                if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
                        rc = kvm_s390_import_bp_data(vcpu, dbg);
        } else {
-               atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
+               atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
                vcpu->arch.guestdbg.last_bp = 0;
        }
 
        if (rc) {
                vcpu->guest_debug = 0;
                kvm_s390_clear_bp_data(vcpu);
-               atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
+               atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
        }
 
        return rc;
@@ -1723,18 +1858,6 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
        return rc;
 }
 
-bool kvm_s390_cmma_enabled(struct kvm *kvm)
-{
-       if (!MACHINE_IS_LPAR)
-               return false;
-       /* only enable for z10 and later */
-       if (!MACHINE_HAS_EDAT1)
-               return false;
-       if (!kvm->arch.use_cmma)
-               return false;
-       return true;
-}
-
 static bool ibs_enabled(struct kvm_vcpu *vcpu)
 {
        return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
@@ -1771,7 +1894,7 @@ retry:
        if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
                if (!ibs_enabled(vcpu)) {
                        trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
-                       atomic_set_mask(CPUSTAT_IBS,
+                       atomic_or(CPUSTAT_IBS,
                                        &vcpu->arch.sie_block->cpuflags);
                }
                goto retry;
@@ -1780,7 +1903,7 @@ retry:
        if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
                if (ibs_enabled(vcpu)) {
                        trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
-                       atomic_clear_mask(CPUSTAT_IBS,
+                       atomic_andnot(CPUSTAT_IBS,
                                          &vcpu->arch.sie_block->cpuflags);
                }
                goto retry;
@@ -2193,8 +2316,21 @@ int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
         * copying in vcpu load/put. Lets update our copies before we save
         * it into the save area
         */
-       save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
-       save_fp_regs(vcpu->arch.guest_fpregs.fprs);
+       save_fpu_regs();
+       if (test_kvm_facility(vcpu->kvm, 129)) {
+               /*
+                * If the vector extension is available, the vector registers
+                * which overlaps with floating-point registers are saved in
+                * the SIE-control block.  Hence, extract the floating-point
+                * registers and the FPC value and store them in the
+                * guest_fpregs structure.
+                */
+               WARN_ON(!is_vx_task(current));    /* XXX remove later */
+               vcpu->arch.guest_fpregs.fpc = current->thread.fpu.fpc;
+               convert_vx_to_fp(vcpu->arch.guest_fpregs.fprs,
+                                current->thread.fpu.vxrs);
+       } else
+               save_fpu_to(&vcpu->arch.guest_fpregs);
        save_access_regs(vcpu->run->s.regs.acrs);
 
        return kvm_s390_store_status_unloaded(vcpu, addr);
@@ -2221,10 +2357,13 @@ int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
 
        /*
         * The guest VXRS are in the host VXRs due to the lazy
-        * copying in vcpu load/put. Let's update our copies before we save
-        * it into the save area.
+        * copying in vcpu load/put. We can simply call save_fpu_regs()
+        * to save the current register state because we are in the
+        * middle of a load/put cycle.
+        *
+        * Let's update our copies before we save it into the save area.
         */
-       save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
+       save_fpu_regs();
 
        return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
 }
@@ -2280,7 +2419,7 @@ void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
                __disable_ibs_on_all_vcpus(vcpu->kvm);
        }
 
-       atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
+       atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
        /*
         * Another VCPU might have used IBS while we were offline.
         * Let's play safe and flush the VCPU at startup.
@@ -2306,7 +2445,7 @@ void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
        /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
        kvm_s390_clear_stop_irq(vcpu);
 
-       atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
+       atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
        __disable_ibs_on_vcpu(vcpu);
 
        for (i = 0; i < online_vcpus; i++) {
@@ -2340,6 +2479,7 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
        case KVM_CAP_S390_CSS_SUPPORT:
                if (!vcpu->kvm->arch.css_support) {
                        vcpu->kvm->arch.css_support = 1;
+                       VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
                        trace_kvm_s390_enable_css(vcpu->kvm);
                }
                r = 0;