Merge remote-tracking branch 'airlied/drm-next' into topic/vblank-rework
[cascardo/linux.git] / drivers / gpu / drm / drm_irq.c
index 5708c05..0342976 100644 (file)
  */
 #define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
 
-/*
- * Clear vblank timestamp buffer for a crtc.
+static bool
+drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
+                         struct timeval *tvblank, unsigned flags);
+
+/**
+ * drm_update_vblank_count - update the master vblank counter
+ * @dev: DRM device
+ * @crtc: counter to update
+ *
+ * Call back into the driver to update the appropriate vblank counter
+ * (specified by @crtc).  Deal with wraparound, if it occurred, and
+ * update the last read value so we can deal with wraparound on the next
+ * call if necessary.
+ *
+ * Only necessary when going from off->on, to account for frames we
+ * didn't get an interrupt for.
+ *
+ * Note: caller must hold dev->vbl_lock since this reads & writes
+ * device vblank fields.
  */
-static void clear_vblank_timestamps(struct drm_device *dev, int crtc)
+static void drm_update_vblank_count(struct drm_device *dev, int crtc)
 {
-       memset(dev->vblank[crtc].time, 0, sizeof(dev->vblank[crtc].time));
+       struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+       u32 cur_vblank, diff, tslot;
+       bool rc;
+       struct timeval t_vblank;
+
+       /*
+        * Interrupts were disabled prior to this call, so deal with counter
+        * wrap if needed.
+        * NOTE!  It's possible we lost a full dev->max_vblank_count events
+        * here if the register is small or we had vblank interrupts off for
+        * a long time.
+        *
+        * We repeat the hardware vblank counter & timestamp query until
+        * we get consistent results. This to prevent races between gpu
+        * updating its hardware counter while we are retrieving the
+        * corresponding vblank timestamp.
+        */
+       do {
+               cur_vblank = dev->driver->get_vblank_counter(dev, crtc);
+               rc = drm_get_last_vbltimestamp(dev, crtc, &t_vblank, 0);
+       } while (cur_vblank != dev->driver->get_vblank_counter(dev, crtc));
+
+       /* Deal with counter wrap */
+       diff = cur_vblank - vblank->last;
+       if (cur_vblank < vblank->last) {
+               diff += dev->max_vblank_count;
+
+               DRM_DEBUG("last_vblank[%d]=0x%x, cur_vblank=0x%x => diff=0x%x\n",
+                         crtc, vblank->last, cur_vblank, diff);
+       }
+
+       DRM_DEBUG("updating vblank count on crtc %d, missed %d\n",
+                 crtc, diff);
+
+       /* Reinitialize corresponding vblank timestamp if high-precision query
+        * available. Skip this step if query unsupported or failed. Will
+        * reinitialize delayed at next vblank interrupt in that case.
+        */
+       if (rc) {
+               tslot = atomic_read(&vblank->count) + diff;
+               vblanktimestamp(dev, crtc, tslot) = t_vblank;
+       }
+
+       smp_mb__before_atomic();
+       atomic_add(diff, &vblank->count);
+       smp_mb__after_atomic();
 }
 
 /*
@@ -71,10 +133,11 @@ static void clear_vblank_timestamps(struct drm_device *dev, int crtc)
  */
 static void vblank_disable_and_save(struct drm_device *dev, int crtc)
 {
+       struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
        unsigned long irqflags;
        u32 vblcount;
        s64 diff_ns;
-       int vblrc;
+       bool vblrc;
        struct timeval tvblank;
        int count = DRM_TIMESTAMP_MAXRETRIES;
 
@@ -84,8 +147,28 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc)
         */
        spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
 
+       /*
+        * If the vblank interrupt was already disbled update the count
+        * and timestamp to maintain the appearance that the counter
+        * has been ticking all along until this time. This makes the
+        * count account for the entire time between drm_vblank_on() and
+        * drm_vblank_off().
+        *
+        * But only do this if precise vblank timestamps are available.
+        * Otherwise we might read a totally bogus timestamp since drivers
+        * lacking precise timestamp support rely upon sampling the system clock
+        * at vblank interrupt time. Which obviously won't work out well if the
+        * vblank interrupt is disabled.
+        */
+       if (!vblank->enabled &&
+           drm_get_last_vbltimestamp(dev, crtc, &tvblank, 0)) {
+               drm_update_vblank_count(dev, crtc);
+               spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
+               return;
+       }
+
        dev->driver->disable_vblank(dev, crtc);
-       dev->vblank[crtc].enabled = false;
+       vblank->enabled = false;
 
        /* No further vblank irq's will be processed after
         * this point. Get current hardware vblank count and
@@ -100,9 +183,9 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc)
         * delayed gpu counter increment.
         */
        do {
-               dev->vblank[crtc].last = dev->driver->get_vblank_counter(dev, crtc);
+               vblank->last = dev->driver->get_vblank_counter(dev, crtc);
                vblrc = drm_get_last_vbltimestamp(dev, crtc, &tvblank, 0);
-       } while (dev->vblank[crtc].last != dev->driver->get_vblank_counter(dev, crtc) && (--count) && vblrc);
+       } while (vblank->last != dev->driver->get_vblank_counter(dev, crtc) && (--count) && vblrc);
 
        if (!count)
                vblrc = 0;
@@ -110,7 +193,7 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc)
        /* Compute time difference to stored timestamp of last vblank
         * as updated by last invocation of drm_handle_vblank() in vblank irq.
         */
-       vblcount = atomic_read(&dev->vblank[crtc].count);
+       vblcount = atomic_read(&vblank->count);
        diff_ns = timeval_to_ns(&tvblank) -
                  timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount));
 
@@ -126,14 +209,18 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc)
         * available. In that case we can't account for this and just
         * hope for the best.
         */
-       if ((vblrc > 0) && (abs64(diff_ns) > 1000000)) {
-               atomic_inc(&dev->vblank[crtc].count);
+       if (vblrc && (abs64(diff_ns) > 1000000)) {
+               /* Store new timestamp in ringbuffer. */
+               vblanktimestamp(dev, crtc, vblcount + 1) = tvblank;
+
+               /* Increment cooked vblank count. This also atomically commits
+                * the timestamp computed above.
+                */
+               smp_mb__before_atomic();
+               atomic_inc(&vblank->count);
                smp_mb__after_atomic();
        }
 
-       /* Invalidate all timestamps while vblank irq's are off. */
-       clear_vblank_timestamps(dev, crtc);
-
        spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
 }
 
@@ -164,14 +251,20 @@ static void vblank_disable_fn(unsigned long arg)
 void drm_vblank_cleanup(struct drm_device *dev)
 {
        int crtc;
+       unsigned long irqflags;
 
        /* Bail if the driver didn't call drm_vblank_init() */
        if (dev->num_crtcs == 0)
                return;
 
        for (crtc = 0; crtc < dev->num_crtcs; crtc++) {
-               del_timer_sync(&dev->vblank[crtc].disable_timer);
-               vblank_disable_fn((unsigned long)&dev->vblank[crtc]);
+               struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+
+               del_timer_sync(&vblank->disable_timer);
+
+               spin_lock_irqsave(&dev->vbl_lock, irqflags);
+               vblank_disable_and_save(dev, crtc);
+               spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
        }
 
        kfree(dev->vblank);
@@ -204,11 +297,13 @@ int drm_vblank_init(struct drm_device *dev, int num_crtcs)
                goto err;
 
        for (i = 0; i < num_crtcs; i++) {
-               dev->vblank[i].dev = dev;
-               dev->vblank[i].crtc = i;
-               init_waitqueue_head(&dev->vblank[i].queue);
-               setup_timer(&dev->vblank[i].disable_timer, vblank_disable_fn,
-                           (unsigned long)&dev->vblank[i]);
+               struct drm_vblank_crtc *vblank = &dev->vblank[i];
+
+               vblank->dev = dev;
+               vblank->crtc = i;
+               init_waitqueue_head(&vblank->queue);
+               setup_timer(&vblank->disable_timer, vblank_disable_fn,
+                           (unsigned long)vblank);
        }
 
        DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
@@ -224,7 +319,7 @@ int drm_vblank_init(struct drm_device *dev, int num_crtcs)
        return 0;
 
 err:
-       drm_vblank_cleanup(dev);
+       dev->num_crtcs = 0;
        return ret;
 }
 EXPORT_SYMBOL(drm_vblank_init);
@@ -360,9 +455,11 @@ int drm_irq_uninstall(struct drm_device *dev)
        if (dev->num_crtcs) {
                spin_lock_irqsave(&dev->vbl_lock, irqflags);
                for (i = 0; i < dev->num_crtcs; i++) {
-                       wake_up(&dev->vblank[i].queue);
-                       dev->vblank[i].enabled = false;
-                       dev->vblank[i].last =
+                       struct drm_vblank_crtc *vblank = &dev->vblank[i];
+
+                       wake_up(&vblank->queue);
+                       vblank->enabled = false;
+                       vblank->last =
                                dev->driver->get_vblank_counter(dev, i);
                }
                spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
@@ -617,7 +714,7 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc,
         * within vblank area, counting down the number of lines until
         * start of scanout.
         */
-       invbl = vbl_status & DRM_SCANOUTPOS_INVBL;
+       invbl = vbl_status & DRM_SCANOUTPOS_IN_VBLANK;
 
        /* Convert scanout position into elapsed time at raw_time query
         * since start of scanout at first display scanline. delta_ns
@@ -647,7 +744,7 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc,
 
        vbl_status = DRM_VBLANKTIME_SCANOUTPOS_METHOD;
        if (invbl)
-               vbl_status |= DRM_VBLANKTIME_INVBL;
+               vbl_status |= DRM_VBLANKTIME_IN_VBLANK;
 
        return vbl_status;
 }
@@ -679,10 +776,11 @@ static struct timeval get_drm_timestamp(void)
  * call, i.e., it isn't very precisely locked to the true vblank.
  *
  * Returns:
- * Non-zero if timestamp is considered to be very precise, zero otherwise.
+ * True if timestamp is considered to be very precise, false otherwise.
  */
-u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
-                             struct timeval *tvblank, unsigned flags)
+static bool
+drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
+                         struct timeval *tvblank, unsigned flags)
 {
        int ret;
 
@@ -694,7 +792,7 @@ u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
                ret = dev->driver->get_vblank_timestamp(dev, crtc, &max_error,
                                                        tvblank, flags);
                if (ret > 0)
-                       return (u32) ret;
+                       return true;
        }
 
        /* GPU high precision timestamp query unsupported or failed.
@@ -702,9 +800,8 @@ u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
         */
        *tvblank = get_drm_timestamp();
 
-       return 0;
+       return false;
 }
-EXPORT_SYMBOL(drm_get_last_vbltimestamp);
 
 /**
  * drm_vblank_count - retrieve "cooked" vblank counter value
@@ -720,9 +817,11 @@ EXPORT_SYMBOL(drm_get_last_vbltimestamp);
  */
 u32 drm_vblank_count(struct drm_device *dev, int crtc)
 {
+       struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+
        if (WARN_ON(crtc >= dev->num_crtcs))
                return 0;
-       return atomic_read(&dev->vblank[crtc].count);
+       return atomic_read(&vblank->count);
 }
 EXPORT_SYMBOL(drm_vblank_count);
 
@@ -742,6 +841,7 @@ EXPORT_SYMBOL(drm_vblank_count);
 u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc,
                              struct timeval *vblanktime)
 {
+       struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
        u32 cur_vblank;
 
        if (WARN_ON(crtc >= dev->num_crtcs))
@@ -753,10 +853,10 @@ u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc,
         * a seqlock.
         */
        do {
-               cur_vblank = atomic_read(&dev->vblank[crtc].count);
+               cur_vblank = atomic_read(&vblank->count);
                *vblanktime = vblanktimestamp(dev, crtc, cur_vblank);
                smp_rmb();
-       } while (cur_vblank != atomic_read(&dev->vblank[crtc].count));
+       } while (cur_vblank != atomic_read(&vblank->count));
 
        return cur_vblank;
 }
@@ -804,70 +904,6 @@ void drm_send_vblank_event(struct drm_device *dev, int crtc,
 }
 EXPORT_SYMBOL(drm_send_vblank_event);
 
-/**
- * drm_update_vblank_count - update the master vblank counter
- * @dev: DRM device
- * @crtc: counter to update
- *
- * Call back into the driver to update the appropriate vblank counter
- * (specified by @crtc).  Deal with wraparound, if it occurred, and
- * update the last read value so we can deal with wraparound on the next
- * call if necessary.
- *
- * Only necessary when going from off->on, to account for frames we
- * didn't get an interrupt for.
- *
- * Note: caller must hold dev->vbl_lock since this reads & writes
- * device vblank fields.
- */
-static void drm_update_vblank_count(struct drm_device *dev, int crtc)
-{
-       u32 cur_vblank, diff, tslot, rc;
-       struct timeval t_vblank;
-
-       /*
-        * Interrupts were disabled prior to this call, so deal with counter
-        * wrap if needed.
-        * NOTE!  It's possible we lost a full dev->max_vblank_count events
-        * here if the register is small or we had vblank interrupts off for
-        * a long time.
-        *
-        * We repeat the hardware vblank counter & timestamp query until
-        * we get consistent results. This to prevent races between gpu
-        * updating its hardware counter while we are retrieving the
-        * corresponding vblank timestamp.
-        */
-       do {
-               cur_vblank = dev->driver->get_vblank_counter(dev, crtc);
-               rc = drm_get_last_vbltimestamp(dev, crtc, &t_vblank, 0);
-       } while (cur_vblank != dev->driver->get_vblank_counter(dev, crtc));
-
-       /* Deal with counter wrap */
-       diff = cur_vblank - dev->vblank[crtc].last;
-       if (cur_vblank < dev->vblank[crtc].last) {
-               diff += dev->max_vblank_count;
-
-               DRM_DEBUG("last_vblank[%d]=0x%x, cur_vblank=0x%x => diff=0x%x\n",
-                         crtc, dev->vblank[crtc].last, cur_vblank, diff);
-       }
-
-       DRM_DEBUG("enabling vblank interrupts on crtc %d, missed %d\n",
-                 crtc, diff);
-
-       /* Reinitialize corresponding vblank timestamp if high-precision query
-        * available. Skip this step if query unsupported or failed. Will
-        * reinitialize delayed at next vblank interrupt in that case.
-        */
-       if (rc) {
-               tslot = atomic_read(&dev->vblank[crtc].count) + diff;
-               vblanktimestamp(dev, crtc, tslot) = t_vblank;
-       }
-
-       smp_mb__before_atomic();
-       atomic_add(diff, &dev->vblank[crtc].count);
-       smp_mb__after_atomic();
-}
-
 /**
  * drm_vblank_enable - enable the vblank interrupt on a CRTC
  * @dev: DRM device
@@ -875,13 +911,14 @@ static void drm_update_vblank_count(struct drm_device *dev, int crtc)
  */
 static int drm_vblank_enable(struct drm_device *dev, int crtc)
 {
+       struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
        int ret = 0;
 
        assert_spin_locked(&dev->vbl_lock);
 
        spin_lock(&dev->vblank_time_lock);
 
-       if (!dev->vblank[crtc].enabled) {
+       if (!vblank->enabled) {
                /*
                 * Enable vblank irqs under vblank_time_lock protection.
                 * All vblank count & timestamp updates are held off
@@ -892,9 +929,9 @@ static int drm_vblank_enable(struct drm_device *dev, int crtc)
                ret = dev->driver->enable_vblank(dev, crtc);
                DRM_DEBUG("enabling vblank on crtc %d, ret: %d\n", crtc, ret);
                if (ret)
-                       atomic_dec(&dev->vblank[crtc].refcount);
+                       atomic_dec(&vblank->refcount);
                else {
-                       dev->vblank[crtc].enabled = true;
+                       vblank->enabled = true;
                        drm_update_vblank_count(dev, crtc);
                }
        }
@@ -919,6 +956,7 @@ static int drm_vblank_enable(struct drm_device *dev, int crtc)
  */
 int drm_vblank_get(struct drm_device *dev, int crtc)
 {
+       struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
        unsigned long irqflags;
        int ret = 0;
 
@@ -927,11 +965,11 @@ int drm_vblank_get(struct drm_device *dev, int crtc)
 
        spin_lock_irqsave(&dev->vbl_lock, irqflags);
        /* Going from 0->1 means we have to enable interrupts again */
-       if (atomic_add_return(1, &dev->vblank[crtc].refcount) == 1) {
+       if (atomic_add_return(1, &vblank->refcount) == 1) {
                ret = drm_vblank_enable(dev, crtc);
        } else {
-               if (!dev->vblank[crtc].enabled) {
-                       atomic_dec(&dev->vblank[crtc].refcount);
+               if (!vblank->enabled) {
+                       atomic_dec(&vblank->refcount);
                        ret = -EINVAL;
                }
        }
@@ -971,16 +1009,23 @@ EXPORT_SYMBOL(drm_crtc_vblank_get);
  */
 void drm_vblank_put(struct drm_device *dev, int crtc)
 {
-       BUG_ON(atomic_read(&dev->vblank[crtc].refcount) == 0);
+       struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+
+       BUG_ON(atomic_read(&vblank->refcount) == 0);
 
        if (WARN_ON(crtc >= dev->num_crtcs))
                return;
 
        /* Last user schedules interrupt disable */
-       if (atomic_dec_and_test(&dev->vblank[crtc].refcount) &&
-           (drm_vblank_offdelay > 0))
-               mod_timer(&dev->vblank[crtc].disable_timer,
-                         jiffies + ((drm_vblank_offdelay * HZ)/1000));
+       if (atomic_dec_and_test(&vblank->refcount)) {
+               if (drm_vblank_offdelay == 0)
+                       return;
+               else if (dev->vblank_disable_immediate || drm_vblank_offdelay < 0)
+                       vblank_disable_fn((unsigned long)vblank);
+               else
+                       mod_timer(&vblank->disable_timer,
+                                 jiffies + ((drm_vblank_offdelay * HZ)/1000));
+       }
 }
 EXPORT_SYMBOL(drm_vblank_put);
 
@@ -1059,6 +1104,7 @@ EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
  */
 void drm_vblank_off(struct drm_device *dev, int crtc)
 {
+       struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
        struct drm_pending_vblank_event *e, *t;
        struct timeval now;
        unsigned long irqflags;
@@ -1067,14 +1113,25 @@ void drm_vblank_off(struct drm_device *dev, int crtc)
        if (WARN_ON(crtc >= dev->num_crtcs))
                return;
 
-       spin_lock_irqsave(&dev->vbl_lock, irqflags);
+       spin_lock_irqsave(&dev->event_lock, irqflags);
+
+       spin_lock(&dev->vbl_lock);
        vblank_disable_and_save(dev, crtc);
-       wake_up(&dev->vblank[crtc].queue);
+       wake_up(&vblank->queue);
+
+       /*
+        * Prevent subsequent drm_vblank_get() from re-enabling
+        * the vblank interrupt by bumping the refcount.
+        */
+       if (!vblank->inmodeset) {
+               atomic_inc(&vblank->refcount);
+               vblank->inmodeset = 1;
+       }
+       spin_unlock(&dev->vbl_lock);
 
        /* Send any queued vblank events, lest the natives grow disquiet */
        seq = drm_vblank_count_and_time(dev, crtc, &now);
 
-       spin_lock(&dev->event_lock);
        list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
                if (e->pipe != crtc)
                        continue;
@@ -1085,9 +1142,7 @@ void drm_vblank_off(struct drm_device *dev, int crtc)
                drm_vblank_put(dev, e->pipe);
                send_vblank_event(dev, e, seq, &now);
        }
-       spin_unlock(&dev->event_lock);
-
-       spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
+       spin_unlock_irqrestore(&dev->event_lock, irqflags);
 }
 EXPORT_SYMBOL(drm_vblank_off);
 
@@ -1124,14 +1179,35 @@ EXPORT_SYMBOL(drm_crtc_vblank_off);
  */
 void drm_vblank_on(struct drm_device *dev, int crtc)
 {
+       struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
        unsigned long irqflags;
 
        if (WARN_ON(crtc >= dev->num_crtcs))
                return;
 
        spin_lock_irqsave(&dev->vbl_lock, irqflags);
-       /* re-enable interrupts if there's are users left */
-       if (atomic_read(&dev->vblank[crtc].refcount) != 0)
+       /* Drop our private "prevent drm_vblank_get" refcount */
+       if (vblank->inmodeset) {
+               atomic_dec(&vblank->refcount);
+               vblank->inmodeset = 0;
+       }
+
+       /*
+        * sample the current counter to avoid random jumps
+        * when drm_vblank_enable() applies the diff
+        *
+        * -1 to make sure user will never see the same
+        * vblank counter value before and after a modeset
+        */
+       vblank->last =
+               (dev->driver->get_vblank_counter(dev, crtc) - 1) &
+               dev->max_vblank_count;
+       /*
+        * re-enable interrupts if there are users left, or the
+        * user wishes vblank interrupts to be enabled all the time.
+        */
+       if (atomic_read(&vblank->refcount) != 0 ||
+           (!dev->vblank_disable_immediate && drm_vblank_offdelay == 0))
                WARN_ON(drm_vblank_enable(dev, crtc));
        spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 }
@@ -1179,6 +1255,8 @@ EXPORT_SYMBOL(drm_crtc_vblank_on);
  */
 void drm_vblank_pre_modeset(struct drm_device *dev, int crtc)
 {
+       struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+
        /* vblank is not initialized (IRQ not installed ?), or has been freed */
        if (!dev->num_crtcs)
                return;
@@ -1193,10 +1271,10 @@ void drm_vblank_pre_modeset(struct drm_device *dev, int crtc)
         * to avoid corrupting the count if multiple, mismatch calls occur),
         * so that interrupts remain enabled in the interim.
         */
-       if (!dev->vblank[crtc].inmodeset) {
-               dev->vblank[crtc].inmodeset = 0x1;
+       if (!vblank->inmodeset) {
+               vblank->inmodeset = 0x1;
                if (drm_vblank_get(dev, crtc) == 0)
-                       dev->vblank[crtc].inmodeset |= 0x2;
+                       vblank->inmodeset |= 0x2;
        }
 }
 EXPORT_SYMBOL(drm_vblank_pre_modeset);
@@ -1211,21 +1289,22 @@ EXPORT_SYMBOL(drm_vblank_pre_modeset);
  */
 void drm_vblank_post_modeset(struct drm_device *dev, int crtc)
 {
+       struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
        unsigned long irqflags;
 
        /* vblank is not initialized (IRQ not installed ?), or has been freed */
        if (!dev->num_crtcs)
                return;
 
-       if (dev->vblank[crtc].inmodeset) {
+       if (vblank->inmodeset) {
                spin_lock_irqsave(&dev->vbl_lock, irqflags);
                dev->vblank_disable_allowed = true;
                spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 
-               if (dev->vblank[crtc].inmodeset & 0x2)
+               if (vblank->inmodeset & 0x2)
                        drm_vblank_put(dev, crtc);
 
-               dev->vblank[crtc].inmodeset = 0;
+               vblank->inmodeset = 0;
        }
 }
 EXPORT_SYMBOL(drm_vblank_post_modeset);
@@ -1277,6 +1356,7 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
                                  union drm_wait_vblank *vblwait,
                                  struct drm_file *file_priv)
 {
+       struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
        struct drm_pending_vblank_event *e;
        struct timeval now;
        unsigned long flags;
@@ -1300,6 +1380,18 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
 
        spin_lock_irqsave(&dev->event_lock, flags);
 
+       /*
+        * drm_vblank_off() might have been called after we called
+        * drm_vblank_get(). drm_vblank_off() holds event_lock
+        * around the vblank disable, so no need for further locking.
+        * The reference from drm_vblank_get() protects against
+        * vblank disable from another source.
+        */
+       if (!vblank->enabled) {
+               ret = -EINVAL;
+               goto err_unlock;
+       }
+
        if (file_priv->event_space < sizeof e->event) {
                ret = -EBUSY;
                goto err_unlock;
@@ -1360,6 +1452,7 @@ err_put:
 int drm_wait_vblank(struct drm_device *dev, void *data,
                    struct drm_file *file_priv)
 {
+       struct drm_vblank_crtc *vblank;
        union drm_wait_vblank *vblwait = data;
        int ret;
        unsigned int flags, seq, crtc, high_crtc;
@@ -1389,6 +1482,8 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
        if (crtc >= dev->num_crtcs)
                return -EINVAL;
 
+       vblank = &dev->vblank[crtc];
+
        ret = drm_vblank_get(dev, crtc);
        if (ret) {
                DRM_DEBUG("failed to acquire vblank counter, %d\n", ret);
@@ -1421,11 +1516,11 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
 
        DRM_DEBUG("waiting on vblank count %d, crtc %d\n",
                  vblwait->request.sequence, crtc);
-       dev->vblank[crtc].last_wait = vblwait->request.sequence;
-       DRM_WAIT_ON(ret, dev->vblank[crtc].queue, 3 * HZ,
+       vblank->last_wait = vblwait->request.sequence;
+       DRM_WAIT_ON(ret, vblank->queue, 3 * HZ,
                    (((drm_vblank_count(dev, crtc) -
                       vblwait->request.sequence) <= (1 << 23)) ||
-                    !dev->vblank[crtc].enabled ||
+                    !vblank->enabled ||
                     !dev->irq_enabled));
 
        if (ret != -EINTR) {
@@ -1450,12 +1545,11 @@ static void drm_handle_vblank_events(struct drm_device *dev, int crtc)
 {
        struct drm_pending_vblank_event *e, *t;
        struct timeval now;
-       unsigned long flags;
        unsigned int seq;
 
-       seq = drm_vblank_count_and_time(dev, crtc, &now);
+       assert_spin_locked(&dev->event_lock);
 
-       spin_lock_irqsave(&dev->event_lock, flags);
+       seq = drm_vblank_count_and_time(dev, crtc, &now);
 
        list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
                if (e->pipe != crtc)
@@ -1471,8 +1565,6 @@ static void drm_handle_vblank_events(struct drm_device *dev, int crtc)
                send_vblank_event(dev, e, seq, &now);
        }
 
-       spin_unlock_irqrestore(&dev->event_lock, flags);
-
        trace_drm_vblank_event(crtc, seq);
 }
 
@@ -1486,6 +1578,7 @@ static void drm_handle_vblank_events(struct drm_device *dev, int crtc)
  */
 bool drm_handle_vblank(struct drm_device *dev, int crtc)
 {
+       struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
        u32 vblcount;
        s64 diff_ns;
        struct timeval tvblank;
@@ -1497,15 +1590,18 @@ bool drm_handle_vblank(struct drm_device *dev, int crtc)
        if (WARN_ON(crtc >= dev->num_crtcs))
                return false;
 
+       spin_lock_irqsave(&dev->event_lock, irqflags);
+
        /* Need timestamp lock to prevent concurrent execution with
         * vblank enable/disable, as this would cause inconsistent
         * or corrupted timestamps and vblank counts.
         */
-       spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
+       spin_lock(&dev->vblank_time_lock);
 
        /* Vblank irq handling disabled. Nothing to do. */
-       if (!dev->vblank[crtc].enabled) {
-               spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
+       if (!vblank->enabled) {
+               spin_unlock(&dev->vblank_time_lock);
+               spin_unlock_irqrestore(&dev->event_lock, irqflags);
                return false;
        }
 
@@ -1514,7 +1610,7 @@ bool drm_handle_vblank(struct drm_device *dev, int crtc)
         */
 
        /* Get current timestamp and count. */
-       vblcount = atomic_read(&dev->vblank[crtc].count);
+       vblcount = atomic_read(&vblank->count);
        drm_get_last_vbltimestamp(dev, crtc, &tvblank, DRM_CALLED_FROM_VBLIRQ);
 
        /* Compute time difference to timestamp of last vblank */
@@ -1538,17 +1634,20 @@ bool drm_handle_vblank(struct drm_device *dev, int crtc)
                 * the timestamp computed above.
                 */
                smp_mb__before_atomic();
-               atomic_inc(&dev->vblank[crtc].count);
+               atomic_inc(&vblank->count);
                smp_mb__after_atomic();
        } else {
                DRM_DEBUG("crtc %d: Redundant vblirq ignored. diff_ns = %d\n",
                          crtc, (int) diff_ns);
        }
 
-       wake_up(&dev->vblank[crtc].queue);
+       spin_unlock(&dev->vblank_time_lock);
+
+       wake_up(&vblank->queue);
        drm_handle_vblank_events(dev, crtc);
 
-       spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
+       spin_unlock_irqrestore(&dev->event_lock, irqflags);
+
        return true;
 }
 EXPORT_SYMBOL(drm_handle_vblank);