Merge tag 'drm-intel-fixes-2014-11-19' into drm-intel-next-queued
authorDaniel Vetter <daniel.vetter@ffwll.ch>
Wed, 19 Nov 2014 17:14:57 +0000 (18:14 +0100)
committerDaniel Vetter <daniel.vetter@ffwll.ch>
Wed, 19 Nov 2014 17:17:38 +0000 (18:17 +0100)
So with all the code movement and extraction in intel_pm.c in -next
git is hopelessly confused with

commit 2208d655a91f9879bd9a39ff9df05dd668b3512c
Author: Daniel Vetter <daniel.vetter@ffwll.ch>
Date:   Fri Nov 14 09:25:29 2014 +0100

    drm/i915: drop WaSetupGtModeTdRowDispatch:snb

from -fixes. Worse even small changes in -next move around the
conflict context so rerere is equally useless. Let's just backmerge
and be done with it.

Conflicts:
drivers/gpu/drm/i915/i915_drv.c
drivers/gpu/drm/i915/intel_pm.c

Except for git getting lost no tricky conflicts really.

Signed-off-by: Daniel Vetter <daniel.vetter@intel.com>
1  2 
drivers/gpu/drm/armada/armada_crtc.c
drivers/gpu/drm/i915/i915_dma.c
drivers/gpu/drm/i915/i915_gem_gtt.c
drivers/gpu/drm/i915/i915_gem_tiling.c
drivers/gpu/drm/i915/intel_display.c
drivers/gpu/drm/i915/intel_dp.c
drivers/gpu/drm/i915/intel_panel.c
drivers/gpu/drm/i915/intel_pm.c
drivers/gpu/drm/nouveau/nv50_display.c
drivers/gpu/drm/tegra/dc.c

Simple merge
Simple merge
Simple merge
Simple merge
Simple merge
@@@ -1123,12 -1102,24 +1124,24 @@@ static u32 get_backlight_min_vbt(struc
  
        WARN_ON(panel->backlight.max == 0);
  
+       /*
+        * XXX: If the vbt value is 255, it makes min equal to max, which leads
+        * to problems. There are such machines out there. Either our
+        * interpretation is wrong or the vbt has bogus data. Or both. Safeguard
+        * against this by letting the minimum be at most (arbitrarily chosen)
+        * 25% of the max.
+        */
+       min = clamp_t(int, dev_priv->vbt.backlight.min_brightness, 0, 64);
+       if (min != dev_priv->vbt.backlight.min_brightness) {
+               DRM_DEBUG_KMS("clamping VBT min backlight %d/255 to %d/255\n",
+                             dev_priv->vbt.backlight.min_brightness, min);
+       }
        /* vbt value is a coefficient in range [0..255] */
-       return scale(dev_priv->vbt.backlight.min_brightness, 0, 255,
-                    0, panel->backlight.max);
+       return scale(min, 0, 255, 0, panel->backlight.max);
  }
  
 -static int bdw_setup_backlight(struct intel_connector *connector)
 +static int bdw_setup_backlight(struct intel_connector *connector, enum pipe unused)
  {
        struct drm_device *dev = connector->base.dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
@@@ -6328,736 -6164,1033 +6328,731 @@@ static void g4x_disable_trickle_feed(st
        }
  }
  
 -static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
 -                                 struct i915_power_well *power_well)
 +static void ilk_init_lp_watermarks(struct drm_device *dev)
  {
 -      hsw_set_power_well(dev_priv, power_well, power_well->count > 0);
 +      struct drm_i915_private *dev_priv = dev->dev_private;
 +
 +      I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
 +      I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
 +      I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
  
        /*
 -       * We're taking over the BIOS, so clear any requests made by it since
 -       * the driver is in charge now.
 +       * Don't touch WM1S_LP_EN here.
 +       * Doing so could cause underruns.
         */
 -      if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST)
 -              I915_WRITE(HSW_PWR_WELL_BIOS, 0);
 -}
 -
 -static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 -                                struct i915_power_well *power_well)
 -{
 -      hsw_set_power_well(dev_priv, power_well, true);
 -}
 -
 -static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 -                                 struct i915_power_well *power_well)
 -{
 -      hsw_set_power_well(dev_priv, power_well, false);
  }
  
 -static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
 -                                         struct i915_power_well *power_well)
 -{
 -}
 -
 -static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
 -                                           struct i915_power_well *power_well)
 -{
 -      return true;
 -}
 -
 -static void vlv_set_power_well(struct drm_i915_private *dev_priv,
 -                             struct i915_power_well *power_well, bool enable)
 +static void ironlake_init_clock_gating(struct drm_device *dev)
  {
 -      enum punit_power_well power_well_id = power_well->data;
 -      u32 mask;
 -      u32 state;
 -      u32 ctrl;
 +      struct drm_i915_private *dev_priv = dev->dev_private;
 +      uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
  
 -      mask = PUNIT_PWRGT_MASK(power_well_id);
 -      state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) :
 -                       PUNIT_PWRGT_PWR_GATE(power_well_id);
 +      /*
 +       * Required for FBC
 +       * WaFbcDisableDpfcClockGating:ilk
 +       */
 +      dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
 +                 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
 +                 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
  
 -      mutex_lock(&dev_priv->rps.hw_lock);
 +      I915_WRITE(PCH_3DCGDIS0,
 +                 MARIUNIT_CLOCK_GATE_DISABLE |
 +                 SVSMUNIT_CLOCK_GATE_DISABLE);
 +      I915_WRITE(PCH_3DCGDIS1,
 +                 VFMUNIT_CLOCK_GATE_DISABLE);
  
 -#define COND \
 -      ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
 +      /*
 +       * According to the spec the following bits should be set in
 +       * order to enable memory self-refresh
 +       * The bit 22/21 of 0x42004
 +       * The bit 5 of 0x42020
 +       * The bit 15 of 0x45000
 +       */
 +      I915_WRITE(ILK_DISPLAY_CHICKEN2,
 +                 (I915_READ(ILK_DISPLAY_CHICKEN2) |
 +                  ILK_DPARB_GATE | ILK_VSDPFD_FULL));
 +      dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
 +      I915_WRITE(DISP_ARB_CTL,
 +                 (I915_READ(DISP_ARB_CTL) |
 +                  DISP_FBC_WM_DIS));
  
 -      if (COND)
 -              goto out;
 +      ilk_init_lp_watermarks(dev);
  
 -      ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
 -      ctrl &= ~mask;
 -      ctrl |= state;
 -      vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
 +      /*
 +       * Based on the document from hardware guys the following bits
 +       * should be set unconditionally in order to enable FBC.
 +       * The bit 22 of 0x42000
 +       * The bit 22 of 0x42004
 +       * The bit 7,8,9 of 0x42020.
 +       */
 +      if (IS_IRONLAKE_M(dev)) {
 +              /* WaFbcAsynchFlipDisableFbcQueue:ilk */
 +              I915_WRITE(ILK_DISPLAY_CHICKEN1,
 +                         I915_READ(ILK_DISPLAY_CHICKEN1) |
 +                         ILK_FBCQ_DIS);
 +              I915_WRITE(ILK_DISPLAY_CHICKEN2,
 +                         I915_READ(ILK_DISPLAY_CHICKEN2) |
 +                         ILK_DPARB_GATE);
 +      }
  
 -      if (wait_for(COND, 100))
 -              DRM_ERROR("timout setting power well state %08x (%08x)\n",
 -                        state,
 -                        vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
 +      I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
  
 -#undef COND
 +      I915_WRITE(ILK_DISPLAY_CHICKEN2,
 +                 I915_READ(ILK_DISPLAY_CHICKEN2) |
 +                 ILK_ELPIN_409_SELECT);
 +      I915_WRITE(_3D_CHICKEN2,
 +                 _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
 +                 _3D_CHICKEN2_WM_READ_PIPELINED);
  
 -out:
 -      mutex_unlock(&dev_priv->rps.hw_lock);
 -}
 +      /* WaDisableRenderCachePipelinedFlush:ilk */
 +      I915_WRITE(CACHE_MODE_0,
 +                 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
  
 -static void vlv_power_well_sync_hw(struct drm_i915_private *dev_priv,
 -                                 struct i915_power_well *power_well)
 -{
 -      vlv_set_power_well(dev_priv, power_well, power_well->count > 0);
 -}
 +      /* WaDisable_RenderCache_OperationalFlush:ilk */
 +      I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
  
 -static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
 -                                struct i915_power_well *power_well)
 -{
 -      vlv_set_power_well(dev_priv, power_well, true);
 -}
 +      g4x_disable_trickle_feed(dev);
  
 -static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
 -                                 struct i915_power_well *power_well)
 -{
 -      vlv_set_power_well(dev_priv, power_well, false);
 +      ibx_init_clock_gating(dev);
  }
  
 -static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
 -                                 struct i915_power_well *power_well)
 +static void cpt_init_clock_gating(struct drm_device *dev)
  {
 -      int power_well_id = power_well->data;
 -      bool enabled = false;
 -      u32 mask;
 -      u32 state;
 -      u32 ctrl;
 -
 -      mask = PUNIT_PWRGT_MASK(power_well_id);
 -      ctrl = PUNIT_PWRGT_PWR_ON(power_well_id);
 -
 -      mutex_lock(&dev_priv->rps.hw_lock);
 +      struct drm_i915_private *dev_priv = dev->dev_private;
 +      int pipe;
 +      uint32_t val;
  
 -      state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
        /*
 -       * We only ever set the power-on and power-gate states, anything
 -       * else is unexpected.
 +       * On Ibex Peak and Cougar Point, we need to disable clock
 +       * gating for the panel power sequencer or it will fail to
 +       * start up when no ports are active.
         */
 -      WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) &&
 -              state != PUNIT_PWRGT_PWR_GATE(power_well_id));
 -      if (state == ctrl)
 -              enabled = true;
 -
 -      /*
 -       * A transient state at this point would mean some unexpected party
 -       * is poking at the power controls too.
 +      I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
 +                 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
 +                 PCH_CPUNIT_CLOCK_GATE_DISABLE);
 +      I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
 +                 DPLS_EDP_PPS_FIX_DIS);
 +      /* The below fixes the weird display corruption, a few pixels shifted
 +       * downward, on (only) LVDS of some HP laptops with IVY.
         */
 -      ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
 -      WARN_ON(ctrl != state);
 +      for_each_pipe(dev_priv, pipe) {
 +              val = I915_READ(TRANS_CHICKEN2(pipe));
 +              val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
 +              val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
 +              if (dev_priv->vbt.fdi_rx_polarity_inverted)
 +                      val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
 +              val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
 +              val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
 +              val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
 +              I915_WRITE(TRANS_CHICKEN2(pipe), val);
 +      }
 +      /* WADP0ClockGatingDisable */
 +      for_each_pipe(dev_priv, pipe) {
 +              I915_WRITE(TRANS_CHICKEN1(pipe),
 +                         TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
 +      }
 +}
  
 -      mutex_unlock(&dev_priv->rps.hw_lock);
 +static void gen6_check_mch_setup(struct drm_device *dev)
 +{
 +      struct drm_i915_private *dev_priv = dev->dev_private;
 +      uint32_t tmp;
  
 -      return enabled;
 +      tmp = I915_READ(MCH_SSKPD);
 +      if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
 +              DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
 +                            tmp);
  }
  
 -static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
 -                                        struct i915_power_well *power_well)
 +static void gen6_init_clock_gating(struct drm_device *dev)
  {
 -      WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D);
 +      struct drm_i915_private *dev_priv = dev->dev_private;
 +      uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
  
 -      vlv_set_power_well(dev_priv, power_well, true);
 +      I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
  
 -      spin_lock_irq(&dev_priv->irq_lock);
 -      valleyview_enable_display_irqs(dev_priv);
 -      spin_unlock_irq(&dev_priv->irq_lock);
 +      I915_WRITE(ILK_DISPLAY_CHICKEN2,
 +                 I915_READ(ILK_DISPLAY_CHICKEN2) |
 +                 ILK_ELPIN_409_SELECT);
  
 -      /*
 -       * During driver initialization/resume we can avoid restoring the
 -       * part of the HW/SW state that will be inited anyway explicitly.
 -       */
 -      if (dev_priv->power_domains.initializing)
 -              return;
 +      /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
 +      I915_WRITE(_3D_CHICKEN,
 +                 _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
  
-       /* WaSetupGtModeTdRowDispatch:snb */
-       if (IS_SNB_GT1(dev))
-               I915_WRITE(GEN6_GT_MODE,
-                          _MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
 -      intel_hpd_init(dev_priv->dev);
 +      /* WaDisable_RenderCache_OperationalFlush:snb */
 +      I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
  
 -      i915_redisable_vga_power_on(dev_priv->dev);
 -}
 +      /*
 +       * BSpec recoomends 8x4 when MSAA is used,
 +       * however in practice 16x4 seems fastest.
 +       *
 +       * Note that PS/WM thread counts depend on the WIZ hashing
 +       * disable bit, which we don't touch here, but it's good
 +       * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 +       */
 +      I915_WRITE(GEN6_GT_MODE,
 +                 GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
  
 -static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
 -                                         struct i915_power_well *power_well)
 -{
 -      WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D);
 +      ilk_init_lp_watermarks(dev);
  
 -      spin_lock_irq(&dev_priv->irq_lock);
 -      valleyview_disable_display_irqs(dev_priv);
 -      spin_unlock_irq(&dev_priv->irq_lock);
 +      I915_WRITE(CACHE_MODE_0,
 +                 _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
  
 -      vlv_set_power_well(dev_priv, power_well, false);
 +      I915_WRITE(GEN6_UCGCTL1,
 +                 I915_READ(GEN6_UCGCTL1) |
 +                 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
 +                 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
  
 -      vlv_power_sequencer_reset(dev_priv);
 -}
 +      /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
 +       * gating disable must be set.  Failure to set it results in
 +       * flickering pixels due to Z write ordering failures after
 +       * some amount of runtime in the Mesa "fire" demo, and Unigine
 +       * Sanctuary and Tropics, and apparently anything else with
 +       * alpha test or pixel discard.
 +       *
 +       * According to the spec, bit 11 (RCCUNIT) must also be set,
 +       * but we didn't debug actual testcases to find it out.
 +       *
 +       * WaDisableRCCUnitClockGating:snb
 +       * WaDisableRCPBUnitClockGating:snb
 +       */
 +      I915_WRITE(GEN6_UCGCTL2,
 +                 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
 +                 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
  
 -static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 -                                         struct i915_power_well *power_well)
 -{
 -      WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC);
 +      /* WaStripsFansDisableFastClipPerformanceFix:snb */
 +      I915_WRITE(_3D_CHICKEN3,
 +                 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
  
        /*
 -       * Enable the CRI clock source so we can get at the
 -       * display and the reference clock for VGA
 -       * hotplug / manual detection.
 +       * Bspec says:
 +       * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
 +       * 3DSTATE_SF number of SF output attributes is more than 16."
         */
 -      I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) |
 -                 DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV);
 -      udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
 -
 -      vlv_set_power_well(dev_priv, power_well, true);
 +      I915_WRITE(_3D_CHICKEN3,
 +                 _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
  
        /*
 -       * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
 -       *  6.  De-assert cmn_reset/side_reset. Same as VLV X0.
 -       *   a. GUnit 0x2110 bit[0] set to 1 (def 0)
 -       *   b. The other bits such as sfr settings / modesel may all
 -       *      be set to 0.
 +       * According to the spec the following bits should be
 +       * set in order to enable memory self-refresh and fbc:
 +       * The bit21 and bit22 of 0x42000
 +       * The bit21 and bit22 of 0x42004
 +       * The bit5 and bit7 of 0x42020
 +       * The bit14 of 0x70180
 +       * The bit14 of 0x71180
         *
 -       * This should only be done on init and resume from S3 with
 -       * both PLLs disabled, or we risk losing DPIO and PLL
 -       * synchronization.
 +       * WaFbcAsynchFlipDisableFbcQueue:snb
         */
 -      I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST);
 -}
 -
 -static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 -                                          struct i915_power_well *power_well)
 -{
 -      enum pipe pipe;
 -
 -      WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC);
 +      I915_WRITE(ILK_DISPLAY_CHICKEN1,
 +                 I915_READ(ILK_DISPLAY_CHICKEN1) |
 +                 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
 +      I915_WRITE(ILK_DISPLAY_CHICKEN2,
 +                 I915_READ(ILK_DISPLAY_CHICKEN2) |
 +                 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
 +      I915_WRITE(ILK_DSPCLK_GATE_D,
 +                 I915_READ(ILK_DSPCLK_GATE_D) |
 +                 ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
 +                 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
  
 -      for_each_pipe(dev_priv, pipe)
 -              assert_pll_disabled(dev_priv, pipe);
 +      g4x_disable_trickle_feed(dev);
  
 -      /* Assert common reset */
 -      I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST);
 +      cpt_init_clock_gating(dev);
  
 -      vlv_set_power_well(dev_priv, power_well, false);
 +      gen6_check_mch_setup(dev);
  }
  
 -static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 -                                         struct i915_power_well *power_well)
 +static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
  {
 -      enum dpio_phy phy;
 -
 -      WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC &&
 -                   power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D);
 +      uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
  
        /*
 -       * Enable the CRI clock source so we can get at the
 -       * display and the reference clock for VGA
 -       * hotplug / manual detection.
 +       * WaVSThreadDispatchOverride:ivb,vlv
 +       *
 +       * This actually overrides the dispatch
 +       * mode for all thread types.
         */
 -      if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) {
 -              phy = DPIO_PHY0;
 -              I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) |
 -                         DPLL_REFA_CLK_ENABLE_VLV);
 -              I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) |
 -                         DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV);
 -      } else {
 -              phy = DPIO_PHY1;
 -              I915_WRITE(DPLL(PIPE_C), I915_READ(DPLL(PIPE_C)) |
 -                         DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV);
 -      }
 -      udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
 -      vlv_set_power_well(dev_priv, power_well, true);
 -
 -      /* Poll for phypwrgood signal */
 -      if (wait_for(I915_READ(DISPLAY_PHY_STATUS) & PHY_POWERGOOD(phy), 1))
 -              DRM_ERROR("Display PHY %d is not power up\n", phy);
 +      reg &= ~GEN7_FF_SCHED_MASK;
 +      reg |= GEN7_FF_TS_SCHED_HW;
 +      reg |= GEN7_FF_VS_SCHED_HW;
 +      reg |= GEN7_FF_DS_SCHED_HW;
  
 -      I915_WRITE(DISPLAY_PHY_CONTROL, I915_READ(DISPLAY_PHY_CONTROL) |
 -                 PHY_COM_LANE_RESET_DEASSERT(phy));
 +      I915_WRITE(GEN7_FF_THREAD_MODE, reg);
  }
  
 -static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 -                                          struct i915_power_well *power_well)
 +static void lpt_init_clock_gating(struct drm_device *dev)
  {
 -      enum dpio_phy phy;
 -
 -      WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC &&
 -                   power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D);
 -
 -      if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) {
 -              phy = DPIO_PHY0;
 -              assert_pll_disabled(dev_priv, PIPE_A);
 -              assert_pll_disabled(dev_priv, PIPE_B);
 -      } else {
 -              phy = DPIO_PHY1;
 -              assert_pll_disabled(dev_priv, PIPE_C);
 -      }
 +      struct drm_i915_private *dev_priv = dev->dev_private;
  
 -      I915_WRITE(DISPLAY_PHY_CONTROL, I915_READ(DISPLAY_PHY_CONTROL) &
 -                 ~PHY_COM_LANE_RESET_DEASSERT(phy));
 +      /*
 +       * TODO: this bit should only be enabled when really needed, then
 +       * disabled when not needed anymore in order to save power.
 +       */
 +      if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
 +              I915_WRITE(SOUTH_DSPCLK_GATE_D,
 +                         I915_READ(SOUTH_DSPCLK_GATE_D) |
 +                         PCH_LP_PARTITION_LEVEL_DISABLE);
  
 -      vlv_set_power_well(dev_priv, power_well, false);
 +      /* WADPOClockGatingDisable:hsw */
 +      I915_WRITE(_TRANSA_CHICKEN1,
 +                 I915_READ(_TRANSA_CHICKEN1) |
 +                 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
  }
  
 -static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
 -                                      struct i915_power_well *power_well)
 +static void lpt_suspend_hw(struct drm_device *dev)
  {
 -      enum pipe pipe = power_well->data;
 -      bool enabled;
 -      u32 state, ctrl;
 -
 -      mutex_lock(&dev_priv->rps.hw_lock);
 -
 -      state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe);
 -      /*
 -       * We only ever set the power-on and power-gate states, anything
 -       * else is unexpected.
 -       */
 -      WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe));
 -      enabled = state == DP_SSS_PWR_ON(pipe);
 -
 -      /*
 -       * A transient state at this point would mean some unexpected party
 -       * is poking at the power controls too.
 -       */
 -      ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe);
 -      WARN_ON(ctrl << 16 != state);
 +      struct drm_i915_private *dev_priv = dev->dev_private;
  
 -      mutex_unlock(&dev_priv->rps.hw_lock);
 +      if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
 +              uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
  
 -      return enabled;
 +              val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
 +              I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
 +      }
  }
  
 -static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
 -                                  struct i915_power_well *power_well,
 -                                  bool enable)
 +static void broadwell_init_clock_gating(struct drm_device *dev)
  {
 -      enum pipe pipe = power_well->data;
 -      u32 state;
 -      u32 ctrl;
 +      struct drm_i915_private *dev_priv = dev->dev_private;
 +      enum pipe pipe;
  
 -      state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
 +      I915_WRITE(WM3_LP_ILK, 0);
 +      I915_WRITE(WM2_LP_ILK, 0);
 +      I915_WRITE(WM1_LP_ILK, 0);
  
 -      mutex_lock(&dev_priv->rps.hw_lock);
 +      /* WaSwitchSolVfFArbitrationPriority:bdw */
 +      I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
  
 -#define COND \
 -      ((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state)
 +      /* WaPsrDPAMaskVBlankInSRD:bdw */
 +      I915_WRITE(CHICKEN_PAR1_1,
 +                 I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
  
 -      if (COND)
 -              goto out;
 +      /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
 +      for_each_pipe(dev_priv, pipe) {
 +              I915_WRITE(CHICKEN_PIPESL_1(pipe),
 +                         I915_READ(CHICKEN_PIPESL_1(pipe)) |
 +                         BDW_DPRS_MASK_VBLANK_SRD);
 +      }
  
 -      ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
 -      ctrl &= ~DP_SSC_MASK(pipe);
 -      ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
 -      vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl);
 +      /* WaVSRefCountFullforceMissDisable:bdw */
 +      /* WaDSRefCountFullforceMissDisable:bdw */
 +      I915_WRITE(GEN7_FF_THREAD_MODE,
 +                 I915_READ(GEN7_FF_THREAD_MODE) &
 +                 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
  
 -      if (wait_for(COND, 100))
 -              DRM_ERROR("timout setting power well state %08x (%08x)\n",
 -                        state,
 -                        vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ));
 +      I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
 +                 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
  
 -#undef COND
 +      /* WaDisableSDEUnitClockGating:bdw */
 +      I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 +                 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  
 -out:
 -      mutex_unlock(&dev_priv->rps.hw_lock);
 +      lpt_init_clock_gating(dev);
  }
  
 -static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv,
 -                                      struct i915_power_well *power_well)
 +static void haswell_init_clock_gating(struct drm_device *dev)
  {
 -      chv_set_pipe_power_well(dev_priv, power_well, power_well->count > 0);
 -}
 +      struct drm_i915_private *dev_priv = dev->dev_private;
  
 -static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
 -                                     struct i915_power_well *power_well)
 -{
 -      WARN_ON_ONCE(power_well->data != PIPE_A &&
 -                   power_well->data != PIPE_B &&
 -                   power_well->data != PIPE_C);
 +      ilk_init_lp_watermarks(dev);
  
 -      chv_set_pipe_power_well(dev_priv, power_well, true);
 -}
 +      /* L3 caching of data atomics doesn't work -- disable it. */
 +      I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
 +      I915_WRITE(HSW_ROW_CHICKEN3,
 +                 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
  
 -static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
 -                                      struct i915_power_well *power_well)
 -{
 -      WARN_ON_ONCE(power_well->data != PIPE_A &&
 -                   power_well->data != PIPE_B &&
 -                   power_well->data != PIPE_C);
 +      /* This is required by WaCatErrorRejectionIssue:hsw */
 +      I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 +                      I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
 +                      GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
  
 -      chv_set_pipe_power_well(dev_priv, power_well, false);
 -}
 +      /* WaVSRefCountFullforceMissDisable:hsw */
 +      I915_WRITE(GEN7_FF_THREAD_MODE,
 +                 I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
  
 -static void check_power_well_state(struct drm_i915_private *dev_priv,
 -                                 struct i915_power_well *power_well)
 -{
 -      bool enabled = power_well->ops->is_enabled(dev_priv, power_well);
 +      /* WaDisable_RenderCache_OperationalFlush:hsw */
 +      I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
  
 -      if (power_well->always_on || !i915.disable_power_well) {
 -              if (!enabled)
 -                      goto mismatch;
 +      /* enable HiZ Raw Stall Optimization */
 +      I915_WRITE(CACHE_MODE_0_GEN7,
 +                 _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
  
 -              return;
 -      }
 +      /* WaDisable4x2SubspanOptimization:hsw */
 +      I915_WRITE(CACHE_MODE_1,
 +                 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
 +
 +      /*
 +       * BSpec recommends 8x4 when MSAA is used,
 +       * however in practice 16x4 seems fastest.
 +       *
 +       * Note that PS/WM thread counts depend on the WIZ hashing
 +       * disable bit, which we don't touch here, but it's good
 +       * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 +       */
 +      I915_WRITE(GEN7_GT_MODE,
 +                 GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
  
 -      if (enabled != (power_well->count > 0))
 -              goto mismatch;
 +      /* WaSwitchSolVfFArbitrationPriority:hsw */
 +      I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
  
 -      return;
 +      /* WaRsPkgCStateDisplayPMReq:hsw */
 +      I915_WRITE(CHICKEN_PAR1_1,
 +                 I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
  
 -mismatch:
 -      WARN(1, "state mismatch for '%s' (always_on %d hw state %d use-count %d disable_power_well %d\n",
 -                power_well->name, power_well->always_on, enabled,
 -                power_well->count, i915.disable_power_well);
 +      lpt_init_clock_gating(dev);
  }
  
 -void intel_display_power_get(struct drm_i915_private *dev_priv,
 -                           enum intel_display_power_domain domain)
 +static void ivybridge_init_clock_gating(struct drm_device *dev)
  {
 -      struct i915_power_domains *power_domains;
 -      struct i915_power_well *power_well;
 -      int i;
 +      struct drm_i915_private *dev_priv = dev->dev_private;
 +      uint32_t snpcr;
  
 -      intel_runtime_pm_get(dev_priv);
 +      ilk_init_lp_watermarks(dev);
  
 -      power_domains = &dev_priv->power_domains;
 +      I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
  
 -      mutex_lock(&power_domains->lock);
 +      /* WaDisableEarlyCull:ivb */
 +      I915_WRITE(_3D_CHICKEN3,
 +                 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
  
 -      for_each_power_well(i, power_well, BIT(domain), power_domains) {
 -              if (!power_well->count++) {
 -                      DRM_DEBUG_KMS("enabling %s\n", power_well->name);
 -                      power_well->ops->enable(dev_priv, power_well);
 -                      power_well->hw_enabled = true;
 -              }
 +      /* WaDisableBackToBackFlipFix:ivb */
 +      I915_WRITE(IVB_CHICKEN3,
 +                 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
 +                 CHICKEN3_DGMG_DONE_FIX_DISABLE);
  
 -              check_power_well_state(dev_priv, power_well);
 -      }
 +      /* WaDisablePSDDualDispatchEnable:ivb */
 +      if (IS_IVB_GT1(dev))
 +              I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
 +                         _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
  
 -      power_domains->domain_use_count[domain]++;
 +      /* WaDisable_RenderCache_OperationalFlush:ivb */
 +      I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
  
 -      mutex_unlock(&power_domains->lock);
 -}
 +      /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
 +      I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
 +                 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
  
 -void intel_display_power_put(struct drm_i915_private *dev_priv,
 -                           enum intel_display_power_domain domain)
 -{
 -      struct i915_power_domains *power_domains;
 -      struct i915_power_well *power_well;
 -      int i;
 +      /* WaApplyL3ControlAndL3ChickenMode:ivb */
 +      I915_WRITE(GEN7_L3CNTLREG1,
 +                      GEN7_WA_FOR_GEN7_L3_CONTROL);
 +      I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
 +                 GEN7_WA_L3_CHICKEN_MODE);
 +      if (IS_IVB_GT1(dev))
 +              I915_WRITE(GEN7_ROW_CHICKEN2,
 +                         _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 +      else {
 +              /* must write both registers */
 +              I915_WRITE(GEN7_ROW_CHICKEN2,
 +                         _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 +              I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
 +                         _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 +      }
  
 -      power_domains = &dev_priv->power_domains;
 +      /* WaForceL3Serialization:ivb */
 +      I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
 +                 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
  
 -      mutex_lock(&power_domains->lock);
 +      /*
 +       * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
 +       * This implements the WaDisableRCZUnitClockGating:ivb workaround.
 +       */
 +      I915_WRITE(GEN6_UCGCTL2,
 +                 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
  
 -      WARN_ON(!power_domains->domain_use_count[domain]);
 -      power_domains->domain_use_count[domain]--;
 +      /* This is required by WaCatErrorRejectionIssue:ivb */
 +      I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 +                      I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
 +                      GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
  
 -      for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
 -              WARN_ON(!power_well->count);
 +      g4x_disable_trickle_feed(dev);
  
 -              if (!--power_well->count && i915.disable_power_well) {
 -                      DRM_DEBUG_KMS("disabling %s\n", power_well->name);
 -                      power_well->hw_enabled = false;
 -                      power_well->ops->disable(dev_priv, power_well);
 -              }
 +      gen7_setup_fixed_func_scheduler(dev_priv);
  
 -              check_power_well_state(dev_priv, power_well);
 +      if (0) { /* causes HiZ corruption on ivb:gt1 */
 +              /* enable HiZ Raw Stall Optimization */
 +              I915_WRITE(CACHE_MODE_0_GEN7,
 +                         _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
        }
  
 -      mutex_unlock(&power_domains->lock);
 -
 -      intel_runtime_pm_put(dev_priv);
 -}
 -
 -static struct i915_power_domains *hsw_pwr;
 -
 -/* Display audio driver power well request */
 -int i915_request_power_well(void)
 -{
 -      struct drm_i915_private *dev_priv;
 -
 -      if (!hsw_pwr)
 -              return -ENODEV;
 +      /* WaDisable4x2SubspanOptimization:ivb */
 +      I915_WRITE(CACHE_MODE_1,
 +                 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
  
 -      dev_priv = container_of(hsw_pwr, struct drm_i915_private,
 -                              power_domains);
 -      intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO);
 -      return 0;
 -}
 -EXPORT_SYMBOL_GPL(i915_request_power_well);
 +      /*
 +       * BSpec recommends 8x4 when MSAA is used,
 +       * however in practice 16x4 seems fastest.
 +       *
 +       * Note that PS/WM thread counts depend on the WIZ hashing
 +       * disable bit, which we don't touch here, but it's good
 +       * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 +       */
 +      I915_WRITE(GEN7_GT_MODE,
 +                 GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
  
 -/* Display audio driver power well release */
 -int i915_release_power_well(void)
 -{
 -      struct drm_i915_private *dev_priv;
 +      snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
 +      snpcr &= ~GEN6_MBC_SNPCR_MASK;
 +      snpcr |= GEN6_MBC_SNPCR_MED;
 +      I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
  
 -      if (!hsw_pwr)
 -              return -ENODEV;
 +      if (!HAS_PCH_NOP(dev))
 +              cpt_init_clock_gating(dev);
  
 -      dev_priv = container_of(hsw_pwr, struct drm_i915_private,
 -                              power_domains);
 -      intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO);
 -      return 0;
 +      gen6_check_mch_setup(dev);
  }
 -EXPORT_SYMBOL_GPL(i915_release_power_well);
  
 -/*
 - * Private interface for the audio driver to get CDCLK in kHz.
 - *
 - * Caller must request power well using i915_request_power_well() prior to
 - * making the call.
 - */
 -int i915_get_cdclk_freq(void)
 +static void valleyview_init_clock_gating(struct drm_device *dev)
  {
 -      struct drm_i915_private *dev_priv;
 -
 -      if (!hsw_pwr)
 -              return -ENODEV;
 -
 -      dev_priv = container_of(hsw_pwr, struct drm_i915_private,
 -                              power_domains);
 -
 -      return intel_ddi_get_cdclk_freq(dev_priv);
 -}
 -EXPORT_SYMBOL_GPL(i915_get_cdclk_freq);
 -
 -
 -#define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1)
 -
 -#define HSW_ALWAYS_ON_POWER_DOMAINS (                 \
 -      BIT(POWER_DOMAIN_PIPE_A) |                      \
 -      BIT(POWER_DOMAIN_TRANSCODER_EDP) |              \
 -      BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) |          \
 -      BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) |          \
 -      BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |          \
 -      BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |          \
 -      BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |          \
 -      BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |          \
 -      BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |          \
 -      BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |          \
 -      BIT(POWER_DOMAIN_PORT_CRT) |                    \
 -      BIT(POWER_DOMAIN_PLLS) |                        \
 -      BIT(POWER_DOMAIN_INIT))
 -#define HSW_DISPLAY_POWER_DOMAINS (                           \
 -      (POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS) |    \
 -      BIT(POWER_DOMAIN_INIT))
 -
 -#define BDW_ALWAYS_ON_POWER_DOMAINS (                 \
 -      HSW_ALWAYS_ON_POWER_DOMAINS |                   \
 -      BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER))
 -#define BDW_DISPLAY_POWER_DOMAINS (                           \
 -      (POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS) |    \
 -      BIT(POWER_DOMAIN_INIT))
 -
 -#define VLV_ALWAYS_ON_POWER_DOMAINS   BIT(POWER_DOMAIN_INIT)
 -#define VLV_DISPLAY_POWER_DOMAINS     POWER_DOMAIN_MASK
 -
 -#define VLV_DPIO_CMN_BC_POWER_DOMAINS (               \
 -      BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |  \
 -      BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
 -      BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |  \
 -      BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
 -      BIT(POWER_DOMAIN_PORT_CRT) |            \
 -      BIT(POWER_DOMAIN_INIT))
 -
 -#define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (        \
 -      BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |  \
 -      BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
 -      BIT(POWER_DOMAIN_INIT))
 -
 -#define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (        \
 -      BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
 -      BIT(POWER_DOMAIN_INIT))
 -
 -#define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (        \
 -      BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |  \
 -      BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
 -      BIT(POWER_DOMAIN_INIT))
 -
 -#define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (        \
 -      BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
 -      BIT(POWER_DOMAIN_INIT))
 -
 -#define CHV_PIPE_A_POWER_DOMAINS (    \
 -      BIT(POWER_DOMAIN_PIPE_A) |      \
 -      BIT(POWER_DOMAIN_INIT))
 -
 -#define CHV_PIPE_B_POWER_DOMAINS (    \
 -      BIT(POWER_DOMAIN_PIPE_B) |      \
 -      BIT(POWER_DOMAIN_INIT))
 -
 -#define CHV_PIPE_C_POWER_DOMAINS (    \
 -      BIT(POWER_DOMAIN_PIPE_C) |      \
 -      BIT(POWER_DOMAIN_INIT))
 -
 -#define CHV_DPIO_CMN_BC_POWER_DOMAINS (               \
 -      BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |  \
 -      BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
 -      BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |  \
 -      BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
 -      BIT(POWER_DOMAIN_INIT))
 -
 -#define CHV_DPIO_CMN_D_POWER_DOMAINS (                \
 -      BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |  \
 -      BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |  \
 -      BIT(POWER_DOMAIN_INIT))
 -
 -#define CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS (        \
 -      BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |  \
 -      BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |  \
 -      BIT(POWER_DOMAIN_INIT))
 -
 -#define CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS (        \
 -      BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |  \
 -      BIT(POWER_DOMAIN_INIT))
 -
 -static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
 -      .sync_hw = i9xx_always_on_power_well_noop,
 -      .enable = i9xx_always_on_power_well_noop,
 -      .disable = i9xx_always_on_power_well_noop,
 -      .is_enabled = i9xx_always_on_power_well_enabled,
 -};
 -
 -static const struct i915_power_well_ops chv_pipe_power_well_ops = {
 -      .sync_hw = chv_pipe_power_well_sync_hw,
 -      .enable = chv_pipe_power_well_enable,
 -      .disable = chv_pipe_power_well_disable,
 -      .is_enabled = chv_pipe_power_well_enabled,
 -};
 -
 -static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
 -      .sync_hw = vlv_power_well_sync_hw,
 -      .enable = chv_dpio_cmn_power_well_enable,
 -      .disable = chv_dpio_cmn_power_well_disable,
 -      .is_enabled = vlv_power_well_enabled,
 -};
 -
 -static struct i915_power_well i9xx_always_on_power_well[] = {
 -      {
 -              .name = "always-on",
 -              .always_on = 1,
 -              .domains = POWER_DOMAIN_MASK,
 -              .ops = &i9xx_always_on_power_well_ops,
 -      },
 -};
 -
 -static const struct i915_power_well_ops hsw_power_well_ops = {
 -      .sync_hw = hsw_power_well_sync_hw,
 -      .enable = hsw_power_well_enable,
 -      .disable = hsw_power_well_disable,
 -      .is_enabled = hsw_power_well_enabled,
 -};
 -
 -static struct i915_power_well hsw_power_wells[] = {
 -      {
 -              .name = "always-on",
 -              .always_on = 1,
 -              .domains = HSW_ALWAYS_ON_POWER_DOMAINS,
 -              .ops = &i9xx_always_on_power_well_ops,
 -      },
 -      {
 -              .name = "display",
 -              .domains = HSW_DISPLAY_POWER_DOMAINS,
 -              .ops = &hsw_power_well_ops,
 -      },
 -};
 +      struct drm_i915_private *dev_priv = dev->dev_private;
  
 -static struct i915_power_well bdw_power_wells[] = {
 -      {
 -              .name = "always-on",
 -              .always_on = 1,
 -              .domains = BDW_ALWAYS_ON_POWER_DOMAINS,
 -              .ops = &i9xx_always_on_power_well_ops,
 -      },
 -      {
 -              .name = "display",
 -              .domains = BDW_DISPLAY_POWER_DOMAINS,
 -              .ops = &hsw_power_well_ops,
 -      },
 -};
 +      I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
  
 -static const struct i915_power_well_ops vlv_display_power_well_ops = {
 -      .sync_hw = vlv_power_well_sync_hw,
 -      .enable = vlv_display_power_well_enable,
 -      .disable = vlv_display_power_well_disable,
 -      .is_enabled = vlv_power_well_enabled,
 -};
 +      /* WaDisableEarlyCull:vlv */
 +      I915_WRITE(_3D_CHICKEN3,
 +                 _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
  
 -static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
 -      .sync_hw = vlv_power_well_sync_hw,
 -      .enable = vlv_dpio_cmn_power_well_enable,
 -      .disable = vlv_dpio_cmn_power_well_disable,
 -      .is_enabled = vlv_power_well_enabled,
 -};
 +      /* WaDisableBackToBackFlipFix:vlv */
 +      I915_WRITE(IVB_CHICKEN3,
 +                 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
 +                 CHICKEN3_DGMG_DONE_FIX_DISABLE);
  
 -static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
 -      .sync_hw = vlv_power_well_sync_hw,
 -      .enable = vlv_power_well_enable,
 -      .disable = vlv_power_well_disable,
 -      .is_enabled = vlv_power_well_enabled,
 -};
 +      /* WaPsdDispatchEnable:vlv */
 +      /* WaDisablePSDDualDispatchEnable:vlv */
 +      I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
 +                 _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
 +                                    GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
  
 -static struct i915_power_well vlv_power_wells[] = {
 -      {
 -              .name = "always-on",
 -              .always_on = 1,
 -              .domains = VLV_ALWAYS_ON_POWER_DOMAINS,
 -              .ops = &i9xx_always_on_power_well_ops,
 -      },
 -      {
 -              .name = "display",
 -              .domains = VLV_DISPLAY_POWER_DOMAINS,
 -              .data = PUNIT_POWER_WELL_DISP2D,
 -              .ops = &vlv_display_power_well_ops,
 -      },
 -      {
 -              .name = "dpio-tx-b-01",
 -              .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
 -                         VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
 -                         VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
 -                         VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
 -              .ops = &vlv_dpio_power_well_ops,
 -              .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
 -      },
 -      {
 -              .name = "dpio-tx-b-23",
 -              .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
 -                         VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
 -                         VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
 -                         VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
 -              .ops = &vlv_dpio_power_well_ops,
 -              .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
 -      },
 -      {
 -              .name = "dpio-tx-c-01",
 -              .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
 -                         VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
 -                         VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
 -                         VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
 -              .ops = &vlv_dpio_power_well_ops,
 -              .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
 -      },
 -      {
 -              .name = "dpio-tx-c-23",
 -              .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
 -                         VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
 -                         VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
 -                         VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
 -              .ops = &vlv_dpio_power_well_ops,
 -              .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
 -      },
 -      {
 -              .name = "dpio-common",
 -              .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
 -              .data = PUNIT_POWER_WELL_DPIO_CMN_BC,
 -              .ops = &vlv_dpio_cmn_power_well_ops,
 -      },
 -};
 +      /* WaDisable_RenderCache_OperationalFlush:vlv */
 +      I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
  
 -static struct i915_power_well chv_power_wells[] = {
 -      {
 -              .name = "always-on",
 -              .always_on = 1,
 -              .domains = VLV_ALWAYS_ON_POWER_DOMAINS,
 -              .ops = &i9xx_always_on_power_well_ops,
 -      },
 -#if 0
 -      {
 -              .name = "display",
 -              .domains = VLV_DISPLAY_POWER_DOMAINS,
 -              .data = PUNIT_POWER_WELL_DISP2D,
 -              .ops = &vlv_display_power_well_ops,
 -      },
 -      {
 -              .name = "pipe-a",
 -              .domains = CHV_PIPE_A_POWER_DOMAINS,
 -              .data = PIPE_A,
 -              .ops = &chv_pipe_power_well_ops,
 -      },
 -      {
 -              .name = "pipe-b",
 -              .domains = CHV_PIPE_B_POWER_DOMAINS,
 -              .data = PIPE_B,
 -              .ops = &chv_pipe_power_well_ops,
 -      },
 -      {
 -              .name = "pipe-c",
 -              .domains = CHV_PIPE_C_POWER_DOMAINS,
 -              .data = PIPE_C,
 -              .ops = &chv_pipe_power_well_ops,
 -      },
 -#endif
 -      {
 -              .name = "dpio-common-bc",
 -              /*
 -               * XXX: cmnreset for one PHY seems to disturb the other.
 -               * As a workaround keep both powered on at the same
 -               * time for now.
 -               */
 -              .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS | CHV_DPIO_CMN_D_POWER_DOMAINS,
 -              .data = PUNIT_POWER_WELL_DPIO_CMN_BC,
 -              .ops = &chv_dpio_cmn_power_well_ops,
 -      },
 -      {
 -              .name = "dpio-common-d",
 -              /*
 -               * XXX: cmnreset for one PHY seems to disturb the other.
 -               * As a workaround keep both powered on at the same
 -               * time for now.
 -               */
 -              .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS | CHV_DPIO_CMN_D_POWER_DOMAINS,
 -              .data = PUNIT_POWER_WELL_DPIO_CMN_D,
 -              .ops = &chv_dpio_cmn_power_well_ops,
 -      },
 -#if 0
 -      {
 -              .name = "dpio-tx-b-01",
 -              .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
 -                         VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS,
 -              .ops = &vlv_dpio_power_well_ops,
 -              .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
 -      },
 -      {
 -              .name = "dpio-tx-b-23",
 -              .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
 -                         VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS,
 -              .ops = &vlv_dpio_power_well_ops,
 -              .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
 -      },
 -      {
 -              .name = "dpio-tx-c-01",
 -              .domains = VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
 -                         VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
 -              .ops = &vlv_dpio_power_well_ops,
 -              .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
 -      },
 -      {
 -              .name = "dpio-tx-c-23",
 -              .domains = VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
 -                         VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
 -              .ops = &vlv_dpio_power_well_ops,
 -              .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
 -      },
 -      {
 -              .name = "dpio-tx-d-01",
 -              .domains = CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS |
 -                         CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS,
 -              .ops = &vlv_dpio_power_well_ops,
 -              .data = PUNIT_POWER_WELL_DPIO_TX_D_LANES_01,
 -      },
 -      {
 -              .name = "dpio-tx-d-23",
 -              .domains = CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS |
 -                         CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS,
 -              .ops = &vlv_dpio_power_well_ops,
 -              .data = PUNIT_POWER_WELL_DPIO_TX_D_LANES_23,
 -      },
 -#endif
 -};
 +      /* WaForceL3Serialization:vlv */
 +      I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
 +                 ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
  
 -static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv,
 -                                               enum punit_power_well power_well_id)
 -{
 -      struct i915_power_domains *power_domains = &dev_priv->power_domains;
 -      struct i915_power_well *power_well;
 -      int i;
 +      /* WaDisableDopClockGating:vlv */
 +      I915_WRITE(GEN7_ROW_CHICKEN2,
 +                 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
  
 -      for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
 -              if (power_well->data == power_well_id)
 -                      return power_well;
 -      }
 +      /* This is required by WaCatErrorRejectionIssue:vlv */
 +      I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 +                 I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
 +                 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
  
 -      return NULL;
 -}
 +      gen7_setup_fixed_func_scheduler(dev_priv);
  
 -#define set_power_wells(power_domains, __power_wells) ({              \
 -      (power_domains)->power_wells = (__power_wells);                 \
 -      (power_domains)->power_well_count = ARRAY_SIZE(__power_wells);  \
 -})
 +      /*
 +       * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
 +       * This implements the WaDisableRCZUnitClockGating:vlv workaround.
 +       */
 +      I915_WRITE(GEN6_UCGCTL2,
 +                 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
  
 -int intel_power_domains_init(struct drm_i915_private *dev_priv)
 -{
 -      struct i915_power_domains *power_domains = &dev_priv->power_domains;
 +      /* WaDisableL3Bank2xClockGate:vlv
 +       * Disabling L3 clock gating- MMIO 940c[25] = 1
 +       * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
 +      I915_WRITE(GEN7_UCGCTL4,
 +                 I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
  
 -      mutex_init(&power_domains->lock);
 +      I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
  
        /*
 -       * The enabling order will be from lower to higher indexed wells,
 -       * the disabling order is reversed.
 +       * BSpec says this must be set, even though
 +       * WaDisable4x2SubspanOptimization isn't listed for VLV.
         */
 -      if (IS_HASWELL(dev_priv->dev)) {
 -              set_power_wells(power_domains, hsw_power_wells);
 -              hsw_pwr = power_domains;
 -      } else if (IS_BROADWELL(dev_priv->dev)) {
 -              set_power_wells(power_domains, bdw_power_wells);
 -              hsw_pwr = power_domains;
 -      } else if (IS_CHERRYVIEW(dev_priv->dev)) {
 -              set_power_wells(power_domains, chv_power_wells);
 -      } else if (IS_VALLEYVIEW(dev_priv->dev)) {
 -              set_power_wells(power_domains, vlv_power_wells);
 -      } else {
 -              set_power_wells(power_domains, i9xx_always_on_power_well);
 -      }
 +      I915_WRITE(CACHE_MODE_1,
 +                 _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
  
 -      return 0;
 -}
 +      /*
 +       * WaIncreaseL3CreditsForVLVB0:vlv
 +       * This is the hardware default actually.
 +       */
 +      I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
  
 -void intel_power_domains_remove(struct drm_i915_private *dev_priv)
 -{
 -      hsw_pwr = NULL;
 +      /*
 +       * WaDisableVLVClockGating_VBIIssue:vlv
 +       * Disable clock gating on th GCFG unit to prevent a delay
 +       * in the reporting of vblank events.
 +       */
 +      I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
  }
  
 -static void intel_power_domains_resume(struct drm_i915_private *dev_priv)
 +static void cherryview_init_clock_gating(struct drm_device *dev)
  {
 -      struct i915_power_domains *power_domains = &dev_priv->power_domains;
 -      struct i915_power_well *power_well;
 -      int i;
 -
 -      mutex_lock(&power_domains->lock);
 -      for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
 -              power_well->ops->sync_hw(dev_priv, power_well);
 -              power_well->hw_enabled = power_well->ops->is_enabled(dev_priv,
 -                                                                   power_well);
 -      }
 -      mutex_unlock(&power_domains->lock);
 -}
 +      struct drm_i915_private *dev_priv = dev->dev_private;
  
 -static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
 -{
 -      struct i915_power_well *cmn =
 -              lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
 -      struct i915_power_well *disp2d =
 -              lookup_power_well(dev_priv, PUNIT_POWER_WELL_DISP2D);
 +      I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
  
 -      /* nothing to do if common lane is already off */
 -      if (!cmn->ops->is_enabled(dev_priv, cmn))
 -              return;
 +      I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
  
 -      /* If the display might be already active skip this */
 -      if (disp2d->ops->is_enabled(dev_priv, disp2d) &&
 -          I915_READ(DPIO_CTL) & DPIO_CMNRST)
 -              return;
 +      /* WaVSRefCountFullforceMissDisable:chv */
 +      /* WaDSRefCountFullforceMissDisable:chv */
 +      I915_WRITE(GEN7_FF_THREAD_MODE,
 +                 I915_READ(GEN7_FF_THREAD_MODE) &
 +                 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
  
 -      DRM_DEBUG_KMS("toggling display PHY side reset\n");
 +      /* WaDisableSemaphoreAndSyncFlipWait:chv */
 +      I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
 +                 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
  
 -      /* cmnlane needs DPLL registers */
 -      disp2d->ops->enable(dev_priv, disp2d);
 +      /* WaDisableCSUnitClockGating:chv */
 +      I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
 +                 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
  
 -      /*
 -       * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
 -       * Need to assert and de-assert PHY SB reset by gating the
 -       * common lane power, then un-gating it.
 -       * Simply ungating isn't enough to reset the PHY enough to get
 -       * ports and lanes running.
 -       */
 -      cmn->ops->disable(dev_priv, cmn);
 +      /* WaDisableSDEUnitClockGating:chv */
 +      I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
 +                 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  }
  
 -void intel_power_domains_init_hw(struct drm_i915_private *dev_priv)
 +static void g4x_init_clock_gating(struct drm_device *dev)
  {
 -      struct drm_device *dev = dev_priv->dev;
 -      struct i915_power_domains *power_domains = &dev_priv->power_domains;
 +      struct drm_i915_private *dev_priv = dev->dev_private;
 +      uint32_t dspclk_gate;
 +
 +      I915_WRITE(RENCLK_GATE_D1, 0);
 +      I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
 +                 GS_UNIT_CLOCK_GATE_DISABLE |
 +                 CL_UNIT_CLOCK_GATE_DISABLE);
 +      I915_WRITE(RAMCLK_GATE_D, 0);
 +      dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
 +              OVRUNIT_CLOCK_GATE_DISABLE |
 +              OVCUNIT_CLOCK_GATE_DISABLE;
 +      if (IS_GM45(dev))
 +              dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
 +      I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
  
 -      power_domains->initializing = true;
 +      /* WaDisableRenderCachePipelinedFlush */
 +      I915_WRITE(CACHE_MODE_0,
 +                 _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
  
 -      if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
 -              mutex_lock(&power_domains->lock);
 -              vlv_cmnlane_wa(dev_priv);
 -              mutex_unlock(&power_domains->lock);
 -      }
 +      /* WaDisable_RenderCache_OperationalFlush:g4x */
 +      I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
  
 -      /* For now, we need the power well to be always enabled. */
 -      intel_display_set_init_power(dev_priv, true);
 -      intel_power_domains_resume(dev_priv);
 -      power_domains->initializing = false;
 +      g4x_disable_trickle_feed(dev);
  }
  
 -void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv)
 +static void crestline_init_clock_gating(struct drm_device *dev)
  {
 -      intel_runtime_pm_get(dev_priv);
 -}
 +      struct drm_i915_private *dev_priv = dev->dev_private;
  
 -void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv)
 -{
 -      intel_runtime_pm_put(dev_priv);
 +      I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
 +      I915_WRITE(RENCLK_GATE_D2, 0);
 +      I915_WRITE(DSPCLK_GATE_D, 0);
 +      I915_WRITE(RAMCLK_GATE_D, 0);
 +      I915_WRITE16(DEUC, 0);
 +      I915_WRITE(MI_ARB_STATE,
 +                 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 +
 +      /* WaDisable_RenderCache_OperationalFlush:gen4 */
 +      I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
  }
  
 -void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
 +static void broadwater_init_clock_gating(struct drm_device *dev)
  {
 -      struct drm_device *dev = dev_priv->dev;
 -      struct device *device = &dev->pdev->dev;
 +      struct drm_i915_private *dev_priv = dev->dev_private;
  
 -      if (!HAS_RUNTIME_PM(dev))
 -              return;
 +      I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
 +                 I965_RCC_CLOCK_GATE_DISABLE |
 +                 I965_RCPB_CLOCK_GATE_DISABLE |
 +                 I965_ISC_CLOCK_GATE_DISABLE |
 +                 I965_FBC_CLOCK_GATE_DISABLE);
 +      I915_WRITE(RENCLK_GATE_D2, 0);
 +      I915_WRITE(MI_ARB_STATE,
 +                 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
  
 -      pm_runtime_get_sync(device);
 -      WARN(dev_priv->pm.suspended, "Device still suspended.\n");
 +      /* WaDisable_RenderCache_OperationalFlush:gen4 */
 +      I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
  }
  
 -void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
 +static void gen3_init_clock_gating(struct drm_device *dev)
  {
 -      struct drm_device *dev = dev_priv->dev;
 -      struct device *device = &dev->pdev->dev;
 +      struct drm_i915_private *dev_priv = dev->dev_private;
 +      u32 dstate = I915_READ(D_STATE);
  
 -      if (!HAS_RUNTIME_PM(dev))
 -              return;
 +      dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
 +              DSTATE_DOT_CLOCK_GATING;
 +      I915_WRITE(D_STATE, dstate);
 +
 +      if (IS_PINEVIEW(dev))
 +              I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
 +
 +      /* IIR "flip pending" means done if this bit is set */
 +      I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
 +
 +      /* interrupts should cause a wake up from C3 */
 +      I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
 +
 +      /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
 +      I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
  
 -      WARN(dev_priv->pm.suspended, "Getting nosync-ref while suspended.\n");
 -      pm_runtime_get_noresume(device);
 +      I915_WRITE(MI_ARB_STATE,
 +                 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
  }
  
 -void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
 +static void i85x_init_clock_gating(struct drm_device *dev)
  {
 -      struct drm_device *dev = dev_priv->dev;
 -      struct device *device = &dev->pdev->dev;
 +      struct drm_i915_private *dev_priv = dev->dev_private;
  
 -      if (!HAS_RUNTIME_PM(dev))
 -              return;
 +      I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
 +
 +      /* interrupts should cause a wake up from C3 */
 +      I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
 +                 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
  
 -      pm_runtime_mark_last_busy(device);
 -      pm_runtime_put_autosuspend(device);
 +      I915_WRITE(MEM_MODE,
 +                 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
  }
  
 -void intel_init_runtime_pm(struct drm_i915_private *dev_priv)
 +static void i830_init_clock_gating(struct drm_device *dev)
  {
 -      struct drm_device *dev = dev_priv->dev;
 -      struct device *device = &dev->pdev->dev;
 -
 -      if (!HAS_RUNTIME_PM(dev))
 -              return;
 +      struct drm_i915_private *dev_priv = dev->dev_private;
  
 -      pm_runtime_set_active(device);
 +      I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
  
 -      /*
 -       * RPM depends on RC6 to save restore the GT HW context, so make RC6 a
 -       * requirement.
 -       */
 -      if (!intel_enable_rc6(dev)) {
 -              DRM_INFO("RC6 disabled, disabling runtime PM support\n");
 -              return;
 -      }
 +      I915_WRITE(MEM_MODE,
 +                 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
 +                 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
 +}
  
 -      pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */
 -      pm_runtime_mark_last_busy(device);
 -      pm_runtime_use_autosuspend(device);
 +void intel_init_clock_gating(struct drm_device *dev)
 +{
 +      struct drm_i915_private *dev_priv = dev->dev_private;
  
 -      pm_runtime_put_autosuspend(device);
 +      dev_priv->display.init_clock_gating(dev);
  }
  
 -void intel_fini_runtime_pm(struct drm_i915_private *dev_priv)
 +void intel_suspend_hw(struct drm_device *dev)
  {
 -      struct drm_device *dev = dev_priv->dev;
 -      struct device *device = &dev->pdev->dev;
 +      if (HAS_PCH_LPT(dev))
 +              lpt_suspend_hw(dev);
 +}
  
 -      if (!HAS_RUNTIME_PM(dev))
 +static void intel_init_fbc(struct drm_i915_private *dev_priv)
 +{
 +      if (!HAS_FBC(dev_priv)) {
 +              dev_priv->fbc.enabled = false;
                return;
 +      }
  
 -      if (!intel_enable_rc6(dev))
 -              return;
 +      if (INTEL_INFO(dev_priv)->gen >= 7) {
 +              dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
 +              dev_priv->display.enable_fbc = gen7_enable_fbc;
 +              dev_priv->display.disable_fbc = ironlake_disable_fbc;
 +      } else if (INTEL_INFO(dev_priv)->gen >= 5) {
 +              dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
 +              dev_priv->display.enable_fbc = ironlake_enable_fbc;
 +              dev_priv->display.disable_fbc = ironlake_disable_fbc;
 +      } else if (IS_GM45(dev_priv)) {
 +              dev_priv->display.fbc_enabled = g4x_fbc_enabled;
 +              dev_priv->display.enable_fbc = g4x_enable_fbc;
 +              dev_priv->display.disable_fbc = g4x_disable_fbc;
 +      } else {
 +              dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
 +              dev_priv->display.enable_fbc = i8xx_enable_fbc;
 +              dev_priv->display.disable_fbc = i8xx_disable_fbc;
 +
 +              /* This value was pulled out of someone's hat */
 +              I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT);
 +      }
  
 -      /* Make sure we're not suspended first. */
 -      pm_runtime_get_sync(device);
 -      pm_runtime_disable(device);
 +      dev_priv->fbc.enabled = dev_priv->display.fbc_enabled(dev_priv->dev);
  }
  
  /* Set up chip specific power management-related functions */
Simple merge