drivers/gpu/drm/i915/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/dmi.h>
  28 #include <linux/module.h>
  29 #include <linux/input.h>
  30 #include <linux/i2c.h>
  31 #include <linux/kernel.h>
  32 #include <linux/slab.h>
  33 #include <linux/vgaarb.h>
  34 #include <drm/drm_edid.h>
  35 #include <drm/drmP.h>
  36 #include "intel_drv.h"
  37 #include <drm/i915_drm.h>
  38 #include "i915_drv.h"
  39 #include "i915_gem_dmabuf.h"
  40 #include "intel_dsi.h"
  41 #include "i915_trace.h"
  42 #include <drm/drm_atomic.h>
  43 #include <drm/drm_atomic_helper.h>
  44 #include <drm/drm_dp_helper.h>
  45 #include <drm/drm_crtc_helper.h>
  46 #include <drm/drm_plane_helper.h>
  47 #include <drm/drm_rect.h>
  48 #include <linux/dma_remapping.h>
  49 #include <linux/reservation.h>
  50
  51 static bool is_mmio_work(struct intel_flip_work *work)
  52 {
  53         return work->mmio_work.func;
  54 }
  55
  56 /* Primary plane formats for gen <= 3 */
  57 static const uint32_t i8xx_primary_formats[] = {
  58         DRM_FORMAT_C8,
  59         DRM_FORMAT_RGB565,
  60         DRM_FORMAT_XRGB1555,
  61         DRM_FORMAT_XRGB8888,
  62 };
  63
  64 /* Primary plane formats for gen >= 4 */
  65 static const uint32_t i965_primary_formats[] = {
  66         DRM_FORMAT_C8,
  67         DRM_FORMAT_RGB565,
  68         DRM_FORMAT_XRGB8888,
  69         DRM_FORMAT_XBGR8888,
  70         DRM_FORMAT_XRGB2101010,
  71         DRM_FORMAT_XBGR2101010,
  72 };
  73
  74 static const uint32_t skl_primary_formats[] = {
  75         DRM_FORMAT_C8,
  76         DRM_FORMAT_RGB565,
  77         DRM_FORMAT_XRGB8888,
  78         DRM_FORMAT_XBGR8888,
  79         DRM_FORMAT_ARGB8888,
  80         DRM_FORMAT_ABGR8888,
  81         DRM_FORMAT_XRGB2101010,
  82         DRM_FORMAT_XBGR2101010,
  83         DRM_FORMAT_YUYV,
  84         DRM_FORMAT_YVYU,
  85         DRM_FORMAT_UYVY,
  86         DRM_FORMAT_VYUY,
  87 };
  88
  89 /* Cursor formats */
  90 static const uint32_t intel_cursor_formats[] = {
  91         DRM_FORMAT_ARGB8888,
  92 };
  93
  94 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
  95                                 struct intel_crtc_state *pipe_config);
  96 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
  97                                    struct intel_crtc_state *pipe_config);
  98
  99 static int intel_framebuffer_init(struct drm_device *dev,
 100                                   struct intel_framebuffer *ifb,
 101                                   struct drm_mode_fb_cmd2 *mode_cmd,
 102                                   struct drm_i915_gem_object *obj);
 103 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc);
 104 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc);
 105 static void intel_set_pipe_src_size(struct intel_crtc *intel_crtc);
 106 static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
 107                                          struct intel_link_m_n *m_n,
 108                                          struct intel_link_m_n *m2_n2);
 109 static void ironlake_set_pipeconf(struct drm_crtc *crtc);
 110 static void haswell_set_pipeconf(struct drm_crtc *crtc);
 111 static void haswell_set_pipemisc(struct drm_crtc *crtc);
 112 static void vlv_prepare_pll(struct intel_crtc *crtc,
 113                             const struct intel_crtc_state *pipe_config);
 114 static void chv_prepare_pll(struct intel_crtc *crtc,
 115                             const struct intel_crtc_state *pipe_config);
 116 static void intel_begin_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 117 static void intel_finish_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
 118 static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc,
 119         struct intel_crtc_state *crtc_state);
 120 static void skylake_pfit_enable(struct intel_crtc *crtc);
 121 static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force);
 122 static void ironlake_pfit_enable(struct intel_crtc *crtc);
 123 static void intel_modeset_setup_hw_state(struct drm_device *dev);
 124 static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc);
 125 static int ilk_max_pixel_rate(struct drm_atomic_state *state);
 126 static int bxt_calc_cdclk(int max_pixclk);
 127
 128 struct intel_limit {
 129         struct {
 130                 int min, max;
 131         } dot, vco, n, m, m1, m2, p, p1;
 132
 133         struct {
 134                 int dot_limit;
 135                 int p2_slow, p2_fast;
 136         } p2;
 137 };
 138
 139 /* returns HPLL frequency in kHz */
 140 static int valleyview_get_vco(struct drm_i915_private *dev_priv)
 141 {
 142         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 143
 144         /* Obtain SKU information */
 145         mutex_lock(&dev_priv->sb_lock);
 146         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 147                 CCK_FUSE_HPLL_FREQ_MASK;
 148         mutex_unlock(&dev_priv->sb_lock);
 149
 150         return vco_freq[hpll_freq] * 1000;
 151 }
 152
 153 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 154                       const char *name, u32 reg, int ref_freq)
 155 {
 156         u32 val;
 157         int divider;
 158
 159         mutex_lock(&dev_priv->sb_lock);
 160         val = vlv_cck_read(dev_priv, reg);
 161         mutex_unlock(&dev_priv->sb_lock);
 162
 163         divider = val & CCK_FREQUENCY_VALUES;
 164
 165         WARN((val & CCK_FREQUENCY_STATUS) !=
 166              (divider << CCK_FREQUENCY_STATUS_SHIFT),
 167              "%s change in progress\n", name);
 168
 169         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 170 }
 171
 172 static int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 173                                   const char *name, u32 reg)
 174 {
 175         if (dev_priv->hpll_freq == 0)
 176                 dev_priv->hpll_freq = valleyview_get_vco(dev_priv);
 177
 178         return vlv_get_cck_clock(dev_priv, name, reg,
 179                                  dev_priv->hpll_freq);
 180 }
 181
 182 static int
 183 intel_pch_rawclk(struct drm_i915_private *dev_priv)
 184 {
 185         return (I915_READ(PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK) * 1000;
 186 }
 187
 188 static int
 189 intel_vlv_hrawclk(struct drm_i915_private *dev_priv)
 190 {
 191         /* RAWCLK_FREQ_VLV register updated from power well code */
 192         return vlv_get_cck_clock_hpll(dev_priv, "hrawclk",
 193                                       CCK_DISPLAY_REF_CLOCK_CONTROL);
 194 }
 195
 196 static int
 197 intel_g4x_hrawclk(struct drm_i915_private *dev_priv)
 198 {
 199         uint32_t clkcfg;
 200
 201         /* hrawclock is 1/4 the FSB frequency */
 202         clkcfg = I915_READ(CLKCFG);
 203         switch (clkcfg & CLKCFG_FSB_MASK) {
 204         case CLKCFG_FSB_400:
 205                 return 100000;
 206         case CLKCFG_FSB_533:
 207                 return 133333;
 208         case CLKCFG_FSB_667:
 209                 return 166667;
 210         case CLKCFG_FSB_800:
 211                 return 200000;
 212         case CLKCFG_FSB_1067:
 213                 return 266667;
 214         case CLKCFG_FSB_1333:
 215                 return 333333;
 216         /* these two are just a guess; one of them might be right */
 217         case CLKCFG_FSB_1600:
 218         case CLKCFG_FSB_1600_ALT:
 219                 return 400000;
 220         default:
 221                 return 133333;
 222         }
 223 }
 224
 225 void intel_update_rawclk(struct drm_i915_private *dev_priv)
 226 {
 227         if (HAS_PCH_SPLIT(dev_priv))
 228                 dev_priv->rawclk_freq = intel_pch_rawclk(dev_priv);
 229         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 230                 dev_priv->rawclk_freq = intel_vlv_hrawclk(dev_priv);
 231         else if (IS_G4X(dev_priv) || IS_PINEVIEW(dev_priv))
 232                 dev_priv->rawclk_freq = intel_g4x_hrawclk(dev_priv);
 233         else
 234                 return; /* no rawclk on other platforms, or no need to know it */
 235
 236         DRM_DEBUG_DRIVER("rawclk rate: %d kHz\n", dev_priv->rawclk_freq);
 237 }
 238
 239 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 240 {
 241         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 242                 return;
 243
 244         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 245                                                       CCK_CZ_CLOCK_CONTROL);
 246
 247         DRM_DEBUG_DRIVER("CZ clock rate: %d kHz\n", dev_priv->czclk_freq);
 248 }
 249
 250 static inline u32 /* units of 100MHz */
 251 intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 252                     const struct intel_crtc_state *pipe_config)
 253 {
 254         if (HAS_DDI(dev_priv))
 255                 return pipe_config->port_clock; /* SPLL */
 256         else if (IS_GEN5(dev_priv))
 257                 return ((I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK) + 2) * 10000;
 258         else
 259                 return 270000;
 260 }
 261
 262 static const struct intel_limit intel_limits_i8xx_dac = {
 263         .dot = { .min = 25000, .max = 350000 },
 264         .vco = { .min = 908000, .max = 1512000 },
 265         .n = { .min = 2, .max = 16 },
 266         .m = { .min = 96, .max = 140 },
 267         .m1 = { .min = 18, .max = 26 },
 268         .m2 = { .min = 6, .max = 16 },
 269         .p = { .min = 4, .max = 128 },
 270         .p1 = { .min = 2, .max = 33 },
 271         .p2 = { .dot_limit = 165000,
 272                 .p2_slow = 4, .p2_fast = 2 },
 273 };
 274
 275 static const struct intel_limit intel_limits_i8xx_dvo = {
 276         .dot = { .min = 25000, .max = 350000 },
 277         .vco = { .min = 908000, .max = 1512000 },
 278         .n = { .min = 2, .max = 16 },
 279         .m = { .min = 96, .max = 140 },
 280         .m1 = { .min = 18, .max = 26 },
 281         .m2 = { .min = 6, .max = 16 },
 282         .p = { .min = 4, .max = 128 },
 283         .p1 = { .min = 2, .max = 33 },
 284         .p2 = { .dot_limit = 165000,
 285                 .p2_slow = 4, .p2_fast = 4 },
 286 };
 287
 288 static const struct intel_limit intel_limits_i8xx_lvds = {
 289         .dot = { .min = 25000, .max = 350000 },
 290         .vco = { .min = 908000, .max = 1512000 },
 291         .n = { .min = 2, .max = 16 },
 292         .m = { .min = 96, .max = 140 },
 293         .m1 = { .min = 18, .max = 26 },
 294         .m2 = { .min = 6, .max = 16 },
 295         .p = { .min = 4, .max = 128 },
 296         .p1 = { .min = 1, .max = 6 },
 297         .p2 = { .dot_limit = 165000,
 298                 .p2_slow = 14, .p2_fast = 7 },
 299 };
 300
 301 static const struct intel_limit intel_limits_i9xx_sdvo = {
 302         .dot = { .min = 20000, .max = 400000 },
 303         .vco = { .min = 1400000, .max = 2800000 },
 304         .n = { .min = 1, .max = 6 },
 305         .m = { .min = 70, .max = 120 },
 306         .m1 = { .min = 8, .max = 18 },
 307         .m2 = { .min = 3, .max = 7 },
 308         .p = { .min = 5, .max = 80 },
 309         .p1 = { .min = 1, .max = 8 },
 310         .p2 = { .dot_limit = 200000,
 311                 .p2_slow = 10, .p2_fast = 5 },
 312 };
 313
 314 static const struct intel_limit intel_limits_i9xx_lvds = {
 315         .dot = { .min = 20000, .max = 400000 },
 316         .vco = { .min = 1400000, .max = 2800000 },
 317         .n = { .min = 1, .max = 6 },
 318         .m = { .min = 70, .max = 120 },
 319         .m1 = { .min = 8, .max = 18 },
 320         .m2 = { .min = 3, .max = 7 },
 321         .p = { .min = 7, .max = 98 },
 322         .p1 = { .min = 1, .max = 8 },
 323         .p2 = { .dot_limit = 112000,
 324                 .p2_slow = 14, .p2_fast = 7 },
 325 };
 326
 327
 328 static const struct intel_limit intel_limits_g4x_sdvo = {
 329         .dot = { .min = 25000, .max = 270000 },
 330         .vco = { .min = 1750000, .max = 3500000},
 331         .n = { .min = 1, .max = 4 },
 332         .m = { .min = 104, .max = 138 },
 333         .m1 = { .min = 17, .max = 23 },
 334         .m2 = { .min = 5, .max = 11 },
 335         .p = { .min = 10, .max = 30 },
 336         .p1 = { .min = 1, .max = 3},
 337         .p2 = { .dot_limit = 270000,
 338                 .p2_slow = 10,
 339                 .p2_fast = 10
 340         },
 341 };
 342
 343 static const struct intel_limit intel_limits_g4x_hdmi = {
 344         .dot = { .min = 22000, .max = 400000 },
 345         .vco = { .min = 1750000, .max = 3500000},
 346         .n = { .min = 1, .max = 4 },
 347         .m = { .min = 104, .max = 138 },
 348         .m1 = { .min = 16, .max = 23 },
 349         .m2 = { .min = 5, .max = 11 },
 350         .p = { .min = 5, .max = 80 },
 351         .p1 = { .min = 1, .max = 8},
 352         .p2 = { .dot_limit = 165000,
 353                 .p2_slow = 10, .p2_fast = 5 },
 354 };
 355
 356 static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
 357         .dot = { .min = 20000, .max = 115000 },
 358         .vco = { .min = 1750000, .max = 3500000 },
 359         .n = { .min = 1, .max = 3 },
 360         .m = { .min = 104, .max = 138 },
 361         .m1 = { .min = 17, .max = 23 },
 362         .m2 = { .min = 5, .max = 11 },
 363         .p = { .min = 28, .max = 112 },
 364         .p1 = { .min = 2, .max = 8 },
 365         .p2 = { .dot_limit = 0,
 366                 .p2_slow = 14, .p2_fast = 14
 367         },
 368 };
 369
 370 static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
 371         .dot = { .min = 80000, .max = 224000 },
 372         .vco = { .min = 1750000, .max = 3500000 },
 373         .n = { .min = 1, .max = 3 },
 374         .m = { .min = 104, .max = 138 },
 375         .m1 = { .min = 17, .max = 23 },
 376         .m2 = { .min = 5, .max = 11 },
 377         .p = { .min = 14, .max = 42 },
 378         .p1 = { .min = 2, .max = 6 },
 379         .p2 = { .dot_limit = 0,
 380                 .p2_slow = 7, .p2_fast = 7
 381         },
 382 };
 383
 384 static const struct intel_limit intel_limits_pineview_sdvo = {
 385         .dot = { .min = 20000, .max = 400000},
 386         .vco = { .min = 1700000, .max = 3500000 },
 387         /* Pineview's Ncounter is a ring counter */
 388         .n = { .min = 3, .max = 6 },
 389         .m = { .min = 2, .max = 256 },
 390         /* Pineview only has one combined m divider, which we treat as m2. */
 391         .m1 = { .min = 0, .max = 0 },
 392         .m2 = { .min = 0, .max = 254 },
 393         .p = { .min = 5, .max = 80 },
 394         .p1 = { .min = 1, .max = 8 },
 395         .p2 = { .dot_limit = 200000,
 396                 .p2_slow = 10, .p2_fast = 5 },
 397 };
 398
 399 static const struct intel_limit intel_limits_pineview_lvds = {
 400         .dot = { .min = 20000, .max = 400000 },
 401         .vco = { .min = 1700000, .max = 3500000 },
 402         .n = { .min = 3, .max = 6 },
 403         .m = { .min = 2, .max = 256 },
 404         .m1 = { .min = 0, .max = 0 },
 405         .m2 = { .min = 0, .max = 254 },
 406         .p = { .min = 7, .max = 112 },
 407         .p1 = { .min = 1, .max = 8 },
 408         .p2 = { .dot_limit = 112000,
 409                 .p2_slow = 14, .p2_fast = 14 },
 410 };
 411
 412 /* Ironlake / Sandybridge
 413  *
 414  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 415  * the range value for them is (actual_value - 2).
 416  */
 417 static const struct intel_limit intel_limits_ironlake_dac = {
 418         .dot = { .min = 25000, .max = 350000 },
 419         .vco = { .min = 1760000, .max = 3510000 },
 420         .n = { .min = 1, .max = 5 },
 421         .m = { .min = 79, .max = 127 },
 422         .m1 = { .min = 12, .max = 22 },
 423         .m2 = { .min = 5, .max = 9 },
 424         .p = { .min = 5, .max = 80 },
 425         .p1 = { .min = 1, .max = 8 },
 426         .p2 = { .dot_limit = 225000,
 427                 .p2_slow = 10, .p2_fast = 5 },
 428 };
 429
 430 static const struct intel_limit intel_limits_ironlake_single_lvds = {
 431         .dot = { .min = 25000, .max = 350000 },
 432         .vco = { .min = 1760000, .max = 3510000 },
 433         .n = { .min = 1, .max = 3 },
 434         .m = { .min = 79, .max = 118 },
 435         .m1 = { .min = 12, .max = 22 },
 436         .m2 = { .min = 5, .max = 9 },
 437         .p = { .min = 28, .max = 112 },
 438         .p1 = { .min = 2, .max = 8 },
 439         .p2 = { .dot_limit = 225000,
 440                 .p2_slow = 14, .p2_fast = 14 },
 441 };
 442
 443 static const struct intel_limit intel_limits_ironlake_dual_lvds = {
 444         .dot = { .min = 25000, .max = 350000 },
 445         .vco = { .min = 1760000, .max = 3510000 },
 446         .n = { .min = 1, .max = 3 },
 447         .m = { .min = 79, .max = 127 },
 448         .m1 = { .min = 12, .max = 22 },
 449         .m2 = { .min = 5, .max = 9 },
 450         .p = { .min = 14, .max = 56 },
 451         .p1 = { .min = 2, .max = 8 },
 452         .p2 = { .dot_limit = 225000,
 453                 .p2_slow = 7, .p2_fast = 7 },
 454 };
 455
 456 /* LVDS 100mhz refclk limits. */
 457 static const struct intel_limit intel_limits_ironlake_single_lvds_100m = {
 458         .dot = { .min = 25000, .max = 350000 },
 459         .vco = { .min = 1760000, .max = 3510000 },
 460         .n = { .min = 1, .max = 2 },
 461         .m = { .min = 79, .max = 126 },
 462         .m1 = { .min = 12, .max = 22 },
 463         .m2 = { .min = 5, .max = 9 },
 464         .p = { .min = 28, .max = 112 },
 465         .p1 = { .min = 2, .max = 8 },
 466         .p2 = { .dot_limit = 225000,
 467                 .p2_slow = 14, .p2_fast = 14 },
 468 };
 469
 470 static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = {
 471         .dot = { .min = 25000, .max = 350000 },
 472         .vco = { .min = 1760000, .max = 3510000 },
 473         .n = { .min = 1, .max = 3 },
 474         .m = { .min = 79, .max = 126 },
 475         .m1 = { .min = 12, .max = 22 },
 476         .m2 = { .min = 5, .max = 9 },
 477         .p = { .min = 14, .max = 42 },
 478         .p1 = { .min = 2, .max = 6 },
 479         .p2 = { .dot_limit = 225000,
 480                 .p2_slow = 7, .p2_fast = 7 },
 481 };
 482
 483 static const struct intel_limit intel_limits_vlv = {
 484          /*
 485           * These are the data rate limits (measured in fast clocks)
 486           * since those are the strictest limits we have. The fast
 487           * clock and actual rate limits are more relaxed, so checking
 488           * them would make no difference.
 489           */
 490         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 491         .vco = { .min = 4000000, .max = 6000000 },
 492         .n = { .min = 1, .max = 7 },
 493         .m1 = { .min = 2, .max = 3 },
 494         .m2 = { .min = 11, .max = 156 },
 495         .p1 = { .min = 2, .max = 3 },
 496         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 497 };
 498
 499 static const struct intel_limit intel_limits_chv = {
 500         /*
 501          * These are the data rate limits (measured in fast clocks)
 502          * since those are the strictest limits we have.  The fast
 503          * clock and actual rate limits are more relaxed, so checking
 504          * them would make no difference.
 505          */
 506         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 507         .vco = { .min = 4800000, .max = 6480000 },
 508         .n = { .min = 1, .max = 1 },
 509         .m1 = { .min = 2, .max = 2 },
 510         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 511         .p1 = { .min = 2, .max = 4 },
 512         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 513 };
 514
 515 static const struct intel_limit intel_limits_bxt = {
 516         /* FIXME: find real dot limits */
 517         .dot = { .min = 0, .max = INT_MAX },
 518         .vco = { .min = 4800000, .max = 6700000 },
 519         .n = { .min = 1, .max = 1 },
 520         .m1 = { .min = 2, .max = 2 },
 521         /* FIXME: find real m2 limits */
 522         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 523         .p1 = { .min = 2, .max = 4 },
 524         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 525 };
 526
 527 static bool
 528 needs_modeset(struct drm_crtc_state *state)
 529 {
 530         return drm_atomic_crtc_needs_modeset(state);
 531 }
 532
 533 /**
 534  * Returns whether any output on the specified pipe is of the specified type
 535  */
 536 bool intel_pipe_has_type(struct intel_crtc *crtc, enum intel_output_type type)
 537 {
 538         struct drm_device *dev = crtc->base.dev;
 539         struct intel_encoder *encoder;
 540
 541         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
 542                 if (encoder->type == type)
 543                         return true;
 544
 545         return false;
 546 }
 547
 548 /**
 549  * Returns whether any output on the specified pipe will have the specified
 550  * type after a staged modeset is complete, i.e., the same as
 551  * intel_pipe_has_type() but looking at encoder->new_crtc instead of
 552  * encoder->crtc.
 553  */
 554 static bool intel_pipe_will_have_type(const struct intel_crtc_state *crtc_state,
 555                                       int type)
 556 {
 557         struct drm_atomic_state *state = crtc_state->base.state;
 558         struct drm_connector *connector;
 559         struct drm_connector_state *connector_state;
 560         struct intel_encoder *encoder;
 561         int i, num_connectors = 0;
 562
 563         for_each_connector_in_state(state, connector, connector_state, i) {
 564                 if (connector_state->crtc != crtc_state->base.crtc)
 565                         continue;
 566
 567                 num_connectors++;
 568
 569                 encoder = to_intel_encoder(connector_state->best_encoder);
 570                 if (encoder->type == type)
 571                         return true;
 572         }
 573
 574         WARN_ON(num_connectors == 0);
 575
 576         return false;
 577 }
 578
 579 /*
 580  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
 581  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
 582  * (clock.dot) clock rates. This fast dot clock is fed to the port's IO logic.
 583  * The helpers' return value is the rate of the clock that is fed to the
 584  * display engine's pipe which can be the above fast dot clock rate or a
 585  * divided-down version of it.
 586  */
 587 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 588 static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 589 {
 590         clock->m = clock->m2 + 2;
 591         clock->p = clock->p1 * clock->p2;
 592         if (WARN_ON(clock->n == 0 || clock->p == 0))
 593                 return 0;
 594         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 595         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 596
 597         return clock->dot;
 598 }
 599
 600 static uint32_t i9xx_dpll_compute_m(struct dpll *dpll)
 601 {
 602         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 603 }
 604
 605 static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 606 {
 607         clock->m = i9xx_dpll_compute_m(clock);
 608         clock->p = clock->p1 * clock->p2;
 609         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 610                 return 0;
 611         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 612         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 613
 614         return clock->dot;
 615 }
 616
 617 static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 618 {
 619         clock->m = clock->m1 * clock->m2;
 620         clock->p = clock->p1 * clock->p2;
 621         if (WARN_ON(clock->n == 0 || clock->p == 0))
 622                 return 0;
 623         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 624         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 625
 626         return clock->dot / 5;
 627 }
 628
 629 int chv_calc_dpll_params(int refclk, struct dpll *clock)
 630 {
 631         clock->m = clock->m1 * clock->m2;
 632         clock->p = clock->p1 * clock->p2;
 633         if (WARN_ON(clock->n == 0 || clock->p == 0))
 634                 return 0;
 635         clock->vco = DIV_ROUND_CLOSEST_ULL((uint64_t)refclk * clock->m,
 636                         clock->n << 22);
 637         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 638
 639         return clock->dot / 5;
 640 }
 641
 642 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 643 /**
 644  * Returns whether the given set of divisors are valid for a given refclk with
 645  * the given connectors.
 646  */
 647
 648 static bool intel_PLL_is_valid(struct drm_device *dev,
 649                                const struct intel_limit *limit,
 650                                const struct dpll *clock)
 651 {
 652         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 653                 INTELPllInvalid("n out of range\n");
 654         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 655                 INTELPllInvalid("p1 out of range\n");
 656         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 657                 INTELPllInvalid("m2 out of range\n");
 658         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 659                 INTELPllInvalid("m1 out of range\n");
 660
 661         if (!IS_PINEVIEW(dev) && !IS_VALLEYVIEW(dev) &&
 662             !IS_CHERRYVIEW(dev) && !IS_BROXTON(dev))
 663                 if (clock->m1 <= clock->m2)
 664                         INTELPllInvalid("m1 <= m2\n");
 665
 666         if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) && !IS_BROXTON(dev)) {
 667                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 668                         INTELPllInvalid("p out of range\n");
 669                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 670                         INTELPllInvalid("m out of range\n");
 671         }
 672
 673         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 674                 INTELPllInvalid("vco out of range\n");
 675         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 676          * connector, etc., rather than just a single range.
 677          */
 678         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 679                 INTELPllInvalid("dot out of range\n");
 680
 681         return true;
 682 }
 683
 684 static int
 685 i9xx_select_p2_div(const struct intel_limit *limit,
 686                    const struct intel_crtc_state *crtc_state,
 687                    int target)
 688 {
 689         struct drm_device *dev = crtc_state->base.crtc->dev;
 690
 691         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 692                 /*
 693                  * For LVDS just rely on its current settings for dual-channel.
 694                  * We haven't figured out how to reliably set up different
 695                  * single/dual channel state, if we even can.
 696                  */
 697                 if (intel_is_dual_link_lvds(dev))
 698                         return limit->p2.p2_fast;
 699                 else
 700                         return limit->p2.p2_slow;
 701         } else {
 702                 if (target < limit->p2.dot_limit)
 703                         return limit->p2.p2_slow;
 704                 else
 705                         return limit->p2.p2_fast;
 706         }
 707 }
 708
 709 /*
 710  * Returns a set of divisors for the desired target clock with the given
 711  * refclk, or FALSE.  The returned values represent the clock equation:
 712  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 713  *
 714  * Target and reference clocks are specified in kHz.
 715  *
 716  * If match_clock is provided, then best_clock P divider must match the P
 717  * divider from @match_clock used for LVDS downclocking.
 718  */
 719 static bool
 720 i9xx_find_best_dpll(const struct intel_limit *limit,
 721                     struct intel_crtc_state *crtc_state,
 722                     int target, int refclk, struct dpll *match_clock,
 723                     struct dpll *best_clock)
 724 {
 725         struct drm_device *dev = crtc_state->base.crtc->dev;
 726         struct dpll clock;
 727         int err = target;
 728
 729         memset(best_clock, 0, sizeof(*best_clock));
 730
 731         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 732
 733         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 734              clock.m1++) {
 735                 for (clock.m2 = limit->m2.min;
 736                      clock.m2 <= limit->m2.max; clock.m2++) {
 737                         if (clock.m2 >= clock.m1)
 738                                 break;
 739                         for (clock.n = limit->n.min;
 740                              clock.n <= limit->n.max; clock.n++) {
 741                                 for (clock.p1 = limit->p1.min;
 742                                         clock.p1 <= limit->p1.max; clock.p1++) {
 743                                         int this_err;
 744
 745                                         i9xx_calc_dpll_params(refclk, &clock);
 746                                         if (!intel_PLL_is_valid(dev, limit,
 747                                                                 &clock))
 748                                                 continue;
 749                                         if (match_clock &&
 750                                             clock.p != match_clock->p)
 751                                                 continue;
 752
 753                                         this_err = abs(clock.dot - target);
 754                                         if (this_err < err) {
 755                                                 *best_clock = clock;
 756                                                 err = this_err;
 757                                         }
 758                                 }
 759                         }
 760                 }
 761         }
 762
 763         return (err != target);
 764 }
 765
 766 /*
 767  * Returns a set of divisors for the desired target clock with the given
 768  * refclk, or FALSE.  The returned values represent the clock equation:
 769  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 770  *
 771  * Target and reference clocks are specified in kHz.
 772  *
 773  * If match_clock is provided, then best_clock P divider must match the P
 774  * divider from @match_clock used for LVDS downclocking.
 775  */
 776 static bool
 777 pnv_find_best_dpll(const struct intel_limit *limit,
 778                    struct intel_crtc_state *crtc_state,
 779                    int target, int refclk, struct dpll *match_clock,
 780                    struct dpll *best_clock)
 781 {
 782         struct drm_device *dev = crtc_state->base.crtc->dev;
 783         struct dpll clock;
 784         int err = target;
 785
 786         memset(best_clock, 0, sizeof(*best_clock));
 787
 788         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 789
 790         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 791              clock.m1++) {
 792                 for (clock.m2 = limit->m2.min;
 793                      clock.m2 <= limit->m2.max; clock.m2++) {
 794                         for (clock.n = limit->n.min;
 795                              clock.n <= limit->n.max; clock.n++) {
 796                                 for (clock.p1 = limit->p1.min;
 797                                         clock.p1 <= limit->p1.max; clock.p1++) {
 798                                         int this_err;
 799
 800                                         pnv_calc_dpll_params(refclk, &clock);
 801                                         if (!intel_PLL_is_valid(dev, limit,
 802                                                                 &clock))
 803                                                 continue;
 804                                         if (match_clock &&
 805                                             clock.p != match_clock->p)
 806                                                 continue;
 807
 808                                         this_err = abs(clock.dot - target);
 809                                         if (this_err < err) {
 810                                                 *best_clock = clock;
 811                                                 err = this_err;
 812                                         }
 813                                 }
 814                         }
 815                 }
 816         }
 817
 818         return (err != target);
 819 }
 820
 821 /*
 822  * Returns a set of divisors for the desired target clock with the given
 823  * refclk, or FALSE.  The returned values represent the clock equation:
 824  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 825  *
 826  * Target and reference clocks are specified in kHz.
 827  *
 828  * If match_clock is provided, then best_clock P divider must match the P
 829  * divider from @match_clock used for LVDS downclocking.
 830  */
 831 static bool
 832 g4x_find_best_dpll(const struct intel_limit *limit,
 833                    struct intel_crtc_state *crtc_state,
 834                    int target, int refclk, struct dpll *match_clock,
 835                    struct dpll *best_clock)
 836 {
 837         struct drm_device *dev = crtc_state->base.crtc->dev;
 838         struct dpll clock;
 839         int max_n;
 840         bool found = false;
 841         /* approximately equals target * 0.00585 */
 842         int err_most = (target >> 8) + (target >> 9);
 843
 844         memset(best_clock, 0, sizeof(*best_clock));
 845
 846         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 847
 848         max_n = limit->n.max;
 849         /* based on hardware requirement, prefer smaller n to precision */
 850         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 851                 /* based on hardware requirement, prefere larger m1,m2 */
 852                 for (clock.m1 = limit->m1.max;
 853                      clock.m1 >= limit->m1.min; clock.m1--) {
 854                         for (clock.m2 = limit->m2.max;
 855                              clock.m2 >= limit->m2.min; clock.m2--) {
 856                                 for (clock.p1 = limit->p1.max;
 857                                      clock.p1 >= limit->p1.min; clock.p1--) {
 858                                         int this_err;
 859
 860                                         i9xx_calc_dpll_params(refclk, &clock);
 861                                         if (!intel_PLL_is_valid(dev, limit,
 862                                                                 &clock))
 863                                                 continue;
 864
 865                                         this_err = abs(clock.dot - target);
 866                                         if (this_err < err_most) {
 867                                                 *best_clock = clock;
 868                                                 err_most = this_err;
 869                                                 max_n = clock.n;
 870                                                 found = true;
 871                                         }
 872                                 }
 873                         }
 874                 }
 875         }
 876         return found;
 877 }
 878
 879 /*
 880  * Check if the calculated PLL configuration is more optimal compared to the
 881  * best configuration and error found so far. Return the calculated error.
 882  */
 883 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 884                                const struct dpll *calculated_clock,
 885                                const struct dpll *best_clock,
 886                                unsigned int best_error_ppm,
 887                                unsigned int *error_ppm)
 888 {
 889         /*
 890          * For CHV ignore the error and consider only the P value.
 891          * Prefer a bigger P value based on HW requirements.
 892          */
 893         if (IS_CHERRYVIEW(dev)) {
 894                 *error_ppm = 0;
 895
 896                 return calculated_clock->p > best_clock->p;
 897         }
 898
 899         if (WARN_ON_ONCE(!target_freq))
 900                 return false;
 901
 902         *error_ppm = div_u64(1000000ULL *
 903                                 abs(target_freq - calculated_clock->dot),
 904                              target_freq);
 905         /*
 906          * Prefer a better P value over a better (smaller) error if the error
 907          * is small. Ensure this preference for future configurations too by
 908          * setting the error to 0.
 909          */
 910         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 911                 *error_ppm = 0;
 912
 913                 return true;
 914         }
 915
 916         return *error_ppm + 10 < best_error_ppm;
 917 }
 918
 919 /*
 920  * Returns a set of divisors for the desired target clock with the given
 921  * refclk, or FALSE.  The returned values represent the clock equation:
 922  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 923  */
 924 static bool
 925 vlv_find_best_dpll(const struct intel_limit *limit,
 926                    struct intel_crtc_state *crtc_state,
 927                    int target, int refclk, struct dpll *match_clock,
 928                    struct dpll *best_clock)
 929 {
 930         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 931         struct drm_device *dev = crtc->base.dev;
 932         struct dpll clock;
 933         unsigned int bestppm = 1000000;
 934         /* min update 19.2 MHz */
 935         int max_n = min(limit->n.max, refclk / 19200);
 936         bool found = false;
 937
 938         target *= 5; /* fast clock */
 939
 940         memset(best_clock, 0, sizeof(*best_clock));
 941
 942         /* based on hardware requirement, prefer smaller n to precision */
 943         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 944                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 945                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 946                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 947                                 clock.p = clock.p1 * clock.p2;
 948                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 949                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 950                                         unsigned int ppm;
 951
 952                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 953                                                                      refclk * clock.m1);
 954
 955                                         vlv_calc_dpll_params(refclk, &clock);
 956
 957                                         if (!intel_PLL_is_valid(dev, limit,
 958                                                                 &clock))
 959                                                 continue;
 960
 961                                         if (!vlv_PLL_is_optimal(dev, target,
 962                                                                 &clock,
 963                                                                 best_clock,
 964                                                                 bestppm, &ppm))
 965                                                 continue;
 966
 967                                         *best_clock = clock;
 968                                         bestppm = ppm;
 969                                         found = true;
 970                                 }
 971                         }
 972                 }
 973         }
 974
 975         return found;
 976 }
 977
 978 /*
 979  * Returns a set of divisors for the desired target clock with the given
 980  * refclk, or FALSE.  The returned values represent the clock equation:
 981  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
 982  */
 983 static bool
 984 chv_find_best_dpll(const struct intel_limit *limit,
 985                    struct intel_crtc_state *crtc_state,
 986                    int target, int refclk, struct dpll *match_clock,
 987                    struct dpll *best_clock)
 988 {
 989         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 990         struct drm_device *dev = crtc->base.dev;
 991         unsigned int best_error_ppm;
 992         struct dpll clock;
 993         uint64_t m2;
 994         int found = false;
 995
 996         memset(best_clock, 0, sizeof(*best_clock));
 997         best_error_ppm = 1000000;
 998
 999         /*
1000          * Based on hardware doc, the n always set to 1, and m1 always
1001          * set to 2.  If requires to support 200Mhz refclk, we need to
1002          * revisit this because n may not 1 anymore.
1003          */
1004         clock.n = 1, clock.m1 = 2;
1005         target *= 5;    /* fast clock */
1006
1007         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
1008                 for (clock.p2 = limit->p2.p2_fast;
1009                                 clock.p2 >= limit->p2.p2_slow;
1010                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
1011                         unsigned int error_ppm;
1012
1013                         clock.p = clock.p1 * clock.p2;
1014
1015                         m2 = DIV_ROUND_CLOSEST_ULL(((uint64_t)target * clock.p *
1016                                         clock.n) << 22, refclk * clock.m1);
1017
1018                         if (m2 > INT_MAX/clock.m1)
1019                                 continue;
1020
1021                         clock.m2 = m2;
1022
1023                         chv_calc_dpll_params(refclk, &clock);
1024
1025                         if (!intel_PLL_is_valid(dev, limit, &clock))
1026                                 continue;
1027
1028                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
1029                                                 best_error_ppm, &error_ppm))
1030                                 continue;
1031
1032                         *best_clock = clock;
1033                         best_error_ppm = error_ppm;
1034                         found = true;
1035                 }
1036         }
1037
1038         return found;
1039 }
1040
1041 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
1042                         struct dpll *best_clock)
1043 {
1044         int refclk = 100000;
1045         const struct intel_limit *limit = &intel_limits_bxt;
1046
1047         return chv_find_best_dpll(limit, crtc_state,
1048                                   target_clock, refclk, NULL, best_clock);
1049 }
1050
1051 bool intel_crtc_active(struct drm_crtc *crtc)
1052 {
1053         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1054
1055         /* Be paranoid as we can arrive here with only partial
1056          * state retrieved from the hardware during setup.
1057          *
1058          * We can ditch the adjusted_mode.crtc_clock check as soon
1059          * as Haswell has gained clock readout/fastboot support.
1060          *
1061          * We can ditch the crtc->primary->fb check as soon as we can
1062          * properly reconstruct framebuffers.
1063          *
1064          * FIXME: The intel_crtc->active here should be switched to
1065          * crtc->state->active once we have proper CRTC states wired up
1066          * for atomic.
1067          */
1068         return intel_crtc->active && crtc->primary->state->fb &&
1069                 intel_crtc->config->base.adjusted_mode.crtc_clock;
1070 }
1071
1072 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
1073                                              enum pipe pipe)
1074 {
1075         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1076         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1077
1078         return intel_crtc->config->cpu_transcoder;
1079 }
1080
1081 static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
1082 {
1083         struct drm_i915_private *dev_priv = dev->dev_private;
1084         i915_reg_t reg = PIPEDSL(pipe);
1085         u32 line1, line2;
1086         u32 line_mask;
1087
1088         if (IS_GEN2(dev))
1089                 line_mask = DSL_LINEMASK_GEN2;
1090         else
1091                 line_mask = DSL_LINEMASK_GEN3;
1092
1093         line1 = I915_READ(reg) & line_mask;
1094         msleep(5);
1095         line2 = I915_READ(reg) & line_mask;
1096
1097         return line1 == line2;
1098 }
1099
1100 /*
1101  * intel_wait_for_pipe_off - wait for pipe to turn off
1102  * @crtc: crtc whose pipe to wait for
1103  *
1104  * After disabling a pipe, we can't wait for vblank in the usual way,
1105  * spinning on the vblank interrupt status bit, since we won't actually
1106  * see an interrupt when the pipe is disabled.
1107  *
1108  * On Gen4 and above:
1109  *   wait for the pipe register state bit to turn off
1110  *
1111  * Otherwise:
1112  *   wait for the display line value to settle (it usually
1113  *   ends up stopping at the start of the next frame).
1114  *
1115  */
1116 static void intel_wait_for_pipe_off(struct intel_crtc *crtc)
1117 {
1118         struct drm_device *dev = crtc->base.dev;
1119         struct drm_i915_private *dev_priv = dev->dev_private;
1120         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
1121         enum pipe pipe = crtc->pipe;
1122
1123         if (INTEL_INFO(dev)->gen >= 4) {
1124                 i915_reg_t reg = PIPECONF(cpu_transcoder);
1125
1126                 /* Wait for the Pipe State to go off */
1127                 if (intel_wait_for_register(dev_priv,
1128                                             reg, I965_PIPECONF_ACTIVE, 0,
1129                                             100))
1130                         WARN(1, "pipe_off wait timed out\n");
1131         } else {
1132                 /* Wait for the display line to settle */
1133                 if (wait_for(pipe_dsl_stopped(dev, pipe), 100))
1134                         WARN(1, "pipe_off wait timed out\n");
1135         }
1136 }
1137
1138 /* Only for pre-ILK configs */
1139 void assert_pll(struct drm_i915_private *dev_priv,
1140                 enum pipe pipe, bool state)
1141 {
1142         u32 val;
1143         bool cur_state;
1144
1145         val = I915_READ(DPLL(pipe));
1146         cur_state = !!(val & DPLL_VCO_ENABLE);
1147         I915_STATE_WARN(cur_state != state,
1148              "PLL state assertion failure (expected %s, current %s)\n",
1149                         onoff(state), onoff(cur_state));
1150 }
1151
1152 /* XXX: the dsi pll is shared between MIPI DSI ports */
1153 void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1154 {
1155         u32 val;
1156         bool cur_state;
1157
1158         mutex_lock(&dev_priv->sb_lock);
1159         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1160         mutex_unlock(&dev_priv->sb_lock);
1161
1162         cur_state = val & DSI_PLL_VCO_EN;
1163         I915_STATE_WARN(cur_state != state,
1164              "DSI PLL state assertion failure (expected %s, current %s)\n",
1165                         onoff(state), onoff(cur_state));
1166 }
1167
1168 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1169                           enum pipe pipe, bool state)
1170 {
1171         bool cur_state;
1172         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1173                                                                       pipe);
1174
1175         if (HAS_DDI(dev_priv)) {
1176                 /* DDI does not have a specific FDI_TX register */
1177                 u32 val = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
1178                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1179         } else {
1180                 u32 val = I915_READ(FDI_TX_CTL(pipe));
1181                 cur_state = !!(val & FDI_TX_ENABLE);
1182         }
1183         I915_STATE_WARN(cur_state != state,
1184              "FDI TX state assertion failure (expected %s, current %s)\n",
1185                         onoff(state), onoff(cur_state));
1186 }
1187 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1188 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1189
1190 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1191                           enum pipe pipe, bool state)
1192 {
1193         u32 val;
1194         bool cur_state;
1195
1196         val = I915_READ(FDI_RX_CTL(pipe));
1197         cur_state = !!(val & FDI_RX_ENABLE);
1198         I915_STATE_WARN(cur_state != state,
1199              "FDI RX state assertion failure (expected %s, current %s)\n",
1200                         onoff(state), onoff(cur_state));
1201 }
1202 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1203 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1204
1205 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1206                                       enum pipe pipe)
1207 {
1208         u32 val;
1209
1210         /* ILK FDI PLL is always enabled */
1211         if (IS_GEN5(dev_priv))
1212                 return;
1213
1214         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1215         if (HAS_DDI(dev_priv))
1216                 return;
1217
1218         val = I915_READ(FDI_TX_CTL(pipe));
1219         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1220 }
1221
1222 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1223                        enum pipe pipe, bool state)
1224 {
1225         u32 val;
1226         bool cur_state;
1227
1228         val = I915_READ(FDI_RX_CTL(pipe));
1229         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1230         I915_STATE_WARN(cur_state != state,
1231              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1232                         onoff(state), onoff(cur_state));
1233 }
1234
1235 void assert_panel_unlocked(struct drm_i915_private *dev_priv,
1236                            enum pipe pipe)
1237 {
1238         struct drm_device *dev = dev_priv->dev;
1239         i915_reg_t pp_reg;
1240         u32 val;
1241         enum pipe panel_pipe = PIPE_A;
1242         bool locked = true;
1243
1244         if (WARN_ON(HAS_DDI(dev)))
1245                 return;
1246
1247         if (HAS_PCH_SPLIT(dev)) {
1248                 u32 port_sel;
1249
1250                 pp_reg = PCH_PP_CONTROL;
1251                 port_sel = I915_READ(PCH_PP_ON_DELAYS) & PANEL_PORT_SELECT_MASK;
1252
1253                 if (port_sel == PANEL_PORT_SELECT_LVDS &&
1254                     I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT)
1255                         panel_pipe = PIPE_B;
1256                 /* XXX: else fix for eDP */
1257         } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
1258                 /* presumably write lock depends on pipe, not port select */
1259                 pp_reg = VLV_PIPE_PP_CONTROL(pipe);
1260                 panel_pipe = pipe;
1261         } else {
1262                 pp_reg = PP_CONTROL;
1263                 if (I915_READ(LVDS) & LVDS_PIPEB_SELECT)
1264                         panel_pipe = PIPE_B;
1265         }
1266
1267         val = I915_READ(pp_reg);
1268         if (!(val & PANEL_POWER_ON) ||
1269             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1270                 locked = false;
1271
1272         I915_STATE_WARN(panel_pipe == pipe && locked,
1273              "panel assertion failure, pipe %c regs locked\n",
1274              pipe_name(pipe));
1275 }
1276
1277 static void assert_cursor(struct drm_i915_private *dev_priv,
1278                           enum pipe pipe, bool state)
1279 {
1280         struct drm_device *dev = dev_priv->dev;
1281         bool cur_state;
1282
1283         if (IS_845G(dev) || IS_I865G(dev))
1284                 cur_state = I915_READ(CURCNTR(PIPE_A)) & CURSOR_ENABLE;
1285         else
1286                 cur_state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE;
1287
1288         I915_STATE_WARN(cur_state != state,
1289              "cursor on pipe %c assertion failure (expected %s, current %s)\n",
1290                         pipe_name(pipe), onoff(state), onoff(cur_state));
1291 }
1292 #define assert_cursor_enabled(d, p) assert_cursor(d, p, true)
1293 #define assert_cursor_disabled(d, p) assert_cursor(d, p, false)
1294
1295 void assert_pipe(struct drm_i915_private *dev_priv,
1296                  enum pipe pipe, bool state)
1297 {
1298         bool cur_state;
1299         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1300                                                                       pipe);
1301         enum intel_display_power_domain power_domain;
1302
1303         /* if we need the pipe quirk it must be always on */
1304         if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
1305             (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
1306                 state = true;
1307
1308         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
1309         if (intel_display_power_get_if_enabled(dev_priv, power_domain)) {
1310                 u32 val = I915_READ(PIPECONF(cpu_transcoder));
1311                 cur_state = !!(val & PIPECONF_ENABLE);
1312
1313                 intel_display_power_put(dev_priv, power_domain);
1314         } else {
1315                 cur_state = false;
1316         }
1317
1318         I915_STATE_WARN(cur_state != state,
1319              "pipe %c assertion failure (expected %s, current %s)\n",
1320                         pipe_name(pipe), onoff(state), onoff(cur_state));
1321 }
1322
1323 static void assert_plane(struct drm_i915_private *dev_priv,
1324                          enum plane plane, bool state)
1325 {
1326         u32 val;
1327         bool cur_state;
1328
1329         val = I915_READ(DSPCNTR(plane));
1330         cur_state = !!(val & DISPLAY_PLANE_ENABLE);
1331         I915_STATE_WARN(cur_state != state,
1332              "plane %c assertion failure (expected %s, current %s)\n",
1333                         plane_name(plane), onoff(state), onoff(cur_state));
1334 }
1335
1336 #define assert_plane_enabled(d, p) assert_plane(d, p, true)
1337 #define assert_plane_disabled(d, p) assert_plane(d, p, false)
1338
1339 static void assert_planes_disabled(struct drm_i915_private *dev_priv,
1340                                    enum pipe pipe)
1341 {
1342         struct drm_device *dev = dev_priv->dev;
1343         int i;
1344
1345         /* Primary planes are fixed to pipes on gen4+ */
1346         if (INTEL_INFO(dev)->gen >= 4) {
1347                 u32 val = I915_READ(DSPCNTR(pipe));
1348                 I915_STATE_WARN(val & DISPLAY_PLANE_ENABLE,
1349                      "plane %c assertion failure, should be disabled but not\n",
1350                      plane_name(pipe));
1351                 return;
1352         }
1353
1354         /* Need to check both planes against the pipe */
1355         for_each_pipe(dev_priv, i) {
1356                 u32 val = I915_READ(DSPCNTR(i));
1357                 enum pipe cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
1358                         DISPPLANE_SEL_PIPE_SHIFT;
1359                 I915_STATE_WARN((val & DISPLAY_PLANE_ENABLE) && pipe == cur_pipe,
1360                      "plane %c assertion failure, should be off on pipe %c but is still active\n",
1361                      plane_name(i), pipe_name(pipe));
1362         }
1363 }
1364
1365 static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
1366                                     enum pipe pipe)
1367 {
1368         struct drm_device *dev = dev_priv->dev;
1369         int sprite;
1370
1371         if (INTEL_INFO(dev)->gen >= 9) {
1372                 for_each_sprite(dev_priv, pipe, sprite) {
1373                         u32 val = I915_READ(PLANE_CTL(pipe, sprite));
1374                         I915_STATE_WARN(val & PLANE_CTL_ENABLE,
1375                              "plane %d assertion failure, should be off on pipe %c but is still active\n",
1376                              sprite, pipe_name(pipe));
1377                 }
1378         } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
1379                 for_each_sprite(dev_priv, pipe, sprite) {
1380                         u32 val = I915_READ(SPCNTR(pipe, sprite));
1381                         I915_STATE_WARN(val & SP_ENABLE,
1382                              "sprite %c assertion failure, should be off on pipe %c but is still active\n",
1383                              sprite_name(pipe, sprite), pipe_name(pipe));
1384                 }
1385         } else if (INTEL_INFO(dev)->gen >= 7) {
1386                 u32 val = I915_READ(SPRCTL(pipe));
1387                 I915_STATE_WARN(val & SPRITE_ENABLE,
1388                      "sprite %c assertion failure, should be off on pipe %c but is still active\n",
1389                      plane_name(pipe), pipe_name(pipe));
1390         } else if (INTEL_INFO(dev)->gen >= 5) {
1391                 u32 val = I915_READ(DVSCNTR(pipe));
1392                 I915_STATE_WARN(val & DVS_ENABLE,
1393                      "sprite %c assertion failure, should be off on pipe %c but is still active\n",
1394                      plane_name(pipe), pipe_name(pipe));
1395         }
1396 }
1397
1398 static void assert_vblank_disabled(struct drm_crtc *crtc)
1399 {
1400         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1401                 drm_crtc_vblank_put(crtc);
1402 }
1403
1404 void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1405                                     enum pipe pipe)
1406 {
1407         u32 val;
1408         bool enabled;
1409
1410         val = I915_READ(PCH_TRANSCONF(pipe));
1411         enabled = !!(val & TRANS_ENABLE);
1412         I915_STATE_WARN(enabled,
1413              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1414              pipe_name(pipe));
1415 }
1416
1417 static bool dp_pipe_enabled(struct drm_i915_private *dev_priv,
1418                             enum pipe pipe, u32 port_sel, u32 val)
1419 {
1420         if ((val & DP_PORT_EN) == 0)
1421                 return false;
1422
1423         if (HAS_PCH_CPT(dev_priv)) {
1424                 u32 trans_dp_ctl = I915_READ(TRANS_DP_CTL(pipe));
1425                 if ((trans_dp_ctl & TRANS_DP_PORT_SEL_MASK) != port_sel)
1426                         return false;
1427         } else if (IS_CHERRYVIEW(dev_priv)) {
1428                 if ((val & DP_PIPE_MASK_CHV) != DP_PIPE_SELECT_CHV(pipe))
1429                         return false;
1430         } else {
1431                 if ((val & DP_PIPE_MASK) != (pipe << 30))
1432                         return false;
1433         }
1434         return true;
1435 }
1436
1437 static bool hdmi_pipe_enabled(struct drm_i915_private *dev_priv,
1438                               enum pipe pipe, u32 val)
1439 {
1440         if ((val & SDVO_ENABLE) == 0)
1441                 return false;
1442
1443         if (HAS_PCH_CPT(dev_priv)) {
1444                 if ((val & SDVO_PIPE_SEL_MASK_CPT) != SDVO_PIPE_SEL_CPT(pipe))
1445                         return false;
1446         } else if (IS_CHERRYVIEW(dev_priv)) {
1447                 if ((val & SDVO_PIPE_SEL_MASK_CHV) != SDVO_PIPE_SEL_CHV(pipe))
1448                         return false;
1449         } else {
1450                 if ((val & SDVO_PIPE_SEL_MASK) != SDVO_PIPE_SEL(pipe))
1451                         return false;
1452         }
1453         return true;
1454 }
1455
1456 static bool lvds_pipe_enabled(struct drm_i915_private *dev_priv,
1457                               enum pipe pipe, u32 val)
1458 {
1459         if ((val & LVDS_PORT_EN) == 0)
1460                 return false;
1461
1462         if (HAS_PCH_CPT(dev_priv)) {
1463                 if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
1464                         return false;
1465         } else {
1466                 if ((val & LVDS_PIPE_MASK) != LVDS_PIPE(pipe))
1467                         return false;
1468         }
1469         return true;
1470 }
1471
1472 static bool adpa_pipe_enabled(struct drm_i915_private *dev_priv,
1473                               enum pipe pipe, u32 val)
1474 {
1475         if ((val & ADPA_DAC_ENABLE) == 0)
1476                 return false;
1477         if (HAS_PCH_CPT(dev_priv)) {
1478                 if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
1479                         return false;
1480         } else {
1481                 if ((val & ADPA_PIPE_SELECT_MASK) != ADPA_PIPE_SELECT(pipe))
1482                         return false;
1483         }
1484         return true;
1485 }
1486
1487 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1488                                    enum pipe pipe, i915_reg_t reg,
1489                                    u32 port_sel)
1490 {
1491         u32 val = I915_READ(reg);
1492         I915_STATE_WARN(dp_pipe_enabled(dev_priv, pipe, port_sel, val),
1493              "PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n",
1494              i915_mmio_reg_offset(reg), pipe_name(pipe));
1495
1496         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && (val & DP_PORT_EN) == 0
1497              && (val & DP_PIPEB_SELECT),
1498              "IBX PCH dp port still using transcoder B\n");
1499 }
1500
1501 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1502                                      enum pipe pipe, i915_reg_t reg)
1503 {
1504         u32 val = I915_READ(reg);
1505         I915_STATE_WARN(hdmi_pipe_enabled(dev_priv, pipe, val),
1506              "PCH HDMI (0x%08x) enabled on transcoder %c, should be disabled\n",
1507              i915_mmio_reg_offset(reg), pipe_name(pipe));
1508
1509         I915_STATE_WARN(HAS_PCH_IBX(dev_priv) && (val & SDVO_ENABLE) == 0
1510              && (val & SDVO_PIPE_B_SELECT),
1511              "IBX PCH hdmi port still using transcoder B\n");
1512 }
1513
1514 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1515                                       enum pipe pipe)
1516 {
1517         u32 val;
1518
1519         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B);
1520         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C);
1521         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D);
1522
1523         val = I915_READ(PCH_ADPA);
1524         I915_STATE_WARN(adpa_pipe_enabled(dev_priv, pipe, val),
1525              "PCH VGA enabled on transcoder %c, should be disabled\n",
1526              pipe_name(pipe));
1527
1528         val = I915_READ(PCH_LVDS);
1529         I915_STATE_WARN(lvds_pipe_enabled(dev_priv, pipe, val),
1530              "PCH LVDS enabled on transcoder %c, should be disabled\n",
1531              pipe_name(pipe));
1532
1533         assert_pch_hdmi_disabled(dev_priv, pipe, PCH_HDMIB);
1534         assert_pch_hdmi_disabled(dev_priv, pipe, PCH_HDMIC);
1535         assert_pch_hdmi_disabled(dev_priv, pipe, PCH_HDMID);
1536 }
1537
1538 static void _vlv_enable_pll(struct intel_crtc *crtc,
1539                             const struct intel_crtc_state *pipe_config)
1540 {
1541         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1542         enum pipe pipe = crtc->pipe;
1543
1544         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1545         POSTING_READ(DPLL(pipe));
1546         udelay(150);
1547
1548         if (intel_wait_for_register(dev_priv,
1549                                     DPLL(pipe),
1550                                     DPLL_LOCK_VLV,
1551                                     DPLL_LOCK_VLV,
1552                                     1))
1553                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
1554 }
1555
1556 static void vlv_enable_pll(struct intel_crtc *crtc,
1557                            const struct intel_crtc_state *pipe_config)
1558 {
1559         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1560         enum pipe pipe = crtc->pipe;
1561
1562         assert_pipe_disabled(dev_priv, pipe);
1563
1564         /* PLL is protected by panel, make sure we can write it */
1565         assert_panel_unlocked(dev_priv, pipe);
1566
1567         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1568                 _vlv_enable_pll(crtc, pipe_config);
1569
1570         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1571         POSTING_READ(DPLL_MD(pipe));
1572 }
1573
1574
1575 static void _chv_enable_pll(struct intel_crtc *crtc,
1576                             const struct intel_crtc_state *pipe_config)
1577 {
1578         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1579         enum pipe pipe = crtc->pipe;
1580         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1581         u32 tmp;
1582
1583         mutex_lock(&dev_priv->sb_lock);
1584
1585         /* Enable back the 10bit clock to display controller */
1586         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1587         tmp |= DPIO_DCLKP_EN;
1588         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1589
1590         mutex_unlock(&dev_priv->sb_lock);
1591
1592         /*
1593          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1594          */
1595         udelay(1);
1596
1597         /* Enable PLL */
1598         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1599
1600         /* Check PLL is locked */
1601         if (intel_wait_for_register(dev_priv,
1602                                     DPLL(pipe), DPLL_LOCK_VLV, DPLL_LOCK_VLV,
1603                                     1))
1604                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1605 }
1606
1607 static void chv_enable_pll(struct intel_crtc *crtc,
1608                            const struct intel_crtc_state *pipe_config)
1609 {
1610         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1611         enum pipe pipe = crtc->pipe;
1612
1613         assert_pipe_disabled(dev_priv, pipe);
1614
1615         /* PLL is protected by panel, make sure we can write it */
1616         assert_panel_unlocked(dev_priv, pipe);
1617
1618         if (pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE)
1619                 _chv_enable_pll(crtc, pipe_config);
1620
1621         if (pipe != PIPE_A) {
1622                 /*
1623                  * WaPixelRepeatModeFixForC0:chv
1624                  *
1625                  * DPLLCMD is AWOL. Use chicken bits to propagate
1626                  * the value from DPLLBMD to either pipe B or C.
1627                  */
1628                 I915_WRITE(CBR4_VLV, pipe == PIPE_B ? CBR_DPLLBMD_PIPE_B : CBR_DPLLBMD_PIPE_C);
1629                 I915_WRITE(DPLL_MD(PIPE_B), pipe_config->dpll_hw_state.dpll_md);
1630                 I915_WRITE(CBR4_VLV, 0);
1631                 dev_priv->chv_dpll_md[pipe] = pipe_config->dpll_hw_state.dpll_md;
1632
1633                 /*
1634                  * DPLLB VGA mode also seems to cause problems.
1635                  * We should always have it disabled.
1636                  */
1637                 WARN_ON((I915_READ(DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0);
1638         } else {
1639                 I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1640                 POSTING_READ(DPLL_MD(pipe));
1641         }
1642 }
1643
1644 static int intel_num_dvo_pipes(struct drm_device *dev)
1645 {
1646         struct intel_crtc *crtc;
1647         int count = 0;
1648
1649         for_each_intel_crtc(dev, crtc)
1650                 count += crtc->base.state->active &&
1651                         intel_pipe_has_type(crtc, INTEL_OUTPUT_DVO);
1652
1653         return count;
1654 }
1655
1656 static void i9xx_enable_pll(struct intel_crtc *crtc)
1657 {
1658         struct drm_device *dev = crtc->base.dev;
1659         struct drm_i915_private *dev_priv = dev->dev_private;
1660         i915_reg_t reg = DPLL(crtc->pipe);
1661         u32 dpll = crtc->config->dpll_hw_state.dpll;
1662
1663         assert_pipe_disabled(dev_priv, crtc->pipe);
1664
1665         /* PLL is protected by panel, make sure we can write it */
1666         if (IS_MOBILE(dev) && !IS_I830(dev))
1667                 assert_panel_unlocked(dev_priv, crtc->pipe);
1668
1669         /* Enable DVO 2x clock on both PLLs if necessary */
1670         if (IS_I830(dev) && intel_num_dvo_pipes(dev) > 0) {
1671                 /*
1672                  * It appears to be important that we don't enable this
1673                  * for the current pipe before otherwise configuring the
1674                  * PLL. No idea how this should be handled if multiple
1675                  * DVO outputs are enabled simultaneosly.
1676                  */
1677                 dpll |= DPLL_DVO_2X_MODE;
1678                 I915_WRITE(DPLL(!crtc->pipe),
1679                            I915_READ(DPLL(!crtc->pipe)) | DPLL_DVO_2X_MODE);
1680         }
1681
1682         /*
1683          * Apparently we need to have VGA mode enabled prior to changing
1684          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
1685          * dividers, even though the register value does change.
1686          */
1687         I915_WRITE(reg, 0);
1688
1689         I915_WRITE(reg, dpll);
1690
1691         /* Wait for the clocks to stabilize. */
1692         POSTING_READ(reg);
1693         udelay(150);
1694
1695         if (INTEL_INFO(dev)->gen >= 4) {
1696                 I915_WRITE(DPLL_MD(crtc->pipe),
1697                            crtc->config->dpll_hw_state.dpll_md);
1698         } else {
1699                 /* The pixel multiplier can only be updated once the
1700                  * DPLL is enabled and the clocks are stable.
1701                  *
1702                  * So write it again.
1703                  */
1704                 I915_WRITE(reg, dpll);
1705         }
1706
1707         /* We do this three times for luck */
1708         I915_WRITE(reg, dpll);
1709         POSTING_READ(reg);
1710         udelay(150); /* wait for warmup */
1711         I915_WRITE(reg, dpll);
1712         POSTING_READ(reg);
1713         udelay(150); /* wait for warmup */
1714         I915_WRITE(reg, dpll);
1715         POSTING_READ(reg);
1716         udelay(150); /* wait for warmup */
1717 }
1718
1719 /**
1720  * i9xx_disable_pll - disable a PLL
1721  * @dev_priv: i915 private structure
1722  * @pipe: pipe PLL to disable
1723  *
1724  * Disable the PLL for @pipe, making sure the pipe is off first.
1725  *
1726  * Note!  This is for pre-ILK only.
1727  */
1728 static void i9xx_disable_pll(struct intel_crtc *crtc)
1729 {
1730         struct drm_device *dev = crtc->base.dev;
1731         struct drm_i915_private *dev_priv = dev->dev_private;
1732         enum pipe pipe = crtc->pipe;
1733
1734         /* Disable DVO 2x clock on both PLLs if necessary */
1735         if (IS_I830(dev) &&
1736             intel_pipe_has_type(crtc, INTEL_OUTPUT_DVO) &&
1737             !intel_num_dvo_pipes(dev)) {
1738                 I915_WRITE(DPLL(PIPE_B),
1739                            I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE);
1740                 I915_WRITE(DPLL(PIPE_A),
1741                            I915_READ(DPLL(PIPE_A)) & ~DPLL_DVO_2X_MODE);
1742         }
1743
1744         /* Don't disable pipe or pipe PLLs if needed */
1745         if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
1746             (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
1747                 return;
1748
1749         /* Make sure the pipe isn't still relying on us */
1750         assert_pipe_disabled(dev_priv, pipe);
1751
1752         I915_WRITE(DPLL(pipe), DPLL_VGA_MODE_DIS);
1753         POSTING_READ(DPLL(pipe));
1754 }
1755
1756 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1757 {
1758         u32 val;
1759
1760         /* Make sure the pipe isn't still relying on us */
1761         assert_pipe_disabled(dev_priv, pipe);
1762
1763         val = DPLL_INTEGRATED_REF_CLK_VLV |
1764                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1765         if (pipe != PIPE_A)
1766                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1767
1768         I915_WRITE(DPLL(pipe), val);
1769         POSTING_READ(DPLL(pipe));
1770 }
1771
1772 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1773 {
1774         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1775         u32 val;
1776
1777         /* Make sure the pipe isn't still relying on us */
1778         assert_pipe_disabled(dev_priv, pipe);
1779
1780         val = DPLL_SSC_REF_CLK_CHV |
1781                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1782         if (pipe != PIPE_A)
1783                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1784
1785         I915_WRITE(DPLL(pipe), val);
1786         POSTING_READ(DPLL(pipe));
1787
1788         mutex_lock(&dev_priv->sb_lock);
1789
1790         /* Disable 10bit clock to display controller */
1791         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1792         val &= ~DPIO_DCLKP_EN;
1793         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1794
1795         mutex_unlock(&dev_priv->sb_lock);
1796 }
1797
1798 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1799                          struct intel_digital_port *dport,
1800                          unsigned int expected_mask)
1801 {
1802         u32 port_mask;
1803         i915_reg_t dpll_reg;
1804
1805         switch (dport->port) {
1806         case PORT_B:
1807                 port_mask = DPLL_PORTB_READY_MASK;
1808                 dpll_reg = DPLL(0);
1809                 break;
1810         case PORT_C:
1811                 port_mask = DPLL_PORTC_READY_MASK;
1812                 dpll_reg = DPLL(0);
1813                 expected_mask <<= 4;
1814                 break;
1815         case PORT_D:
1816                 port_mask = DPLL_PORTD_READY_MASK;
1817                 dpll_reg = DPIO_PHY_STATUS;
1818                 break;
1819         default:
1820                 BUG();
1821         }
1822
1823         if (intel_wait_for_register(dev_priv,
1824                                     dpll_reg, port_mask, expected_mask,
1825                                     1000))
1826                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
1827                      port_name(dport->port), I915_READ(dpll_reg) & port_mask, expected_mask);
1828 }
1829
1830 static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1831                                            enum pipe pipe)
1832 {
1833         struct drm_device *dev = dev_priv->dev;
1834         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1835         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1836         i915_reg_t reg;
1837         uint32_t val, pipeconf_val;
1838
1839         /* Make sure PCH DPLL is enabled */
1840         assert_shared_dpll_enabled(dev_priv, intel_crtc->config->shared_dpll);
1841
1842         /* FDI must be feeding us bits for PCH ports */
1843         assert_fdi_tx_enabled(dev_priv, pipe);
1844         assert_fdi_rx_enabled(dev_priv, pipe);
1845
1846         if (HAS_PCH_CPT(dev)) {
1847                 /* Workaround: Set the timing override bit before enabling the
1848                  * pch transcoder. */
1849                 reg = TRANS_CHICKEN2(pipe);
1850                 val = I915_READ(reg);
1851                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1852                 I915_WRITE(reg, val);
1853         }
1854
1855         reg = PCH_TRANSCONF(pipe);
1856         val = I915_READ(reg);
1857         pipeconf_val = I915_READ(PIPECONF(pipe));
1858
1859         if (HAS_PCH_IBX(dev_priv)) {
1860                 /*
1861                  * Make the BPC in transcoder be consistent with
1862                  * that in pipeconf reg. For HDMI we must use 8bpc
1863                  * here for both 8bpc and 12bpc.
1864                  */
1865                 val &= ~PIPECONF_BPC_MASK;
1866                 if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_HDMI))
1867                         val |= PIPECONF_8BPC;
1868                 else
1869                         val |= pipeconf_val & PIPECONF_BPC_MASK;
1870         }
1871
1872         val &= ~TRANS_INTERLACE_MASK;
1873         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK)
1874                 if (HAS_PCH_IBX(dev_priv) &&
1875                     intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
1876                         val |= TRANS_LEGACY_INTERLACED_ILK;
1877                 else
1878                         val |= TRANS_INTERLACED;
1879         else
1880                 val |= TRANS_PROGRESSIVE;
1881
1882         I915_WRITE(reg, val | TRANS_ENABLE);
1883         if (wait_for(I915_READ(reg) & TRANS_STATE_ENABLE, 100))
1884                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
1885 }
1886
1887 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1888                                       enum transcoder cpu_transcoder)
1889 {
1890         u32 val, pipeconf_val;
1891
1892         /* FDI must be feeding us bits for PCH ports */
1893         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
1894         assert_fdi_rx_enabled(dev_priv, TRANSCODER_A);
1895
1896         /* Workaround: set timing override bit. */
1897         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1898         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1899         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1900
1901         val = TRANS_ENABLE;
1902         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
1903
1904         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
1905             PIPECONF_INTERLACED_ILK)
1906                 val |= TRANS_INTERLACED;
1907         else
1908                 val |= TRANS_PROGRESSIVE;
1909
1910         I915_WRITE(LPT_TRANSCONF, val);
1911         if (wait_for(I915_READ(LPT_TRANSCONF) & TRANS_STATE_ENABLE, 100))
1912                 DRM_ERROR("Failed to enable PCH transcoder\n");
1913 }
1914
1915 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
1916                                             enum pipe pipe)
1917 {
1918         struct drm_device *dev = dev_priv->dev;
1919         i915_reg_t reg;
1920         uint32_t val;
1921
1922         /* FDI relies on the transcoder */
1923         assert_fdi_tx_disabled(dev_priv, pipe);
1924         assert_fdi_rx_disabled(dev_priv, pipe);
1925
1926         /* Ports must be off as well */
1927         assert_pch_ports_disabled(dev_priv, pipe);
1928
1929         reg = PCH_TRANSCONF(pipe);
1930         val = I915_READ(reg);
1931         val &= ~TRANS_ENABLE;
1932         I915_WRITE(reg, val);
1933         /* wait for PCH transcoder off, transcoder state */
1934         if (wait_for((I915_READ(reg) & TRANS_STATE_ENABLE) == 0, 50))
1935                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
1936
1937         if (HAS_PCH_CPT(dev)) {
1938                 /* Workaround: Clear the timing override chicken bit again. */
1939                 reg = TRANS_CHICKEN2(pipe);
1940                 val = I915_READ(reg);
1941                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1942                 I915_WRITE(reg, val);
1943         }
1944 }
1945
1946 static void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
1947 {
1948         u32 val;
1949
1950         val = I915_READ(LPT_TRANSCONF);
1951         val &= ~TRANS_ENABLE;
1952         I915_WRITE(LPT_TRANSCONF, val);
1953         /* wait for PCH transcoder off, transcoder state */
1954         if (wait_for((I915_READ(LPT_TRANSCONF) & TRANS_STATE_ENABLE) == 0, 50))
1955                 DRM_ERROR("Failed to disable PCH transcoder\n");
1956
1957         /* Workaround: clear timing override bit. */
1958         val = I915_READ(TRANS_CHICKEN2(PIPE_A));
1959         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
1960         I915_WRITE(TRANS_CHICKEN2(PIPE_A), val);
1961 }
1962
1963 /**
1964  * intel_enable_pipe - enable a pipe, asserting requirements
1965  * @crtc: crtc responsible for the pipe
1966  *
1967  * Enable @crtc's pipe, making sure that various hardware specific requirements
1968  * are met, if applicable, e.g. PLL enabled, LVDS pairs enabled, etc.
1969  */
1970 static void intel_enable_pipe(struct intel_crtc *crtc)
1971 {
1972         struct drm_device *dev = crtc->base.dev;
1973         struct drm_i915_private *dev_priv = dev->dev_private;
1974         enum pipe pipe = crtc->pipe;
1975         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
1976         enum pipe pch_transcoder;
1977         i915_reg_t reg;
1978         u32 val;
1979
1980         DRM_DEBUG_KMS("enabling pipe %c\n", pipe_name(pipe));
1981
1982         assert_planes_disabled(dev_priv, pipe);
1983         assert_cursor_disabled(dev_priv, pipe);
1984         assert_sprites_disabled(dev_priv, pipe);
1985
1986         if (HAS_PCH_LPT(dev_priv))
1987                 pch_transcoder = TRANSCODER_A;
1988         else
1989                 pch_transcoder = pipe;
1990
1991         /*
1992          * A pipe without a PLL won't actually be able to drive bits from
1993          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
1994          * need the check.
1995          */
1996         if (HAS_GMCH_DISPLAY(dev_priv))
1997                 if (crtc->config->has_dsi_encoder)
1998                         assert_dsi_pll_enabled(dev_priv);
1999                 else
2000                         assert_pll_enabled(dev_priv, pipe);
2001         else {
2002                 if (crtc->config->has_pch_encoder) {
2003                         /* if driving the PCH, we need FDI enabled */
2004                         assert_fdi_rx_pll_enabled(dev_priv, pch_transcoder);
2005                         assert_fdi_tx_pll_enabled(dev_priv,
2006                                                   (enum pipe) cpu_transcoder);
2007                 }
2008                 /* FIXME: assert CPU port conditions for SNB+ */
2009         }
2010
2011         reg = PIPECONF(cpu_transcoder);
2012         val = I915_READ(reg);
2013         if (val & PIPECONF_ENABLE) {
2014                 WARN_ON(!((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
2015                           (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE)));
2016                 return;
2017         }
2018
2019         I915_WRITE(reg, val | PIPECONF_ENABLE);
2020         POSTING_READ(reg);
2021
2022         /*
2023          * Until the pipe starts DSL will read as 0, which would cause
2024          * an apparent vblank timestamp jump, which messes up also the
2025          * frame count when it's derived from the timestamps. So let's
2026          * wait for the pipe to start properly before we call
2027          * drm_crtc_vblank_on()
2028          */
2029         if (dev->max_vblank_count == 0 &&
2030             wait_for(intel_get_crtc_scanline(crtc) != crtc->scanline_offset, 50))
2031                 DRM_ERROR("pipe %c didn't start\n", pipe_name(pipe));
2032 }
2033
2034 /**
2035  * intel_disable_pipe - disable a pipe, asserting requirements
2036  * @crtc: crtc whose pipes is to be disabled
2037  *
2038  * Disable the pipe of @crtc, making sure that various hardware
2039  * specific requirements are met, if applicable, e.g. plane
2040  * disabled, panel fitter off, etc.
2041  *
2042  * Will wait until the pipe has shut down before returning.
2043  */
2044 static void intel_disable_pipe(struct intel_crtc *crtc)
2045 {
2046         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
2047         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
2048         enum pipe pipe = crtc->pipe;
2049         i915_reg_t reg;
2050         u32 val;
2051
2052         DRM_DEBUG_KMS("disabling pipe %c\n", pipe_name(pipe));
2053
2054         /*
2055          * Make sure planes won't keep trying to pump pixels to us,
2056          * or we might hang the display.
2057          */
2058         assert_planes_disabled(dev_priv, pipe);
2059         assert_cursor_disabled(dev_priv, pipe);
2060         assert_sprites_disabled(dev_priv, pipe);
2061
2062         reg = PIPECONF(cpu_transcoder);
2063         val = I915_READ(reg);
2064         if ((val & PIPECONF_ENABLE) == 0)
2065                 return;
2066
2067         /*
2068          * Double wide has implications for planes
2069          * so best keep it disabled when not needed.
2070          */
2071         if (crtc->config->double_wide)
2072                 val &= ~PIPECONF_DOUBLE_WIDE;
2073
2074         /* Don't disable pipe or pipe PLLs if needed */
2075         if (!(pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) &&
2076             !(pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
2077                 val &= ~PIPECONF_ENABLE;
2078
2079         I915_WRITE(reg, val);
2080         if ((val & PIPECONF_ENABLE) == 0)
2081                 intel_wait_for_pipe_off(crtc);
2082 }
2083
2084 static bool need_vtd_wa(struct drm_device *dev)
2085 {
2086 #ifdef CONFIG_INTEL_IOMMU
2087         if (INTEL_INFO(dev)->gen >= 6 && intel_iommu_gfx_mapped)
2088                 return true;
2089 #endif
2090         return false;
2091 }
2092
2093 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
2094 {
2095         return IS_GEN2(dev_priv) ? 2048 : 4096;
2096 }
2097
2098 static unsigned int intel_tile_width_bytes(const struct drm_i915_private *dev_priv,
2099                                            uint64_t fb_modifier, unsigned int cpp)
2100 {
2101         switch (fb_modifier) {
2102         case DRM_FORMAT_MOD_NONE:
2103                 return cpp;
2104         case I915_FORMAT_MOD_X_TILED:
2105                 if (IS_GEN2(dev_priv))
2106                         return 128;
2107                 else
2108                         return 512;
2109         case I915_FORMAT_MOD_Y_TILED:
2110                 if (IS_GEN2(dev_priv) || HAS_128_BYTE_Y_TILING(dev_priv))
2111                         return 128;
2112                 else
2113                         return 512;
2114         case I915_FORMAT_MOD_Yf_TILED:
2115                 switch (cpp) {
2116                 case 1:
2117                         return 64;
2118                 case 2:
2119                 case 4:
2120                         return 128;
2121                 case 8:
2122                 case 16:
2123                         return 256;
2124                 default:
2125                         MISSING_CASE(cpp);
2126                         return cpp;
2127                 }
2128                 break;
2129         default:
2130                 MISSING_CASE(fb_modifier);
2131                 return cpp;
2132         }
2133 }
2134
2135 unsigned int intel_tile_height(const struct drm_i915_private *dev_priv,
2136                                uint64_t fb_modifier, unsigned int cpp)
2137 {
2138         if (fb_modifier == DRM_FORMAT_MOD_NONE)
2139                 return 1;
2140         else
2141                 return intel_tile_size(dev_priv) /
2142                         intel_tile_width_bytes(dev_priv, fb_modifier, cpp);
2143 }
2144
2145 /* Return the tile dimensions in pixel units */
2146 static void intel_tile_dims(const struct drm_i915_private *dev_priv,
2147                             unsigned int *tile_width,
2148                             unsigned int *tile_height,
2149                             uint64_t fb_modifier,
2150                             unsigned int cpp)
2151 {
2152         unsigned int tile_width_bytes =
2153                 intel_tile_width_bytes(dev_priv, fb_modifier, cpp);
2154
2155         *tile_width = tile_width_bytes / cpp;
2156         *tile_height = intel_tile_size(dev_priv) / tile_width_bytes;
2157 }
2158
2159 unsigned int
2160 intel_fb_align_height(struct drm_device *dev, unsigned int height,
2161                       uint32_t pixel_format, uint64_t fb_modifier)
2162 {
2163         unsigned int cpp = drm_format_plane_cpp(pixel_format, 0);
2164         unsigned int tile_height = intel_tile_height(to_i915(dev), fb_modifier, cpp);
2165
2166         return ALIGN(height, tile_height);
2167 }
2168
2169 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
2170 {
2171         unsigned int size = 0;
2172         int i;
2173
2174         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
2175                 size += rot_info->plane[i].width * rot_info->plane[i].height;
2176
2177         return size;
2178 }
2179
2180 static void
2181 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view,
2182                         const struct drm_framebuffer *fb,
2183                         unsigned int rotation)
2184 {
2185         if (intel_rotation_90_or_270(rotation)) {
2186                 *view = i915_ggtt_view_rotated;
2187                 view->params.rotated = to_intel_framebuffer(fb)->rot_info;
2188         } else {
2189                 *view = i915_ggtt_view_normal;
2190         }
2191 }
2192
2193 static void
2194 intel_fill_fb_info(struct drm_i915_private *dev_priv,
2195                    struct drm_framebuffer *fb)
2196 {
2197         struct intel_rotation_info *info = &to_intel_framebuffer(fb)->rot_info;
2198         unsigned int tile_size, tile_width, tile_height, cpp;
2199
2200         tile_size = intel_tile_size(dev_priv);
2201
2202         cpp = drm_format_plane_cpp(fb->pixel_format, 0);
2203         intel_tile_dims(dev_priv, &tile_width, &tile_height,
2204                         fb->modifier[0], cpp);
2205
2206         info->plane[0].width = DIV_ROUND_UP(fb->pitches[0], tile_width * cpp);
2207         info->plane[0].height = DIV_ROUND_UP(fb->height, tile_height);
2208
2209         if (info->pixel_format == DRM_FORMAT_NV12) {
2210                 cpp = drm_format_plane_cpp(fb->pixel_format, 1);
2211                 intel_tile_dims(dev_priv, &tile_width, &tile_height,
2212                                 fb->modifier[1], cpp);
2213
2214                 info->uv_offset = fb->offsets[1];
2215                 info->plane[1].width = DIV_ROUND_UP(fb->pitches[1], tile_width * cpp);
2216                 info->plane[1].height = DIV_ROUND_UP(fb->height / 2, tile_height);
2217         }
2218 }
2219
2220 static unsigned int intel_linear_alignment(const struct drm_i915_private *dev_priv)
2221 {
2222         if (INTEL_INFO(dev_priv)->gen >= 9)
2223                 return 256 * 1024;
2224         else if (IS_BROADWATER(dev_priv) || IS_CRESTLINE(dev_priv) ||
2225                  IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2226                 return 128 * 1024;
2227         else if (INTEL_INFO(dev_priv)->gen >= 4)
2228                 return 4 * 1024;
2229         else
2230                 return 0;
2231 }
2232
2233 static unsigned int intel_surf_alignment(const struct drm_i915_private *dev_priv,
2234                                          uint64_t fb_modifier)
2235 {
2236         switch (fb_modifier) {
2237         case DRM_FORMAT_MOD_NONE:
2238                 return intel_linear_alignment(dev_priv);
2239         case I915_FORMAT_MOD_X_TILED:
2240                 if (INTEL_INFO(dev_priv)->gen >= 9)
2241                         return 256 * 1024;
2242                 return 0;
2243         case I915_FORMAT_MOD_Y_TILED:
2244         case I915_FORMAT_MOD_Yf_TILED:
2245                 return 1 * 1024 * 1024;
2246         default:
2247                 MISSING_CASE(fb_modifier);
2248                 return 0;
2249         }
2250 }
2251
2252 int
2253 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
2254                            unsigned int rotation)
2255 {
2256         struct drm_device *dev = fb->dev;
2257         struct drm_i915_private *dev_priv = dev->dev_private;
2258         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2259         struct i915_ggtt_view view;
2260         u32 alignment;
2261         int ret;
2262
2263         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2264
2265         alignment = intel_surf_alignment(dev_priv, fb->modifier[0]);
2266
2267         intel_fill_fb_ggtt_view(&view, fb, rotation);
2268
2269         /* Note that the w/a also requires 64 PTE of padding following the
2270          * bo. We currently fill all unused PTE with the shadow page and so
2271          * we should always have valid PTE following the scanout preventing
2272          * the VT-d warning.
2273          */
2274         if (need_vtd_wa(dev) && alignment < 256 * 1024)
2275                 alignment = 256 * 1024;
2276
2277         /*
2278          * Global gtt pte registers are special registers which actually forward
2279          * writes to a chunk of system memory. Which means that there is no risk
2280          * that the register values disappear as soon as we call
2281          * intel_runtime_pm_put(), so it is correct to wrap only the
2282          * pin/unpin/fence and not more.
2283          */
2284         intel_runtime_pm_get(dev_priv);
2285
2286         ret = i915_gem_object_pin_to_display_plane(obj, alignment,
2287                                                    &view);
2288         if (ret)
2289                 goto err_pm;
2290
2291         /* Install a fence for tiled scan-out. Pre-i965 always needs a
2292          * fence, whereas 965+ only requires a fence if using
2293          * framebuffer compression.  For simplicity, we always install
2294          * a fence as the cost is not that onerous.
2295          */
2296         if (view.type == I915_GGTT_VIEW_NORMAL) {
2297                 ret = i915_gem_object_get_fence(obj);
2298                 if (ret == -EDEADLK) {
2299                         /*
2300                          * -EDEADLK means there are no free fences
2301                          * no pending flips.
2302                          *
2303                          * This is propagated to atomic, but it uses
2304                          * -EDEADLK to force a locking recovery, so
2305                          * change the returned error to -EBUSY.
2306                          */
2307                         ret = -EBUSY;
2308                         goto err_unpin;
2309                 } else if (ret)
2310                         goto err_unpin;
2311
2312                 i915_gem_object_pin_fence(obj);
2313         }
2314
2315         intel_runtime_pm_put(dev_priv);
2316         return 0;
2317
2318 err_unpin:
2319         i915_gem_object_unpin_from_display_plane(obj, &view);
2320 err_pm:
2321         intel_runtime_pm_put(dev_priv);
2322         return ret;
2323 }
2324
2325 void intel_unpin_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
2326 {
2327         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2328         struct i915_ggtt_view view;
2329
2330         WARN_ON(!mutex_is_locked(&obj->base.dev->struct_mutex));
2331
2332         intel_fill_fb_ggtt_view(&view, fb, rotation);
2333
2334         if (view.type == I915_GGTT_VIEW_NORMAL)
2335                 i915_gem_object_unpin_fence(obj);
2336
2337         i915_gem_object_unpin_from_display_plane(obj, &view);
2338 }
2339
2340 /*
2341  * Adjust the tile offset by moving the difference into
2342  * the x/y offsets.
2343  *
2344  * Input tile dimensions and pitch must already be
2345  * rotated to match x and y, and in pixel units.
2346  */
2347 static u32 intel_adjust_tile_offset(int *x, int *y,
2348                                     unsigned int tile_width,
2349                                     unsigned int tile_height,
2350                                     unsigned int tile_size,
2351                                     unsigned int pitch_tiles,
2352                                     u32 old_offset,
2353                                     u32 new_offset)
2354 {
2355         unsigned int tiles;
2356
2357         WARN_ON(old_offset & (tile_size - 1));
2358         WARN_ON(new_offset & (tile_size - 1));
2359         WARN_ON(new_offset > old_offset);
2360
2361         tiles = (old_offset - new_offset) / tile_size;
2362
2363         *y += tiles / pitch_tiles * tile_height;
2364         *x += tiles % pitch_tiles * tile_width;
2365
2366         return new_offset;
2367 }
2368
2369 /*
2370  * Computes the linear offset to the base tile and adjusts
2371  * x, y. bytes per pixel is assumed to be a power-of-two.
2372  *
2373  * In the 90/270 rotated case, x and y are assumed
2374  * to be already rotated to match the rotated GTT view, and
2375  * pitch is the tile_height aligned framebuffer height.
2376  */
2377 u32 intel_compute_tile_offset(int *x, int *y,
2378                               const struct drm_framebuffer *fb, int plane,
2379                               unsigned int pitch,
2380                               unsigned int rotation)
2381 {
2382         const struct drm_i915_private *dev_priv = to_i915(fb->dev);
2383         uint64_t fb_modifier = fb->modifier[plane];
2384         unsigned int cpp = drm_format_plane_cpp(fb->pixel_format, plane);
2385         u32 offset, offset_aligned, alignment;
2386
2387         alignment = intel_surf_alignment(dev_priv, fb_modifier);
2388         if (alignment)
2389                 alignment--;
2390
2391         if (fb_modifier != DRM_FORMAT_MOD_NONE) {
2392                 unsigned int tile_size, tile_width, tile_height;
2393                 unsigned int tile_rows, tiles, pitch_tiles;
2394
2395                 tile_size = intel_tile_size(dev_priv);
2396                 intel_tile_dims(dev_priv, &tile_width, &tile_height,
2397                                 fb_modifier, cpp);
2398
2399                 if (intel_rotation_90_or_270(rotation)) {
2400                         pitch_tiles = pitch / tile_height;
2401                         swap(tile_width, tile_height);
2402                 } else {
2403                         pitch_tiles = pitch / (tile_width * cpp);
2404                 }
2405
2406                 tile_rows = *y / tile_height;
2407                 *y %= tile_height;
2408
2409                 tiles = *x / tile_width;
2410                 *x %= tile_width;
2411
2412                 offset = (tile_rows * pitch_tiles + tiles) * tile_size;
2413                 offset_aligned = offset & ~alignment;
2414
2415                 intel_adjust_tile_offset(x, y, tile_width, tile_height,
2416                                          tile_size, pitch_tiles,
2417                                          offset, offset_aligned);
2418         } else {
2419                 offset = *y * pitch + *x * cpp;
2420                 offset_aligned = offset & ~alignment;
2421
2422                 *y = (offset & alignment) / pitch;
2423                 *x = ((offset & alignment) - *y * pitch) / cpp;
2424         }
2425
2426         return offset_aligned;
2427 }
2428
2429 static int i9xx_format_to_fourcc(int format)
2430 {
2431         switch (format) {
2432         case DISPPLANE_8BPP:
2433                 return DRM_FORMAT_C8;
2434         case DISPPLANE_BGRX555:
2435                 return DRM_FORMAT_XRGB1555;
2436         case DISPPLANE_BGRX565:
2437                 return DRM_FORMAT_RGB565;
2438         default:
2439         case DISPPLANE_BGRX888:
2440                 return DRM_FORMAT_XRGB8888;
2441         case DISPPLANE_RGBX888:
2442                 return DRM_FORMAT_XBGR8888;
2443         case DISPPLANE_BGRX101010:
2444                 return DRM_FORMAT_XRGB2101010;
2445         case DISPPLANE_RGBX101010:
2446                 return DRM_FORMAT_XBGR2101010;
2447         }
2448 }
2449
2450 static int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
2451 {
2452         switch (format) {
2453         case PLANE_CTL_FORMAT_RGB_565:
2454                 return DRM_FORMAT_RGB565;
2455         default:
2456         case PLANE_CTL_FORMAT_XRGB_8888:
2457                 if (rgb_order) {
2458                         if (alpha)
2459                                 return DRM_FORMAT_ABGR8888;
2460                         else
2461                                 return DRM_FORMAT_XBGR8888;
2462                 } else {
2463                         if (alpha)
2464                                 return DRM_FORMAT_ARGB8888;
2465                         else
2466                                 return DRM_FORMAT_XRGB8888;
2467                 }
2468         case PLANE_CTL_FORMAT_XRGB_2101010:
2469                 if (rgb_order)
2470                         return DRM_FORMAT_XBGR2101010;
2471                 else
2472                         return DRM_FORMAT_XRGB2101010;
2473         }
2474 }
2475
2476 static bool
2477 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
2478                               struct intel_initial_plane_config *plane_config)
2479 {
2480         struct drm_device *dev = crtc->base.dev;
2481         struct drm_i915_private *dev_priv = to_i915(dev);
2482         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2483         struct drm_i915_gem_object *obj = NULL;
2484         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
2485         struct drm_framebuffer *fb = &plane_config->fb->base;
2486         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
2487         u32 size_aligned = round_up(plane_config->base + plane_config->size,
2488                                     PAGE_SIZE);
2489
2490         size_aligned -= base_aligned;
2491
2492         if (plane_config->size == 0)
2493                 return false;
2494
2495         /* If the FB is too big, just don't use it since fbdev is not very
2496          * important and we should probably use that space with FBC or other
2497          * features. */
2498         if (size_aligned * 2 > ggtt->stolen_usable_size)
2499                 return false;
2500
2501         mutex_lock(&dev->struct_mutex);
2502
2503         obj = i915_gem_object_create_stolen_for_preallocated(dev,
2504                                                              base_aligned,
2505                                                              base_aligned,
2506                                                              size_aligned);
2507         if (!obj) {
2508                 mutex_unlock(&dev->struct_mutex);
2509                 return false;
2510         }
2511
2512         obj->tiling_mode = plane_config->tiling;
2513         if (obj->tiling_mode == I915_TILING_X)
2514                 obj->stride = fb->pitches[0];
2515
2516         mode_cmd.pixel_format = fb->pixel_format;
2517         mode_cmd.width = fb->width;
2518         mode_cmd.height = fb->height;
2519         mode_cmd.pitches[0] = fb->pitches[0];
2520         mode_cmd.modifier[0] = fb->modifier[0];
2521         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
2522
2523         if (intel_framebuffer_init(dev, to_intel_framebuffer(fb),
2524                                    &mode_cmd, obj)) {
2525                 DRM_DEBUG_KMS("intel fb init failed\n");
2526                 goto out_unref_obj;
2527         }
2528
2529         mutex_unlock(&dev->struct_mutex);
2530
2531         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
2532         return true;
2533
2534 out_unref_obj:
2535         drm_gem_object_unreference(&obj->base);
2536         mutex_unlock(&dev->struct_mutex);
2537         return false;
2538 }
2539
2540 /* Update plane->state->fb to match plane->fb after driver-internal updates */
2541 static void
2542 update_state_fb(struct drm_plane *plane)
2543 {
2544         if (plane->fb == plane->state->fb)
2545                 return;
2546
2547         if (plane->state->fb)
2548                 drm_framebuffer_unreference(plane->state->fb);
2549         plane->state->fb = plane->fb;
2550         if (plane->state->fb)
2551                 drm_framebuffer_reference(plane->state->fb);
2552 }
2553
2554 static void
2555 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
2556                              struct intel_initial_plane_config *plane_config)
2557 {
2558         struct drm_device *dev = intel_crtc->base.dev;
2559         struct drm_i915_private *dev_priv = dev->dev_private;
2560         struct drm_crtc *c;
2561         struct intel_crtc *i;
2562         struct drm_i915_gem_object *obj;
2563         struct drm_plane *primary = intel_crtc->base.primary;
2564         struct drm_plane_state *plane_state = primary->state;
2565         struct drm_crtc_state *crtc_state = intel_crtc->base.state;
2566         struct intel_plane *intel_plane = to_intel_plane(primary);
2567         struct intel_plane_state *intel_state =
2568                 to_intel_plane_state(plane_state);
2569         struct drm_framebuffer *fb;
2570
2571         if (!plane_config->fb)
2572                 return;
2573
2574         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
2575                 fb = &plane_config->fb->base;
2576                 goto valid_fb;
2577         }
2578
2579         kfree(plane_config->fb);
2580
2581         /*
2582          * Failed to alloc the obj, check to see if we should share
2583          * an fb with another CRTC instead
2584          */
2585         for_each_crtc(dev, c) {
2586                 i = to_intel_crtc(c);
2587
2588                 if (c == &intel_crtc->base)
2589                         continue;
2590
2591                 if (!i->active)
2592                         continue;
2593
2594                 fb = c->primary->fb;
2595                 if (!fb)
2596                         continue;
2597
2598                 obj = intel_fb_obj(fb);
2599                 if (i915_gem_obj_ggtt_offset(obj) == plane_config->base) {
2600                         drm_framebuffer_reference(fb);
2601                         goto valid_fb;
2602                 }
2603         }
2604
2605         /*
2606          * We've failed to reconstruct the BIOS FB.  Current display state
2607          * indicates that the primary plane is visible, but has a NULL FB,
2608          * which will lead to problems later if we don't fix it up.  The
2609          * simplest solution is to just disable the primary plane now and
2610          * pretend the BIOS never had it enabled.
2611          */
2612         to_intel_plane_state(plane_state)->visible = false;
2613         crtc_state->plane_mask &= ~(1 << drm_plane_index(primary));
2614         intel_pre_disable_primary_noatomic(&intel_crtc->base);
2615         intel_plane->disable_plane(primary, &intel_crtc->base);
2616
2617         return;
2618
2619 valid_fb:
2620         plane_state->src_x = 0;
2621         plane_state->src_y = 0;
2622         plane_state->src_w = fb->width << 16;
2623         plane_state->src_h = fb->height << 16;
2624
2625         plane_state->crtc_x = 0;
2626         plane_state->crtc_y = 0;
2627         plane_state->crtc_w = fb->width;
2628         plane_state->crtc_h = fb->height;
2629
2630         intel_state->src.x1 = plane_state->src_x;
2631         intel_state->src.y1 = plane_state->src_y;
2632         intel_state->src.x2 = plane_state->src_x + plane_state->src_w;
2633         intel_state->src.y2 = plane_state->src_y + plane_state->src_h;
2634         intel_state->dst.x1 = plane_state->crtc_x;
2635         intel_state->dst.y1 = plane_state->crtc_y;
2636         intel_state->dst.x2 = plane_state->crtc_x + plane_state->crtc_w;
2637         intel_state->dst.y2 = plane_state->crtc_y + plane_state->crtc_h;
2638
2639         obj = intel_fb_obj(fb);
2640         if (obj->tiling_mode != I915_TILING_NONE)
2641                 dev_priv->preserve_bios_swizzle = true;
2642
2643         drm_framebuffer_reference(fb);
2644         primary->fb = primary->state->fb = fb;
2645         primary->crtc = primary->state->crtc = &intel_crtc->base;
2646         intel_crtc->base.state->plane_mask |= (1 << drm_plane_index(primary));
2647         obj->frontbuffer_bits |= to_intel_plane(primary)->frontbuffer_bit;
2648 }
2649
2650 static void i9xx_update_primary_plane(struct drm_plane *primary,
2651                                       const struct intel_crtc_state *crtc_state,
2652                                       const struct intel_plane_state *plane_state)
2653 {
2654         struct drm_device *dev = primary->dev;
2655         struct drm_i915_private *dev_priv = dev->dev_private;
2656         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
2657         struct drm_framebuffer *fb = plane_state->base.fb;
2658         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2659         int plane = intel_crtc->plane;
2660         u32 linear_offset;
2661         u32 dspcntr;
2662         i915_reg_t reg = DSPCNTR(plane);
2663         unsigned int rotation = plane_state->base.rotation;
2664         int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
2665         int x = plane_state->src.x1 >> 16;
2666         int y = plane_state->src.y1 >> 16;
2667
2668         dspcntr = DISPPLANE_GAMMA_ENABLE;
2669
2670         dspcntr |= DISPLAY_PLANE_ENABLE;
2671
2672         if (INTEL_INFO(dev)->gen < 4) {
2673                 if (intel_crtc->pipe == PIPE_B)
2674                         dspcntr |= DISPPLANE_SEL_PIPE_B;
2675
2676                 /* pipesrc and dspsize control the size that is scaled from,
2677                  * which should always be the user's requested size.
2678                  */
2679                 I915_WRITE(DSPSIZE(plane),
2680                            ((crtc_state->pipe_src_h - 1) << 16) |
2681                            (crtc_state->pipe_src_w - 1));
2682                 I915_WRITE(DSPPOS(plane), 0);
2683         } else if (IS_CHERRYVIEW(dev) && plane == PLANE_B) {
2684                 I915_WRITE(PRIMSIZE(plane),
2685                            ((crtc_state->pipe_src_h - 1) << 16) |
2686                            (crtc_state->pipe_src_w - 1));
2687                 I915_WRITE(PRIMPOS(plane), 0);
2688                 I915_WRITE(PRIMCNSTALPHA(plane), 0);
2689         }
2690
2691         switch (fb->pixel_format) {
2692         case DRM_FORMAT_C8:
2693                 dspcntr |= DISPPLANE_8BPP;
2694                 break;
2695         case DRM_FORMAT_XRGB1555:
2696                 dspcntr |= DISPPLANE_BGRX555;
2697                 break;
2698         case DRM_FORMAT_RGB565:
2699                 dspcntr |= DISPPLANE_BGRX565;
2700                 break;
2701         case DRM_FORMAT_XRGB8888:
2702                 dspcntr |= DISPPLANE_BGRX888;
2703                 break;
2704         case DRM_FORMAT_XBGR8888:
2705                 dspcntr |= DISPPLANE_RGBX888;
2706                 break;
2707         case DRM_FORMAT_XRGB2101010:
2708                 dspcntr |= DISPPLANE_BGRX101010;
2709                 break;
2710         case DRM_FORMAT_XBGR2101010:
2711                 dspcntr |= DISPPLANE_RGBX101010;
2712                 break;
2713         default:
2714                 BUG();
2715         }
2716
2717         if (INTEL_INFO(dev)->gen >= 4 &&
2718             obj->tiling_mode != I915_TILING_NONE)
2719                 dspcntr |= DISPPLANE_TILED;
2720
2721         if (IS_G4X(dev))
2722                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
2723
2724         linear_offset = y * fb->pitches[0] + x * cpp;
2725
2726         if (INTEL_INFO(dev)->gen >= 4) {
2727                 intel_crtc->dspaddr_offset =
2728                         intel_compute_tile_offset(&x, &y, fb, 0,
2729                                                   fb->pitches[0], rotation);
2730                 linear_offset -= intel_crtc->dspaddr_offset;
2731         } else {
2732                 intel_crtc->dspaddr_offset = linear_offset;
2733         }
2734
2735         if (rotation == BIT(DRM_ROTATE_180)) {
2736                 dspcntr |= DISPPLANE_ROTATE_180;
2737
2738                 x += (crtc_state->pipe_src_w - 1);
2739                 y += (crtc_state->pipe_src_h - 1);
2740
2741                 /* Finding the last pixel of the last line of the display
2742                 data and adding to linear_offset*/
2743                 linear_offset +=
2744                         (crtc_state->pipe_src_h - 1) * fb->pitches[0] +
2745                         (crtc_state->pipe_src_w - 1) * cpp;
2746         }
2747
2748         intel_crtc->adjusted_x = x;
2749         intel_crtc->adjusted_y = y;
2750
2751         I915_WRITE(reg, dspcntr);
2752
2753         I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
2754         if (INTEL_INFO(dev)->gen >= 4) {
2755                 I915_WRITE(DSPSURF(plane),
2756                            i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset);
2757                 I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
2758                 I915_WRITE(DSPLINOFF(plane), linear_offset);
2759         } else
2760                 I915_WRITE(DSPADDR(plane), i915_gem_obj_ggtt_offset(obj) + linear_offset);
2761         POSTING_READ(reg);
2762 }
2763
2764 static void i9xx_disable_primary_plane(struct drm_plane *primary,
2765                                        struct drm_crtc *crtc)
2766 {
2767         struct drm_device *dev = crtc->dev;
2768         struct drm_i915_private *dev_priv = dev->dev_private;
2769         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2770         int plane = intel_crtc->plane;
2771
2772         I915_WRITE(DSPCNTR(plane), 0);
2773         if (INTEL_INFO(dev_priv)->gen >= 4)
2774                 I915_WRITE(DSPSURF(plane), 0);
2775         else
2776                 I915_WRITE(DSPADDR(plane), 0);
2777         POSTING_READ(DSPCNTR(plane));
2778 }
2779
2780 static void ironlake_update_primary_plane(struct drm_plane *primary,
2781                                           const struct intel_crtc_state *crtc_state,
2782                                           const struct intel_plane_state *plane_state)
2783 {
2784         struct drm_device *dev = primary->dev;
2785         struct drm_i915_private *dev_priv = dev->dev_private;
2786         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
2787         struct drm_framebuffer *fb = plane_state->base.fb;
2788         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2789         int plane = intel_crtc->plane;
2790         u32 linear_offset;
2791         u32 dspcntr;
2792         i915_reg_t reg = DSPCNTR(plane);
2793         unsigned int rotation = plane_state->base.rotation;
2794         int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
2795         int x = plane_state->src.x1 >> 16;
2796         int y = plane_state->src.y1 >> 16;
2797
2798         dspcntr = DISPPLANE_GAMMA_ENABLE;
2799         dspcntr |= DISPLAY_PLANE_ENABLE;
2800
2801         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2802                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
2803
2804         switch (fb->pixel_format) {
2805         case DRM_FORMAT_C8:
2806                 dspcntr |= DISPPLANE_8BPP;
2807                 break;
2808         case DRM_FORMAT_RGB565:
2809                 dspcntr |= DISPPLANE_BGRX565;
2810                 break;
2811         case DRM_FORMAT_XRGB8888:
2812                 dspcntr |= DISPPLANE_BGRX888;
2813                 break;
2814         case DRM_FORMAT_XBGR8888:
2815                 dspcntr |= DISPPLANE_RGBX888;
2816                 break;
2817         case DRM_FORMAT_XRGB2101010:
2818                 dspcntr |= DISPPLANE_BGRX101010;
2819                 break;
2820         case DRM_FORMAT_XBGR2101010:
2821                 dspcntr |= DISPPLANE_RGBX101010;
2822                 break;
2823         default:
2824                 BUG();
2825         }
2826
2827         if (obj->tiling_mode != I915_TILING_NONE)
2828                 dspcntr |= DISPPLANE_TILED;
2829
2830         if (!IS_HASWELL(dev) && !IS_BROADWELL(dev))
2831                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
2832
2833         linear_offset = y * fb->pitches[0] + x * cpp;
2834         intel_crtc->dspaddr_offset =
2835                 intel_compute_tile_offset(&x, &y, fb, 0,
2836                                           fb->pitches[0], rotation);
2837         linear_offset -= intel_crtc->dspaddr_offset;
2838         if (rotation == BIT(DRM_ROTATE_180)) {
2839                 dspcntr |= DISPPLANE_ROTATE_180;
2840
2841                 if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
2842                         x += (crtc_state->pipe_src_w - 1);
2843                         y += (crtc_state->pipe_src_h - 1);
2844
2845                         /* Finding the last pixel of the last line of the display
2846                         data and adding to linear_offset*/
2847                         linear_offset +=
2848                                 (crtc_state->pipe_src_h - 1) * fb->pitches[0] +
2849                                 (crtc_state->pipe_src_w - 1) * cpp;
2850                 }
2851         }
2852
2853         intel_crtc->adjusted_x = x;
2854         intel_crtc->adjusted_y = y;
2855
2856         I915_WRITE(reg, dspcntr);
2857
2858         I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
2859         I915_WRITE(DSPSURF(plane),
2860                    i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset);
2861         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
2862                 I915_WRITE(DSPOFFSET(plane), (y << 16) | x);
2863         } else {
2864                 I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
2865                 I915_WRITE(DSPLINOFF(plane), linear_offset);
2866         }
2867         POSTING_READ(reg);
2868 }
2869
2870 u32 intel_fb_stride_alignment(const struct drm_i915_private *dev_priv,
2871                               uint64_t fb_modifier, uint32_t pixel_format)
2872 {
2873         if (fb_modifier == DRM_FORMAT_MOD_NONE) {
2874                 return 64;
2875         } else {
2876                 int cpp = drm_format_plane_cpp(pixel_format, 0);
2877
2878                 return intel_tile_width_bytes(dev_priv, fb_modifier, cpp);
2879         }
2880 }
2881
2882 u32 intel_plane_obj_offset(struct intel_plane *intel_plane,
2883                            struct drm_i915_gem_object *obj,
2884                            unsigned int plane)
2885 {
2886         struct i915_ggtt_view view;
2887         struct i915_vma *vma;
2888         u64 offset;
2889
2890         intel_fill_fb_ggtt_view(&view, intel_plane->base.state->fb,
2891                                 intel_plane->base.state->rotation);
2892
2893         vma = i915_gem_obj_to_ggtt_view(obj, &view);
2894         if (WARN(!vma, "ggtt vma for display object not found! (view=%u)\n",
2895                 view.type))
2896                 return -1;
2897
2898         offset = vma->node.start;
2899
2900         if (plane == 1) {
2901                 offset += vma->ggtt_view.params.rotated.uv_start_page *
2902                           PAGE_SIZE;
2903         }
2904
2905         WARN_ON(upper_32_bits(offset));
2906
2907         return lower_32_bits(offset);
2908 }
2909
2910 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
2911 {
2912         struct drm_device *dev = intel_crtc->base.dev;
2913         struct drm_i915_private *dev_priv = dev->dev_private;
2914
2915         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
2916         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
2917         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0);
2918 }
2919
2920 /*
2921  * This function detaches (aka. unbinds) unused scalers in hardware
2922  */
2923 static void skl_detach_scalers(struct intel_crtc *intel_crtc)
2924 {
2925         struct intel_crtc_scaler_state *scaler_state;
2926         int i;
2927
2928         scaler_state = &intel_crtc->config->scaler_state;
2929
2930         /* loop through and disable scalers that aren't in use */
2931         for (i = 0; i < intel_crtc->num_scalers; i++) {
2932                 if (!scaler_state->scalers[i].in_use)
2933                         skl_detach_scaler(intel_crtc, i);
2934         }
2935 }
2936
2937 u32 skl_plane_ctl_format(uint32_t pixel_format)
2938 {
2939         switch (pixel_format) {
2940         case DRM_FORMAT_C8:
2941                 return PLANE_CTL_FORMAT_INDEXED;
2942         case DRM_FORMAT_RGB565:
2943                 return PLANE_CTL_FORMAT_RGB_565;
2944         case DRM_FORMAT_XBGR8888:
2945                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
2946         case DRM_FORMAT_XRGB8888:
2947                 return PLANE_CTL_FORMAT_XRGB_8888;
2948         /*
2949          * XXX: For ARBG/ABGR formats we default to expecting scanout buffers
2950          * to be already pre-multiplied. We need to add a knob (or a different
2951          * DRM_FORMAT) for user-space to configure that.
2952          */
2953         case DRM_FORMAT_ABGR8888:
2954                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX |
2955                         PLANE_CTL_ALPHA_SW_PREMULTIPLY;
2956         case DRM_FORMAT_ARGB8888:
2957                 return PLANE_CTL_FORMAT_XRGB_8888 |
2958                         PLANE_CTL_ALPHA_SW_PREMULTIPLY;
2959         case DRM_FORMAT_XRGB2101010:
2960                 return PLANE_CTL_FORMAT_XRGB_2101010;
2961         case DRM_FORMAT_XBGR2101010:
2962                 return PLANE_CTL_ORDER_RGBX | PLANE_CTL_FORMAT_XRGB_2101010;
2963         case DRM_FORMAT_YUYV:
2964                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
2965         case DRM_FORMAT_YVYU:
2966                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
2967         case DRM_FORMAT_UYVY:
2968                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
2969         case DRM_FORMAT_VYUY:
2970                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
2971         default:
2972                 MISSING_CASE(pixel_format);
2973         }
2974
2975         return 0;
2976 }
2977
2978 u32 skl_plane_ctl_tiling(uint64_t fb_modifier)
2979 {
2980         switch (fb_modifier) {
2981         case DRM_FORMAT_MOD_NONE:
2982                 break;
2983         case I915_FORMAT_MOD_X_TILED:
2984                 return PLANE_CTL_TILED_X;
2985         case I915_FORMAT_MOD_Y_TILED:
2986                 return PLANE_CTL_TILED_Y;
2987         case I915_FORMAT_MOD_Yf_TILED:
2988                 return PLANE_CTL_TILED_YF;
2989         default:
2990                 MISSING_CASE(fb_modifier);
2991         }
2992
2993         return 0;
2994 }
2995
2996 u32 skl_plane_ctl_rotation(unsigned int rotation)
2997 {
2998         switch (rotation) {
2999         case BIT(DRM_ROTATE_0):
3000                 break;
3001         /*
3002          * DRM_ROTATE_ is counter clockwise to stay compatible with Xrandr
3003          * while i915 HW rotation is clockwise, thats why this swapping.
3004          */
3005         case BIT(DRM_ROTATE_90):
3006                 return PLANE_CTL_ROTATE_270;
3007         case BIT(DRM_ROTATE_180):
3008                 return PLANE_CTL_ROTATE_180;
3009         case BIT(DRM_ROTATE_270):
3010                 return PLANE_CTL_ROTATE_90;
3011         default:
3012                 MISSING_CASE(rotation);
3013         }
3014
3015         return 0;
3016 }
3017
3018 static void skylake_update_primary_plane(struct drm_plane *plane,
3019                                          const struct intel_crtc_state *crtc_state,
3020                                          const struct intel_plane_state *plane_state)
3021 {
3022         struct drm_device *dev = plane->dev;
3023         struct drm_i915_private *dev_priv = dev->dev_private;
3024         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
3025         struct drm_framebuffer *fb = plane_state->base.fb;
3026         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
3027         int pipe = intel_crtc->pipe;
3028         u32 plane_ctl, stride_div, stride;
3029         u32 tile_height, plane_offset, plane_size;
3030         unsigned int rotation = plane_state->base.rotation;
3031         int x_offset, y_offset;
3032         u32 surf_addr;
3033         int scaler_id = plane_state->scaler_id;
3034         int src_x = plane_state->src.x1 >> 16;
3035         int src_y = plane_state->src.y1 >> 16;
3036         int src_w = drm_rect_width(&plane_state->src) >> 16;
3037         int src_h = drm_rect_height(&plane_state->src) >> 16;
3038         int dst_x = plane_state->dst.x1;
3039         int dst_y = plane_state->dst.y1;
3040         int dst_w = drm_rect_width(&plane_state->dst);
3041         int dst_h = drm_rect_height(&plane_state->dst);
3042
3043         plane_ctl = PLANE_CTL_ENABLE |
3044                     PLANE_CTL_PIPE_GAMMA_ENABLE |
3045                     PLANE_CTL_PIPE_CSC_ENABLE;
3046
3047         plane_ctl |= skl_plane_ctl_format(fb->pixel_format);
3048         plane_ctl |= skl_plane_ctl_tiling(fb->modifier[0]);
3049         plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
3050         plane_ctl |= skl_plane_ctl_rotation(rotation);
3051
3052         stride_div = intel_fb_stride_alignment(dev_priv, fb->modifier[0],
3053                                                fb->pixel_format);
3054         surf_addr = intel_plane_obj_offset(to_intel_plane(plane), obj, 0);
3055
3056         WARN_ON(drm_rect_width(&plane_state->src) == 0);
3057
3058         if (intel_rotation_90_or_270(rotation)) {
3059                 int cpp = drm_format_plane_cpp(fb->pixel_format, 0);
3060
3061                 /* stride = Surface height in tiles */
3062                 tile_height = intel_tile_height(dev_priv, fb->modifier[0], cpp);
3063                 stride = DIV_ROUND_UP(fb->height, tile_height);
3064                 x_offset = stride * tile_height - src_y - src_h;
3065                 y_offset = src_x;
3066                 plane_size = (src_w - 1) << 16 | (src_h - 1);
3067         } else {
3068                 stride = fb->pitches[0] / stride_div;
3069                 x_offset = src_x;
3070                 y_offset = src_y;
3071                 plane_size = (src_h - 1) << 16 | (src_w - 1);
3072         }
3073         plane_offset = y_offset << 16 | x_offset;
3074
3075         intel_crtc->adjusted_x = x_offset;
3076         intel_crtc->adjusted_y = y_offset;
3077
3078         I915_WRITE(PLANE_CTL(pipe, 0), plane_ctl);
3079         I915_WRITE(PLANE_OFFSET(pipe, 0), plane_offset);
3080         I915_WRITE(PLANE_SIZE(pipe, 0), plane_size);
3081         I915_WRITE(PLANE_STRIDE(pipe, 0), stride);
3082
3083         if (scaler_id >= 0) {
3084                 uint32_t ps_ctrl = 0;
3085
3086                 WARN_ON(!dst_w || !dst_h);
3087                 ps_ctrl = PS_SCALER_EN | PS_PLANE_SEL(0) |
3088                         crtc_state->scaler_state.scalers[scaler_id].mode;
3089                 I915_WRITE(SKL_PS_CTRL(pipe, scaler_id), ps_ctrl);
3090                 I915_WRITE(SKL_PS_PWR_GATE(pipe, scaler_id), 0);
3091                 I915_WRITE(SKL_PS_WIN_POS(pipe, scaler_id), (dst_x << 16) | dst_y);
3092                 I915_WRITE(SKL_PS_WIN_SZ(pipe, scaler_id), (dst_w << 16) | dst_h);
3093                 I915_WRITE(PLANE_POS(pipe, 0), 0);
3094         } else {
3095                 I915_WRITE(PLANE_POS(pipe, 0), (dst_y << 16) | dst_x);
3096         }
3097
3098         I915_WRITE(PLANE_SURF(pipe, 0), surf_addr);
3099
3100         POSTING_READ(PLANE_SURF(pipe, 0));
3101 }
3102
3103 static void skylake_disable_primary_plane(struct drm_plane *primary,
3104                                           struct drm_crtc *crtc)
3105 {
3106         struct drm_device *dev = crtc->dev;
3107         struct drm_i915_private *dev_priv = dev->dev_private;
3108         int pipe = to_intel_crtc(crtc)->pipe;
3109
3110         I915_WRITE(PLANE_CTL(pipe, 0), 0);
3111         I915_WRITE(PLANE_SURF(pipe, 0), 0);
3112         POSTING_READ(PLANE_SURF(pipe, 0));
3113 }
3114
3115 /* Assume fb object is pinned & idle & fenced and just update base pointers */
3116 static int
3117 intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
3118                            int x, int y, enum mode_set_atomic state)
3119 {
3120         /* Support for kgdboc is disabled, this needs a major rework. */
3121         DRM_ERROR("legacy panic handler not supported any more.\n");
3122
3123         return -ENODEV;
3124 }
3125
3126 static void intel_complete_page_flips(struct drm_i915_private *dev_priv)
3127 {
3128         struct intel_crtc *crtc;
3129
3130         for_each_intel_crtc(dev_priv->dev, crtc)
3131                 intel_finish_page_flip_cs(dev_priv, crtc->pipe);
3132 }
3133
3134 static void intel_update_primary_planes(struct drm_device *dev)
3135 {
3136         struct drm_crtc *crtc;
3137
3138         for_each_crtc(dev, crtc) {
3139                 struct intel_plane *plane = to_intel_plane(crtc->primary);
3140                 struct intel_plane_state *plane_state;
3141
3142                 drm_modeset_lock_crtc(crtc, &plane->base);
3143                 plane_state = to_intel_plane_state(plane->base.state);
3144
3145                 if (plane_state->visible)
3146                         plane->update_plane(&plane->base,
3147                                             to_intel_crtc_state(crtc->state),
3148                                             plane_state);
3149
3150                 drm_modeset_unlock_crtc(crtc);
3151         }
3152 }
3153
3154 void intel_prepare_reset(struct drm_i915_private *dev_priv)
3155 {
3156         /* no reset support for gen2 */
3157         if (IS_GEN2(dev_priv))
3158                 return;
3159
3160         /* reset doesn't touch the display */
3161         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
3162                 return;
3163
3164         drm_modeset_lock_all(dev_priv->dev);
3165         /*
3166          * Disabling the crtcs gracefully seems nicer. Also the
3167          * g33 docs say we should at least disable all the planes.
3168          */
3169         intel_display_suspend(dev_priv->dev);
3170 }
3171
3172 void intel_finish_reset(struct drm_i915_private *dev_priv)
3173 {
3174         /*
3175          * Flips in the rings will be nuked by the reset,
3176          * so complete all pending flips so that user space
3177          * will get its events and not get stuck.
3178          */
3179         intel_complete_page_flips(dev_priv);
3180
3181         /* no reset support for gen2 */
3182         if (IS_GEN2(dev_priv))
3183                 return;
3184
3185         /* reset doesn't touch the display */
3186         if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) {
3187                 /*
3188                  * Flips in the rings have been nuked by the reset,
3189                  * so update the base address of all primary
3190                  * planes to the the last fb to make sure we're
3191                  * showing the correct fb after a reset.
3192                  *
3193                  * FIXME: Atomic will make this obsolete since we won't schedule
3194                  * CS-based flips (which might get lost in gpu resets) any more.
3195                  */
3196                 intel_update_primary_planes(dev_priv->dev);
3197                 return;
3198         }
3199
3200         /*
3201          * The display has been reset as well,
3202          * so need a full re-initialization.
3203          */
3204         intel_runtime_pm_disable_interrupts(dev_priv);
3205         intel_runtime_pm_enable_interrupts(dev_priv);
3206
3207         intel_modeset_init_hw(dev_priv->dev);
3208
3209         spin_lock_irq(&dev_priv->irq_lock);
3210         if (dev_priv->display.hpd_irq_setup)
3211                 dev_priv->display.hpd_irq_setup(dev_priv);
3212         spin_unlock_irq(&dev_priv->irq_lock);
3213
3214         intel_display_resume(dev_priv->dev);
3215
3216         intel_hpd_init(dev_priv);
3217
3218         drm_modeset_unlock_all(dev_priv->dev);
3219 }
3220
3221 static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc)
3222 {
3223         struct drm_device *dev = crtc->dev;
3224         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3225         unsigned reset_counter;
3226         bool pending;
3227
3228         reset_counter = i915_reset_counter(&to_i915(dev)->gpu_error);
3229         if (intel_crtc->reset_counter != reset_counter)
3230                 return false;
3231
3232         spin_lock_irq(&dev->event_lock);
3233         pending = to_intel_crtc(crtc)->flip_work != NULL;
3234         spin_unlock_irq(&dev->event_lock);
3235
3236         return pending;
3237 }
3238
3239 static void intel_update_pipe_config(struct intel_crtc *crtc,
3240                                      struct intel_crtc_state *old_crtc_state)
3241 {
3242         struct drm_device *dev = crtc->base.dev;
3243         struct drm_i915_private *dev_priv = dev->dev_private;
3244         struct intel_crtc_state *pipe_config =
3245                 to_intel_crtc_state(crtc->base.state);
3246
3247         /* drm_atomic_helper_update_legacy_modeset_state might not be called. */
3248         crtc->base.mode = crtc->base.state->mode;
3249
3250         DRM_DEBUG_KMS("Updating pipe size %ix%i -> %ix%i\n",
3251                       old_crtc_state->pipe_src_w, old_crtc_state->pipe_src_h,
3252                       pipe_config->pipe_src_w, pipe_config->pipe_src_h);
3253
3254         /*
3255          * Update pipe size and adjust fitter if needed: the reason for this is
3256          * that in compute_mode_changes we check the native mode (not the pfit
3257          * mode) to see if we can flip rather than do a full mode set. In the
3258          * fastboot case, we'll flip, but if we don't update the pipesrc and
3259          * pfit state, we'll end up with a big fb scanned out into the wrong
3260          * sized surface.
3261          */
3262
3263         I915_WRITE(PIPESRC(crtc->pipe),
3264                    ((pipe_config->pipe_src_w - 1) << 16) |
3265                    (pipe_config->pipe_src_h - 1));
3266
3267         /* on skylake this is done by detaching scalers */
3268         if (INTEL_INFO(dev)->gen >= 9) {
3269                 skl_detach_scalers(crtc);
3270
3271                 if (pipe_config->pch_pfit.enabled)
3272                         skylake_pfit_enable(crtc);
3273         } else if (HAS_PCH_SPLIT(dev)) {
3274                 if (pipe_config->pch_pfit.enabled)
3275                         ironlake_pfit_enable(crtc);
3276                 else if (old_crtc_state->pch_pfit.enabled)
3277                         ironlake_pfit_disable(crtc, true);
3278         }
3279 }
3280
3281 static void intel_fdi_normal_train(struct drm_crtc *crtc)
3282 {
3283         struct drm_device *dev = crtc->dev;
3284         struct drm_i915_private *dev_priv = dev->dev_private;
3285         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3286         int pipe = intel_crtc->pipe;
3287         i915_reg_t reg;
3288         u32 temp;
3289
3290         /* enable normal train */
3291         reg = FDI_TX_CTL(pipe);
3292         temp = I915_READ(reg);
3293         if (IS_IVYBRIDGE(dev)) {
3294                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
3295                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
3296         } else {
3297                 temp &= ~FDI_LINK_TRAIN_NONE;
3298                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
3299         }
3300         I915_WRITE(reg, temp);
3301
3302         reg = FDI_RX_CTL(pipe);
3303         temp = I915_READ(reg);
3304         if (HAS_PCH_CPT(dev)) {
3305                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3306                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
3307         } else {
3308                 temp &= ~FDI_LINK_TRAIN_NONE;
3309                 temp |= FDI_LINK_TRAIN_NONE;
3310         }
3311         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
3312
3313         /* wait one idle pattern time */
3314         POSTING_READ(reg);
3315         udelay(1000);
3316
3317         /* IVB wants error correction enabled */
3318         if (IS_IVYBRIDGE(dev))
3319                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
3320                            FDI_FE_ERRC_ENABLE);
3321 }
3322
3323 /* The FDI link training functions for ILK/Ibexpeak. */
3324 static void ironlake_fdi_link_train(struct drm_crtc *crtc)
3325 {
3326         struct drm_device *dev = crtc->dev;
3327         struct drm_i915_private *dev_priv = dev->dev_private;
3328         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3329         int pipe = intel_crtc->pipe;
3330         i915_reg_t reg;
3331         u32 temp, tries;
3332
3333         /* FDI needs bits from pipe first */
3334         assert_pipe_enabled(dev_priv, pipe);
3335
3336         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3337            for train result */
3338         reg = FDI_RX_IMR(pipe);
3339         temp = I915_READ(reg);
3340         temp &= ~FDI_RX_SYMBOL_LOCK;
3341         temp &= ~FDI_RX_BIT_LOCK;
3342         I915_WRITE(reg, temp);
3343         I915_READ(reg);
3344         udelay(150);
3345
3346         /* enable CPU FDI TX and PCH FDI RX */
3347         reg = FDI_TX_CTL(pipe);
3348         temp = I915_READ(reg);
3349         temp &= ~FDI_DP_PORT_WIDTH_MASK;
3350         temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3351         temp &= ~FDI_LINK_TRAIN_NONE;
3352         temp |= FDI_LINK_TRAIN_PATTERN_1;
3353         I915_WRITE(reg, temp | FDI_TX_ENABLE);
3354
3355         reg = FDI_RX_CTL(pipe);
3356         temp = I915_READ(reg);
3357         temp &= ~FDI_LINK_TRAIN_NONE;
3358         temp |= FDI_LINK_TRAIN_PATTERN_1;
3359         I915_WRITE(reg, temp | FDI_RX_ENABLE);
3360
3361         POSTING_READ(reg);
3362         udelay(150);
3363
3364         /* Ironlake workaround, enable clock pointer after FDI enable*/
3365         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
3366         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
3367                    FDI_RX_PHASE_SYNC_POINTER_EN);
3368
3369         reg = FDI_RX_IIR(pipe);
3370         for (tries = 0; tries < 5; tries++) {
3371                 temp = I915_READ(reg);
3372                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3373
3374                 if ((temp & FDI_RX_BIT_LOCK)) {
3375                         DRM_DEBUG_KMS("FDI train 1 done.\n");
3376                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
3377                         break;
3378                 }
3379         }
3380         if (tries == 5)
3381                 DRM_ERROR("FDI train 1 fail!\n");
3382
3383         /* Train 2 */
3384         reg = FDI_TX_CTL(pipe);
3385         temp = I915_READ(reg);
3386         temp &= ~FDI_LINK_TRAIN_NONE;
3387         temp |= FDI_LINK_TRAIN_PATTERN_2;
3388         I915_WRITE(reg, temp);
3389
3390         reg = FDI_RX_CTL(pipe);
3391         temp = I915_READ(reg);
3392         temp &= ~FDI_LINK_TRAIN_NONE;
3393         temp |= FDI_LINK_TRAIN_PATTERN_2;
3394         I915_WRITE(reg, temp);
3395
3396         POSTING_READ(reg);
3397         udelay(150);
3398
3399         reg = FDI_RX_IIR(pipe);
3400         for (tries = 0; tries < 5; tries++) {
3401                 temp = I915_READ(reg);
3402                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3403
3404                 if (temp & FDI_RX_SYMBOL_LOCK) {
3405                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
3406                         DRM_DEBUG_KMS("FDI train 2 done.\n");
3407                         break;
3408                 }
3409         }
3410         if (tries == 5)
3411                 DRM_ERROR("FDI train 2 fail!\n");
3412
3413         DRM_DEBUG_KMS("FDI train done\n");
3414
3415 }
3416
3417 static const int snb_b_fdi_train_param[] = {
3418         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
3419         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
3420         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
3421         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
3422 };
3423
3424 /* The FDI link training functions for SNB/Cougarpoint. */
3425 static void gen6_fdi_link_train(struct drm_crtc *crtc)
3426 {
3427         struct drm_device *dev = crtc->dev;
3428         struct drm_i915_private *dev_priv = dev->dev_private;
3429         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3430         int pipe = intel_crtc->pipe;
3431         i915_reg_t reg;
3432         u32 temp, i, retry;
3433
3434         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3435            for train result */
3436         reg = FDI_RX_IMR(pipe);
3437         temp = I915_READ(reg);
3438         temp &= ~FDI_RX_SYMBOL_LOCK;
3439         temp &= ~FDI_RX_BIT_LOCK;
3440         I915_WRITE(reg, temp);
3441
3442         POSTING_READ(reg);
3443         udelay(150);
3444
3445         /* enable CPU FDI TX and PCH FDI RX */
3446         reg = FDI_TX_CTL(pipe);
3447         temp = I915_READ(reg);
3448         temp &= ~FDI_DP_PORT_WIDTH_MASK;
3449         temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3450         temp &= ~FDI_LINK_TRAIN_NONE;
3451         temp |= FDI_LINK_TRAIN_PATTERN_1;
3452         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3453         /* SNB-B */
3454         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
3455         I915_WRITE(reg, temp | FDI_TX_ENABLE);
3456
3457         I915_WRITE(FDI_RX_MISC(pipe),
3458                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
3459
3460         reg = FDI_RX_CTL(pipe);
3461         temp = I915_READ(reg);
3462         if (HAS_PCH_CPT(dev)) {
3463                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3464                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
3465         } else {
3466                 temp &= ~FDI_LINK_TRAIN_NONE;
3467                 temp |= FDI_LINK_TRAIN_PATTERN_1;
3468         }
3469         I915_WRITE(reg, temp | FDI_RX_ENABLE);
3470
3471         POSTING_READ(reg);
3472         udelay(150);
3473
3474         for (i = 0; i < 4; i++) {
3475                 reg = FDI_TX_CTL(pipe);
3476                 temp = I915_READ(reg);
3477                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3478                 temp |= snb_b_fdi_train_param[i];
3479                 I915_WRITE(reg, temp);
3480
3481                 POSTING_READ(reg);
3482                 udelay(500);
3483
3484                 for (retry = 0; retry < 5; retry++) {
3485                         reg = FDI_RX_IIR(pipe);
3486                         temp = I915_READ(reg);
3487                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3488                         if (temp & FDI_RX_BIT_LOCK) {
3489                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
3490                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
3491                                 break;
3492                         }
3493                         udelay(50);
3494                 }
3495                 if (retry < 5)
3496                         break;
3497         }
3498         if (i == 4)
3499                 DRM_ERROR("FDI train 1 fail!\n");
3500
3501         /* Train 2 */
3502         reg = FDI_TX_CTL(pipe);
3503         temp = I915_READ(reg);
3504         temp &= ~FDI_LINK_TRAIN_NONE;
3505         temp |= FDI_LINK_TRAIN_PATTERN_2;
3506         if (IS_GEN6(dev)) {
3507                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3508                 /* SNB-B */
3509                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
3510         }
3511         I915_WRITE(reg, temp);
3512
3513         reg = FDI_RX_CTL(pipe);
3514         temp = I915_READ(reg);
3515         if (HAS_PCH_CPT(dev)) {
3516                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3517                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
3518         } else {
3519                 temp &= ~FDI_LINK_TRAIN_NONE;
3520                 temp |= FDI_LINK_TRAIN_PATTERN_2;
3521         }
3522         I915_WRITE(reg, temp);
3523
3524         POSTING_READ(reg);
3525         udelay(150);
3526
3527         for (i = 0; i < 4; i++) {
3528                 reg = FDI_TX_CTL(pipe);
3529                 temp = I915_READ(reg);
3530                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3531                 temp |= snb_b_fdi_train_param[i];
3532                 I915_WRITE(reg, temp);
3533
3534                 POSTING_READ(reg);
3535                 udelay(500);
3536
3537                 for (retry = 0; retry < 5; retry++) {
3538                         reg = FDI_RX_IIR(pipe);
3539                         temp = I915_READ(reg);
3540                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3541                         if (temp & FDI_RX_SYMBOL_LOCK) {
3542                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
3543                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
3544                                 break;
3545                         }
3546                         udelay(50);
3547                 }
3548                 if (retry < 5)
3549                         break;
3550         }
3551         if (i == 4)
3552                 DRM_ERROR("FDI train 2 fail!\n");
3553
3554         DRM_DEBUG_KMS("FDI train done.\n");
3555 }
3556
3557 /* Manual link training for Ivy Bridge A0 parts */
3558 static void ivb_manual_fdi_link_train(struct drm_crtc *crtc)
3559 {
3560         struct drm_device *dev = crtc->dev;
3561         struct drm_i915_private *dev_priv = dev->dev_private;
3562         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3563         int pipe = intel_crtc->pipe;
3564         i915_reg_t reg;
3565         u32 temp, i, j;
3566
3567         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3568            for train result */
3569         reg = FDI_RX_IMR(pipe);
3570         temp = I915_READ(reg);
3571         temp &= ~FDI_RX_SYMBOL_LOCK;
3572         temp &= ~FDI_RX_BIT_LOCK;
3573         I915_WRITE(reg, temp);
3574
3575         POSTING_READ(reg);
3576         udelay(150);
3577
3578         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
3579                       I915_READ(FDI_RX_IIR(pipe)));
3580
3581         /* Try each vswing and preemphasis setting twice before moving on */
3582         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
3583                 /* disable first in case we need to retry */
3584                 reg = FDI_TX_CTL(pipe);
3585                 temp = I915_READ(reg);
3586                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
3587                 temp &= ~FDI_TX_ENABLE;
3588                 I915_WRITE(reg, temp);
3589
3590                 reg = FDI_RX_CTL(pipe);
3591                 temp = I915_READ(reg);
3592                 temp &= ~FDI_LINK_TRAIN_AUTO;
3593                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3594                 temp &= ~FDI_RX_ENABLE;
3595                 I915_WRITE(reg, temp);
3596
3597                 /* enable CPU FDI TX and PCH FDI RX */
3598                 reg = FDI_TX_CTL(pipe);
3599                 temp = I915_READ(reg);
3600                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
3601                 temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3602                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
3603                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3604                 temp |= snb_b_fdi_train_param[j/2];
3605                 temp |= FDI_COMPOSITE_SYNC;
3606                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
3607
3608                 I915_WRITE(FDI_RX_MISC(pipe),
3609                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
3610
3611                 reg = FDI_RX_CTL(pipe);
3612                 temp = I915_READ(reg);
3613                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
3614                 temp |= FDI_COMPOSITE_SYNC;
3615                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
3616
3617                 POSTING_READ(reg);
3618                 udelay(1); /* should be 0.5us */
3619
3620                 for (i = 0; i < 4; i++) {
3621                         reg = FDI_RX_IIR(pipe);
3622                         temp = I915_READ(reg);
3623                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3624
3625                         if (temp & FDI_RX_BIT_LOCK ||
3626                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
3627                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
3628                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
3629                                               i);
3630                                 break;
3631                         }
3632                         udelay(1); /* should be 0.5us */
3633                 }
3634                 if (i == 4) {
3635                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
3636                         continue;
3637                 }
3638
3639                 /* Train 2 */
3640                 reg = FDI_TX_CTL(pipe);
3641                 temp = I915_READ(reg);
3642                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
3643                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
3644                 I915_WRITE(reg, temp);
3645
3646                 reg = FDI_RX_CTL(pipe);
3647                 temp = I915_READ(reg);
3648                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3649                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
3650                 I915_WRITE(reg, temp);
3651
3652                 POSTING_READ(reg);
3653                 udelay(2); /* should be 1.5us */
3654
3655                 for (i = 0; i < 4; i++) {
3656                         reg = FDI_RX_IIR(pipe);
3657                         temp = I915_READ(reg);
3658                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3659
3660                         if (temp & FDI_RX_SYMBOL_LOCK ||
3661                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
3662                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
3663                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
3664                                               i);
3665                                 goto train_done;
3666                         }
3667                         udelay(2); /* should be 1.5us */
3668                 }
3669                 if (i == 4)
3670                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
3671         }
3672
3673 train_done:
3674         DRM_DEBUG_KMS("FDI train done.\n");
3675 }
3676
3677 static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc)
3678 {
3679         struct drm_device *dev = intel_crtc->base.dev;
3680         struct drm_i915_private *dev_priv = dev->dev_private;
3681         int pipe = intel_crtc->pipe;
3682         i915_reg_t reg;
3683         u32 temp;
3684
3685         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
3686         reg = FDI_RX_CTL(pipe);
3687         temp = I915_READ(reg);
3688         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
3689         temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3690         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
3691         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
3692
3693         POSTING_READ(reg);
3694         udelay(200);
3695
3696         /* Switch from Rawclk to PCDclk */
3697         temp = I915_READ(reg);
3698         I915_WRITE(reg, temp | FDI_PCDCLK);
3699
3700         POSTING_READ(reg);
3701         udelay(200);
3702
3703         /* Enable CPU FDI TX PLL, always on for Ironlake */
3704         reg = FDI_TX_CTL(pipe);
3705         temp = I915_READ(reg);
3706         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
3707                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
3708
3709                 POSTING_READ(reg);
3710                 udelay(100);
3711         }
3712 }
3713
3714 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
3715 {
3716         struct drm_device *dev = intel_crtc->base.dev;
3717         struct drm_i915_private *dev_priv = dev->dev_private;
3718         int pipe = intel_crtc->pipe;
3719         i915_reg_t reg;
3720         u32 temp;
3721
3722         /* Switch from PCDclk to Rawclk */
3723         reg = FDI_RX_CTL(pipe);
3724         temp = I915_READ(reg);
3725         I915_WRITE(reg, temp & ~FDI_PCDCLK);
3726
3727         /* Disable CPU FDI TX PLL */
3728         reg = FDI_TX_CTL(pipe);
3729         temp = I915_READ(reg);
3730         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
3731
3732         POSTING_READ(reg);
3733         udelay(100);
3734
3735         reg = FDI_RX_CTL(pipe);
3736         temp = I915_READ(reg);
3737         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
3738
3739         /* Wait for the clocks to turn off. */
3740         POSTING_READ(reg);
3741         udelay(100);
3742 }
3743
3744 static void ironlake_fdi_disable(struct drm_crtc *crtc)
3745 {
3746         struct drm_device *dev = crtc->dev;
3747         struct drm_i915_private *dev_priv = dev->dev_private;
3748         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3749         int pipe = intel_crtc->pipe;
3750         i915_reg_t reg;
3751         u32 temp;
3752
3753         /* disable CPU FDI tx and PCH FDI rx */
3754         reg = FDI_TX_CTL(pipe);
3755         temp = I915_READ(reg);
3756         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
3757         POSTING_READ(reg);
3758
3759         reg = FDI_RX_CTL(pipe);
3760         temp = I915_READ(reg);
3761         temp &= ~(0x7 << 16);
3762         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
3763         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
3764
3765         POSTING_READ(reg);
3766         udelay(100);
3767
3768         /* Ironlake workaround, disable clock pointer after downing FDI */
3769         if (HAS_PCH_IBX(dev))
3770                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
3771
3772         /* still set train pattern 1 */
3773         reg = FDI_TX_CTL(pipe);
3774         temp = I915_READ(reg);
3775         temp &= ~FDI_LINK_TRAIN_NONE;
3776         temp |= FDI_LINK_TRAIN_PATTERN_1;
3777         I915_WRITE(reg, temp);
3778
3779         reg = FDI_RX_CTL(pipe);
3780         temp = I915_READ(reg);
3781         if (HAS_PCH_CPT(dev)) {
3782                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3783                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
3784         } else {
3785                 temp &= ~FDI_LINK_TRAIN_NONE;
3786                 temp |= FDI_LINK_TRAIN_PATTERN_1;
3787         }
3788         /* BPC in FDI rx is consistent with that in PIPECONF */
3789         temp &= ~(0x07 << 16);
3790         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
3791         I915_WRITE(reg, temp);
3792
3793         POSTING_READ(reg);
3794         udelay(100);
3795 }
3796
3797 bool intel_has_pending_fb_unpin(struct drm_device *dev)
3798 {
3799         struct intel_crtc *crtc;
3800
3801         /* Note that we don't need to be called with mode_config.lock here
3802          * as our list of CRTC objects is static for the lifetime of the
3803          * device and so cannot disappear as we iterate. Similarly, we can
3804          * happily treat the predicates as racy, atomic checks as userspace
3805          * cannot claim and pin a new fb without at least acquring the
3806          * struct_mutex and so serialising with us.
3807          */
3808         for_each_intel_crtc(dev, crtc) {
3809                 if (atomic_read(&crtc->unpin_work_count) == 0)
3810                         continue;
3811
3812                 if (crtc->flip_work)
3813                         intel_wait_for_vblank(dev, crtc->pipe);
3814
3815                 return true;
3816         }
3817
3818         return false;
3819 }
3820
3821 static void page_flip_completed(struct intel_crtc *intel_crtc)
3822 {
3823         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3824         struct intel_flip_work *work = intel_crtc->flip_work;
3825
3826         intel_crtc->flip_work = NULL;
3827
3828         if (work->event)
3829                 drm_crtc_send_vblank_event(&intel_crtc->base, work->event);
3830
3831         drm_crtc_vblank_put(&intel_crtc->base);
3832
3833         wake_up_all(&dev_priv->pending_flip_queue);
3834         queue_work(dev_priv->wq, &work->unpin_work);
3835
3836         trace_i915_flip_complete(intel_crtc->plane,
3837                                  work->pending_flip_obj);
3838 }
3839
3840 static int intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
3841 {
3842         struct drm_device *dev = crtc->dev;
3843         struct drm_i915_private *dev_priv = dev->dev_private;
3844         long ret;
3845
3846         WARN_ON(waitqueue_active(&dev_priv->pending_flip_queue));
3847
3848         ret = wait_event_interruptible_timeout(
3849                                         dev_priv->pending_flip_queue,
3850                                         !intel_crtc_has_pending_flip(crtc),
3851                                         60*HZ);
3852
3853         if (ret < 0)
3854                 return ret;
3855
3856         if (ret == 0) {
3857                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3858                 struct intel_flip_work *work;
3859
3860                 spin_lock_irq(&dev->event_lock);
3861                 work = intel_crtc->flip_work;
3862                 if (work && !is_mmio_work(work)) {
3863                         WARN_ONCE(1, "Removing stuck page flip\n");
3864                         page_flip_completed(intel_crtc);
3865                 }
3866                 spin_unlock_irq(&dev->event_lock);
3867         }
3868
3869         return 0;
3870 }
3871
3872 static void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
3873 {
3874         u32 temp;
3875
3876         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
3877
3878         mutex_lock(&dev_priv->sb_lock);
3879
3880         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
3881         temp |= SBI_SSCCTL_DISABLE;
3882         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
3883
3884         mutex_unlock(&dev_priv->sb_lock);
3885 }
3886
3887 /* Program iCLKIP clock to the desired frequency */
3888 static void lpt_program_iclkip(struct drm_crtc *crtc)
3889 {
3890         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
3891         int clock = to_intel_crtc(crtc)->config->base.adjusted_mode.crtc_clock;
3892         u32 divsel, phaseinc, auxdiv, phasedir = 0;
3893         u32 temp;
3894
3895         lpt_disable_iclkip(dev_priv);
3896
3897         /* The iCLK virtual clock root frequency is in MHz,
3898          * but the adjusted_mode->crtc_clock in in KHz. To get the
3899          * divisors, it is necessary to divide one by another, so we
3900          * convert the virtual clock precision to KHz here for higher
3901          * precision.
3902          */
3903         for (auxdiv = 0; auxdiv < 2; auxdiv++) {
3904                 u32 iclk_virtual_root_freq = 172800 * 1000;
3905                 u32 iclk_pi_range = 64;
3906                 u32 desired_divisor;
3907
3908                 desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
3909                                                     clock << auxdiv);
3910                 divsel = (desired_divisor / iclk_pi_range) - 2;
3911                 phaseinc = desired_divisor % iclk_pi_range;
3912
3913                 /*
3914                  * Near 20MHz is a corner case which is
3915                  * out of range for the 7-bit divisor
3916                  */
3917                 if (divsel <= 0x7f)
3918                         break;
3919         }
3920
3921         /* This should not happen with any sane values */
3922         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
3923                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
3924         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
3925                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
3926
3927         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
3928                         clock,
3929                         auxdiv,
3930                         divsel,
3931                         phasedir,
3932                         phaseinc);
3933
3934         mutex_lock(&dev_priv->sb_lock);
3935
3936         /* Program SSCDIVINTPHASE6 */
3937         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
3938         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
3939         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
3940         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
3941         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
3942         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
3943         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
3944         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
3945
3946         /* Program SSCAUXDIV */
3947         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
3948         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
3949         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
3950         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
3951
3952         /* Enable modulator and associated divider */
3953         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
3954         temp &= ~SBI_SSCCTL_DISABLE;
3955         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
3956
3957         mutex_unlock(&dev_priv->sb_lock);
3958
3959         /* Wait for initialization time */
3960         udelay(24);
3961
3962         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
3963 }
3964
3965 int lpt_get_iclkip(struct drm_i915_private *dev_priv)
3966 {
3967         u32 divsel, phaseinc, auxdiv;
3968         u32 iclk_virtual_root_freq = 172800 * 1000;
3969         u32 iclk_pi_range = 64;
3970         u32 desired_divisor;
3971         u32 temp;
3972
3973         if ((I915_READ(PIXCLK_GATE) & PIXCLK_GATE_UNGATE) == 0)
3974                 return 0;
3975
3976         mutex_lock(&dev_priv->sb_lock);
3977
3978         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
3979         if (temp & SBI_SSCCTL_DISABLE) {
3980                 mutex_unlock(&dev_priv->sb_lock);
3981                 return 0;
3982         }
3983
3984         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
3985         divsel = (temp & SBI_SSCDIVINTPHASE_DIVSEL_MASK) >>
3986                 SBI_SSCDIVINTPHASE_DIVSEL_SHIFT;
3987         phaseinc = (temp & SBI_SSCDIVINTPHASE_INCVAL_MASK) >>
3988                 SBI_SSCDIVINTPHASE_INCVAL_SHIFT;
3989
3990         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
3991         auxdiv = (temp & SBI_SSCAUXDIV_FINALDIV2SEL_MASK) >>
3992                 SBI_SSCAUXDIV_FINALDIV2SEL_SHIFT;
3993
3994         mutex_unlock(&dev_priv->sb_lock);
3995
3996         desired_divisor = (divsel + 2) * iclk_pi_range + phaseinc;
3997
3998         return DIV_ROUND_CLOSEST(iclk_virtual_root_freq,
3999                                  desired_divisor << auxdiv);
4000 }
4001
4002 static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc,
4003                                                 enum pipe pch_transcoder)
4004 {
4005         struct drm_device *dev = crtc->base.dev;
4006         struct drm_i915_private *dev_priv = dev->dev_private;
4007         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
4008
4009         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
4010                    I915_READ(HTOTAL(cpu_transcoder)));
4011         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
4012                    I915_READ(HBLANK(cpu_transcoder)));
4013         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
4014                    I915_READ(HSYNC(cpu_transcoder)));
4015
4016         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
4017                    I915_READ(VTOTAL(cpu_transcoder)));
4018         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
4019                    I915_READ(VBLANK(cpu_transcoder)));
4020         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
4021                    I915_READ(VSYNC(cpu_transcoder)));
4022         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
4023                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
4024 }
4025
4026 static void cpt_set_fdi_bc_bifurcation(struct drm_device *dev, bool enable)
4027 {
4028         struct drm_i915_private *dev_priv = dev->dev_private;
4029         uint32_t temp;
4030
4031         temp = I915_READ(SOUTH_CHICKEN1);
4032         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
4033                 return;
4034
4035         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
4036         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
4037
4038         temp &= ~FDI_BC_BIFURCATION_SELECT;
4039         if (enable)
4040                 temp |= FDI_BC_BIFURCATION_SELECT;
4041
4042         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
4043         I915_WRITE(SOUTH_CHICKEN1, temp);
4044         POSTING_READ(SOUTH_CHICKEN1);
4045 }
4046
4047 static void ivybridge_update_fdi_bc_bifurcation(struct intel_crtc *intel_crtc)
4048 {
4049         struct drm_device *dev = intel_crtc->base.dev;
4050
4051         switch (intel_crtc->pipe) {
4052         case PIPE_A:
4053                 break;
4054         case PIPE_B:
4055                 if (intel_crtc->config->fdi_lanes > 2)
4056                         cpt_set_fdi_bc_bifurcation(dev, false);
4057                 else
4058                         cpt_set_fdi_bc_bifurcation(dev, true);
4059
4060                 break;
4061         case PIPE_C:
4062                 cpt_set_fdi_bc_bifurcation(dev, true);
4063
4064                 break;
4065         default:
4066                 BUG();
4067         }
4068 }
4069
4070 /* Return which DP Port should be selected for Transcoder DP control */
4071 static enum port
4072 intel_trans_dp_port_sel(struct drm_crtc *crtc)
4073 {
4074         struct drm_device *dev = crtc->dev;
4075         struct intel_encoder *encoder;
4076
4077         for_each_encoder_on_crtc(dev, crtc, encoder) {
4078                 if (encoder->type == INTEL_OUTPUT_DISPLAYPORT ||
4079                     encoder->type == INTEL_OUTPUT_EDP)
4080                         return enc_to_dig_port(&encoder->base)->port;
4081         }
4082
4083         return -1;
4084 }
4085
4086 /*
4087  * Enable PCH resources required for PCH ports:
4088  *   - PCH PLLs
4089  *   - FDI training & RX/TX
4090  *   - update transcoder timings
4091  *   - DP transcoding bits
4092  *   - transcoder
4093  */
4094 static void ironlake_pch_enable(struct drm_crtc *crtc)
4095 {
4096         struct drm_device *dev = crtc->dev;
4097         struct drm_i915_private *dev_priv = dev->dev_private;
4098         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4099         int pipe = intel_crtc->pipe;
4100         u32 temp;
4101
4102         assert_pch_transcoder_disabled(dev_priv, pipe);
4103
4104         if (IS_IVYBRIDGE(dev))
4105                 ivybridge_update_fdi_bc_bifurcation(intel_crtc);
4106
4107         /* Write the TU size bits before fdi link training, so that error
4108          * detection works. */
4109         I915_WRITE(FDI_RX_TUSIZE1(pipe),
4110                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
4111
4112         /* For PCH output, training FDI link */
4113         dev_priv->display.fdi_link_train(crtc);
4114
4115         /* We need to program the right clock selection before writing the pixel
4116          * mutliplier into the DPLL. */
4117         if (HAS_PCH_CPT(dev)) {
4118                 u32 sel;
4119
4120                 temp = I915_READ(PCH_DPLL_SEL);
4121                 temp |= TRANS_DPLL_ENABLE(pipe);
4122                 sel = TRANS_DPLLB_SEL(pipe);
4123                 if (intel_crtc->config->shared_dpll ==
4124                     intel_get_shared_dpll_by_id(dev_priv, DPLL_ID_PCH_PLL_B))
4125                         temp |= sel;
4126                 else
4127                         temp &= ~sel;
4128                 I915_WRITE(PCH_DPLL_SEL, temp);
4129         }
4130
4131         /* XXX: pch pll's can be enabled any time before we enable the PCH
4132          * transcoder, and we actually should do this to not upset any PCH
4133          * transcoder that already use the clock when we share it.
4134          *
4135          * Note that enable_shared_dpll tries to do the right thing, but
4136          * get_shared_dpll unconditionally resets the pll - we need that to have
4137          * the right LVDS enable sequence. */
4138         intel_enable_shared_dpll(intel_crtc);
4139
4140         /* set transcoder timing, panel must allow it */
4141         assert_panel_unlocked(dev_priv, pipe);
4142         ironlake_pch_transcoder_set_timings(intel_crtc, pipe);
4143
4144         intel_fdi_normal_train(crtc);
4145
4146         /* For PCH DP, enable TRANS_DP_CTL */
4147         if (HAS_PCH_CPT(dev) && intel_crtc->config->has_dp_encoder) {
4148                 const struct drm_display_mode *adjusted_mode =
4149                         &intel_crtc->config->base.adjusted_mode;
4150                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
4151                 i915_reg_t reg = TRANS_DP_CTL(pipe);
4152                 temp = I915_READ(reg);
4153                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
4154                           TRANS_DP_SYNC_MASK |
4155                           TRANS_DP_BPC_MASK);
4156                 temp |= TRANS_DP_OUTPUT_ENABLE;
4157                 temp |= bpc << 9; /* same format but at 11:9 */
4158
4159                 if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC)
4160                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
4161                 if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC)
4162                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
4163
4164                 switch (intel_trans_dp_port_sel(crtc)) {
4165                 case PORT_B:
4166                         temp |= TRANS_DP_PORT_SEL_B;
4167                         break;
4168                 case PORT_C:
4169                         temp |= TRANS_DP_PORT_SEL_C;
4170                         break;
4171                 case PORT_D:
4172                         temp |= TRANS_DP_PORT_SEL_D;
4173                         break;
4174                 default:
4175                         BUG();
4176                 }
4177
4178                 I915_WRITE(reg, temp);
4179         }
4180
4181         ironlake_enable_pch_transcoder(dev_priv, pipe);
4182 }
4183
4184 static void lpt_pch_enable(struct drm_crtc *crtc)
4185 {
4186         struct drm_device *dev = crtc->dev;
4187         struct drm_i915_private *dev_priv = dev->dev_private;
4188         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4189         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
4190
4191         assert_pch_transcoder_disabled(dev_priv, TRANSCODER_A);
4192
4193         lpt_program_iclkip(crtc);
4194
4195         /* Set transcoder timing. */
4196         ironlake_pch_transcoder_set_timings(intel_crtc, PIPE_A);
4197
4198         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
4199 }
4200
4201 static void cpt_verify_modeset(struct drm_device *dev, int pipe)
4202 {
4203         struct drm_i915_private *dev_priv = dev->dev_private;
4204         i915_reg_t dslreg = PIPEDSL(pipe);
4205         u32 temp;
4206
4207         temp = I915_READ(dslreg);
4208         udelay(500);
4209         if (wait_for(I915_READ(dslreg) != temp, 5)) {
4210                 if (wait_for(I915_READ(dslreg) != temp, 5))
4211                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
4212         }
4213 }
4214
4215 static int
4216 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
4217                   unsigned scaler_user, int *scaler_id, unsigned int rotation,
4218                   int src_w, int src_h, int dst_w, int dst_h)
4219 {
4220         struct intel_crtc_scaler_state *scaler_state =
4221                 &crtc_state->scaler_state;
4222         struct intel_crtc *intel_crtc =
4223                 to_intel_crtc(crtc_state->base.crtc);
4224         int need_scaling;
4225
4226         need_scaling = intel_rotation_90_or_270(rotation) ?
4227                 (src_h != dst_w || src_w != dst_h):
4228                 (src_w != dst_w || src_h != dst_h);
4229
4230         /*
4231          * if plane is being disabled or scaler is no more required or force detach
4232          *  - free scaler binded to this plane/crtc
4233          *  - in order to do this, update crtc->scaler_usage
4234          *
4235          * Here scaler state in crtc_state is set free so that
4236          * scaler can be assigned to other user. Actual register
4237          * update to free the scaler is done in plane/panel-fit programming.
4238          * For this purpose crtc/plane_state->scaler_id isn't reset here.
4239          */
4240         if (force_detach || !need_scaling) {
4241                 if (*scaler_id >= 0) {
4242                         scaler_state->scaler_users &= ~(1 << scaler_user);
4243                         scaler_state->scalers[*scaler_id].in_use = 0;
4244
4245                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4246                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
4247                                 intel_crtc->pipe, scaler_user, *scaler_id,
4248                                 scaler_state->scaler_users);
4249                         *scaler_id = -1;
4250                 }
4251                 return 0;
4252         }
4253
4254         /* range checks */
4255         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
4256                 dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
4257
4258                 src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
4259                 dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H) {
4260                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
4261                         "size is out of scaler range\n",
4262                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
4263                 return -EINVAL;
4264         }
4265
4266         /* mark this plane as a scaler user in crtc_state */
4267         scaler_state->scaler_users |= (1 << scaler_user);
4268         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4269                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
4270                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
4271                 scaler_state->scaler_users);
4272
4273         return 0;
4274 }
4275
4276 /**
4277  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
4278  *
4279  * @state: crtc's scaler state
4280  *
4281  * Return
4282  *     0 - scaler_usage updated successfully
4283  *    error - requested scaling cannot be supported or other error condition
4284  */
4285 int skl_update_scaler_crtc(struct intel_crtc_state *state)
4286 {
4287         struct intel_crtc *intel_crtc = to_intel_crtc(state->base.crtc);
4288         const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
4289
4290         DRM_DEBUG_KMS("Updating scaler for [CRTC:%d:%s] scaler_user index %u.%u\n",
4291                       intel_crtc->base.base.id, intel_crtc->base.name,
4292                       intel_crtc->pipe, SKL_CRTC_INDEX);
4293
4294         return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
4295                 &state->scaler_state.scaler_id, BIT(DRM_ROTATE_0),
4296                 state->pipe_src_w, state->pipe_src_h,
4297                 adjusted_mode->crtc_hdisplay, adjusted_mode->crtc_vdisplay);
4298 }
4299
4300 /**
4301  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
4302  *
4303  * @state: crtc's scaler state
4304  * @plane_state: atomic plane state to update
4305  *
4306  * Return
4307  *     0 - scaler_usage updated successfully
4308  *    error - requested scaling cannot be supported or other error condition
4309  */
4310 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
4311                                    struct intel_plane_state *plane_state)
4312 {
4313
4314         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
4315         struct intel_plane *intel_plane =
4316                 to_intel_plane(plane_state->base.plane);
4317         struct drm_framebuffer *fb = plane_state->base.fb;
4318         int ret;
4319
4320         bool force_detach = !fb || !plane_state->visible;
4321
4322         DRM_DEBUG_KMS("Updating scaler for [PLANE:%d:%s] scaler_user index %u.%u\n",
4323                       intel_plane->base.base.id, intel_plane->base.name,
4324                       intel_crtc->pipe, drm_plane_index(&intel_plane->base));
4325
4326         ret = skl_update_scaler(crtc_state, force_detach,
4327                                 drm_plane_index(&intel_plane->base),
4328                                 &plane_state->scaler_id,
4329                                 plane_state->base.rotation,
4330                                 drm_rect_width(&plane_state->src) >> 16,
4331                                 drm_rect_height(&plane_state->src) >> 16,
4332                                 drm_rect_width(&plane_state->dst),
4333                                 drm_rect_height(&plane_state->dst));
4334
4335         if (ret || plane_state->scaler_id < 0)
4336                 return ret;
4337
4338         /* check colorkey */
4339         if (plane_state->ckey.flags != I915_SET_COLORKEY_NONE) {
4340                 DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed",
4341                               intel_plane->base.base.id,
4342                               intel_plane->base.name);
4343                 return -EINVAL;
4344         }
4345
4346         /* Check src format */
4347         switch (fb->pixel_format) {
4348         case DRM_FORMAT_RGB565:
4349         case DRM_FORMAT_XBGR8888:
4350         case DRM_FORMAT_XRGB8888:
4351         case DRM_FORMAT_ABGR8888:
4352         case DRM_FORMAT_ARGB8888:
4353         case DRM_FORMAT_XRGB2101010:
4354         case DRM_FORMAT_XBGR2101010:
4355         case DRM_FORMAT_YUYV:
4356         case DRM_FORMAT_YVYU:
4357         case DRM_FORMAT_UYVY:
4358         case DRM_FORMAT_VYUY:
4359                 break;
4360         default:
4361                 DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
4362                               intel_plane->base.base.id, intel_plane->base.name,
4363                               fb->base.id, fb->pixel_format);
4364                 return -EINVAL;
4365         }
4366
4367         return 0;
4368 }
4369
4370 static void skylake_scaler_disable(struct intel_crtc *crtc)
4371 {
4372         int i;
4373
4374         for (i = 0; i < crtc->num_scalers; i++)
4375                 skl_detach_scaler(crtc, i);
4376 }
4377
4378 static void skylake_pfit_enable(struct intel_crtc *crtc)
4379 {
4380         struct drm_device *dev = crtc->base.dev;
4381         struct drm_i915_private *dev_priv = dev->dev_private;
4382         int pipe = crtc->pipe;
4383         struct intel_crtc_scaler_state *scaler_state =
4384                 &crtc->config->scaler_state;
4385
4386         DRM_DEBUG_KMS("for crtc_state = %p\n", crtc->config);
4387
4388         if (crtc->config->pch_pfit.enabled) {
4389                 int id;
4390
4391                 if (WARN_ON(crtc->config->scaler_state.scaler_id < 0)) {
4392                         DRM_ERROR("Requesting pfit without getting a scaler first\n");
4393                         return;
4394                 }
4395
4396                 id = scaler_state->scaler_id;
4397                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
4398                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
4399                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc->config->pch_pfit.pos);
4400                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc->config->pch_pfit.size);
4401
4402                 DRM_DEBUG_KMS("for crtc_state = %p scaler_id = %d\n", crtc->config, id);
4403         }
4404 }
4405
4406 static void ironlake_pfit_enable(struct intel_crtc *crtc)
4407 {
4408         struct drm_device *dev = crtc->base.dev;
4409         struct drm_i915_private *dev_priv = dev->dev_private;
4410         int pipe = crtc->pipe;
4411
4412         if (crtc->config->pch_pfit.enabled) {
4413                 /* Force use of hard-coded filter coefficients
4414                  * as some pre-programmed values are broken,
4415                  * e.g. x201.
4416                  */
4417                 if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
4418                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
4419                                                  PF_PIPE_SEL_IVB(pipe));
4420                 else
4421                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
4422                 I915_WRITE(PF_WIN_POS(pipe), crtc->config->pch_pfit.pos);
4423                 I915_WRITE(PF_WIN_SZ(pipe), crtc->config->pch_pfit.size);
4424         }
4425 }
4426
4427 void hsw_enable_ips(struct intel_crtc *crtc)
4428 {
4429         struct drm_device *dev = crtc->base.dev;
4430         struct drm_i915_private *dev_priv = dev->dev_private;
4431
4432         if (!crtc->config->ips_enabled)
4433                 return;
4434
4435         /*
4436          * We can only enable IPS after we enable a plane and wait for a vblank
4437          * This function is called from post_plane_update, which is run after
4438          * a vblank wait.
4439          */
4440
4441         assert_plane_enabled(dev_priv, crtc->plane);
4442         if (IS_BROADWELL(dev)) {
4443                 mutex_lock(&dev_priv->rps.hw_lock);
4444                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0xc0000000));
4445                 mutex_unlock(&dev_priv->rps.hw_lock);
4446                 /* Quoting Art Runyan: "its not safe to expect any particular
4447                  * value in IPS_CTL bit 31 after enabling IPS through the
4448                  * mailbox." Moreover, the mailbox may return a bogus state,
4449                  * so we need to just enable it and continue on.
4450                  */
4451         } else {
4452                 I915_WRITE(IPS_CTL, IPS_ENABLE);
4453                 /* The bit only becomes 1 in the next vblank, so this wait here
4454                  * is essentially intel_wait_for_vblank. If we don't have this
4455                  * and don't wait for vblanks until the end of crtc_enable, then
4456                  * the HW state readout code will complain that the expected
4457                  * IPS_CTL value is not the one we read. */
4458                 if (wait_for(I915_READ_NOTRACE(IPS_CTL) & IPS_ENABLE, 50))
4459                         DRM_ERROR("Timed out waiting for IPS enable\n");
4460         }
4461 }
4462
4463 void hsw_disable_ips(struct intel_crtc *crtc)
4464 {
4465         struct drm_device *dev = crtc->base.dev;
4466         struct drm_i915_private *dev_priv = dev->dev_private;
4467
4468         if (!crtc->config->ips_enabled)
4469                 return;
4470
4471         assert_plane_enabled(dev_priv, crtc->plane);
4472         if (IS_BROADWELL(dev)) {
4473                 mutex_lock(&dev_priv->rps.hw_lock);
4474                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
4475                 mutex_unlock(&dev_priv->rps.hw_lock);
4476                 /* wait for pcode to finish disabling IPS, which may take up to 42ms */
4477                 if (wait_for((I915_READ(IPS_CTL) & IPS_ENABLE) == 0, 42))
4478                         DRM_ERROR("Timed out waiting for IPS disable\n");
4479         } else {
4480                 I915_WRITE(IPS_CTL, 0);
4481                 POSTING_READ(IPS_CTL);
4482         }
4483
4484         /* We need to wait for a vblank before we can disable the plane. */
4485         intel_wait_for_vblank(dev, crtc->pipe);
4486 }
4487
4488 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
4489 {
4490         if (intel_crtc->overlay) {
4491                 struct drm_device *dev = intel_crtc->base.dev;
4492                 struct drm_i915_private *dev_priv = dev->dev_private;
4493
4494                 mutex_lock(&dev->struct_mutex);
4495                 dev_priv->mm.interruptible = false;
4496                 (void) intel_overlay_switch_off(intel_crtc->overlay);
4497                 dev_priv->mm.interruptible = true;
4498                 mutex_unlock(&dev->struct_mutex);
4499         }
4500
4501         /* Let userspace switch the overlay on again. In most cases userspace
4502          * has to recompute where to put it anyway.
4503          */
4504 }
4505
4506 /**
4507  * intel_post_enable_primary - Perform operations after enabling primary plane
4508  * @crtc: the CRTC whose primary plane was just enabled
4509  *
4510  * Performs potentially sleeping operations that must be done after the primary
4511  * plane is enabled, such as updating FBC and IPS.  Note that this may be
4512  * called due to an explicit primary plane update, or due to an implicit
4513  * re-enable that is caused when a sprite plane is updated to no longer
4514  * completely hide the primary plane.
4515  */
4516 static void
4517 intel_post_enable_primary(struct drm_crtc *crtc)
4518 {
4519         struct drm_device *dev = crtc->dev;
4520         struct drm_i915_private *dev_priv = dev->dev_private;
4521         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4522         int pipe = intel_crtc->pipe;
4523
4524         /*
4525          * FIXME IPS should be fine as long as one plane is
4526          * enabled, but in practice it seems to have problems
4527          * when going from primary only to sprite only and vice
4528          * versa.
4529          */
4530         hsw_enable_ips(intel_crtc);
4531
4532         /*
4533          * Gen2 reports pipe underruns whenever all planes are disabled.
4534          * So don't enable underrun reporting before at least some planes
4535          * are enabled.
4536          * FIXME: Need to fix the logic to work when we turn off all planes
4537          * but leave the pipe running.
4538          */
4539         if (IS_GEN2(dev))
4540                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
4541
4542         /* Underruns don't always raise interrupts, so check manually. */
4543         intel_check_cpu_fifo_underruns(dev_priv);
4544         intel_check_pch_fifo_underruns(dev_priv);
4545 }
4546
4547 /* FIXME move all this to pre_plane_update() with proper state tracking */
4548 static void
4549 intel_pre_disable_primary(struct drm_crtc *crtc)
4550 {
4551         struct drm_device *dev = crtc->dev;
4552         struct drm_i915_private *dev_priv = dev->dev_private;
4553         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4554         int pipe = intel_crtc->pipe;
4555
4556         /*
4557          * Gen2 reports pipe underruns whenever all planes are disabled.
4558          * So diasble underrun reporting before all the planes get disabled.
4559          * FIXME: Need to fix the logic to work when we turn off all planes
4560          * but leave the pipe running.
4561          */
4562         if (IS_GEN2(dev))
4563                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
4564
4565         /*
4566          * FIXME IPS should be fine as long as one plane is
4567          * enabled, but in practice it seems to have problems
4568          * when going from primary only to sprite only and vice
4569          * versa.
4570          */
4571         hsw_disable_ips(intel_crtc);
4572 }
4573
4574 /* FIXME get rid of this and use pre_plane_update */
4575 static void
4576 intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
4577 {
4578         struct drm_device *dev = crtc->dev;
4579         struct drm_i915_private *dev_priv = dev->dev_private;
4580         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4581         int pipe = intel_crtc->pipe;
4582
4583         intel_pre_disable_primary(crtc);
4584
4585         /*
4586          * Vblank time updates from the shadow to live plane control register
4587          * are blocked if the memory self-refresh mode is active at that
4588          * moment. So to make sure the plane gets truly disabled, disable
4589          * first the self-refresh mode. The self-refresh enable bit in turn
4590          * will be checked/applied by the HW only at the next frame start
4591          * event which is after the vblank start event, so we need to have a
4592          * wait-for-vblank between disabling the plane and the pipe.
4593          */
4594         if (HAS_GMCH_DISPLAY(dev)) {
4595                 intel_set_memory_cxsr(dev_priv, false);
4596                 dev_priv->wm.vlv.cxsr = false;
4597                 intel_wait_for_vblank(dev, pipe);
4598         }
4599 }
4600
4601 static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
4602 {
4603         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
4604         struct drm_atomic_state *old_state = old_crtc_state->base.state;
4605         struct intel_crtc_state *pipe_config =
4606                 to_intel_crtc_state(crtc->base.state);
4607         struct drm_device *dev = crtc->base.dev;
4608         struct drm_plane *primary = crtc->base.primary;
4609         struct drm_plane_state *old_pri_state =
4610                 drm_atomic_get_existing_plane_state(old_state, primary);
4611
4612         intel_frontbuffer_flip(dev, pipe_config->fb_bits);
4613
4614         crtc->wm.cxsr_allowed = true;
4615
4616         if (pipe_config->update_wm_post && pipe_config->base.active)
4617                 intel_update_watermarks(&crtc->base);
4618
4619         if (old_pri_state) {
4620                 struct intel_plane_state *primary_state =
4621                         to_intel_plane_state(primary->state);
4622                 struct intel_plane_state *old_primary_state =
4623                         to_intel_plane_state(old_pri_state);
4624
4625                 intel_fbc_post_update(crtc);
4626
4627                 if (primary_state->visible &&
4628                     (needs_modeset(&pipe_config->base) ||
4629                      !old_primary_state->visible))
4630                         intel_post_enable_primary(&crtc->base);
4631         }
4632 }
4633
4634 static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state)
4635 {
4636         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
4637         struct drm_device *dev = crtc->base.dev;
4638         struct drm_i915_private *dev_priv = dev->dev_private;
4639         struct intel_crtc_state *pipe_config =
4640                 to_intel_crtc_state(crtc->base.state);
4641         struct drm_atomic_state *old_state = old_crtc_state->base.state;
4642         struct drm_plane *primary = crtc->base.primary;
4643         struct drm_plane_state *old_pri_state =
4644                 drm_atomic_get_existing_plane_state(old_state, primary);
4645         bool modeset = needs_modeset(&pipe_config->base);
4646
4647         if (old_pri_state) {
4648                 struct intel_plane_state *primary_state =
4649                         to_intel_plane_state(primary->state);
4650                 struct intel_plane_state *old_primary_state =
4651                         to_intel_plane_state(old_pri_state);
4652
4653                 intel_fbc_pre_update(crtc, pipe_config, primary_state);
4654
4655                 if (old_primary_state->visible &&
4656                     (modeset || !primary_state->visible))
4657                         intel_pre_disable_primary(&crtc->base);
4658         }
4659
4660         if (pipe_config->disable_cxsr && HAS_GMCH_DISPLAY(dev)) {
4661                 crtc->wm.cxsr_allowed = false;
4662
4663                 /*
4664                  * Vblank time updates from the shadow to live plane control register
4665                  * are blocked if the memory self-refresh mode is active at that
4666                  * moment. So to make sure the plane gets truly disabled, disable
4667                  * first the self-refresh mode. The self-refresh enable bit in turn
4668                  * will be checked/applied by the HW only at the next frame start
4669                  * event which is after the vblank start event, so we need to have a
4670                  * wait-for-vblank between disabling the plane and the pipe.
4671                  */
4672                 if (old_crtc_state->base.active) {
4673                         intel_set_memory_cxsr(dev_priv, false);
4674                         dev_priv->wm.vlv.cxsr = false;
4675                         intel_wait_for_vblank(dev, crtc->pipe);
4676                 }
4677         }
4678
4679         /*
4680          * IVB workaround: must disable low power watermarks for at least
4681          * one frame before enabling scaling.  LP watermarks can be re-enabled
4682          * when scaling is disabled.
4683          *
4684          * WaCxSRDisabledForSpriteScaling:ivb
4685          */
4686         if (pipe_config->disable_lp_wm) {
4687                 ilk_disable_lp_wm(dev);
4688                 intel_wait_for_vblank(dev, crtc->pipe);
4689         }
4690
4691         /*
4692          * If we're doing a modeset, we're done.  No need to do any pre-vblank
4693          * watermark programming here.
4694          */
4695         if (needs_modeset(&pipe_config->base))
4696                 return;
4697
4698         /*
4699          * For platforms that support atomic watermarks, program the
4700          * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
4701          * will be the intermediate values that are safe for both pre- and
4702          * post- vblank; when vblank happens, the 'active' values will be set
4703          * to the final 'target' values and we'll do this again to get the
4704          * optimal watermarks.  For gen9+ platforms, the values we program here
4705          * will be the final target values which will get automatically latched
4706          * at vblank time; no further programming will be necessary.
4707          *
4708          * If a platform hasn't been transitioned to atomic watermarks yet,
4709          * we'll continue to update watermarks the old way, if flags tell
4710          * us to.
4711          */
4712         if (dev_priv->display.initial_watermarks != NULL)
4713                 dev_priv->display.initial_watermarks(pipe_config);
4714         else if (pipe_config->update_wm_pre)
4715                 intel_update_watermarks(&crtc->base);
4716 }
4717
4718 static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask)
4719 {
4720         struct drm_device *dev = crtc->dev;
4721         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4722         struct drm_plane *p;
4723         int pipe = intel_crtc->pipe;
4724
4725         intel_crtc_dpms_overlay_disable(intel_crtc);
4726
4727         drm_for_each_plane_mask(p, dev, plane_mask)
4728                 to_intel_plane(p)->disable_plane(p, crtc);
4729
4730         /*
4731          * FIXME: Once we grow proper nuclear flip support out of this we need
4732          * to compute the mask of flip planes precisely. For the time being
4733          * consider this a flip to a NULL plane.
4734          */
4735         intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_ALL_MASK(pipe));
4736 }
4737
4738 static void ironlake_crtc_enable(struct drm_crtc *crtc)
4739 {
4740         struct drm_device *dev = crtc->dev;
4741         struct drm_i915_private *dev_priv = dev->dev_private;
4742         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4743         struct intel_encoder *encoder;
4744         int pipe = intel_crtc->pipe;
4745         struct intel_crtc_state *pipe_config =
4746                 to_intel_crtc_state(crtc->state);
4747
4748         if (WARN_ON(intel_crtc->active))
4749                 return;
4750
4751         /*
4752          * Sometimes spurious CPU pipe underruns happen during FDI
4753          * training, at least with VGA+HDMI cloning. Suppress them.
4754          *
4755          * On ILK we get an occasional spurious CPU pipe underruns
4756          * between eDP port A enable and vdd enable. Also PCH port
4757          * enable seems to result in the occasional CPU pipe underrun.
4758          *
4759          * Spurious PCH underruns also occur during PCH enabling.
4760          */
4761         if (intel_crtc->config->has_pch_encoder || IS_GEN5(dev_priv))
4762                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
4763         if (intel_crtc->config->has_pch_encoder)
4764                 intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
4765
4766         if (intel_crtc->config->has_pch_encoder)
4767                 intel_prepare_shared_dpll(intel_crtc);
4768
4769         if (intel_crtc->config->has_dp_encoder)
4770                 intel_dp_set_m_n(intel_crtc, M1_N1);
4771
4772         intel_set_pipe_timings(intel_crtc);
4773         intel_set_pipe_src_size(intel_crtc);
4774
4775         if (intel_crtc->config->has_pch_encoder) {
4776                 intel_cpu_transcoder_set_m_n(intel_crtc,
4777                                      &intel_crtc->config->fdi_m_n, NULL);
4778         }
4779
4780         ironlake_set_pipeconf(crtc);
4781
4782         intel_crtc->active = true;
4783
4784         for_each_encoder_on_crtc(dev, crtc, encoder)
4785                 if (encoder->pre_enable)
4786                         encoder->pre_enable(encoder);
4787
4788         if (intel_crtc->config->has_pch_encoder) {
4789                 /* Note: FDI PLL enabling _must_ be done before we enable the
4790                  * cpu pipes, hence this is separate from all the other fdi/pch
4791                  * enabling. */
4792                 ironlake_fdi_pll_enable(intel_crtc);
4793         } else {
4794                 assert_fdi_tx_disabled(dev_priv, pipe);
4795                 assert_fdi_rx_disabled(dev_priv, pipe);
4796         }
4797
4798         ironlake_pfit_enable(intel_crtc);
4799
4800         /*
4801          * On ILK+ LUT must be loaded before the pipe is running but with
4802          * clocks enabled
4803          */
4804         intel_color_load_luts(&pipe_config->base);
4805
4806         if (dev_priv->display.initial_watermarks != NULL)
4807                 dev_priv->display.initial_watermarks(intel_crtc->config);
4808         intel_enable_pipe(intel_crtc);
4809
4810         if (intel_crtc->config->has_pch_encoder)
4811                 ironlake_pch_enable(crtc);
4812
4813         assert_vblank_disabled(crtc);
4814         drm_crtc_vblank_on(crtc);
4815
4816         for_each_encoder_on_crtc(dev, crtc, encoder)
4817                 encoder->enable(encoder);
4818
4819         if (HAS_PCH_CPT(dev))
4820                 cpt_verify_modeset(dev, intel_crtc->pipe);
4821
4822         /* Must wait for vblank to avoid spurious PCH FIFO underruns */
4823         if (intel_crtc->config->has_pch_encoder)
4824                 intel_wait_for_vblank(dev, pipe);
4825         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
4826         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
4827 }
4828
4829 /* IPS only exists on ULT machines and is tied to pipe A. */
4830 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
4831 {
4832         return HAS_IPS(crtc->base.dev) && crtc->pipe == PIPE_A;
4833 }
4834
4835 static void haswell_crtc_enable(struct drm_crtc *crtc)
4836 {
4837         struct drm_device *dev = crtc->dev;
4838         struct drm_i915_private *dev_priv = dev->dev_private;
4839         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4840         struct intel_encoder *encoder;
4841         int pipe = intel_crtc->pipe, hsw_workaround_pipe;
4842         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
4843         struct intel_crtc_state *pipe_config =
4844                 to_intel_crtc_state(crtc->state);
4845
4846         if (WARN_ON(intel_crtc->active))
4847                 return;
4848
4849         if (intel_crtc->config->has_pch_encoder)
4850                 intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
4851                                                       false);
4852
4853         for_each_encoder_on_crtc(dev, crtc, encoder)
4854                 if (encoder->pre_pll_enable)
4855                         encoder->pre_pll_enable(encoder);
4856
4857         if (intel_crtc->config->shared_dpll)
4858                 intel_enable_shared_dpll(intel_crtc);
4859
4860         if (intel_crtc->config->has_dp_encoder)
4861                 intel_dp_set_m_n(intel_crtc, M1_N1);
4862
4863         if (!intel_crtc->config->has_dsi_encoder)
4864                 intel_set_pipe_timings(intel_crtc);
4865
4866         intel_set_pipe_src_size(intel_crtc);
4867
4868         if (cpu_transcoder != TRANSCODER_EDP &&
4869             !transcoder_is_dsi(cpu_transcoder)) {
4870                 I915_WRITE(PIPE_MULT(cpu_transcoder),
4871                            intel_crtc->config->pixel_multiplier - 1);
4872         }
4873
4874         if (intel_crtc->config->has_pch_encoder) {
4875                 intel_cpu_transcoder_set_m_n(intel_crtc,
4876                                      &intel_crtc->config->fdi_m_n, NULL);
4877         }
4878
4879         if (!intel_crtc->config->has_dsi_encoder)
4880                 haswell_set_pipeconf(crtc);
4881
4882         haswell_set_pipemisc(crtc);
4883
4884         intel_color_set_csc(&pipe_config->base);
4885
4886         intel_crtc->active = true;
4887
4888         if (intel_crtc->config->has_pch_encoder)
4889                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
4890         else
4891                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
4892
4893         for_each_encoder_on_crtc(dev, crtc, encoder) {
4894                 if (encoder->pre_enable)
4895                         encoder->pre_enable(encoder);
4896         }
4897
4898         if (intel_crtc->config->has_pch_encoder)
4899                 dev_priv->display.fdi_link_train(crtc);
4900
4901         if (!intel_crtc->config->has_dsi_encoder)
4902                 intel_ddi_enable_pipe_clock(intel_crtc);
4903
4904         if (INTEL_INFO(dev)->gen >= 9)
4905                 skylake_pfit_enable(intel_crtc);
4906         else
4907                 ironlake_pfit_enable(intel_crtc);
4908
4909         /*
4910          * On ILK+ LUT must be loaded before the pipe is running but with
4911          * clocks enabled
4912          */
4913         intel_color_load_luts(&pipe_config->base);
4914
4915         intel_ddi_set_pipe_settings(crtc);
4916         if (!intel_crtc->config->has_dsi_encoder)
4917                 intel_ddi_enable_transcoder_func(crtc);
4918
4919         if (dev_priv->display.initial_watermarks != NULL)
4920                 dev_priv->display.initial_watermarks(pipe_config);
4921         else
4922                 intel_update_watermarks(crtc);
4923
4924         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
4925         if (!intel_crtc->config->has_dsi_encoder)
4926                 intel_enable_pipe(intel_crtc);
4927
4928         if (intel_crtc->config->has_pch_encoder)
4929                 lpt_pch_enable(crtc);
4930
4931         if (intel_crtc->config->dp_encoder_is_mst)
4932                 intel_ddi_set_vc_payload_alloc(crtc, true);
4933
4934         assert_vblank_disabled(crtc);
4935         drm_crtc_vblank_on(crtc);
4936
4937         for_each_encoder_on_crtc(dev, crtc, encoder) {
4938                 encoder->enable(encoder);
4939                 intel_opregion_notify_encoder(encoder, true);
4940         }
4941
4942         if (intel_crtc->config->has_pch_encoder) {
4943                 intel_wait_for_vblank(dev, pipe);
4944                 intel_wait_for_vblank(dev, pipe);
4945                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
4946                 intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
4947                                                       true);
4948         }
4949
4950         /* If we change the relative order between pipe/planes enabling, we need
4951          * to change the workaround. */
4952         hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
4953         if (IS_HASWELL(dev) && hsw_workaround_pipe != INVALID_PIPE) {
4954                 intel_wait_for_vblank(dev, hsw_workaround_pipe);
4955                 intel_wait_for_vblank(dev, hsw_workaround_pipe);
4956         }
4957 }
4958
4959 static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force)
4960 {
4961         struct drm_device *dev = crtc->base.dev;
4962         struct drm_i915_private *dev_priv = dev->dev_private;
4963         int pipe = crtc->pipe;
4964
4965         /* To avoid upsetting the power well on haswell only disable the pfit if
4966          * it's in use. The hw state code will make sure we get this right. */
4967         if (force || crtc->config->pch_pfit.enabled) {
4968                 I915_WRITE(PF_CTL(pipe), 0);
4969                 I915_WRITE(PF_WIN_POS(pipe), 0);
4970                 I915_WRITE(PF_WIN_SZ(pipe), 0);
4971         }
4972 }
4973
4974 static void ironlake_crtc_disable(struct drm_crtc *crtc)
4975 {
4976         struct drm_device *dev = crtc->dev;
4977         struct drm_i915_private *dev_priv = dev->dev_private;
4978         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4979         struct intel_encoder *encoder;
4980         int pipe = intel_crtc->pipe;
4981
4982         /*
4983          * Sometimes spurious CPU pipe underruns happen when the
4984          * pipe is already disabled, but FDI RX/TX is still enabled.
4985          * Happens at least with VGA+HDMI cloning. Suppress them.
4986          */
4987         if (intel_crtc->config->has_pch_encoder) {
4988                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
4989                 intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
4990         }
4991
4992         for_each_encoder_on_crtc(dev, crtc, encoder)
4993                 encoder->disable(encoder);
4994
4995         drm_crtc_vblank_off(crtc);
4996         assert_vblank_disabled(crtc);
4997
4998         intel_disable_pipe(intel_crtc);
4999
5000         ironlake_pfit_disable(intel_crtc, false);
5001
5002         if (intel_crtc->config->has_pch_encoder)
5003                 ironlake_fdi_disable(crtc);
5004
5005         for_each_encoder_on_crtc(dev, crtc, encoder)
5006                 if (encoder->post_disable)
5007                         encoder->post_disable(encoder);
5008
5009         if (intel_crtc->config->has_pch_encoder) {
5010                 ironlake_disable_pch_transcoder(dev_priv, pipe);
5011
5012                 if (HAS_PCH_CPT(dev)) {
5013                         i915_reg_t reg;
5014                         u32 temp;
5015
5016                         /* disable TRANS_DP_CTL */
5017                         reg = TRANS_DP_CTL(pipe);
5018                         temp = I915_READ(reg);
5019                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
5020                                   TRANS_DP_PORT_SEL_MASK);
5021                         temp |= TRANS_DP_PORT_SEL_NONE;
5022                         I915_WRITE(reg, temp);
5023
5024                         /* disable DPLL_SEL */
5025                         temp = I915_READ(PCH_DPLL_SEL);
5026                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
5027                         I915_WRITE(PCH_DPLL_SEL, temp);
5028                 }
5029
5030                 ironlake_fdi_pll_disable(intel_crtc);
5031         }
5032
5033         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
5034         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
5035 }
5036
5037 static void haswell_crtc_disable(struct drm_crtc *crtc)
5038 {
5039         struct drm_device *dev = crtc->dev;
5040         struct drm_i915_private *dev_priv = dev->dev_private;
5041         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5042         struct intel_encoder *encoder;
5043         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
5044
5045         if (intel_crtc->config->has_pch_encoder)
5046                 intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
5047                                                       false);
5048
5049         for_each_encoder_on_crtc(dev, crtc, encoder) {
5050                 intel_opregion_notify_encoder(encoder, false);
5051                 encoder->disable(encoder);
5052         }
5053
5054         drm_crtc_vblank_off(crtc);
5055         assert_vblank_disabled(crtc);
5056
5057         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
5058         if (!intel_crtc->config->has_dsi_encoder)
5059                 intel_disable_pipe(intel_crtc);
5060
5061         if (intel_crtc->config->dp_encoder_is_mst)
5062                 intel_ddi_set_vc_payload_alloc(crtc, false);
5063
5064         if (!intel_crtc->config->has_dsi_encoder)
5065                 intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder);
5066
5067         if (INTEL_INFO(dev)->gen >= 9)
5068                 skylake_scaler_disable(intel_crtc);
5069         else
5070                 ironlake_pfit_disable(intel_crtc, false);
5071
5072         if (!intel_crtc->config->has_dsi_encoder)
5073                 intel_ddi_disable_pipe_clock(intel_crtc);
5074
5075         for_each_encoder_on_crtc(dev, crtc, encoder)
5076                 if (encoder->post_disable)
5077                         encoder->post_disable(encoder);
5078
5079         if (intel_crtc->config->has_pch_encoder) {
5080                 lpt_disable_pch_transcoder(dev_priv);
5081                 lpt_disable_iclkip(dev_priv);
5082                 intel_ddi_fdi_disable(crtc);
5083
5084                 intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
5085                                                       true);
5086         }
5087 }
5088
5089 static void i9xx_pfit_enable(struct intel_crtc *crtc)
5090 {
5091         struct drm_device *dev = crtc->base.dev;
5092         struct drm_i915_private *dev_priv = dev->dev_private;
5093         struct intel_crtc_state *pipe_config = crtc->config;
5094
5095         if (!pipe_config->gmch_pfit.control)
5096                 return;
5097
5098         /*
5099          * The panel fitter should only be adjusted whilst the pipe is disabled,
5100          * according to register description and PRM.
5101          */
5102         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
5103         assert_pipe_disabled(dev_priv, crtc->pipe);
5104
5105         I915_WRITE(PFIT_PGM_RATIOS, pipe_config->gmch_pfit.pgm_ratios);
5106         I915_WRITE(PFIT_CONTROL, pipe_config->gmch_pfit.control);
5107
5108         /* Border color in case we don't scale up to the full screen. Black by
5109          * default, change to something else for debugging. */
5110         I915_WRITE(BCLRPAT(crtc->pipe), 0);
5111 }
5112
5113 static enum intel_display_power_domain port_to_power_domain(enum port port)
5114 {
5115         switch (port) {
5116         case PORT_A:
5117                 return POWER_DOMAIN_PORT_DDI_A_LANES;
5118         case PORT_B:
5119                 return POWER_DOMAIN_PORT_DDI_B_LANES;
5120         case PORT_C:
5121                 return POWER_DOMAIN_PORT_DDI_C_LANES;
5122         case PORT_D:
5123                 return POWER_DOMAIN_PORT_DDI_D_LANES;
5124         case PORT_E:
5125                 return POWER_DOMAIN_PORT_DDI_E_LANES;
5126         default:
5127                 MISSING_CASE(port);
5128                 return POWER_DOMAIN_PORT_OTHER;
5129         }
5130 }
5131
5132 static enum intel_display_power_domain port_to_aux_power_domain(enum port port)
5133 {
5134         switch (port) {
5135         case PORT_A:
5136                 return POWER_DOMAIN_AUX_A;
5137         case PORT_B:
5138                 return POWER_DOMAIN_AUX_B;
5139         case PORT_C:
5140                 return POWER_DOMAIN_AUX_C;
5141         case PORT_D:
5142                 return POWER_DOMAIN_AUX_D;
5143         case PORT_E:
5144                 /* FIXME: Check VBT for actual wiring of PORT E */
5145                 return POWER_DOMAIN_AUX_D;
5146         default:
5147                 MISSING_CASE(port);
5148                 return POWER_DOMAIN_AUX_A;
5149         }
5150 }
5151
5152 enum intel_display_power_domain
5153 intel_display_port_power_domain(struct intel_encoder *intel_encoder)
5154 {
5155         struct drm_device *dev = intel_encoder->base.dev;
5156         struct intel_digital_port *intel_dig_port;
5157
5158         switch (intel_encoder->type) {
5159         case INTEL_OUTPUT_UNKNOWN:
5160                 /* Only DDI platforms should ever use this output type */
5161                 WARN_ON_ONCE(!HAS_DDI(dev));
5162         case INTEL_OUTPUT_DISPLAYPORT:
5163         case INTEL_OUTPUT_HDMI:
5164         case INTEL_OUTPUT_EDP:
5165                 intel_dig_port = enc_to_dig_port(&intel_encoder->base);
5166                 return port_to_power_domain(intel_dig_port->port);
5167         case INTEL_OUTPUT_DP_MST:
5168                 intel_dig_port = enc_to_mst(&intel_encoder->base)->primary;
5169                 return port_to_power_domain(intel_dig_port->port);
5170         case INTEL_OUTPUT_ANALOG:
5171                 return POWER_DOMAIN_PORT_CRT;
5172         case INTEL_OUTPUT_DSI:
5173                 return POWER_DOMAIN_PORT_DSI;
5174         default:
5175                 return POWER_DOMAIN_PORT_OTHER;
5176         }
5177 }
5178
5179 enum intel_display_power_domain
5180 intel_display_port_aux_power_domain(struct intel_encoder *intel_encoder)
5181 {
5182         struct drm_device *dev = intel_encoder->base.dev;
5183         struct intel_digital_port *intel_dig_port;
5184
5185         switch (intel_encoder->type) {
5186         case INTEL_OUTPUT_UNKNOWN:
5187         case INTEL_OUTPUT_HDMI:
5188                 /*
5189                  * Only DDI platforms should ever use these output types.
5190                  * We can get here after the HDMI detect code has already set
5191                  * the type of the shared encoder. Since we can't be sure
5192                  * what's the status of the given connectors, play safe and
5193                  * run the DP detection too.
5194                  */
5195                 WARN_ON_ONCE(!HAS_DDI(dev));
5196         case INTEL_OUTPUT_DISPLAYPORT:
5197         case INTEL_OUTPUT_EDP:
5198                 intel_dig_port = enc_to_dig_port(&intel_encoder->base);
5199                 return port_to_aux_power_domain(intel_dig_port->port);
5200         case INTEL_OUTPUT_DP_MST:
5201                 intel_dig_port = enc_to_mst(&intel_encoder->base)->primary;
5202                 return port_to_aux_power_domain(intel_dig_port->port);
5203         default:
5204                 MISSING_CASE(intel_encoder->type);
5205                 return POWER_DOMAIN_AUX_A;
5206         }
5207 }
5208
5209 static unsigned long get_crtc_power_domains(struct drm_crtc *crtc,
5210                                             struct intel_crtc_state *crtc_state)
5211 {
5212         struct drm_device *dev = crtc->dev;
5213         struct drm_encoder *encoder;
5214         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5215         enum pipe pipe = intel_crtc->pipe;
5216         unsigned long mask;
5217         enum transcoder transcoder = crtc_state->cpu_transcoder;
5218
5219         if (!crtc_state->base.active)
5220                 return 0;
5221
5222         mask = BIT(POWER_DOMAIN_PIPE(pipe));
5223         mask |= BIT(POWER_DOMAIN_TRANSCODER(transcoder));
5224         if (crtc_state->pch_pfit.enabled ||
5225             crtc_state->pch_pfit.force_thru)
5226                 mask |= BIT(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
5227
5228         drm_for_each_encoder_mask(encoder, dev, crtc_state->base.encoder_mask) {
5229                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
5230
5231                 mask |= BIT(intel_display_port_power_domain(intel_encoder));
5232         }
5233
5234         if (crtc_state->shared_dpll)
5235                 mask |= BIT(POWER_DOMAIN_PLLS);
5236
5237         return mask;
5238 }
5239
5240 static unsigned long
5241 modeset_get_crtc_power_domains(struct drm_crtc *crtc,
5242                                struct intel_crtc_state *crtc_state)
5243 {
5244         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
5245         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5246         enum intel_display_power_domain domain;
5247         unsigned long domains, new_domains, old_domains;
5248
5249         old_domains = intel_crtc->enabled_power_domains;
5250         intel_crtc->enabled_power_domains = new_domains =
5251                 get_crtc_power_domains(crtc, crtc_state);
5252
5253         domains = new_domains & ~old_domains;
5254
5255         for_each_power_domain(domain, domains)
5256                 intel_display_power_get(dev_priv, domain);
5257
5258         return old_domains & ~new_domains;
5259 }
5260
5261 static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
5262                                       unsigned long domains)
5263 {
5264         enum intel_display_power_domain domain;
5265
5266         for_each_power_domain(domain, domains)
5267                 intel_display_power_put(dev_priv, domain);
5268 }
5269
5270 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
5271 {
5272         int max_cdclk_freq = dev_priv->max_cdclk_freq;
5273
5274         if (INTEL_INFO(dev_priv)->gen >= 9 ||
5275             IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
5276                 return max_cdclk_freq;
5277         else if (IS_CHERRYVIEW(dev_priv))
5278                 return max_cdclk_freq*95/100;
5279         else if (INTEL_INFO(dev_priv)->gen < 4)
5280                 return 2*max_cdclk_freq*90/100;
5281         else
5282                 return max_cdclk_freq*90/100;
5283 }
5284
5285 static int skl_calc_cdclk(int max_pixclk, int vco);
5286
5287 static void intel_update_max_cdclk(struct drm_device *dev)
5288 {
5289         struct drm_i915_private *dev_priv = dev->dev_private;
5290
5291         if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
5292                 u32 limit = I915_READ(SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
5293                 int max_cdclk, vco;
5294
5295                 vco = dev_priv->skl_preferred_vco_freq;
5296                 WARN_ON(vco != 8100000 && vco != 8640000);
5297
5298                 /*
5299                  * Use the lower (vco 8640) cdclk values as a
5300                  * first guess. skl_calc_cdclk() will correct it
5301                  * if the preferred vco is 8100 instead.
5302                  */
5303                 if (limit == SKL_DFSM_CDCLK_LIMIT_675)
5304                         max_cdclk = 617143;
5305                 else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
5306                         max_cdclk = 540000;
5307                 else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
5308                         max_cdclk = 432000;
5309                 else
5310                         max_cdclk = 308571;
5311
5312                 dev_priv->max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
5313         } else if (IS_BROXTON(dev)) {
5314                 dev_priv->max_cdclk_freq = 624000;
5315         } else if (IS_BROADWELL(dev))  {
5316                 /*
5317                  * FIXME with extra cooling we can allow
5318                  * 540 MHz for ULX and 675 Mhz for ULT.
5319                  * How can we know if extra cooling is
5320                  * available? PCI ID, VTB, something else?
5321                  */
5322                 if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
5323                         dev_priv->max_cdclk_freq = 450000;
5324                 else if (IS_BDW_ULX(dev))
5325                         dev_priv->max_cdclk_freq = 450000;
5326                 else if (IS_BDW_ULT(dev))
5327                         dev_priv->max_cdclk_freq = 540000;
5328                 else
5329                         dev_priv->max_cdclk_freq = 675000;
5330         } else if (IS_CHERRYVIEW(dev)) {
5331                 dev_priv->max_cdclk_freq = 320000;
5332         } else if (IS_VALLEYVIEW(dev)) {
5333                 dev_priv->max_cdclk_freq = 400000;
5334         } else {
5335                 /* otherwise assume cdclk is fixed */
5336                 dev_priv->max_cdclk_freq = dev_priv->cdclk_freq;
5337         }
5338
5339         dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
5340
5341         DRM_DEBUG_DRIVER("Max CD clock rate: %d kHz\n",
5342                          dev_priv->max_cdclk_freq);
5343
5344         DRM_DEBUG_DRIVER("Max dotclock rate: %d kHz\n",
5345                          dev_priv->max_dotclk_freq);
5346 }
5347
5348 static void intel_update_cdclk(struct drm_device *dev)
5349 {
5350         struct drm_i915_private *dev_priv = dev->dev_private;
5351
5352         dev_priv->cdclk_freq = dev_priv->display.get_display_clock_speed(dev);
5353
5354         if (INTEL_GEN(dev_priv) >= 9)
5355                 DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz, VCO: %d kHz, ref: %d kHz\n",
5356                                  dev_priv->cdclk_freq, dev_priv->cdclk_pll.vco,
5357                                  dev_priv->cdclk_pll.ref);
5358         else
5359                 DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz\n",
5360                                  dev_priv->cdclk_freq);
5361
5362         /*
5363          * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
5364          * Programmng [sic] note: bit[9:2] should be programmed to the number
5365          * of cdclk that generates 4MHz reference clock freq which is used to
5366          * generate GMBus clock. This will vary with the cdclk freq.
5367          */
5368         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
5369                 I915_WRITE(GMBUSFREQ_VLV, DIV_ROUND_UP(dev_priv->cdclk_freq, 1000));
5370 }
5371
5372 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
5373 static int skl_cdclk_decimal(int cdclk)
5374 {
5375         return DIV_ROUND_CLOSEST(cdclk - 1000, 500);
5376 }
5377
5378 static int bxt_de_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
5379 {
5380         int ratio;
5381
5382         if (cdclk == dev_priv->cdclk_pll.ref)
5383                 return 0;
5384
5385         switch (cdclk) {
5386         default:
5387                 MISSING_CASE(cdclk);
5388         case 144000:
5389         case 288000:
5390         case 384000:
5391         case 576000:
5392                 ratio = 60;
5393                 break;
5394         case 624000:
5395                 ratio = 65;
5396                 break;
5397         }
5398
5399         return dev_priv->cdclk_pll.ref * ratio;
5400 }
5401
5402 static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
5403 {
5404         I915_WRITE(BXT_DE_PLL_ENABLE, 0);
5405
5406         /* Timeout 200us */
5407         if (wait_for((I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK) == 0, 1))
5408                 DRM_ERROR("timeout waiting for DE PLL unlock\n");
5409
5410         dev_priv->cdclk_pll.vco = 0;
5411 }
5412
5413 static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
5414 {
5415         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk_pll.ref);
5416         u32 val;
5417
5418         val = I915_READ(BXT_DE_PLL_CTL);
5419         val &= ~BXT_DE_PLL_RATIO_MASK;
5420         val |= BXT_DE_PLL_RATIO(ratio);
5421         I915_WRITE(BXT_DE_PLL_CTL, val);
5422
5423         I915_WRITE(BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
5424
5425         /* Timeout 200us */
5426         if (wait_for((I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK) != 0, 1))
5427                 DRM_ERROR("timeout waiting for DE PLL lock\n");
5428
5429         dev_priv->cdclk_pll.vco = vco;
5430 }
5431
5432 static void bxt_set_cdclk(struct drm_i915_private *dev_priv, int cdclk)
5433 {
5434         u32 val, divider;
5435         int vco, ret;
5436
5437         vco = bxt_de_pll_vco(dev_priv, cdclk);
5438
5439         DRM_DEBUG_DRIVER("Changing CDCLK to %d kHz (VCO %d kHz)\n", cdclk, vco);
5440
5441         /* cdclk = vco / 2 / div{1,1.5,2,4} */
5442         switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
5443         case 8:
5444                 divider = BXT_CDCLK_CD2X_DIV_SEL_4;
5445                 break;
5446         case 4:
5447                 divider = BXT_CDCLK_CD2X_DIV_SEL_2;
5448                 break;
5449         case 3:
5450                 divider = BXT_CDCLK_CD2X_DIV_SEL_1_5;
5451                 break;
5452         case 2:
5453                 divider = BXT_CDCLK_CD2X_DIV_SEL_1;
5454                 break;
5455         default:
5456                 WARN_ON(cdclk != dev_priv->cdclk_pll.ref);
5457                 WARN_ON(vco != 0);
5458
5459                 divider = BXT_CDCLK_CD2X_DIV_SEL_1;
5460                 break;
5461         }
5462
5463         /* Inform power controller of upcoming frequency change */
5464         mutex_lock(&dev_priv->rps.hw_lock);
5465         ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
5466                                       0x80000000);
5467         mutex_unlock(&dev_priv->rps.hw_lock);
5468
5469         if (ret) {
5470                 DRM_ERROR("PCode CDCLK freq change notify failed (err %d, freq %d)\n",
5471                           ret, cdclk);
5472                 return;
5473         }
5474
5475         if (dev_priv->cdclk_pll.vco != 0 &&
5476             dev_priv->cdclk_pll.vco != vco)
5477                 bxt_de_pll_disable(dev_priv);
5478
5479         if (dev_priv->cdclk_pll.vco != vco)
5480                 bxt_de_pll_enable(dev_priv, vco);
5481
5482         val = divider | skl_cdclk_decimal(cdclk);
5483         /*
5484          * FIXME if only the cd2x divider needs changing, it could be done
5485          * without shutting off the pipe (if only one pipe is active).
5486          */
5487         val |= BXT_CDCLK_CD2X_PIPE_NONE;
5488         /*
5489          * Disable SSA Precharge when CD clock frequency < 500 MHz,
5490          * enable otherwise.
5491          */
5492         if (cdclk >= 500000)
5493                 val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
5494         I915_WRITE(CDCLK_CTL, val);
5495
5496         mutex_lock(&dev_priv->rps.hw_lock);
5497         ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
5498                                       DIV_ROUND_UP(cdclk, 25000));
5499         mutex_unlock(&dev_priv->rps.hw_lock);
5500
5501         if (ret) {
5502                 DRM_ERROR("PCode CDCLK freq set failed, (err %d, freq %d)\n",
5503                           ret, cdclk);
5504                 return;
5505         }
5506
5507         intel_update_cdclk(dev_priv->dev);
5508 }
5509
5510 static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
5511 {
5512         u32 cdctl, expected;
5513
5514         intel_update_cdclk(dev_priv->dev);
5515
5516         if (dev_priv->cdclk_pll.vco == 0 ||
5517             dev_priv->cdclk_freq == dev_priv->cdclk_pll.ref)
5518                 goto sanitize;
5519
5520         /* DPLL okay; verify the cdclock
5521          *
5522          * Some BIOS versions leave an incorrect decimal frequency value and
5523          * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4,
5524          * so sanitize this register.
5525          */
5526         cdctl = I915_READ(CDCLK_CTL);
5527         /*
5528          * Let's ignore the pipe field, since BIOS could have configured the
5529          * dividers both synching to an active pipe, or asynchronously
5530          * (PIPE_NONE).
5531          */
5532         cdctl &= ~BXT_CDCLK_CD2X_PIPE_NONE;
5533
5534         expected = (cdctl & BXT_CDCLK_CD2X_DIV_SEL_MASK) |
5535                    skl_cdclk_decimal(dev_priv->cdclk_freq);
5536         /*
5537          * Disable SSA Precharge when CD clock frequency < 500 MHz,
5538          * enable otherwise.
5539          */
5540         if (dev_priv->cdclk_freq >= 500000)
5541                 expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
5542
5543         if (cdctl == expected)
5544                 /* All well; nothing to sanitize */
5545                 return;
5546
5547 sanitize:
5548         DRM_DEBUG_KMS("Sanitizing cdclk programmed by pre-os\n");
5549
5550         /* force cdclk programming */
5551         dev_priv->cdclk_freq = 0;
5552
5553         /* force full PLL disable + enable */
5554         dev_priv->cdclk_pll.vco = -1;
5555 }
5556
5557 void bxt_init_cdclk(struct drm_i915_private *dev_priv)
5558 {
5559         bxt_sanitize_cdclk(dev_priv);
5560
5561         if (dev_priv->cdclk_freq != 0 && dev_priv->cdclk_pll.vco != 0)
5562                 return;
5563
5564         /*
5565          * FIXME:
5566          * - The initial CDCLK needs to be read from VBT.
5567          *   Need to make this change after VBT has changes for BXT.
5568          */
5569         bxt_set_cdclk(dev_priv, bxt_calc_cdclk(0));
5570 }
5571
5572 void bxt_uninit_cdclk(struct drm_i915_private *dev_priv)
5573 {
5574         bxt_set_cdclk(dev_priv, dev_priv->cdclk_pll.ref);
5575 }
5576
5577 static int skl_calc_cdclk(int max_pixclk, int vco)
5578 {
5579         if (vco == 8640000) {
5580                 if (max_pixclk > 540000)
5581                         return 617143;
5582                 else if (max_pixclk > 432000)
5583                         return 540000;
5584                 else if (max_pixclk > 308571)
5585                         return 432000;
5586                 else
5587                         return 308571;
5588         } else {
5589                 if (max_pixclk > 540000)
5590                         return 675000;
5591                 else if (max_pixclk > 450000)
5592                         return 540000;
5593                 else if (max_pixclk > 337500)
5594                         return 450000;
5595                 else
5596                         return 337500;
5597         }
5598 }
5599
5600 static void
5601 skl_dpll0_update(struct drm_i915_private *dev_priv)
5602 {
5603         u32 val;
5604
5605         dev_priv->cdclk_pll.ref = 24000;
5606         dev_priv->cdclk_pll.vco = 0;
5607
5608         val = I915_READ(LCPLL1_CTL);
5609         if ((val & LCPLL_PLL_ENABLE) == 0)
5610                 return;
5611
5612         if (WARN_ON((val & LCPLL_PLL_LOCK) == 0))
5613                 return;
5614
5615         val = I915_READ(DPLL_CTRL1);
5616
5617         if (WARN_ON((val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
5618                             DPLL_CTRL1_SSC(SKL_DPLL0) |
5619                             DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) !=
5620                     DPLL_CTRL1_OVERRIDE(SKL_DPLL0)))
5621                 return;
5622
5623         switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) {
5624         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0):
5625         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0):
5626         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0):
5627         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0):
5628                 dev_priv->cdclk_pll.vco = 8100000;
5629                 break;
5630         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0):
5631         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0):
5632                 dev_priv->cdclk_pll.vco = 8640000;
5633                 break;
5634         default:
5635                 MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
5636                 break;
5637         }
5638 }
5639
5640 void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv, int vco)
5641 {
5642         bool changed = dev_priv->skl_preferred_vco_freq != vco;
5643
5644         dev_priv->skl_preferred_vco_freq = vco;
5645
5646         if (changed)
5647                 intel_update_max_cdclk(dev_priv->dev);
5648 }
5649
5650 static void
5651 skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
5652 {
5653         int min_cdclk = skl_calc_cdclk(0, vco);
5654         u32 val;
5655
5656         WARN_ON(vco != 8100000 && vco != 8640000);
5657
5658         /* select the minimum CDCLK before enabling DPLL 0 */
5659         val = CDCLK_FREQ_337_308 | skl_cdclk_decimal(min_cdclk);
5660         I915_WRITE(CDCLK_CTL, val);
5661         POSTING_READ(CDCLK_CTL);
5662
5663         /*
5664          * We always enable DPLL0 with the lowest link rate possible, but still
5665          * taking into account the VCO required to operate the eDP panel at the
5666          * desired frequency. The usual DP link rates operate with a VCO of
5667          * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
5668          * The modeset code is responsible for the selection of the exact link
5669          * rate later on, with the constraint of choosing a frequency that
5670          * works with vco.
5671          */
5672         val = I915_READ(DPLL_CTRL1);
5673
5674         val &= ~(DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) | DPLL_CTRL1_SSC(SKL_DPLL0) |
5675                  DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
5676         val |= DPLL_CTRL1_OVERRIDE(SKL_DPLL0);
5677         if (vco == 8640000)
5678                 val |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080,
5679                                             SKL_DPLL0);
5680         else
5681                 val |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810,
5682                                             SKL_DPLL0);
5683
5684         I915_WRITE(DPLL_CTRL1, val);
5685         POSTING_READ(DPLL_CTRL1);
5686
5687         I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) | LCPLL_PLL_ENABLE);
5688
5689         if (wait_for(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK, 5))
5690                 DRM_ERROR("DPLL0 not locked\n");
5691
5692         dev_priv->cdclk_pll.vco = vco;
5693
5694         /* We'll want to keep using the current vco from now on. */
5695         skl_set_preferred_cdclk_vco(dev_priv, vco);
5696 }
5697
5698 static void
5699 skl_dpll0_disable(struct drm_i915_private *dev_priv)
5700 {
5701         I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) & ~LCPLL_PLL_ENABLE);
5702         if (wait_for(!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK), 1))
5703                 DRM_ERROR("Couldn't disable DPLL0\n");
5704
5705         dev_priv->cdclk_pll.vco = 0;
5706 }
5707
5708 static bool skl_cdclk_pcu_ready(struct drm_i915_private *dev_priv)
5709 {
5710         int ret;
5711         u32 val;
5712
5713         /* inform PCU we want to change CDCLK */
5714         val = SKL_CDCLK_PREPARE_FOR_CHANGE;
5715         mutex_lock(&dev_priv->rps.hw_lock);
5716         ret = sandybridge_pcode_read(dev_priv, SKL_PCODE_CDCLK_CONTROL, &val);
5717         mutex_unlock(&dev_priv->rps.hw_lock);
5718
5719         return ret == 0 && (val & SKL_CDCLK_READY_FOR_CHANGE);
5720 }
5721
5722 static bool skl_cdclk_wait_for_pcu_ready(struct drm_i915_private *dev_priv)
5723 {
5724         unsigned int i;
5725
5726         for (i = 0; i < 15; i++) {
5727                 if (skl_cdclk_pcu_ready(dev_priv))
5728                         return true;
5729                 udelay(10);
5730         }
5731
5732         return false;
5733 }
5734
5735 static void skl_set_cdclk(struct drm_i915_private *dev_priv, int cdclk, int vco)
5736 {
5737         struct drm_device *dev = dev_priv->dev;
5738         u32 freq_select, pcu_ack;
5739
5740         WARN_ON((cdclk == 24000) != (vco == 0));
5741
5742         DRM_DEBUG_DRIVER("Changing CDCLK to %d kHz (VCO %d kHz)\n", cdclk, vco);
5743
5744         if (!skl_cdclk_wait_for_pcu_ready(dev_priv)) {
5745                 DRM_ERROR("failed to inform PCU about cdclk change\n");
5746                 return;
5747         }
5748
5749         /* set CDCLK_CTL */
5750         switch (cdclk) {
5751         case 450000:
5752         case 432000:
5753                 freq_select = CDCLK_FREQ_450_432;
5754                 pcu_ack = 1;
5755                 break;
5756         case 540000:
5757                 freq_select = CDCLK_FREQ_540;
5758                 pcu_ack = 2;
5759                 break;
5760         case 308571:
5761         case 337500:
5762         default:
5763                 freq_select = CDCLK_FREQ_337_308;
5764                 pcu_ack = 0;
5765                 break;
5766         case 617143:
5767         case 675000:
5768                 freq_select = CDCLK_FREQ_675_617;
5769                 pcu_ack = 3;
5770                 break;
5771         }
5772
5773         if (dev_priv->cdclk_pll.vco != 0 &&
5774             dev_priv->cdclk_pll.vco != vco)
5775                 skl_dpll0_disable(dev_priv);
5776
5777         if (dev_priv->cdclk_pll.vco != vco)
5778                 skl_dpll0_enable(dev_priv, vco);
5779
5780         I915_WRITE(CDCLK_CTL, freq_select | skl_cdclk_decimal(cdclk));
5781         POSTING_READ(CDCLK_CTL);
5782
5783         /* inform PCU of the change */
5784         mutex_lock(&dev_priv->rps.hw_lock);
5785         sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, pcu_ack);
5786         mutex_unlock(&dev_priv->rps.hw_lock);
5787
5788         intel_update_cdclk(dev);
5789 }
5790
5791 static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv);
5792
5793 void skl_uninit_cdclk(struct drm_i915_private *dev_priv)
5794 {
5795         skl_set_cdclk(dev_priv, dev_priv->cdclk_pll.ref, 0);
5796 }
5797
5798 void skl_init_cdclk(struct drm_i915_private *dev_priv)
5799 {
5800         int cdclk, vco;
5801
5802         skl_sanitize_cdclk(dev_priv);
5803
5804         if (dev_priv->cdclk_freq != 0 && dev_priv->cdclk_pll.vco != 0) {
5805                 /*
5806                  * Use the current vco as our initial
5807                  * guess as to what the preferred vco is.
5808                  */
5809                 if (dev_priv->skl_preferred_vco_freq == 0)
5810                         skl_set_preferred_cdclk_vco(dev_priv,
5811                                                     dev_priv->cdclk_pll.vco);
5812                 return;
5813         }
5814
5815         vco = dev_priv->skl_preferred_vco_freq;
5816         if (vco == 0)
5817                 vco = 8100000;
5818         cdclk = skl_calc_cdclk(0, vco);
5819
5820         skl_set_cdclk(dev_priv, cdclk, vco);
5821 }
5822
5823 static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
5824 {
5825         uint32_t cdctl, expected;
5826
5827         /*
5828          * check if the pre-os intialized the display
5829          * There is SWF18 scratchpad register defined which is set by the
5830          * pre-os which can be used by the OS drivers to check the status
5831          */
5832         if ((I915_READ(SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
5833                 goto sanitize;
5834
5835         intel_update_cdclk(dev_priv->dev);
5836         /* Is PLL enabled and locked ? */
5837         if (dev_priv->cdclk_pll.vco == 0 ||
5838             dev_priv->cdclk_freq == dev_priv->cdclk_pll.ref)
5839                 goto sanitize;
5840
5841         /* DPLL okay; verify the cdclock
5842          *
5843          * Noticed in some instances that the freq selection is correct but
5844          * decimal part is programmed wrong from BIOS where pre-os does not
5845          * enable display. Verify the same as well.
5846          */
5847         cdctl = I915_READ(CDCLK_CTL);
5848         expected = (cdctl & CDCLK_FREQ_SEL_MASK) |
5849                 skl_cdclk_decimal(dev_priv->cdclk_freq);
5850         if (cdctl == expected)
5851                 /* All well; nothing to sanitize */
5852                 return;
5853
5854 sanitize:
5855         DRM_DEBUG_KMS("Sanitizing cdclk programmed by pre-os\n");
5856
5857         /* force cdclk programming */
5858         dev_priv->cdclk_freq = 0;
5859         /* force full PLL disable + enable */
5860         dev_priv->cdclk_pll.vco = -1;
5861 }
5862
5863 /* Adjust CDclk dividers to allow high res or save power if possible */
5864 static void valleyview_set_cdclk(struct drm_device *dev, int cdclk)
5865 {
5866         struct drm_i915_private *dev_priv = dev->dev_private;
5867         u32 val, cmd;
5868
5869         WARN_ON(dev_priv->display.get_display_clock_speed(dev)
5870                                         != dev_priv->cdclk_freq);
5871
5872         if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
5873                 cmd = 2;
5874         else if (cdclk == 266667)
5875                 cmd = 1;
5876         else
5877                 cmd = 0;
5878
5879         mutex_lock(&dev_priv->rps.hw_lock);
5880         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
5881         val &= ~DSPFREQGUAR_MASK;
5882         val |= (cmd << DSPFREQGUAR_SHIFT);
5883         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
5884         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) &
5885                       DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
5886                      50)) {
5887                 DRM_ERROR("timed out waiting for CDclk change\n");
5888         }
5889         mutex_unlock(&dev_priv->rps.hw_lock);
5890
5891         mutex_lock(&dev_priv->sb_lock);
5892
5893         if (cdclk == 400000) {
5894                 u32 divider;
5895
5896                 divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
5897
5898                 /* adjust cdclk divider */
5899                 val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
5900                 val &= ~CCK_FREQUENCY_VALUES;
5901                 val |= divider;
5902                 vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
5903
5904                 if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
5905                               CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
5906                              50))
5907                         DRM_ERROR("timed out waiting for CDclk change\n");
5908         }
5909
5910         /* adjust self-refresh exit latency value */
5911         val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
5912         val &= ~0x7f;
5913
5914         /*
5915          * For high bandwidth configs, we set a higher latency in the bunit
5916          * so that the core display fetch happens in time to avoid underruns.
5917          */
5918         if (cdclk == 400000)
5919                 val |= 4500 / 250; /* 4.5 usec */
5920         else
5921                 val |= 3000 / 250; /* 3.0 usec */
5922         vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
5923
5924         mutex_unlock(&dev_priv->sb_lock);
5925
5926         intel_update_cdclk(dev);
5927 }
5928
5929 static void cherryview_set_cdclk(struct drm_device *dev, int cdclk)
5930 {
5931         struct drm_i915_private *dev_priv = dev->dev_private;
5932         u32 val, cmd;
5933
5934         WARN_ON(dev_priv->display.get_display_clock_speed(dev)
5935                                                 != dev_priv->cdclk_freq);
5936
5937         switch (cdclk) {
5938         case 333333:
5939         case 320000:
5940         case 266667:
5941         case 200000:
5942                 break;
5943         default:
5944                 MISSING_CASE(cdclk);
5945                 return;
5946         }
5947
5948         /*
5949          * Specs are full of misinformation, but testing on actual
5950          * hardware has shown that we just need to write the desired
5951          * CCK divider into the Punit register.
5952          */
5953         cmd = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
5954
5955         mutex_lock(&dev_priv->rps.hw_lock);
5956         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
5957         val &= ~DSPFREQGUAR_MASK_CHV;
5958         val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
5959         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
5960         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) &
5961                       DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
5962                      50)) {
5963                 DRM_ERROR("timed out waiting for CDclk change\n");
5964         }
5965         mutex_unlock(&dev_priv->rps.hw_lock);
5966
5967         intel_update_cdclk(dev);
5968 }
5969
5970 static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv,
5971                                  int max_pixclk)
5972 {
5973         int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ? 333333 : 320000;
5974         int limit = IS_CHERRYVIEW(dev_priv) ? 95 : 90;
5975
5976         /*
5977          * Really only a few cases to deal with, as only 4 CDclks are supported:
5978          *   200MHz
5979          *   267MHz
5980          *   320/333MHz (depends on HPLL freq)
5981          *   400MHz (VLV only)
5982          * So we check to see whether we're above 90% (VLV) or 95% (CHV)
5983          * of the lower bin and adjust if needed.
5984          *
5985          * We seem to get an unstable or solid color picture at 200MHz.
5986          * Not sure what's wrong. For now use 200MHz only when all pipes
5987          * are off.
5988          */
5989         if (!IS_CHERRYVIEW(dev_priv) &&
5990             max_pixclk > freq_320*limit/100)
5991                 return 400000;
5992         else if (max_pixclk > 266667*limit/100)
5993                 return freq_320;
5994         else if (max_pixclk > 0)
5995                 return 266667;
5996         else
5997                 return 200000;
5998 }
5999
6000 static int bxt_calc_cdclk(int max_pixclk)
6001 {
6002         if (max_pixclk > 576000)
6003                 return 624000;
6004         else if (max_pixclk > 384000)
6005                 return 576000;
6006         else if (max_pixclk > 288000)
6007                 return 384000;
6008         else if (max_pixclk > 144000)
6009                 return 288000;
6010         else
6011                 return 144000;
6012 }
6013
6014 /* Compute the max pixel clock for new configuration. */
6015 static int intel_mode_max_pixclk(struct drm_device *dev,
6016                                  struct drm_atomic_state *state)
6017 {
6018         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
6019         struct drm_i915_private *dev_priv = dev->dev_private;
6020         struct drm_crtc *crtc;
6021         struct drm_crtc_state *crtc_state;
6022         unsigned max_pixclk = 0, i;
6023         enum pipe pipe;
6024
6025         memcpy(intel_state->min_pixclk, dev_priv->min_pixclk,
6026                sizeof(intel_state->min_pixclk));
6027
6028         for_each_crtc_in_state(state, crtc, crtc_state, i) {
6029                 int pixclk = 0;
6030
6031                 if (crtc_state->enable)
6032                         pixclk = crtc_state->adjusted_mode.crtc_clock;
6033
6034                 intel_state->min_pixclk[i] = pixclk;
6035         }
6036
6037         for_each_pipe(dev_priv, pipe)
6038                 max_pixclk = max(intel_state->min_pixclk[pipe], max_pixclk);
6039
6040         return max_pixclk;
6041 }
6042
6043 static int valleyview_modeset_calc_cdclk(struct drm_atomic_state *state)
6044 {
6045         struct drm_device *dev = state->dev;
6046         struct drm_i915_private *dev_priv = dev->dev_private;
6047         int max_pixclk = intel_mode_max_pixclk(dev, state);
6048         struct intel_atomic_state *intel_state =
6049                 to_intel_atomic_state(state);
6050
6051         intel_state->cdclk = intel_state->dev_cdclk =
6052                 valleyview_calc_cdclk(dev_priv, max_pixclk);
6053
6054         if (!intel_state->active_crtcs)
6055                 intel_state->dev_cdclk = valleyview_calc_cdclk(dev_priv, 0);
6056
6057         return 0;
6058 }
6059
6060 static int bxt_modeset_calc_cdclk(struct drm_atomic_state *state)
6061 {
6062         int max_pixclk = ilk_max_pixel_rate(state);
6063         struct intel_atomic_state *intel_state =
6064                 to_intel_atomic_state(state);
6065
6066         intel_state->cdclk = intel_state->dev_cdclk =
6067                 bxt_calc_cdclk(max_pixclk);
6068
6069         if (!intel_state->active_crtcs)
6070                 intel_state->dev_cdclk = bxt_calc_cdclk(0);
6071
6072         return 0;
6073 }
6074
6075 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
6076 {
6077         unsigned int credits, default_credits;
6078
6079         if (IS_CHERRYVIEW(dev_priv))
6080                 default_credits = PFI_CREDIT(12);
6081         else
6082                 default_credits = PFI_CREDIT(8);
6083
6084         if (dev_priv->cdclk_freq >= dev_priv->czclk_freq) {
6085                 /* CHV suggested value is 31 or 63 */
6086                 if (IS_CHERRYVIEW(dev_priv))
6087                         credits = PFI_CREDIT_63;
6088                 else
6089                         credits = PFI_CREDIT(15);
6090         } else {
6091                 credits = default_credits;
6092         }
6093
6094         /*
6095          * WA - write default credits before re-programming
6096          * FIXME: should we also set the resend bit here?
6097          */
6098         I915_WRITE(GCI_CONTROL, VGA_FAST_MODE_DISABLE |
6099                    default_credits);
6100
6101         I915_WRITE(GCI_CONTROL, VGA_FAST_MODE_DISABLE |
6102                    credits | PFI_CREDIT_RESEND);
6103
6104         /*
6105          * FIXME is this guaranteed to clear
6106          * immediately or should we poll for it?
6107          */
6108         WARN_ON(I915_READ(GCI_CONTROL) & PFI_CREDIT_RESEND);
6109 }
6110
6111 static void valleyview_modeset_commit_cdclk(struct drm_atomic_state *old_state)
6112 {
6113         struct drm_device *dev = old_state->dev;
6114         struct drm_i915_private *dev_priv = dev->dev_private;
6115         struct intel_atomic_state *old_intel_state =
6116                 to_intel_atomic_state(old_state);
6117         unsigned req_cdclk = old_intel_state->dev_cdclk;
6118
6119         /*
6120          * FIXME: We can end up here with all power domains off, yet
6121          * with a CDCLK frequency other than the minimum. To account
6122          * for this take the PIPE-A power domain, which covers the HW
6123          * blocks needed for the following programming. This can be
6124          * removed once it's guaranteed that we get here either with
6125          * the minimum CDCLK set, or the required power domains
6126          * enabled.
6127          */
6128         intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A);
6129
6130         if (IS_CHERRYVIEW(dev))
6131                 cherryview_set_cdclk(dev, req_cdclk);
6132         else
6133                 valleyview_set_cdclk(dev, req_cdclk);
6134
6135         vlv_program_pfi_credits(dev_priv);
6136
6137         intel_display_power_put(dev_priv, POWER_DOMAIN_PIPE_A);
6138 }
6139
6140 static void valleyview_crtc_enable(struct drm_crtc *crtc)
6141 {
6142         struct drm_device *dev = crtc->dev;
6143         struct drm_i915_private *dev_priv = to_i915(dev);
6144         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6145         struct intel_encoder *encoder;
6146         struct intel_crtc_state *pipe_config =
6147                 to_intel_crtc_state(crtc->state);
6148         int pipe = intel_crtc->pipe;
6149
6150         if (WARN_ON(intel_crtc->active))
6151                 return;
6152
6153         if (intel_crtc->config->has_dp_encoder)
6154                 intel_dp_set_m_n(intel_crtc, M1_N1);
6155
6156         intel_set_pipe_timings(intel_crtc);
6157         intel_set_pipe_src_size(intel_crtc);
6158
6159         if (IS_CHERRYVIEW(dev) && pipe == PIPE_B) {
6160                 struct drm_i915_private *dev_priv = dev->dev_private;
6161
6162                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
6163                 I915_WRITE(CHV_CANVAS(pipe), 0);
6164         }
6165
6166         i9xx_set_pipeconf(intel_crtc);
6167
6168         intel_crtc->active = true;
6169
6170         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6171
6172         for_each_encoder_on_crtc(dev, crtc, encoder)
6173                 if (encoder->pre_pll_enable)
6174                         encoder->pre_pll_enable(encoder);
6175
6176         if (IS_CHERRYVIEW(dev)) {
6177                 chv_prepare_pll(intel_crtc, intel_crtc->config);
6178                 chv_enable_pll(intel_crtc, intel_crtc->config);
6179         } else {
6180                 vlv_prepare_pll(intel_crtc, intel_crtc->config);
6181                 vlv_enable_pll(intel_crtc, intel_crtc->config);
6182         }
6183
6184         for_each_encoder_on_crtc(dev, crtc, encoder)
6185                 if (encoder->pre_enable)
6186                         encoder->pre_enable(encoder);
6187
6188         i9xx_pfit_enable(intel_crtc);
6189
6190         intel_color_load_luts(&pipe_config->base);
6191
6192         intel_update_watermarks(crtc);
6193         intel_enable_pipe(intel_crtc);
6194
6195         assert_vblank_disabled(crtc);
6196         drm_crtc_vblank_on(crtc);
6197
6198         for_each_encoder_on_crtc(dev, crtc, encoder)
6199                 encoder->enable(encoder);
6200 }
6201
6202 static void i9xx_set_pll_dividers(struct intel_crtc *crtc)
6203 {
6204         struct drm_device *dev = crtc->base.dev;
6205         struct drm_i915_private *dev_priv = dev->dev_private;
6206
6207         I915_WRITE(FP0(crtc->pipe), crtc->config->dpll_hw_state.fp0);
6208         I915_WRITE(FP1(crtc->pipe), crtc->config->dpll_hw_state.fp1);
6209 }
6210
6211 static void i9xx_crtc_enable(struct drm_crtc *crtc)
6212 {
6213         struct drm_device *dev = crtc->dev;
6214         struct drm_i915_private *dev_priv = to_i915(dev);
6215         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6216         struct intel_encoder *encoder;
6217         struct intel_crtc_state *pipe_config =
6218                 to_intel_crtc_state(crtc->state);
6219         enum pipe pipe = intel_crtc->pipe;
6220
6221         if (WARN_ON(intel_crtc->active))
6222                 return;
6223
6224         i9xx_set_pll_dividers(intel_crtc);
6225
6226         if (intel_crtc->config->has_dp_encoder)
6227                 intel_dp_set_m_n(intel_crtc, M1_N1);
6228
6229         intel_set_pipe_timings(intel_crtc);
6230         intel_set_pipe_src_size(intel_crtc);
6231
6232         i9xx_set_pipeconf(intel_crtc);
6233
6234         intel_crtc->active = true;
6235
6236         if (!IS_GEN2(dev))
6237                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6238
6239         for_each_encoder_on_crtc(dev, crtc, encoder)
6240                 if (encoder->pre_enable)
6241                         encoder->pre_enable(encoder);
6242
6243         i9xx_enable_pll(intel_crtc);
6244
6245         i9xx_pfit_enable(intel_crtc);
6246
6247         intel_color_load_luts(&pipe_config->base);
6248
6249         intel_update_watermarks(crtc);
6250         intel_enable_pipe(intel_crtc);
6251
6252         assert_vblank_disabled(crtc);
6253         drm_crtc_vblank_on(crtc);
6254
6255         for_each_encoder_on_crtc(dev, crtc, encoder)
6256                 encoder->enable(encoder);
6257 }
6258
6259 static void i9xx_pfit_disable(struct intel_crtc *crtc)
6260 {
6261         struct drm_device *dev = crtc->base.dev;
6262         struct drm_i915_private *dev_priv = dev->dev_private;
6263
6264         if (!crtc->config->gmch_pfit.control)
6265                 return;
6266
6267         assert_pipe_disabled(dev_priv, crtc->pipe);
6268
6269         DRM_DEBUG_DRIVER("disabling pfit, current: 0x%08x\n",
6270                          I915_READ(PFIT_CONTROL));
6271         I915_WRITE(PFIT_CONTROL, 0);
6272 }
6273
6274 static void i9xx_crtc_disable(struct drm_crtc *crtc)
6275 {
6276         struct drm_device *dev = crtc->dev;
6277         struct drm_i915_private *dev_priv = dev->dev_private;
6278         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6279         struct intel_encoder *encoder;
6280         int pipe = intel_crtc->pipe;
6281
6282         /*
6283          * On gen2 planes are double buffered but the pipe isn't, so we must
6284          * wait for planes to fully turn off before disabling the pipe.
6285          */
6286         if (IS_GEN2(dev))
6287                 intel_wait_for_vblank(dev, pipe);
6288
6289         for_each_encoder_on_crtc(dev, crtc, encoder)
6290                 encoder->disable(encoder);
6291
6292         drm_crtc_vblank_off(crtc);
6293         assert_vblank_disabled(crtc);
6294
6295         intel_disable_pipe(intel_crtc);
6296
6297         i9xx_pfit_disable(intel_crtc);
6298
6299         for_each_encoder_on_crtc(dev, crtc, encoder)
6300                 if (encoder->post_disable)
6301                         encoder->post_disable(encoder);
6302
6303         if (!intel_crtc->config->has_dsi_encoder) {
6304                 if (IS_CHERRYVIEW(dev))
6305                         chv_disable_pll(dev_priv, pipe);
6306                 else if (IS_VALLEYVIEW(dev))
6307                         vlv_disable_pll(dev_priv, pipe);
6308                 else
6309                         i9xx_disable_pll(intel_crtc);
6310         }
6311
6312         for_each_encoder_on_crtc(dev, crtc, encoder)
6313                 if (encoder->post_pll_disable)
6314                         encoder->post_pll_disable(encoder);
6315
6316         if (!IS_GEN2(dev))
6317                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6318 }
6319
6320 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
6321 {
6322         struct intel_encoder *encoder;
6323         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6324         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6325         enum intel_display_power_domain domain;
6326         unsigned long domains;
6327
6328         if (!intel_crtc->active)
6329                 return;
6330
6331         if (to_intel_plane_state(crtc->primary->state)->visible) {
6332                 WARN_ON(intel_crtc->flip_work);
6333
6334                 intel_pre_disable_primary_noatomic(crtc);
6335
6336                 intel_crtc_disable_planes(crtc, 1 << drm_plane_index(crtc->primary));
6337                 to_intel_plane_state(crtc->primary->state)->visible = false;
6338         }
6339
6340         dev_priv->display.crtc_disable(crtc);
6341
6342         DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
6343                       crtc->base.id, crtc->name);
6344
6345         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->state, NULL) < 0);
6346         crtc->state->active = false;
6347         intel_crtc->active = false;
6348         crtc->enabled = false;
6349         crtc->state->connector_mask = 0;
6350         crtc->state->encoder_mask = 0;
6351
6352         for_each_encoder_on_crtc(crtc->dev, crtc, encoder)
6353                 encoder->base.crtc = NULL;
6354
6355         intel_fbc_disable(intel_crtc);
6356         intel_update_watermarks(crtc);
6357         intel_disable_shared_dpll(intel_crtc);
6358
6359         domains = intel_crtc->enabled_power_domains;
6360         for_each_power_domain(domain, domains)
6361                 intel_display_power_put(dev_priv, domain);
6362         intel_crtc->enabled_power_domains = 0;
6363
6364         dev_priv->active_crtcs &= ~(1 << intel_crtc->pipe);
6365         dev_priv->min_pixclk[intel_crtc->pipe] = 0;
6366 }
6367
6368 /*
6369  * turn all crtc's off, but do not adjust state
6370  * This has to be paired with a call to intel_modeset_setup_hw_state.
6371  */
6372 int intel_display_suspend(struct drm_device *dev)
6373 {
6374         struct drm_i915_private *dev_priv = to_i915(dev);
6375         struct drm_atomic_state *state;
6376         int ret;
6377
6378         state = drm_atomic_helper_suspend(dev);
6379         ret = PTR_ERR_OR_ZERO(state);
6380         if (ret)
6381                 DRM_ERROR("Suspending crtc's failed with %i\n", ret);
6382         else
6383                 dev_priv->modeset_restore_state = state;
6384         return ret;
6385 }
6386
6387 void intel_encoder_destroy(struct drm_encoder *encoder)
6388 {
6389         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6390
6391         drm_encoder_cleanup(encoder);
6392         kfree(intel_encoder);
6393 }
6394
6395 /* Cross check the actual hw state with our own modeset state tracking (and it's
6396  * internal consistency). */
6397 static void intel_connector_verify_state(struct intel_connector *connector)
6398 {
6399         struct drm_crtc *crtc = connector->base.state->crtc;
6400
6401         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
6402                       connector->base.base.id,
6403                       connector->base.name);
6404
6405         if (connector->get_hw_state(connector)) {
6406                 struct intel_encoder *encoder = connector->encoder;
6407                 struct drm_connector_state *conn_state = connector->base.state;
6408
6409                 I915_STATE_WARN(!crtc,
6410                          "connector enabled without attached crtc\n");
6411
6412                 if (!crtc)
6413                         return;
6414
6415                 I915_STATE_WARN(!crtc->state->active,
6416                       "connector is active, but attached crtc isn't\n");
6417
6418                 if (!encoder || encoder->type == INTEL_OUTPUT_DP_MST)
6419                         return;
6420
6421                 I915_STATE_WARN(conn_state->best_encoder != &encoder->base,
6422                         "atomic encoder doesn't match attached encoder\n");
6423
6424                 I915_STATE_WARN(conn_state->crtc != encoder->base.crtc,
6425                         "attached encoder crtc differs from connector crtc\n");
6426         } else {
6427                 I915_STATE_WARN(crtc && crtc->state->active,
6428                         "attached crtc is active, but connector isn't\n");
6429                 I915_STATE_WARN(!crtc && connector->base.state->best_encoder,
6430                         "best encoder set without crtc!\n");
6431         }
6432 }
6433
6434 int intel_connector_init(struct intel_connector *connector)
6435 {
6436         drm_atomic_helper_connector_reset(&connector->base);
6437
6438         if (!connector->base.state)
6439                 return -ENOMEM;
6440
6441         return 0;
6442 }
6443
6444 struct intel_connector *intel_connector_alloc(void)
6445 {
6446         struct intel_connector *connector;
6447
6448         connector = kzalloc(sizeof *connector, GFP_KERNEL);
6449         if (!connector)
6450                 return NULL;
6451
6452         if (intel_connector_init(connector) < 0) {
6453                 kfree(connector);
6454                 return NULL;
6455         }
6456
6457         return connector;
6458 }
6459
6460 /* Simple connector->get_hw_state implementation for encoders that support only
6461  * one connector and no cloning and hence the encoder state determines the state
6462  * of the connector. */
6463 bool intel_connector_get_hw_state(struct intel_connector *connector)
6464 {
6465         enum pipe pipe = 0;
6466         struct intel_encoder *encoder = connector->encoder;
6467
6468         return encoder->get_hw_state(encoder, &pipe);
6469 }
6470
6471 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
6472 {
6473         if (crtc_state->base.enable && crtc_state->has_pch_encoder)
6474                 return crtc_state->fdi_lanes;
6475
6476         return 0;
6477 }
6478
6479 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
6480                                      struct intel_crtc_state *pipe_config)
6481 {
6482         struct drm_atomic_state *state = pipe_config->base.state;
6483         struct intel_crtc *other_crtc;
6484         struct intel_crtc_state *other_crtc_state;
6485
6486         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
6487                       pipe_name(pipe), pipe_config->fdi_lanes);
6488         if (pipe_config->fdi_lanes > 4) {
6489                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
6490                               pipe_name(pipe), pipe_config->fdi_lanes);
6491                 return -EINVAL;
6492         }
6493
6494         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
6495                 if (pipe_config->fdi_lanes > 2) {
6496                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
6497                                       pipe_config->fdi_lanes);
6498                         return -EINVAL;
6499                 } else {
6500                         return 0;
6501                 }
6502         }
6503
6504         if (INTEL_INFO(dev)->num_pipes == 2)
6505                 return 0;
6506
6507         /* Ivybridge 3 pipe is really complicated */
6508         switch (pipe) {
6509         case PIPE_A:
6510                 return 0;
6511         case PIPE_B:
6512                 if (pipe_config->fdi_lanes <= 2)
6513                         return 0;
6514
6515                 other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_C));
6516                 other_crtc_state =
6517                         intel_atomic_get_crtc_state(state, other_crtc);
6518                 if (IS_ERR(other_crtc_state))
6519                         return PTR_ERR(other_crtc_state);
6520
6521                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
6522                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
6523                                       pipe_name(pipe), pipe_config->fdi_lanes);
6524                         return -EINVAL;
6525                 }
6526                 return 0;
6527         case PIPE_C:
6528                 if (pipe_config->fdi_lanes > 2) {
6529                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
6530                                       pipe_name(pipe), pipe_config->fdi_lanes);
6531                         return -EINVAL;
6532                 }
6533
6534                 other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_B));
6535                 other_crtc_state =
6536                         intel_atomic_get_crtc_state(state, other_crtc);
6537                 if (IS_ERR(other_crtc_state))
6538                         return PTR_ERR(other_crtc_state);
6539
6540                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
6541                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
6542                         return -EINVAL;
6543                 }
6544                 return 0;
6545         default:
6546                 BUG();
6547         }
6548 }
6549
6550 #define RETRY 1
6551 static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
6552                                        struct intel_crtc_state *pipe_config)
6553 {
6554         struct drm_device *dev = intel_crtc->base.dev;
6555         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6556         int lane, link_bw, fdi_dotclock, ret;
6557         bool needs_recompute = false;
6558
6559 retry:
6560         /* FDI is a binary signal running at ~2.7GHz, encoding
6561          * each output octet as 10 bits. The actual frequency
6562          * is stored as a divider into a 100MHz clock, and the
6563          * mode pixel clock is stored in units of 1KHz.
6564          * Hence the bw of each lane in terms of the mode signal
6565          * is:
6566          */
6567         link_bw = intel_fdi_link_freq(to_i915(dev), pipe_config);
6568
6569         fdi_dotclock = adjusted_mode->crtc_clock;
6570
6571         lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
6572                                            pipe_config->pipe_bpp);
6573
6574         pipe_config->fdi_lanes = lane;
6575
6576         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
6577                                link_bw, &pipe_config->fdi_m_n);
6578
6579         ret = ironlake_check_fdi_lanes(dev, intel_crtc->pipe, pipe_config);
6580         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
6581                 pipe_config->pipe_bpp -= 2*3;
6582                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
6583                               pipe_config->pipe_bpp);
6584                 needs_recompute = true;
6585                 pipe_config->bw_constrained = true;
6586
6587                 goto retry;
6588         }
6589
6590         if (needs_recompute)
6591                 return RETRY;
6592
6593         return ret;
6594 }
6595
6596 static bool pipe_config_supports_ips(struct drm_i915_private *dev_priv,
6597                                      struct intel_crtc_state *pipe_config)
6598 {
6599         if (pipe_config->pipe_bpp > 24)
6600                 return false;
6601
6602         /* HSW can handle pixel rate up to cdclk? */
6603         if (IS_HASWELL(dev_priv))
6604                 return true;
6605
6606         /*
6607          * We compare against max which means we must take
6608          * the increased cdclk requirement into account when
6609          * calculating the new cdclk.
6610          *
6611          * Should measure whether using a lower cdclk w/o IPS
6612          */
6613         return ilk_pipe_pixel_rate(pipe_config) <=
6614                 dev_priv->max_cdclk_freq * 95 / 100;
6615 }
6616
6617 static void hsw_compute_ips_config(struct intel_crtc *crtc,
6618                                    struct intel_crtc_state *pipe_config)
6619 {
6620         struct drm_device *dev = crtc->base.dev;
6621         struct drm_i915_private *dev_priv = dev->dev_private;
6622
6623         pipe_config->ips_enabled = i915.enable_ips &&
6624                 hsw_crtc_supports_ips(crtc) &&
6625                 pipe_config_supports_ips(dev_priv, pipe_config);
6626 }
6627
6628 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
6629 {
6630         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6631
6632         /* GDG double wide on either pipe, otherwise pipe A only */
6633         return INTEL_INFO(dev_priv)->gen < 4 &&
6634                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
6635 }
6636
6637 static int intel_crtc_compute_config(struct intel_crtc *crtc,
6638                                      struct intel_crtc_state *pipe_config)
6639 {
6640         struct drm_device *dev = crtc->base.dev;
6641         struct drm_i915_private *dev_priv = dev->dev_private;
6642         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6643         int clock_limit = dev_priv->max_dotclk_freq;
6644
6645         if (INTEL_INFO(dev)->gen < 4) {
6646                 clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
6647
6648                 /*
6649                  * Enable double wide mode when the dot clock
6650                  * is > 90% of the (display) core speed.
6651                  */
6652                 if (intel_crtc_supports_double_wide(crtc) &&
6653                     adjusted_mode->crtc_clock > clock_limit) {
6654                         clock_limit = dev_priv->max_dotclk_freq;
6655                         pipe_config->double_wide = true;
6656                 }
6657         }
6658
6659         if (adjusted_mode->crtc_clock > clock_limit) {
6660                 DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
6661                               adjusted_mode->crtc_clock, clock_limit,
6662                               yesno(pipe_config->double_wide));
6663                 return -EINVAL;
6664         }
6665
6666         /*
6667          * Pipe horizontal size must be even in:
6668          * - DVO ganged mode
6669          * - LVDS dual channel mode
6670          * - Double wide pipe
6671          */
6672         if ((intel_pipe_will_have_type(pipe_config, INTEL_OUTPUT_LVDS) &&
6673              intel_is_dual_link_lvds(dev)) || pipe_config->double_wide)
6674                 pipe_config->pipe_src_w &= ~1;
6675
6676         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
6677          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
6678          */
6679         if ((INTEL_INFO(dev)->gen > 4 || IS_G4X(dev)) &&
6680                 adjusted_mode->crtc_hsync_start == adjusted_mode->crtc_hdisplay)
6681                 return -EINVAL;
6682
6683         if (HAS_IPS(dev))
6684                 hsw_compute_ips_config(crtc, pipe_config);
6685
6686         if (pipe_config->has_pch_encoder)
6687                 return ironlake_fdi_compute_config(crtc, pipe_config);
6688
6689         return 0;
6690 }
6691
6692 static int skylake_get_display_clock_speed(struct drm_device *dev)
6693 {
6694         struct drm_i915_private *dev_priv = to_i915(dev);
6695         uint32_t cdctl;
6696
6697         skl_dpll0_update(dev_priv);
6698
6699         if (dev_priv->cdclk_pll.vco == 0)
6700                 return dev_priv->cdclk_pll.ref;
6701
6702         cdctl = I915_READ(CDCLK_CTL);
6703
6704         if (dev_priv->cdclk_pll.vco == 8640000) {
6705                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
6706                 case CDCLK_FREQ_450_432:
6707                         return 432000;
6708                 case CDCLK_FREQ_337_308:
6709                         return 308571;
6710                 case CDCLK_FREQ_540:
6711                         return 540000;
6712                 case CDCLK_FREQ_675_617:
6713                         return 617143;
6714                 default:
6715                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
6716                 }
6717         } else {
6718                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
6719                 case CDCLK_FREQ_450_432:
6720                         return 450000;
6721                 case CDCLK_FREQ_337_308:
6722                         return 337500;
6723                 case CDCLK_FREQ_540:
6724                         return 540000;
6725                 case CDCLK_FREQ_675_617:
6726                         return 675000;
6727                 default:
6728                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
6729                 }
6730         }
6731
6732         return dev_priv->cdclk_pll.ref;
6733 }
6734
6735 static void bxt_de_pll_update(struct drm_i915_private *dev_priv)
6736 {
6737         u32 val;
6738
6739         dev_priv->cdclk_pll.ref = 19200;
6740         dev_priv->cdclk_pll.vco = 0;
6741
6742         val = I915_READ(BXT_DE_PLL_ENABLE);
6743         if ((val & BXT_DE_PLL_PLL_ENABLE) == 0)
6744                 return;
6745
6746         if (WARN_ON((val & BXT_DE_PLL_LOCK) == 0))
6747                 return;
6748
6749         val = I915_READ(BXT_DE_PLL_CTL);
6750         dev_priv->cdclk_pll.vco = (val & BXT_DE_PLL_RATIO_MASK) *
6751                 dev_priv->cdclk_pll.ref;
6752 }
6753
6754 static int broxton_get_display_clock_speed(struct drm_device *dev)
6755 {
6756         struct drm_i915_private *dev_priv = to_i915(dev);
6757         u32 divider;
6758         int div, vco;
6759
6760         bxt_de_pll_update(dev_priv);
6761
6762         vco = dev_priv->cdclk_pll.vco;
6763         if (vco == 0)
6764                 return dev_priv->cdclk_pll.ref;
6765
6766         divider = I915_READ(CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
6767
6768         switch (divider) {
6769         case BXT_CDCLK_CD2X_DIV_SEL_1:
6770                 div = 2;
6771                 break;
6772         case BXT_CDCLK_CD2X_DIV_SEL_1_5:
6773                 div = 3;
6774                 break;
6775         case BXT_CDCLK_CD2X_DIV_SEL_2:
6776                 div = 4;
6777                 break;
6778         case BXT_CDCLK_CD2X_DIV_SEL_4:
6779                 div = 8;
6780                 break;
6781         default:
6782                 MISSING_CASE(divider);
6783                 return dev_priv->cdclk_pll.ref;
6784         }
6785
6786         return DIV_ROUND_CLOSEST(vco, div);
6787 }
6788
6789 static int broadwell_get_display_clock_speed(struct drm_device *dev)
6790 {
6791         struct drm_i915_private *dev_priv = dev->dev_private;
6792         uint32_t lcpll = I915_READ(LCPLL_CTL);
6793         uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
6794
6795         if (lcpll & LCPLL_CD_SOURCE_FCLK)
6796                 return 800000;
6797         else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
6798                 return 450000;
6799         else if (freq == LCPLL_CLK_FREQ_450)
6800                 return 450000;
6801         else if (freq == LCPLL_CLK_FREQ_54O_BDW)
6802                 return 540000;
6803         else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
6804                 return 337500;
6805         else
6806                 return 675000;
6807 }
6808
6809 static int haswell_get_display_clock_speed(struct drm_device *dev)
6810 {
6811         struct drm_i915_private *dev_priv = dev->dev_private;
6812         uint32_t lcpll = I915_READ(LCPLL_CTL);
6813         uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
6814
6815         if (lcpll & LCPLL_CD_SOURCE_FCLK)
6816                 return 800000;
6817         else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
6818                 return 450000;
6819         else if (freq == LCPLL_CLK_FREQ_450)
6820                 return 450000;
6821         else if (IS_HSW_ULT(dev))
6822                 return 337500;
6823         else
6824                 return 540000;
6825 }
6826
6827 static int valleyview_get_display_clock_speed(struct drm_device *dev)
6828 {
6829         return vlv_get_cck_clock_hpll(to_i915(dev), "cdclk",
6830                                       CCK_DISPLAY_CLOCK_CONTROL);
6831 }
6832
6833 static int ilk_get_display_clock_speed(struct drm_device *dev)
6834 {
6835         return 450000;
6836 }
6837
6838 static int i945_get_display_clock_speed(struct drm_device *dev)
6839 {
6840         return 400000;
6841 }
6842
6843 static int i915_get_display_clock_speed(struct drm_device *dev)
6844 {
6845         return 333333;
6846 }
6847
6848 static int i9xx_misc_get_display_clock_speed(struct drm_device *dev)
6849 {
6850         return 200000;
6851 }
6852
6853 static int pnv_get_display_clock_speed(struct drm_device *dev)
6854 {
6855         u16 gcfgc = 0;
6856
6857         pci_read_config_word(dev->pdev, GCFGC, &gcfgc);
6858
6859         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
6860         case GC_DISPLAY_CLOCK_267_MHZ_PNV:
6861                 return 266667;
6862         case GC_DISPLAY_CLOCK_333_MHZ_PNV:
6863                 return 333333;
6864         case GC_DISPLAY_CLOCK_444_MHZ_PNV:
6865                 return 444444;
6866         case GC_DISPLAY_CLOCK_200_MHZ_PNV:
6867                 return 200000;
6868         default:
6869                 DRM_ERROR("Unknown pnv display core clock 0x%04x\n", gcfgc);
6870         case GC_DISPLAY_CLOCK_133_MHZ_PNV:
6871                 return 133333;
6872         case GC_DISPLAY_CLOCK_167_MHZ_PNV:
6873                 return 166667;
6874         }
6875 }
6876
6877 static int i915gm_get_display_clock_speed(struct drm_device *dev)
6878 {
6879         u16 gcfgc = 0;
6880
6881         pci_read_config_word(dev->pdev, GCFGC, &gcfgc);
6882
6883         if (gcfgc & GC_LOW_FREQUENCY_ENABLE)
6884                 return 133333;
6885         else {
6886                 switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
6887                 case GC_DISPLAY_CLOCK_333_MHZ:
6888                         return 333333;
6889                 default:
6890                 case GC_DISPLAY_CLOCK_190_200_MHZ:
6891                         return 190000;
6892                 }
6893         }
6894 }
6895
6896 static int i865_get_display_clock_speed(struct drm_device *dev)
6897 {
6898         return 266667;
6899 }
6900
6901 static int i85x_get_display_clock_speed(struct drm_device *dev)
6902 {
6903         u16 hpllcc = 0;
6904
6905         /*
6906          * 852GM/852GMV only supports 133 MHz and the HPLLCC
6907          * encoding is different :(
6908          * FIXME is this the right way to detect 852GM/852GMV?
6909          */
6910         if (dev->pdev->revision == 0x1)
6911                 return 133333;
6912
6913         pci_bus_read_config_word(dev->pdev->bus,
6914                                  PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
6915
6916         /* Assume that the hardware is in the high speed state.  This
6917          * should be the default.
6918          */
6919         switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
6920         case GC_CLOCK_133_200:
6921         case GC_CLOCK_133_200_2:
6922         case GC_CLOCK_100_200:
6923                 return 200000;
6924         case GC_CLOCK_166_250:
6925                 return 250000;
6926         case GC_CLOCK_100_133:
6927                 return 133333;
6928         case GC_CLOCK_133_266:
6929         case GC_CLOCK_133_266_2:
6930         case GC_CLOCK_166_266:
6931                 return 266667;
6932         }
6933
6934         /* Shouldn't happen */
6935         return 0;
6936 }
6937
6938 static int i830_get_display_clock_speed(struct drm_device *dev)
6939 {
6940         return 133333;
6941 }
6942
6943 static unsigned int intel_hpll_vco(struct drm_device *dev)
6944 {
6945         struct drm_i915_private *dev_priv = dev->dev_private;
6946         static const unsigned int blb_vco[8] = {
6947                 [0] = 3200000,
6948                 [1] = 4000000,
6949                 [2] = 5333333,
6950                 [3] = 4800000,
6951                 [4] = 6400000,
6952         };
6953         static const unsigned int pnv_vco[8] = {
6954                 [0] = 3200000,
6955                 [1] = 4000000,
6956                 [2] = 5333333,
6957                 [3] = 4800000,
6958                 [4] = 2666667,
6959         };
6960         static const unsigned int cl_vco[8] = {
6961                 [0] = 3200000,
6962                 [1] = 4000000,
6963                 [2] = 5333333,
6964                 [3] = 6400000,
6965                 [4] = 3333333,
6966                 [5] = 3566667,
6967                 [6] = 4266667,
6968         };
6969         static const unsigned int elk_vco[8] = {
6970                 [0] = 3200000,
6971                 [1] = 4000000,
6972                 [2] = 5333333,
6973                 [3] = 4800000,
6974         };
6975         static const unsigned int ctg_vco[8] = {
6976                 [0] = 3200000,
6977                 [1] = 4000000,
6978                 [2] = 5333333,
6979                 [3] = 6400000,
6980                 [4] = 2666667,
6981                 [5] = 4266667,
6982         };
6983         const unsigned int *vco_table;
6984         unsigned int vco;
6985         uint8_t tmp = 0;
6986
6987         /* FIXME other chipsets? */
6988         if (IS_GM45(dev))
6989                 vco_table = ctg_vco;
6990         else if (IS_G4X(dev))
6991                 vco_table = elk_vco;
6992         else if (IS_CRESTLINE(dev))
6993                 vco_table = cl_vco;
6994         else if (IS_PINEVIEW(dev))
6995                 vco_table = pnv_vco;
6996         else if (IS_G33(dev))
6997                 vco_table = blb_vco;
6998         else
6999                 return 0;
7000
7001         tmp = I915_READ(IS_MOBILE(dev) ? HPLLVCO_MOBILE : HPLLVCO);
7002
7003         vco = vco_table[tmp & 0x7];
7004         if (vco == 0)
7005                 DRM_ERROR("Bad HPLL VCO (HPLLVCO=0x%02x)\n", tmp);
7006         else
7007                 DRM_DEBUG_KMS("HPLL VCO %u kHz\n", vco);
7008
7009         return vco;
7010 }
7011
7012 static int gm45_get_display_clock_speed(struct drm_device *dev)
7013 {
7014         unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
7015         uint16_t tmp = 0;
7016
7017         pci_read_config_word(dev->pdev, GCFGC, &tmp);
7018
7019         cdclk_sel = (tmp >> 12) & 0x1;
7020
7021         switch (vco) {
7022         case 2666667:
7023         case 4000000:
7024         case 5333333:
7025                 return cdclk_sel ? 333333 : 222222;
7026         case 3200000:
7027                 return cdclk_sel ? 320000 : 228571;
7028         default:
7029                 DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n", vco, tmp);
7030                 return 222222;
7031         }
7032 }
7033
7034 static int i965gm_get_display_clock_speed(struct drm_device *dev)
7035 {
7036         static const uint8_t div_3200[] = { 16, 10,  8 };
7037         static const uint8_t div_4000[] = { 20, 12, 10 };
7038         static const uint8_t div_5333[] = { 24, 16, 14 };
7039         const uint8_t *div_table;
7040         unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
7041         uint16_t tmp = 0;
7042
7043         pci_read_config_word(dev->pdev, GCFGC, &tmp);
7044
7045         cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
7046
7047         if (cdclk_sel >= ARRAY_SIZE(div_3200))
7048                 goto fail;
7049
7050         switch (vco) {
7051         case 3200000:
7052                 div_table = div_3200;
7053                 break;
7054         case 4000000:
7055                 div_table = div_4000;
7056                 break;
7057         case 5333333:
7058                 div_table = div_5333;
7059                 break;
7060         default:
7061                 goto fail;
7062         }
7063
7064         return DIV_ROUND_CLOSEST(vco, div_table[cdclk_sel]);
7065
7066 fail:
7067         DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n", vco, tmp);
7068         return 200000;
7069 }
7070
7071 static int g33_get_display_clock_speed(struct drm_device *dev)
7072 {
7073         static const uint8_t div_3200[] = { 12, 10,  8,  7, 5, 16 };
7074         static const uint8_t div_4000[] = { 14, 12, 10,  8, 6, 20 };
7075         static const uint8_t div_4800[] = { 20, 14, 12, 10, 8, 24 };
7076         static const uint8_t div_5333[] = { 20, 16, 12, 12, 8, 28 };
7077         const uint8_t *div_table;
7078         unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
7079         uint16_t tmp = 0;
7080
7081         pci_read_config_word(dev->pdev, GCFGC, &tmp);
7082
7083         cdclk_sel = (tmp >> 4) & 0x7;
7084
7085         if (cdclk_sel >= ARRAY_SIZE(div_3200))
7086                 goto fail;
7087
7088         switch (vco) {
7089         case 3200000:
7090                 div_table = div_3200;
7091                 break;
7092         case 4000000:
7093                 div_table = div_4000;
7094                 break;
7095         case 4800000:
7096                 div_table = div_4800;
7097                 break;
7098         case 5333333:
7099                 div_table = div_5333;
7100                 break;
7101         default:
7102                 goto fail;
7103         }
7104
7105         return DIV_ROUND_CLOSEST(vco, div_table[cdclk_sel]);
7106
7107 fail:
7108         DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n", vco, tmp);
7109         return 190476;
7110 }
7111
7112 static void
7113 intel_reduce_m_n_ratio(uint32_t *num, uint32_t *den)
7114 {
7115         while (*num > DATA_LINK_M_N_MASK ||
7116                *den > DATA_LINK_M_N_MASK) {
7117                 *num >>= 1;
7118                 *den >>= 1;
7119         }
7120 }
7121
7122 static void compute_m_n(unsigned int m, unsigned int n,
7123                         uint32_t *ret_m, uint32_t *ret_n)
7124 {
7125         *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
7126         *ret_m = div_u64((uint64_t) m * *ret_n, n);
7127         intel_reduce_m_n_ratio(ret_m, ret_n);
7128 }
7129
7130 void
7131 intel_link_compute_m_n(int bits_per_pixel, int nlanes,
7132                        int pixel_clock, int link_clock,
7133                        struct intel_link_m_n *m_n)
7134 {
7135         m_n->tu = 64;
7136
7137         compute_m_n(bits_per_pixel * pixel_clock,
7138                     link_clock * nlanes * 8,
7139                     &m_n->gmch_m, &m_n->gmch_n);
7140
7141         compute_m_n(pixel_clock, link_clock,
7142                     &m_n->link_m, &m_n->link_n);
7143 }
7144
7145 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
7146 {
7147         if (i915.panel_use_ssc >= 0)
7148                 return i915.panel_use_ssc != 0;
7149         return dev_priv->vbt.lvds_use_ssc
7150                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
7151 }
7152
7153 static uint32_t pnv_dpll_compute_fp(struct dpll *dpll)
7154 {
7155         return (1 << dpll->n) << 16 | dpll->m2;
7156 }
7157
7158 static uint32_t i9xx_dpll_compute_fp(struct dpll *dpll)
7159 {
7160         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
7161 }
7162
7163 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
7164                                      struct intel_crtc_state *crtc_state,
7165                                      struct dpll *reduced_clock)
7166 {
7167         struct drm_device *dev = crtc->base.dev;
7168         u32 fp, fp2 = 0;
7169
7170         if (IS_PINEVIEW(dev)) {
7171                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
7172                 if (reduced_clock)
7173                         fp2 = pnv_dpll_compute_fp(reduced_clock);
7174         } else {
7175                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
7176                 if (reduced_clock)
7177                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
7178         }
7179
7180         crtc_state->dpll_hw_state.fp0 = fp;
7181
7182         crtc->lowfreq_avail = false;
7183         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7184             reduced_clock) {
7185                 crtc_state->dpll_hw_state.fp1 = fp2;
7186                 crtc->lowfreq_avail = true;
7187         } else {
7188                 crtc_state->dpll_hw_state.fp1 = fp;
7189         }
7190 }
7191
7192 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
7193                 pipe)
7194 {
7195         u32 reg_val;
7196
7197         /*
7198          * PLLB opamp always calibrates to max value of 0x3f, force enable it
7199          * and set it to a reasonable value instead.
7200          */
7201         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7202         reg_val &= 0xffffff00;
7203         reg_val |= 0x00000030;
7204         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7205
7206         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7207         reg_val &= 0x8cffffff;
7208         reg_val = 0x8c000000;
7209         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7210
7211         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7212         reg_val &= 0xffffff00;
7213         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7214
7215         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7216         reg_val &= 0x00ffffff;
7217         reg_val |= 0xb0000000;
7218         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7219 }
7220
7221 static void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc,
7222                                          struct intel_link_m_n *m_n)
7223 {
7224         struct drm_device *dev = crtc->base.dev;
7225         struct drm_i915_private *dev_priv = dev->dev_private;
7226         int pipe = crtc->pipe;
7227
7228         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7229         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
7230         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
7231         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
7232 }
7233
7234 static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
7235                                          struct intel_link_m_n *m_n,
7236                                          struct intel_link_m_n *m2_n2)
7237 {
7238         struct drm_device *dev = crtc->base.dev;
7239         struct drm_i915_private *dev_priv = dev->dev_private;
7240         int pipe = crtc->pipe;
7241         enum transcoder transcoder = crtc->config->cpu_transcoder;
7242
7243         if (INTEL_INFO(dev)->gen >= 5) {
7244                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
7245                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
7246                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
7247                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
7248                 /* M2_N2 registers to be set only for gen < 8 (M2_N2 available
7249                  * for gen < 8) and if DRRS is supported (to make sure the
7250                  * registers are not unnecessarily accessed).
7251                  */
7252                 if (m2_n2 && (IS_CHERRYVIEW(dev) || INTEL_INFO(dev)->gen < 8) &&
7253                         crtc->config->has_drrs) {
7254                         I915_WRITE(PIPE_DATA_M2(transcoder),
7255                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
7256                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
7257                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
7258                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
7259                 }
7260         } else {
7261                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7262                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
7263                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
7264                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
7265         }
7266 }
7267
7268 void intel_dp_set_m_n(struct intel_crtc *crtc, enum link_m_n_set m_n)
7269 {
7270         struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
7271
7272         if (m_n == M1_N1) {
7273                 dp_m_n = &crtc->config->dp_m_n;
7274                 dp_m2_n2 = &crtc->config->dp_m2_n2;
7275         } else if (m_n == M2_N2) {
7276
7277                 /*
7278                  * M2_N2 registers are not supported. Hence m2_n2 divider value
7279                  * needs to be programmed into M1_N1.
7280                  */
7281                 dp_m_n = &crtc->config->dp_m2_n2;
7282         } else {
7283                 DRM_ERROR("Unsupported divider value\n");
7284                 return;
7285         }
7286
7287         if (crtc->config->has_pch_encoder)
7288                 intel_pch_transcoder_set_m_n(crtc, &crtc->config->dp_m_n);
7289         else
7290                 intel_cpu_transcoder_set_m_n(crtc, dp_m_n, dp_m2_n2);
7291 }
7292
7293 static void vlv_compute_dpll(struct intel_crtc *crtc,
7294                              struct intel_crtc_state *pipe_config)
7295 {
7296         pipe_config->dpll_hw_state.dpll = DPLL_INTEGRATED_REF_CLK_VLV |
7297                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
7298         if (crtc->pipe != PIPE_A)
7299                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7300
7301         /* DPLL not used with DSI, but still need the rest set up */
7302         if (!pipe_config->has_dsi_encoder)
7303                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
7304                         DPLL_EXT_BUFFER_ENABLE_VLV;
7305
7306         pipe_config->dpll_hw_state.dpll_md =
7307                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7308 }
7309
7310 static void chv_compute_dpll(struct intel_crtc *crtc,
7311                              struct intel_crtc_state *pipe_config)
7312 {
7313         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLK_CHV |
7314                 DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
7315         if (crtc->pipe != PIPE_A)
7316                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7317
7318         /* DPLL not used with DSI, but still need the rest set up */
7319         if (!pipe_config->has_dsi_encoder)
7320                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
7321
7322         pipe_config->dpll_hw_state.dpll_md =
7323                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7324 }
7325
7326 static void vlv_prepare_pll(struct intel_crtc *crtc,
7327                             const struct intel_crtc_state *pipe_config)
7328 {
7329         struct drm_device *dev = crtc->base.dev;
7330         struct drm_i915_private *dev_priv = dev->dev_private;
7331         enum pipe pipe = crtc->pipe;
7332         u32 mdiv;
7333         u32 bestn, bestm1, bestm2, bestp1, bestp2;
7334         u32 coreclk, reg_val;
7335
7336         /* Enable Refclk */
7337         I915_WRITE(DPLL(pipe),
7338                    pipe_config->dpll_hw_state.dpll &
7339                    ~(DPLL_VCO_ENABLE | DPLL_EXT_BUFFER_ENABLE_VLV));
7340
7341         /* No need to actually set up the DPLL with DSI */
7342         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7343                 return;
7344
7345         mutex_lock(&dev_priv->sb_lock);
7346
7347         bestn = pipe_config->dpll.n;
7348         bestm1 = pipe_config->dpll.m1;
7349         bestm2 = pipe_config->dpll.m2;
7350         bestp1 = pipe_config->dpll.p1;
7351         bestp2 = pipe_config->dpll.p2;
7352
7353         /* See eDP HDMI DPIO driver vbios notes doc */
7354
7355         /* PLL B needs special handling */
7356         if (pipe == PIPE_B)
7357                 vlv_pllb_recal_opamp(dev_priv, pipe);
7358
7359         /* Set up Tx target for periodic Rcomp update */
7360         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
7361
7362         /* Disable target IRef on PLL */
7363         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
7364         reg_val &= 0x00ffffff;
7365         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
7366
7367         /* Disable fast lock */
7368         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
7369
7370         /* Set idtafcrecal before PLL is enabled */
7371         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
7372         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
7373         mdiv |= ((bestn << DPIO_N_SHIFT));
7374         mdiv |= (1 << DPIO_K_SHIFT);
7375
7376         /*
7377          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
7378          * but we don't support that).
7379          * Note: don't use the DAC post divider as it seems unstable.
7380          */
7381         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
7382         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7383
7384         mdiv |= DPIO_ENABLE_CALIBRATION;
7385         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7386
7387         /* Set HBR and RBR LPF coefficients */
7388         if (pipe_config->port_clock == 162000 ||
7389             intel_pipe_has_type(crtc, INTEL_OUTPUT_ANALOG) ||
7390             intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI))
7391                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7392                                  0x009f0003);
7393         else
7394                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7395                                  0x00d0000f);
7396
7397         if (pipe_config->has_dp_encoder) {
7398                 /* Use SSC source */
7399                 if (pipe == PIPE_A)
7400                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7401                                          0x0df40000);
7402                 else
7403                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7404                                          0x0df70000);
7405         } else { /* HDMI or VGA */
7406                 /* Use bend source */
7407                 if (pipe == PIPE_A)
7408                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7409                                          0x0df70000);
7410                 else
7411                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7412                                          0x0df40000);
7413         }
7414
7415         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
7416         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
7417         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) ||
7418             intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))
7419                 coreclk |= 0x01000000;
7420         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
7421
7422         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
7423         mutex_unlock(&dev_priv->sb_lock);
7424 }
7425
7426 static void chv_prepare_pll(struct intel_crtc *crtc,
7427                             const struct intel_crtc_state *pipe_config)
7428 {
7429         struct drm_device *dev = crtc->base.dev;
7430         struct drm_i915_private *dev_priv = dev->dev_private;
7431         enum pipe pipe = crtc->pipe;
7432         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7433         u32 loopfilter, tribuf_calcntr;
7434         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
7435         u32 dpio_val;
7436         int vco;
7437
7438         /* Enable Refclk and SSC */
7439         I915_WRITE(DPLL(pipe),
7440                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
7441
7442         /* No need to actually set up the DPLL with DSI */
7443         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
7444                 return;
7445
7446         bestn = pipe_config->dpll.n;
7447         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
7448         bestm1 = pipe_config->dpll.m1;
7449         bestm2 = pipe_config->dpll.m2 >> 22;
7450         bestp1 = pipe_config->dpll.p1;
7451         bestp2 = pipe_config->dpll.p2;
7452         vco = pipe_config->dpll.vco;
7453         dpio_val = 0;
7454         loopfilter = 0;
7455
7456         mutex_lock(&dev_priv->sb_lock);
7457
7458         /* p1 and p2 divider */
7459         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
7460                         5 << DPIO_CHV_S1_DIV_SHIFT |
7461                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
7462                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
7463                         1 << DPIO_CHV_K_DIV_SHIFT);
7464
7465         /* Feedback post-divider - m2 */
7466         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
7467
7468         /* Feedback refclk divider - n and m1 */
7469         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
7470                         DPIO_CHV_M1_DIV_BY_2 |
7471                         1 << DPIO_CHV_N_DIV_SHIFT);
7472
7473         /* M2 fraction division */
7474         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
7475
7476         /* M2 fraction division enable */
7477         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7478         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
7479         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
7480         if (bestm2_frac)
7481                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
7482         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
7483
7484         /* Program digital lock detect threshold */
7485         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
7486         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
7487                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
7488         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
7489         if (!bestm2_frac)
7490                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
7491         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
7492
7493         /* Loop filter */
7494         if (vco == 5400000) {
7495                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
7496                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
7497                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
7498                 tribuf_calcntr = 0x9;
7499         } else if (vco <= 6200000) {
7500                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
7501                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
7502                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7503                 tribuf_calcntr = 0x9;
7504         } else if (vco <= 6480000) {
7505                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7506                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7507                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7508                 tribuf_calcntr = 0x8;
7509         } else {
7510                 /* Not supported. Apply the same limits as in the max case */
7511                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7512                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7513                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7514                 tribuf_calcntr = 0;
7515         }
7516         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
7517
7518         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
7519         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
7520         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
7521         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
7522
7523         /* AFC Recal */
7524         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
7525                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
7526                         DPIO_AFC_RECAL);
7527
7528         mutex_unlock(&dev_priv->sb_lock);
7529 }
7530
7531 /**
7532  * vlv_force_pll_on - forcibly enable just the PLL
7533  * @dev_priv: i915 private structure
7534  * @pipe: pipe PLL to enable
7535  * @dpll: PLL configuration
7536  *
7537  * Enable the PLL for @pipe using the supplied @dpll config. To be used
7538  * in cases where we need the PLL enabled even when @pipe is not going to
7539  * be enabled.
7540  */
7541 int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe,
7542                      const struct dpll *dpll)
7543 {
7544         struct intel_crtc *crtc =
7545                 to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe));
7546         struct intel_crtc_state *pipe_config;
7547
7548         pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL);
7549         if (!pipe_config)
7550                 return -ENOMEM;
7551
7552         pipe_config->base.crtc = &crtc->base;
7553         pipe_config->pixel_multiplier = 1;
7554         pipe_config->dpll = *dpll;
7555
7556         if (IS_CHERRYVIEW(dev)) {
7557                 chv_compute_dpll(crtc, pipe_config);
7558                 chv_prepare_pll(crtc, pipe_config);
7559                 chv_enable_pll(crtc, pipe_config);
7560         } else {
7561                 vlv_compute_dpll(crtc, pipe_config);
7562                 vlv_prepare_pll(crtc, pipe_config);
7563                 vlv_enable_pll(crtc, pipe_config);
7564         }
7565
7566         kfree(pipe_config);
7567
7568         return 0;
7569 }
7570
7571 /**
7572  * vlv_force_pll_off - forcibly disable just the PLL
7573  * @dev_priv: i915 private structure
7574  * @pipe: pipe PLL to disable
7575  *
7576  * Disable the PLL for @pipe. To be used in cases where we need
7577  * the PLL enabled even when @pipe is not going to be enabled.
7578  */
7579 void vlv_force_pll_off(struct drm_device *dev, enum pipe pipe)
7580 {
7581         if (IS_CHERRYVIEW(dev))
7582                 chv_disable_pll(to_i915(dev), pipe);
7583         else
7584                 vlv_disable_pll(to_i915(dev), pipe);
7585 }
7586
7587 static void i9xx_compute_dpll(struct intel_crtc *crtc,
7588                               struct intel_crtc_state *crtc_state,
7589                               struct dpll *reduced_clock)
7590 {
7591         struct drm_device *dev = crtc->base.dev;
7592         struct drm_i915_private *dev_priv = dev->dev_private;
7593         u32 dpll;
7594         bool is_sdvo;
7595         struct dpll *clock = &crtc_state->dpll;
7596
7597         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7598
7599         is_sdvo = intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_SDVO) ||
7600                 intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_HDMI);
7601
7602         dpll = DPLL_VGA_MODE_DIS;
7603
7604         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS))
7605                 dpll |= DPLLB_MODE_LVDS;
7606         else
7607                 dpll |= DPLLB_MODE_DAC_SERIAL;
7608
7609         if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
7610                 dpll |= (crtc_state->pixel_multiplier - 1)
7611                         << SDVO_MULTIPLIER_SHIFT_HIRES;
7612         }
7613
7614         if (is_sdvo)
7615                 dpll |= DPLL_SDVO_HIGH_SPEED;
7616
7617         if (crtc_state->has_dp_encoder)
7618                 dpll |= DPLL_SDVO_HIGH_SPEED;
7619
7620         /* compute bitmask from p1 value */
7621         if (IS_PINEVIEW(dev))
7622                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
7623         else {
7624                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7625                 if (IS_G4X(dev) && reduced_clock)
7626                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
7627         }
7628         switch (clock->p2) {
7629         case 5:
7630                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
7631                 break;
7632         case 7:
7633                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
7634                 break;
7635         case 10:
7636                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
7637                 break;
7638         case 14:
7639                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
7640                 break;
7641         }
7642         if (INTEL_INFO(dev)->gen >= 4)
7643                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
7644
7645         if (crtc_state->sdvo_tv_clock)
7646                 dpll |= PLL_REF_INPUT_TVCLKINBC;
7647         else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7648                  intel_panel_use_ssc(dev_priv))
7649                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7650         else
7651                 dpll |= PLL_REF_INPUT_DREFCLK;
7652
7653         dpll |= DPLL_VCO_ENABLE;
7654         crtc_state->dpll_hw_state.dpll = dpll;
7655
7656         if (INTEL_INFO(dev)->gen >= 4) {
7657                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
7658                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7659                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
7660         }
7661 }
7662
7663 static void i8xx_compute_dpll(struct intel_crtc *crtc,
7664                               struct intel_crtc_state *crtc_state,
7665                               struct dpll *reduced_clock)
7666 {
7667         struct drm_device *dev = crtc->base.dev;
7668         struct drm_i915_private *dev_priv = dev->dev_private;
7669         u32 dpll;
7670         struct dpll *clock = &crtc_state->dpll;
7671
7672         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7673
7674         dpll = DPLL_VGA_MODE_DIS;
7675
7676         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7677                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7678         } else {
7679                 if (clock->p1 == 2)
7680                         dpll |= PLL_P1_DIVIDE_BY_TWO;
7681                 else
7682                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7683                 if (clock->p2 == 4)
7684                         dpll |= PLL_P2_DIVIDE_BY_4;
7685         }
7686
7687         if (!IS_I830(dev) && intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_DVO))
7688                 dpll |= DPLL_DVO_2X_MODE;
7689
7690         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7691             intel_panel_use_ssc(dev_priv))
7692                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7693         else
7694                 dpll |= PLL_REF_INPUT_DREFCLK;
7695
7696         dpll |= DPLL_VCO_ENABLE;
7697         crtc_state->dpll_hw_state.dpll = dpll;
7698 }
7699
7700 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
7701 {
7702         struct drm_device *dev = intel_crtc->base.dev;
7703         struct drm_i915_private *dev_priv = dev->dev_private;
7704         enum pipe pipe = intel_crtc->pipe;
7705         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
7706         const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
7707         uint32_t crtc_vtotal, crtc_vblank_end;
7708         int vsyncshift = 0;
7709
7710         /* We need to be careful not to changed the adjusted mode, for otherwise
7711          * the hw state checker will get angry at the mismatch. */
7712         crtc_vtotal = adjusted_mode->crtc_vtotal;
7713         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
7714
7715         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
7716                 /* the chip adds 2 halflines automatically */
7717                 crtc_vtotal -= 1;
7718                 crtc_vblank_end -= 1;
7719
7720                 if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
7721                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
7722                 else
7723                         vsyncshift = adjusted_mode->crtc_hsync_start -
7724                                 adjusted_mode->crtc_htotal / 2;
7725                 if (vsyncshift < 0)
7726                         vsyncshift += adjusted_mode->crtc_htotal;
7727         }
7728
7729         if (INTEL_INFO(dev)->gen > 3)
7730                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
7731
7732         I915_WRITE(HTOTAL(cpu_transcoder),
7733                    (adjusted_mode->crtc_hdisplay - 1) |
7734                    ((adjusted_mode->crtc_htotal - 1) << 16));
7735         I915_WRITE(HBLANK(cpu_transcoder),
7736                    (adjusted_mode->crtc_hblank_start - 1) |
7737                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
7738         I915_WRITE(HSYNC(cpu_transcoder),
7739                    (adjusted_mode->crtc_hsync_start - 1) |
7740                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
7741
7742         I915_WRITE(VTOTAL(cpu_transcoder),
7743                    (adjusted_mode->crtc_vdisplay - 1) |
7744                    ((crtc_vtotal - 1) << 16));
7745         I915_WRITE(VBLANK(cpu_transcoder),
7746                    (adjusted_mode->crtc_vblank_start - 1) |
7747                    ((crtc_vblank_end - 1) << 16));
7748         I915_WRITE(VSYNC(cpu_transcoder),
7749                    (adjusted_mode->crtc_vsync_start - 1) |
7750                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
7751
7752         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
7753          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
7754          * documented on the DDI_FUNC_CTL register description, EDP Input Select
7755          * bits. */
7756         if (IS_HASWELL(dev) && cpu_transcoder == TRANSCODER_EDP &&
7757             (pipe == PIPE_B || pipe == PIPE_C))
7758                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
7759
7760 }
7761
7762 static void intel_set_pipe_src_size(struct intel_crtc *intel_crtc)
7763 {
7764         struct drm_device *dev = intel_crtc->base.dev;
7765         struct drm_i915_private *dev_priv = dev->dev_private;
7766         enum pipe pipe = intel_crtc->pipe;
7767
7768         /* pipesrc controls the size that is scaled from, which should
7769          * always be the user's requested size.
7770          */
7771         I915_WRITE(PIPESRC(pipe),
7772                    ((intel_crtc->config->pipe_src_w - 1) << 16) |
7773                    (intel_crtc->config->pipe_src_h - 1));
7774 }
7775
7776 static void intel_get_pipe_timings(struct intel_crtc *crtc,
7777                                    struct intel_crtc_state *pipe_config)
7778 {
7779         struct drm_device *dev = crtc->base.dev;
7780         struct drm_i915_private *dev_priv = dev->dev_private;
7781         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
7782         uint32_t tmp;
7783
7784         tmp = I915_READ(HTOTAL(cpu_transcoder));
7785         pipe_config->base.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
7786         pipe_config->base.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
7787         tmp = I915_READ(HBLANK(cpu_transcoder));
7788         pipe_config->base.adjusted_mode.crtc_hblank_start = (tmp & 0xffff) + 1;
7789         pipe_config->base.adjusted_mode.crtc_hblank_end = ((tmp >> 16) & 0xffff) + 1;
7790         tmp = I915_READ(HSYNC(cpu_transcoder));
7791         pipe_config->base.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
7792         pipe_config->base.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
7793
7794         tmp = I915_READ(VTOTAL(cpu_transcoder));
7795         pipe_config->base.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
7796         pipe_config->base.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
7797         tmp = I915_READ(VBLANK(cpu_transcoder));
7798         pipe_config->base.adjusted_mode.crtc_vblank_start = (tmp & 0xffff) + 1;
7799         pipe_config->base.adjusted_mode.crtc_vblank_end = ((tmp >> 16) & 0xffff) + 1;
7800         tmp = I915_READ(VSYNC(cpu_transcoder));
7801         pipe_config->base.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
7802         pipe_config->base.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
7803
7804         if (I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK) {
7805                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
7806                 pipe_config->base.adjusted_mode.crtc_vtotal += 1;
7807                 pipe_config->base.adjusted_mode.crtc_vblank_end += 1;
7808         }
7809 }
7810
7811 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
7812                                     struct intel_crtc_state *pipe_config)
7813 {
7814         struct drm_device *dev = crtc->base.dev;
7815         struct drm_i915_private *dev_priv = dev->dev_private;
7816         u32 tmp;
7817
7818         tmp = I915_READ(PIPESRC(crtc->pipe));
7819         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
7820         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
7821
7822         pipe_config->base.mode.vdisplay = pipe_config->pipe_src_h;
7823         pipe_config->base.mode.hdisplay = pipe_config->pipe_src_w;
7824 }
7825
7826 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
7827                                  struct intel_crtc_state *pipe_config)
7828 {
7829         mode->hdisplay = pipe_config->base.adjusted_mode.crtc_hdisplay;
7830         mode->htotal = pipe_config->base.adjusted_mode.crtc_htotal;
7831         mode->hsync_start = pipe_config->base.adjusted_mode.crtc_hsync_start;
7832         mode->hsync_end = pipe_config->base.adjusted_mode.crtc_hsync_end;
7833
7834         mode->vdisplay = pipe_config->base.adjusted_mode.crtc_vdisplay;
7835         mode->vtotal = pipe_config->base.adjusted_mode.crtc_vtotal;
7836         mode->vsync_start = pipe_config->base.adjusted_mode.crtc_vsync_start;
7837         mode->vsync_end = pipe_config->base.adjusted_mode.crtc_vsync_end;
7838
7839         mode->flags = pipe_config->base.adjusted_mode.flags;
7840         mode->type = DRM_MODE_TYPE_DRIVER;
7841
7842         mode->clock = pipe_config->base.adjusted_mode.crtc_clock;
7843         mode->flags |= pipe_config->base.adjusted_mode.flags;
7844
7845         mode->hsync = drm_mode_hsync(mode);
7846         mode->vrefresh = drm_mode_vrefresh(mode);
7847         drm_mode_set_name(mode);
7848 }
7849
7850 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
7851 {
7852         struct drm_device *dev = intel_crtc->base.dev;
7853         struct drm_i915_private *dev_priv = dev->dev_private;
7854         uint32_t pipeconf;
7855
7856         pipeconf = 0;
7857
7858         if ((intel_crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
7859             (intel_crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
7860                 pipeconf |= I915_READ(PIPECONF(intel_crtc->pipe)) & PIPECONF_ENABLE;
7861
7862         if (intel_crtc->config->double_wide)
7863                 pipeconf |= PIPECONF_DOUBLE_WIDE;
7864
7865         /* only g4x and later have fancy bpc/dither controls */
7866         if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
7867                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
7868                 if (intel_crtc->config->dither && intel_crtc->config->pipe_bpp != 30)
7869                         pipeconf |= PIPECONF_DITHER_EN |
7870                                     PIPECONF_DITHER_TYPE_SP;
7871
7872                 switch (intel_crtc->config->pipe_bpp) {
7873                 case 18:
7874                         pipeconf |= PIPECONF_6BPC;
7875                         break;
7876                 case 24:
7877                         pipeconf |= PIPECONF_8BPC;
7878                         break;
7879                 case 30:
7880                         pipeconf |= PIPECONF_10BPC;
7881                         break;
7882                 default:
7883                         /* Case prevented by intel_choose_pipe_bpp_dither. */
7884                         BUG();
7885                 }
7886         }
7887
7888         if (HAS_PIPE_CXSR(dev)) {
7889                 if (intel_crtc->lowfreq_avail) {
7890                         DRM_DEBUG_KMS("enabling CxSR downclocking\n");
7891                         pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
7892                 } else {
7893                         DRM_DEBUG_KMS("disabling CxSR downclocking\n");
7894                 }
7895         }
7896
7897         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
7898                 if (INTEL_INFO(dev)->gen < 4 ||
7899                     intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
7900                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
7901                 else
7902                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
7903         } else
7904                 pipeconf |= PIPECONF_PROGRESSIVE;
7905
7906         if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
7907              intel_crtc->config->limited_color_range)
7908                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
7909
7910         I915_WRITE(PIPECONF(intel_crtc->pipe), pipeconf);
7911         POSTING_READ(PIPECONF(intel_crtc->pipe));
7912 }
7913
7914 static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
7915                                    struct intel_crtc_state *crtc_state)
7916 {
7917         struct drm_device *dev = crtc->base.dev;
7918         struct drm_i915_private *dev_priv = dev->dev_private;
7919         const struct intel_limit *limit;
7920         int refclk = 48000;
7921
7922         memset(&crtc_state->dpll_hw_state, 0,
7923                sizeof(crtc_state->dpll_hw_state));
7924
7925         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7926                 if (intel_panel_use_ssc(dev_priv)) {
7927                         refclk = dev_priv->vbt.lvds_ssc_freq;
7928                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7929                 }
7930
7931                 limit = &intel_limits_i8xx_lvds;
7932         } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_DVO)) {
7933                 limit = &intel_limits_i8xx_dvo;
7934         } else {
7935                 limit = &intel_limits_i8xx_dac;
7936         }
7937
7938         if (!crtc_state->clock_set &&
7939             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7940                                  refclk, NULL, &crtc_state->dpll)) {
7941                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7942                 return -EINVAL;
7943         }
7944
7945         i8xx_compute_dpll(crtc, crtc_state, NULL);
7946
7947         return 0;
7948 }
7949
7950 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
7951                                   struct intel_crtc_state *crtc_state)
7952 {
7953         struct drm_device *dev = crtc->base.dev;
7954         struct drm_i915_private *dev_priv = dev->dev_private;
7955         const struct intel_limit *limit;
7956         int refclk = 96000;
7957
7958         memset(&crtc_state->dpll_hw_state, 0,
7959                sizeof(crtc_state->dpll_hw_state));
7960
7961         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7962                 if (intel_panel_use_ssc(dev_priv)) {
7963                         refclk = dev_priv->vbt.lvds_ssc_freq;
7964                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7965                 }
7966
7967                 if (intel_is_dual_link_lvds(dev))
7968                         limit = &intel_limits_g4x_dual_channel_lvds;
7969                 else
7970                         limit = &intel_limits_g4x_single_channel_lvds;
7971         } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_HDMI) ||
7972                    intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
7973                 limit = &intel_limits_g4x_hdmi;
7974         } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_SDVO)) {
7975                 limit = &intel_limits_g4x_sdvo;
7976         } else {
7977                 /* The option is for other outputs */
7978                 limit = &intel_limits_i9xx_sdvo;
7979         }
7980
7981         if (!crtc_state->clock_set &&
7982             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
7983                                 refclk, NULL, &crtc_state->dpll)) {
7984                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
7985                 return -EINVAL;
7986         }
7987
7988         i9xx_compute_dpll(crtc, crtc_state, NULL);
7989
7990         return 0;
7991 }
7992
7993 static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
7994                                   struct intel_crtc_state *crtc_state)
7995 {
7996         struct drm_device *dev = crtc->base.dev;
7997         struct drm_i915_private *dev_priv = dev->dev_private;
7998         const struct intel_limit *limit;
7999         int refclk = 96000;
8000
8001         memset(&crtc_state->dpll_hw_state, 0,
8002                sizeof(crtc_state->dpll_hw_state));
8003
8004         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8005                 if (intel_panel_use_ssc(dev_priv)) {
8006                         refclk = dev_priv->vbt.lvds_ssc_freq;
8007                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8008                 }
8009
8010                 limit = &intel_limits_pineview_lvds;
8011         } else {
8012                 limit = &intel_limits_pineview_sdvo;
8013         }
8014
8015         if (!crtc_state->clock_set &&
8016             !pnv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8017                                 refclk, NULL, &crtc_state->dpll)) {
8018                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8019                 return -EINVAL;
8020         }
8021
8022         i9xx_compute_dpll(crtc, crtc_state, NULL);
8023
8024         return 0;
8025 }
8026
8027 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
8028                                    struct intel_crtc_state *crtc_state)
8029 {
8030         struct drm_device *dev = crtc->base.dev;
8031         struct drm_i915_private *dev_priv = dev->dev_private;
8032         const struct intel_limit *limit;
8033         int refclk = 96000;
8034
8035         memset(&crtc_state->dpll_hw_state, 0,
8036                sizeof(crtc_state->dpll_hw_state));
8037
8038         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
8039                 if (intel_panel_use_ssc(dev_priv)) {
8040                         refclk = dev_priv->vbt.lvds_ssc_freq;
8041                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
8042                 }
8043
8044                 limit = &intel_limits_i9xx_lvds;
8045         } else {
8046                 limit = &intel_limits_i9xx_sdvo;
8047         }
8048
8049         if (!crtc_state->clock_set &&
8050             !i9xx_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8051                                  refclk, NULL, &crtc_state->dpll)) {
8052                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8053                 return -EINVAL;
8054         }
8055
8056         i9xx_compute_dpll(crtc, crtc_state, NULL);
8057
8058         return 0;
8059 }
8060
8061 static int chv_crtc_compute_clock(struct intel_crtc *crtc,
8062                                   struct intel_crtc_state *crtc_state)
8063 {
8064         int refclk = 100000;
8065         const struct intel_limit *limit = &intel_limits_chv;
8066
8067         memset(&crtc_state->dpll_hw_state, 0,
8068                sizeof(crtc_state->dpll_hw_state));
8069
8070         if (!crtc_state->clock_set &&
8071             !chv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8072                                 refclk, NULL, &crtc_state->dpll)) {
8073                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8074                 return -EINVAL;
8075         }
8076
8077         chv_compute_dpll(crtc, crtc_state);
8078
8079         return 0;
8080 }
8081
8082 static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
8083                                   struct intel_crtc_state *crtc_state)
8084 {
8085         int refclk = 100000;
8086         const struct intel_limit *limit = &intel_limits_vlv;
8087
8088         memset(&crtc_state->dpll_hw_state, 0,
8089                sizeof(crtc_state->dpll_hw_state));
8090
8091         if (!crtc_state->clock_set &&
8092             !vlv_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
8093                                 refclk, NULL, &crtc_state->dpll)) {
8094                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8095                 return -EINVAL;
8096         }
8097
8098         vlv_compute_dpll(crtc, crtc_state);
8099
8100         return 0;
8101 }
8102
8103 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
8104                                  struct intel_crtc_state *pipe_config)
8105 {
8106         struct drm_device *dev = crtc->base.dev;
8107         struct drm_i915_private *dev_priv = dev->dev_private;
8108         uint32_t tmp;
8109
8110         if (INTEL_INFO(dev)->gen <= 3 && (IS_I830(dev) || !IS_MOBILE(dev)))
8111                 return;
8112
8113         tmp = I915_READ(PFIT_CONTROL);
8114         if (!(tmp & PFIT_ENABLE))
8115                 return;
8116
8117         /* Check whether the pfit is attached to our pipe. */
8118         if (INTEL_INFO(dev)->gen < 4) {
8119                 if (crtc->pipe != PIPE_B)
8120                         return;
8121         } else {
8122                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
8123                         return;
8124         }
8125
8126         pipe_config->gmch_pfit.control = tmp;
8127         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
8128 }
8129
8130 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
8131                                struct intel_crtc_state *pipe_config)
8132 {
8133         struct drm_device *dev = crtc->base.dev;
8134         struct drm_i915_private *dev_priv = dev->dev_private;
8135         int pipe = pipe_config->cpu_transcoder;
8136         struct dpll clock;
8137         u32 mdiv;
8138         int refclk = 100000;
8139
8140         /* In case of DSI, DPLL will not be used */
8141         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8142                 return;
8143
8144         mutex_lock(&dev_priv->sb_lock);
8145         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
8146         mutex_unlock(&dev_priv->sb_lock);
8147
8148         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
8149         clock.m2 = mdiv & DPIO_M2DIV_MASK;
8150         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
8151         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
8152         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
8153
8154         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
8155 }
8156
8157 static void
8158 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
8159                               struct intel_initial_plane_config *plane_config)
8160 {
8161         struct drm_device *dev = crtc->base.dev;
8162         struct drm_i915_private *dev_priv = dev->dev_private;
8163         u32 val, base, offset;
8164         int pipe = crtc->pipe, plane = crtc->plane;
8165         int fourcc, pixel_format;
8166         unsigned int aligned_height;
8167         struct drm_framebuffer *fb;
8168         struct intel_framebuffer *intel_fb;
8169
8170         val = I915_READ(DSPCNTR(plane));
8171         if (!(val & DISPLAY_PLANE_ENABLE))
8172                 return;
8173
8174         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8175         if (!intel_fb) {
8176                 DRM_DEBUG_KMS("failed to alloc fb\n");
8177                 return;
8178         }
8179
8180         fb = &intel_fb->base;
8181
8182         if (INTEL_INFO(dev)->gen >= 4) {
8183                 if (val & DISPPLANE_TILED) {
8184                         plane_config->tiling = I915_TILING_X;
8185                         fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
8186                 }
8187         }
8188
8189         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
8190         fourcc = i9xx_format_to_fourcc(pixel_format);
8191         fb->pixel_format = fourcc;
8192         fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
8193
8194         if (INTEL_INFO(dev)->gen >= 4) {
8195                 if (plane_config->tiling)
8196                         offset = I915_READ(DSPTILEOFF(plane));
8197                 else
8198                         offset = I915_READ(DSPLINOFF(plane));
8199                 base = I915_READ(DSPSURF(plane)) & 0xfffff000;
8200         } else {
8201                 base = I915_READ(DSPADDR(plane));
8202         }
8203         plane_config->base = base;
8204
8205         val = I915_READ(PIPESRC(pipe));
8206         fb->width = ((val >> 16) & 0xfff) + 1;
8207         fb->height = ((val >> 0) & 0xfff) + 1;
8208
8209         val = I915_READ(DSPSTRIDE(pipe));
8210         fb->pitches[0] = val & 0xffffffc0;
8211
8212         aligned_height = intel_fb_align_height(dev, fb->height,
8213                                                fb->pixel_format,
8214                                                fb->modifier[0]);
8215
8216         plane_config->size = fb->pitches[0] * aligned_height;
8217
8218         DRM_DEBUG_KMS("pipe/plane %c/%d with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
8219                       pipe_name(pipe), plane, fb->width, fb->height,
8220                       fb->bits_per_pixel, base, fb->pitches[0],
8221                       plane_config->size);
8222
8223         plane_config->fb = intel_fb;
8224 }
8225
8226 static void chv_crtc_clock_get(struct intel_crtc *crtc,
8227                                struct intel_crtc_state *pipe_config)
8228 {
8229         struct drm_device *dev = crtc->base.dev;
8230         struct drm_i915_private *dev_priv = dev->dev_private;
8231         int pipe = pipe_config->cpu_transcoder;
8232         enum dpio_channel port = vlv_pipe_to_channel(pipe);
8233         struct dpll clock;
8234         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
8235         int refclk = 100000;
8236
8237         /* In case of DSI, DPLL will not be used */
8238         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
8239                 return;
8240
8241         mutex_lock(&dev_priv->sb_lock);
8242         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
8243         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
8244         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
8245         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
8246         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
8247         mutex_unlock(&dev_priv->sb_lock);
8248
8249         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
8250         clock.m2 = (pll_dw0 & 0xff) << 22;
8251         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
8252                 clock.m2 |= pll_dw2 & 0x3fffff;
8253         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
8254         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
8255         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
8256
8257         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
8258 }
8259
8260 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
8261                                  struct intel_crtc_state *pipe_config)
8262 {
8263         struct drm_device *dev = crtc->base.dev;
8264         struct drm_i915_private *dev_priv = dev->dev_private;
8265         enum intel_display_power_domain power_domain;
8266         uint32_t tmp;
8267         bool ret;
8268
8269         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
8270         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
8271                 return false;
8272
8273         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
8274         pipe_config->shared_dpll = NULL;
8275
8276         ret = false;
8277
8278         tmp = I915_READ(PIPECONF(crtc->pipe));
8279         if (!(tmp & PIPECONF_ENABLE))
8280                 goto out;
8281
8282         if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
8283                 switch (tmp & PIPECONF_BPC_MASK) {
8284                 case PIPECONF_6BPC:
8285                         pipe_config->pipe_bpp = 18;
8286                         break;
8287                 case PIPECONF_8BPC:
8288                         pipe_config->pipe_bpp = 24;
8289                         break;
8290                 case PIPECONF_10BPC:
8291                         pipe_config->pipe_bpp = 30;
8292                         break;
8293                 default:
8294                         break;
8295                 }
8296         }
8297
8298         if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
8299             (tmp & PIPECONF_COLOR_RANGE_SELECT))
8300                 pipe_config->limited_color_range = true;
8301
8302         if (INTEL_INFO(dev)->gen < 4)
8303                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
8304
8305         intel_get_pipe_timings(crtc, pipe_config);
8306         intel_get_pipe_src_size(crtc, pipe_config);
8307
8308         i9xx_get_pfit_config(crtc, pipe_config);
8309
8310         if (INTEL_INFO(dev)->gen >= 4) {
8311                 /* No way to read it out on pipes B and C */
8312                 if (IS_CHERRYVIEW(dev) && crtc->pipe != PIPE_A)
8313                         tmp = dev_priv->chv_dpll_md[crtc->pipe];
8314                 else
8315                         tmp = I915_READ(DPLL_MD(crtc->pipe));
8316                 pipe_config->pixel_multiplier =
8317                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
8318                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
8319                 pipe_config->dpll_hw_state.dpll_md = tmp;
8320         } else if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
8321                 tmp = I915_READ(DPLL(crtc->pipe));
8322                 pipe_config->pixel_multiplier =
8323                         ((tmp & SDVO_MULTIPLIER_MASK)
8324                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
8325         } else {
8326                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
8327                  * port and will be fixed up in the encoder->get_config
8328                  * function. */
8329                 pipe_config->pixel_multiplier = 1;
8330         }
8331         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
8332         if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
8333                 /*
8334                  * DPLL_DVO_2X_MODE must be enabled for both DPLLs
8335                  * on 830. Filter it out here so that we don't
8336                  * report errors due to that.
8337                  */
8338                 if (IS_I830(dev))
8339                         pipe_config->dpll_hw_state.dpll &= ~DPLL_DVO_2X_MODE;
8340
8341                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
8342                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
8343         } else {
8344                 /* Mask out read-only status bits. */
8345                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
8346                                                      DPLL_PORTC_READY_MASK |
8347                                                      DPLL_PORTB_READY_MASK);
8348         }
8349
8350         if (IS_CHERRYVIEW(dev))
8351                 chv_crtc_clock_get(crtc, pipe_config);
8352         else if (IS_VALLEYVIEW(dev))
8353                 vlv_crtc_clock_get(crtc, pipe_config);
8354         else
8355                 i9xx_crtc_clock_get(crtc, pipe_config);
8356
8357         /*
8358          * Normally the dotclock is filled in by the encoder .get_config()
8359          * but in case the pipe is enabled w/o any ports we need a sane
8360          * default.
8361          */
8362         pipe_config->base.adjusted_mode.crtc_clock =
8363                 pipe_config->port_clock / pipe_config->pixel_multiplier;
8364
8365         ret = true;
8366
8367 out:
8368         intel_display_power_put(dev_priv, power_domain);
8369
8370         return ret;
8371 }
8372
8373 static void ironlake_init_pch_refclk(struct drm_device *dev)
8374 {
8375         struct drm_i915_private *dev_priv = dev->dev_private;
8376         struct intel_encoder *encoder;
8377         int i;
8378         u32 val, final;
8379         bool has_lvds = false;
8380         bool has_cpu_edp = false;
8381         bool has_panel = false;
8382         bool has_ck505 = false;
8383         bool can_ssc = false;
8384         bool using_ssc_source = false;
8385
8386         /* We need to take the global config into account */
8387         for_each_intel_encoder(dev, encoder) {
8388                 switch (encoder->type) {
8389                 case INTEL_OUTPUT_LVDS:
8390                         has_panel = true;
8391                         has_lvds = true;
8392                         break;
8393                 case INTEL_OUTPUT_EDP:
8394                         has_panel = true;
8395                         if (enc_to_dig_port(&encoder->base)->port == PORT_A)
8396                                 has_cpu_edp = true;
8397                         break;
8398                 default:
8399                         break;
8400                 }
8401         }
8402
8403         if (HAS_PCH_IBX(dev)) {
8404                 has_ck505 = dev_priv->vbt.display_clock_mode;
8405                 can_ssc = has_ck505;
8406         } else {
8407                 has_ck505 = false;
8408                 can_ssc = true;
8409         }
8410
8411         /* Check if any DPLLs are using the SSC source */
8412         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
8413                 u32 temp = I915_READ(PCH_DPLL(i));
8414
8415                 if (!(temp & DPLL_VCO_ENABLE))
8416                         continue;
8417
8418                 if ((temp & PLL_REF_INPUT_MASK) ==
8419                     PLLB_REF_INPUT_SPREADSPECTRUMIN) {
8420                         using_ssc_source = true;
8421                         break;
8422                 }
8423         }
8424
8425         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d using_ssc_source %d\n",
8426                       has_panel, has_lvds, has_ck505, using_ssc_source);
8427
8428         /* Ironlake: try to setup display ref clock before DPLL
8429          * enabling. This is only under driver's control after
8430          * PCH B stepping, previous chipset stepping should be
8431          * ignoring this setting.
8432          */
8433         val = I915_READ(PCH_DREF_CONTROL);
8434
8435         /* As we must carefully and slowly disable/enable each source in turn,
8436          * compute the final state we want first and check if we need to
8437          * make any changes at all.
8438          */
8439         final = val;
8440         final &= ~DREF_NONSPREAD_SOURCE_MASK;
8441         if (has_ck505)
8442                 final |= DREF_NONSPREAD_CK505_ENABLE;
8443         else
8444                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
8445
8446         final &= ~DREF_SSC_SOURCE_MASK;
8447         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8448         final &= ~DREF_SSC1_ENABLE;
8449
8450         if (has_panel) {
8451                 final |= DREF_SSC_SOURCE_ENABLE;
8452
8453                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
8454                         final |= DREF_SSC1_ENABLE;
8455
8456                 if (has_cpu_edp) {
8457                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
8458                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8459                         else
8460                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8461                 } else
8462                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8463         } else if (using_ssc_source) {
8464                 final |= DREF_SSC_SOURCE_ENABLE;
8465                 final |= DREF_SSC1_ENABLE;
8466         }
8467
8468         if (final == val)
8469                 return;
8470
8471         /* Always enable nonspread source */
8472         val &= ~DREF_NONSPREAD_SOURCE_MASK;
8473
8474         if (has_ck505)
8475                 val |= DREF_NONSPREAD_CK505_ENABLE;
8476         else
8477                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
8478
8479         if (has_panel) {
8480                 val &= ~DREF_SSC_SOURCE_MASK;
8481                 val |= DREF_SSC_SOURCE_ENABLE;
8482
8483                 /* SSC must be turned on before enabling the CPU output  */
8484                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8485                         DRM_DEBUG_KMS("Using SSC on panel\n");
8486                         val |= DREF_SSC1_ENABLE;
8487                 } else
8488                         val &= ~DREF_SSC1_ENABLE;
8489
8490                 /* Get SSC going before enabling the outputs */
8491                 I915_WRITE(PCH_DREF_CONTROL, val);
8492                 POSTING_READ(PCH_DREF_CONTROL);
8493                 udelay(200);
8494
8495                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8496
8497                 /* Enable CPU source on CPU attached eDP */
8498                 if (has_cpu_edp) {
8499                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8500                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
8501                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8502                         } else
8503                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8504                 } else
8505                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8506
8507                 I915_WRITE(PCH_DREF_CONTROL, val);
8508                 POSTING_READ(PCH_DREF_CONTROL);
8509                 udelay(200);
8510         } else {
8511                 DRM_DEBUG_KMS("Disabling CPU source output\n");
8512
8513                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8514
8515                 /* Turn off CPU output */
8516                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8517
8518                 I915_WRITE(PCH_DREF_CONTROL, val);
8519                 POSTING_READ(PCH_DREF_CONTROL);
8520                 udelay(200);
8521
8522                 if (!using_ssc_source) {
8523                         DRM_DEBUG_KMS("Disabling SSC source\n");
8524
8525                         /* Turn off the SSC source */
8526                         val &= ~DREF_SSC_SOURCE_MASK;
8527                         val |= DREF_SSC_SOURCE_DISABLE;
8528
8529                         /* Turn off SSC1 */
8530                         val &= ~DREF_SSC1_ENABLE;
8531
8532                         I915_WRITE(PCH_DREF_CONTROL, val);
8533                         POSTING_READ(PCH_DREF_CONTROL);
8534                         udelay(200);
8535                 }
8536         }
8537
8538         BUG_ON(val != final);
8539 }
8540
8541 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
8542 {
8543         uint32_t tmp;
8544
8545         tmp = I915_READ(SOUTH_CHICKEN2);
8546         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
8547         I915_WRITE(SOUTH_CHICKEN2, tmp);
8548
8549         if (wait_for_us(I915_READ(SOUTH_CHICKEN2) &
8550                         FDI_MPHY_IOSFSB_RESET_STATUS, 100))
8551                 DRM_ERROR("FDI mPHY reset assert timeout\n");
8552
8553         tmp = I915_READ(SOUTH_CHICKEN2);
8554         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
8555         I915_WRITE(SOUTH_CHICKEN2, tmp);
8556
8557         if (wait_for_us((I915_READ(SOUTH_CHICKEN2) &
8558                          FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
8559                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
8560 }
8561
8562 /* WaMPhyProgramming:hsw */
8563 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
8564 {
8565         uint32_t tmp;
8566
8567         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
8568         tmp &= ~(0xFF << 24);
8569         tmp |= (0x12 << 24);
8570         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
8571
8572         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
8573         tmp |= (1 << 11);
8574         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
8575
8576         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
8577         tmp |= (1 << 11);
8578         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
8579
8580         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
8581         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8582         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
8583
8584         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
8585         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8586         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
8587
8588         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
8589         tmp &= ~(7 << 13);
8590         tmp |= (5 << 13);
8591         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
8592
8593         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
8594         tmp &= ~(7 << 13);
8595         tmp |= (5 << 13);
8596         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
8597
8598         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
8599         tmp &= ~0xFF;
8600         tmp |= 0x1C;
8601         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
8602
8603         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
8604         tmp &= ~0xFF;
8605         tmp |= 0x1C;
8606         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
8607
8608         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
8609         tmp &= ~(0xFF << 16);
8610         tmp |= (0x1C << 16);
8611         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
8612
8613         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
8614         tmp &= ~(0xFF << 16);
8615         tmp |= (0x1C << 16);
8616         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
8617
8618         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
8619         tmp |= (1 << 27);
8620         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
8621
8622         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
8623         tmp |= (1 << 27);
8624         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
8625
8626         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
8627         tmp &= ~(0xF << 28);
8628         tmp |= (4 << 28);
8629         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
8630
8631         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
8632         tmp &= ~(0xF << 28);
8633         tmp |= (4 << 28);
8634         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
8635 }
8636
8637 /* Implements 3 different sequences from BSpec chapter "Display iCLK
8638  * Programming" based on the parameters passed:
8639  * - Sequence to enable CLKOUT_DP
8640  * - Sequence to enable CLKOUT_DP without spread
8641  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
8642  */
8643 static void lpt_enable_clkout_dp(struct drm_device *dev, bool with_spread,
8644                                  bool with_fdi)
8645 {
8646         struct drm_i915_private *dev_priv = dev->dev_private;
8647         uint32_t reg, tmp;
8648
8649         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
8650                 with_spread = true;
8651         if (WARN(HAS_PCH_LPT_LP(dev) && with_fdi, "LP PCH doesn't have FDI\n"))
8652                 with_fdi = false;
8653
8654         mutex_lock(&dev_priv->sb_lock);
8655
8656         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8657         tmp &= ~SBI_SSCCTL_DISABLE;
8658         tmp |= SBI_SSCCTL_PATHALT;
8659         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8660
8661         udelay(24);
8662
8663         if (with_spread) {
8664                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8665                 tmp &= ~SBI_SSCCTL_PATHALT;
8666                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8667
8668                 if (with_fdi) {
8669                         lpt_reset_fdi_mphy(dev_priv);
8670                         lpt_program_fdi_mphy(dev_priv);
8671                 }
8672         }
8673
8674         reg = HAS_PCH_LPT_LP(dev) ? SBI_GEN0 : SBI_DBUFF0;
8675         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8676         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8677         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8678
8679         mutex_unlock(&dev_priv->sb_lock);
8680 }
8681
8682 /* Sequence to disable CLKOUT_DP */
8683 static void lpt_disable_clkout_dp(struct drm_device *dev)
8684 {
8685         struct drm_i915_private *dev_priv = dev->dev_private;
8686         uint32_t reg, tmp;
8687
8688         mutex_lock(&dev_priv->sb_lock);
8689
8690         reg = HAS_PCH_LPT_LP(dev) ? SBI_GEN0 : SBI_DBUFF0;
8691         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8692         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8693         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8694
8695         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8696         if (!(tmp & SBI_SSCCTL_DISABLE)) {
8697                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
8698                         tmp |= SBI_SSCCTL_PATHALT;
8699                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8700                         udelay(32);
8701                 }
8702                 tmp |= SBI_SSCCTL_DISABLE;
8703                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8704         }
8705
8706         mutex_unlock(&dev_priv->sb_lock);
8707 }
8708
8709 #define BEND_IDX(steps) ((50 + (steps)) / 5)
8710
8711 static const uint16_t sscdivintphase[] = {
8712         [BEND_IDX( 50)] = 0x3B23,
8713         [BEND_IDX( 45)] = 0x3B23,
8714         [BEND_IDX( 40)] = 0x3C23,
8715         [BEND_IDX( 35)] = 0x3C23,
8716         [BEND_IDX( 30)] = 0x3D23,
8717         [BEND_IDX( 25)] = 0x3D23,
8718         [BEND_IDX( 20)] = 0x3E23,
8719         [BEND_IDX( 15)] = 0x3E23,
8720         [BEND_IDX( 10)] = 0x3F23,
8721         [BEND_IDX(  5)] = 0x3F23,
8722         [BEND_IDX(  0)] = 0x0025,
8723         [BEND_IDX( -5)] = 0x0025,
8724         [BEND_IDX(-10)] = 0x0125,
8725         [BEND_IDX(-15)] = 0x0125,
8726         [BEND_IDX(-20)] = 0x0225,
8727         [BEND_IDX(-25)] = 0x0225,
8728         [BEND_IDX(-30)] = 0x0325,
8729         [BEND_IDX(-35)] = 0x0325,
8730         [BEND_IDX(-40)] = 0x0425,
8731         [BEND_IDX(-45)] = 0x0425,
8732         [BEND_IDX(-50)] = 0x0525,
8733 };
8734
8735 /*
8736  * Bend CLKOUT_DP
8737  * steps -50 to 50 inclusive, in steps of 5
8738  * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
8739  * change in clock period = -(steps / 10) * 5.787 ps
8740  */
8741 static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
8742 {
8743         uint32_t tmp;
8744         int idx = BEND_IDX(steps);
8745
8746         if (WARN_ON(steps % 5 != 0))
8747                 return;
8748
8749         if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
8750                 return;
8751
8752         mutex_lock(&dev_priv->sb_lock);
8753
8754         if (steps % 10 != 0)
8755                 tmp = 0xAAAAAAAB;
8756         else
8757                 tmp = 0x00000000;
8758         intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
8759
8760         tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
8761         tmp &= 0xffff0000;
8762         tmp |= sscdivintphase[idx];
8763         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
8764
8765         mutex_unlock(&dev_priv->sb_lock);
8766 }
8767
8768 #undef BEND_IDX
8769
8770 static void lpt_init_pch_refclk(struct drm_device *dev)
8771 {
8772         struct intel_encoder *encoder;
8773         bool has_vga = false;
8774
8775         for_each_intel_encoder(dev, encoder) {
8776                 switch (encoder->type) {
8777                 case INTEL_OUTPUT_ANALOG:
8778                         has_vga = true;
8779                         break;
8780                 default:
8781                         break;
8782                 }
8783         }
8784
8785         if (has_vga) {
8786                 lpt_bend_clkout_dp(to_i915(dev), 0);
8787                 lpt_enable_clkout_dp(dev, true, true);
8788         } else {
8789                 lpt_disable_clkout_dp(dev);
8790         }
8791 }
8792
8793 /*
8794  * Initialize reference clocks when the driver loads
8795  */
8796 void intel_init_pch_refclk(struct drm_device *dev)
8797 {
8798         if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
8799                 ironlake_init_pch_refclk(dev);
8800         else if (HAS_PCH_LPT(dev))
8801                 lpt_init_pch_refclk(dev);
8802 }
8803
8804 static void ironlake_set_pipeconf(struct drm_crtc *crtc)
8805 {
8806         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
8807         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8808         int pipe = intel_crtc->pipe;
8809         uint32_t val;
8810
8811         val = 0;
8812
8813         switch (intel_crtc->config->pipe_bpp) {
8814         case 18:
8815                 val |= PIPECONF_6BPC;
8816                 break;
8817         case 24:
8818                 val |= PIPECONF_8BPC;
8819                 break;
8820         case 30:
8821                 val |= PIPECONF_10BPC;
8822                 break;
8823         case 36:
8824                 val |= PIPECONF_12BPC;
8825                 break;
8826         default:
8827                 /* Case prevented by intel_choose_pipe_bpp_dither. */
8828                 BUG();
8829         }
8830
8831         if (intel_crtc->config->dither)
8832                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8833
8834         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8835                 val |= PIPECONF_INTERLACED_ILK;
8836         else
8837                 val |= PIPECONF_PROGRESSIVE;
8838
8839         if (intel_crtc->config->limited_color_range)
8840                 val |= PIPECONF_COLOR_RANGE_SELECT;
8841
8842         I915_WRITE(PIPECONF(pipe), val);
8843         POSTING_READ(PIPECONF(pipe));
8844 }
8845
8846 static void haswell_set_pipeconf(struct drm_crtc *crtc)
8847 {
8848         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
8849         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8850         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
8851         u32 val = 0;
8852
8853         if (IS_HASWELL(dev_priv) && intel_crtc->config->dither)
8854                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8855
8856         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8857                 val |= PIPECONF_INTERLACED_ILK;
8858         else
8859                 val |= PIPECONF_PROGRESSIVE;
8860
8861         I915_WRITE(PIPECONF(cpu_transcoder), val);
8862         POSTING_READ(PIPECONF(cpu_transcoder));
8863 }
8864
8865 static void haswell_set_pipemisc(struct drm_crtc *crtc)
8866 {
8867         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
8868         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8869
8870         if (IS_BROADWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 9) {
8871                 u32 val = 0;
8872
8873                 switch (intel_crtc->config->pipe_bpp) {
8874                 case 18:
8875                         val |= PIPEMISC_DITHER_6_BPC;
8876                         break;
8877                 case 24:
8878                         val |= PIPEMISC_DITHER_8_BPC;
8879                         break;
8880                 case 30:
8881                         val |= PIPEMISC_DITHER_10_BPC;
8882                         break;
8883                 case 36:
8884                         val |= PIPEMISC_DITHER_12_BPC;
8885                         break;
8886                 default:
8887                         /* Case prevented by pipe_config_set_bpp. */
8888                         BUG();
8889                 }
8890
8891                 if (intel_crtc->config->dither)
8892                         val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
8893
8894                 I915_WRITE(PIPEMISC(intel_crtc->pipe), val);
8895         }
8896 }
8897
8898 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
8899 {
8900         /*
8901          * Account for spread spectrum to avoid
8902          * oversubscribing the link. Max center spread
8903          * is 2.5%; use 5% for safety's sake.
8904          */
8905         u32 bps = target_clock * bpp * 21 / 20;
8906         return DIV_ROUND_UP(bps, link_bw * 8);
8907 }
8908
8909 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
8910 {
8911         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
8912 }
8913
8914 static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
8915                                   struct intel_crtc_state *crtc_state,
8916                                   struct dpll *reduced_clock)
8917 {
8918         struct drm_crtc *crtc = &intel_crtc->base;
8919         struct drm_device *dev = crtc->dev;
8920         struct drm_i915_private *dev_priv = dev->dev_private;
8921         struct drm_atomic_state *state = crtc_state->base.state;
8922         struct drm_connector *connector;
8923         struct drm_connector_state *connector_state;
8924         struct intel_encoder *encoder;
8925         u32 dpll, fp, fp2;
8926         int factor, i;
8927         bool is_lvds = false, is_sdvo = false;
8928
8929         for_each_connector_in_state(state, connector, connector_state, i) {
8930                 if (connector_state->crtc != crtc_state->base.crtc)
8931                         continue;
8932
8933                 encoder = to_intel_encoder(connector_state->best_encoder);
8934
8935                 switch (encoder->type) {
8936                 case INTEL_OUTPUT_LVDS:
8937                         is_lvds = true;
8938                         break;
8939                 case INTEL_OUTPUT_SDVO:
8940                 case INTEL_OUTPUT_HDMI:
8941                         is_sdvo = true;
8942                         break;
8943                 default:
8944                         break;
8945                 }
8946         }
8947
8948         /* Enable autotuning of the PLL clock (if permissible) */
8949         factor = 21;
8950         if (is_lvds) {
8951                 if ((intel_panel_use_ssc(dev_priv) &&
8952                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
8953                     (HAS_PCH_IBX(dev) && intel_is_dual_link_lvds(dev)))
8954                         factor = 25;
8955         } else if (crtc_state->sdvo_tv_clock)
8956                 factor = 20;
8957
8958         fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
8959
8960         if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor))
8961                 fp |= FP_CB_TUNE;
8962
8963         if (reduced_clock) {
8964                 fp2 = i9xx_dpll_compute_fp(reduced_clock);
8965
8966                 if (reduced_clock->m < factor * reduced_clock->n)
8967                         fp2 |= FP_CB_TUNE;
8968         } else {
8969                 fp2 = fp;
8970         }
8971
8972         dpll = 0;
8973
8974         if (is_lvds)
8975                 dpll |= DPLLB_MODE_LVDS;
8976         else
8977                 dpll |= DPLLB_MODE_DAC_SERIAL;
8978
8979         dpll |= (crtc_state->pixel_multiplier - 1)
8980                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
8981
8982         if (is_sdvo)
8983                 dpll |= DPLL_SDVO_HIGH_SPEED;
8984         if (crtc_state->has_dp_encoder)
8985                 dpll |= DPLL_SDVO_HIGH_SPEED;
8986
8987         /* compute bitmask from p1 value */
8988         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8989         /* also FPA1 */
8990         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8991
8992         switch (crtc_state->dpll.p2) {
8993         case 5:
8994                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8995                 break;
8996         case 7:
8997                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8998                 break;
8999         case 10:
9000                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
9001                 break;
9002         case 14:
9003                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
9004                 break;
9005         }
9006
9007         if (is_lvds && intel_panel_use_ssc(dev_priv))
9008                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
9009         else
9010                 dpll |= PLL_REF_INPUT_DREFCLK;
9011
9012         dpll |= DPLL_VCO_ENABLE;
9013
9014         crtc_state->dpll_hw_state.dpll = dpll;
9015         crtc_state->dpll_hw_state.fp0 = fp;
9016         crtc_state->dpll_hw_state.fp1 = fp2;
9017 }
9018
9019 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
9020                                        struct intel_crtc_state *crtc_state)
9021 {
9022         struct drm_device *dev = crtc->base.dev;
9023         struct drm_i915_private *dev_priv = dev->dev_private;
9024         struct dpll reduced_clock;
9025         bool has_reduced_clock = false;
9026         struct intel_shared_dpll *pll;
9027         const struct intel_limit *limit;
9028         int refclk = 120000;
9029
9030         memset(&crtc_state->dpll_hw_state, 0,
9031                sizeof(crtc_state->dpll_hw_state));
9032
9033         crtc->lowfreq_avail = false;
9034
9035         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
9036         if (!crtc_state->has_pch_encoder)
9037                 return 0;
9038
9039         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
9040                 if (intel_panel_use_ssc(dev_priv)) {
9041                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
9042                                       dev_priv->vbt.lvds_ssc_freq);
9043                         refclk = dev_priv->vbt.lvds_ssc_freq;
9044                 }
9045
9046                 if (intel_is_dual_link_lvds(dev)) {
9047                         if (refclk == 100000)
9048                                 limit = &intel_limits_ironlake_dual_lvds_100m;
9049                         else
9050                                 limit = &intel_limits_ironlake_dual_lvds;
9051                 } else {
9052                         if (refclk == 100000)
9053                                 limit = &intel_limits_ironlake_single_lvds_100m;
9054                         else
9055                                 limit = &intel_limits_ironlake_single_lvds;
9056                 }
9057         } else {
9058                 limit = &intel_limits_ironlake_dac;
9059         }
9060
9061         if (!crtc_state->clock_set &&
9062             !g4x_find_best_dpll(limit, crtc_state, crtc_state->port_clock,
9063                                 refclk, NULL, &crtc_state->dpll)) {
9064                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
9065                 return -EINVAL;
9066         }
9067
9068         ironlake_compute_dpll(crtc, crtc_state,
9069                               has_reduced_clock ? &reduced_clock : NULL);
9070
9071         pll = intel_get_shared_dpll(crtc, crtc_state, NULL);
9072         if (pll == NULL) {
9073                 DRM_DEBUG_DRIVER("failed to find PLL for pipe %c\n",
9074                                  pipe_name(crtc->pipe));
9075                 return -EINVAL;
9076         }
9077
9078         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
9079             has_reduced_clock)
9080                 crtc->lowfreq_avail = true;
9081
9082         return 0;
9083 }
9084
9085 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
9086                                          struct intel_link_m_n *m_n)
9087 {
9088         struct drm_device *dev = crtc->base.dev;
9089         struct drm_i915_private *dev_priv = dev->dev_private;
9090         enum pipe pipe = crtc->pipe;
9091
9092         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
9093         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
9094         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
9095                 & ~TU_SIZE_MASK;
9096         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
9097         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
9098                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9099 }
9100
9101 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
9102                                          enum transcoder transcoder,
9103                                          struct intel_link_m_n *m_n,
9104                                          struct intel_link_m_n *m2_n2)
9105 {
9106         struct drm_device *dev = crtc->base.dev;
9107         struct drm_i915_private *dev_priv = dev->dev_private;
9108         enum pipe pipe = crtc->pipe;
9109
9110         if (INTEL_INFO(dev)->gen >= 5) {
9111                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
9112                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
9113                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
9114                         & ~TU_SIZE_MASK;
9115                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
9116                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
9117                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9118                 /* Read M2_N2 registers only for gen < 8 (M2_N2 available for
9119                  * gen < 8) and if DRRS is supported (to make sure the
9120                  * registers are not unnecessarily read).
9121                  */
9122                 if (m2_n2 && INTEL_INFO(dev)->gen < 8 &&
9123                         crtc->config->has_drrs) {
9124                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
9125                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
9126                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
9127                                         & ~TU_SIZE_MASK;
9128                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
9129                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
9130                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9131                 }
9132         } else {
9133                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
9134                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
9135                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
9136                         & ~TU_SIZE_MASK;
9137                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
9138                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
9139                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
9140         }
9141 }
9142
9143 void intel_dp_get_m_n(struct intel_crtc *crtc,
9144                       struct intel_crtc_state *pipe_config)
9145 {
9146         if (pipe_config->has_pch_encoder)
9147                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
9148         else
9149                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
9150                                              &pipe_config->dp_m_n,
9151                                              &pipe_config->dp_m2_n2);
9152 }
9153
9154 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
9155                                         struct intel_crtc_state *pipe_config)
9156 {
9157         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
9158                                      &pipe_config->fdi_m_n, NULL);
9159 }
9160
9161 static void skylake_get_pfit_config(struct intel_crtc *crtc,
9162                                     struct intel_crtc_state *pipe_config)
9163 {
9164         struct drm_device *dev = crtc->base.dev;
9165         struct drm_i915_private *dev_priv = dev->dev_private;
9166         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
9167         uint32_t ps_ctrl = 0;
9168         int id = -1;
9169         int i;
9170
9171         /* find scaler attached to this pipe */
9172         for (i = 0; i < crtc->num_scalers; i++) {
9173                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
9174                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
9175                         id = i;
9176                         pipe_config->pch_pfit.enabled = true;
9177                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
9178                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
9179                         break;
9180                 }
9181         }
9182
9183         scaler_state->scaler_id = id;
9184         if (id >= 0) {
9185                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
9186         } else {
9187                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
9188         }
9189 }
9190
9191 static void
9192 skylake_get_initial_plane_config(struct intel_crtc *crtc,
9193                                  struct intel_initial_plane_config *plane_config)
9194 {
9195         struct drm_device *dev = crtc->base.dev;
9196         struct drm_i915_private *dev_priv = dev->dev_private;
9197         u32 val, base, offset, stride_mult, tiling;
9198         int pipe = crtc->pipe;
9199         int fourcc, pixel_format;
9200         unsigned int aligned_height;
9201         struct drm_framebuffer *fb;
9202         struct intel_framebuffer *intel_fb;
9203
9204         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
9205         if (!intel_fb) {
9206                 DRM_DEBUG_KMS("failed to alloc fb\n");
9207                 return;
9208         }
9209
9210         fb = &intel_fb->base;
9211
9212         val = I915_READ(PLANE_CTL(pipe, 0));
9213         if (!(val & PLANE_CTL_ENABLE))
9214                 goto error;
9215
9216         pixel_format = val & PLANE_CTL_FORMAT_MASK;
9217         fourcc = skl_format_to_fourcc(pixel_format,
9218                                       val & PLANE_CTL_ORDER_RGBX,
9219                                       val & PLANE_CTL_ALPHA_MASK);
9220         fb->pixel_format = fourcc;
9221         fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
9222
9223         tiling = val & PLANE_CTL_TILED_MASK;
9224         switch (tiling) {
9225         case PLANE_CTL_TILED_LINEAR:
9226                 fb->modifier[0] = DRM_FORMAT_MOD_NONE;
9227                 break;
9228         case PLANE_CTL_TILED_X:
9229                 plane_config->tiling = I915_TILING_X;
9230                 fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
9231                 break;
9232         case PLANE_CTL_TILED_Y:
9233                 fb->modifier[0] = I915_FORMAT_MOD_Y_TILED;
9234                 break;
9235         case PLANE_CTL_TILED_YF:
9236                 fb->modifier[0] = I915_FORMAT_MOD_Yf_TILED;
9237                 break;
9238         default:
9239                 MISSING_CASE(tiling);
9240                 goto error;
9241         }
9242
9243         base = I915_READ(PLANE_SURF(pipe, 0)) & 0xfffff000;
9244         plane_config->base = base;
9245
9246         offset = I915_READ(PLANE_OFFSET(pipe, 0));
9247
9248         val = I915_READ(PLANE_SIZE(pipe, 0));
9249         fb->height = ((val >> 16) & 0xfff) + 1;
9250         fb->width = ((val >> 0) & 0x1fff) + 1;
9251
9252         val = I915_READ(PLANE_STRIDE(pipe, 0));
9253         stride_mult = intel_fb_stride_alignment(dev_priv, fb->modifier[0],
9254                                                 fb->pixel_format);
9255         fb->pitches[0] = (val & 0x3ff) * stride_mult;
9256
9257         aligned_height = intel_fb_align_height(dev, fb->height,
9258                                                fb->pixel_format,
9259                                                fb->modifier[0]);
9260
9261         plane_config->size = fb->pitches[0] * aligned_height;
9262
9263         DRM_DEBUG_KMS("pipe %c with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9264                       pipe_name(pipe), fb->width, fb->height,
9265                       fb->bits_per_pixel, base, fb->pitches[0],
9266                       plane_config->size);
9267
9268         plane_config->fb = intel_fb;
9269         return;
9270
9271 error:
9272         kfree(fb);
9273 }
9274
9275 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
9276                                      struct intel_crtc_state *pipe_config)
9277 {
9278         struct drm_device *dev = crtc->base.dev;
9279         struct drm_i915_private *dev_priv = dev->dev_private;
9280         uint32_t tmp;
9281
9282         tmp = I915_READ(PF_CTL(crtc->pipe));
9283
9284         if (tmp & PF_ENABLE) {
9285                 pipe_config->pch_pfit.enabled = true;
9286                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
9287                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
9288
9289                 /* We currently do not free assignements of panel fitters on
9290                  * ivb/hsw (since we don't use the higher upscaling modes which
9291                  * differentiates them) so just WARN about this case for now. */
9292                 if (IS_GEN7(dev)) {
9293                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
9294                                 PF_PIPE_SEL_IVB(crtc->pipe));
9295                 }
9296         }
9297 }
9298
9299 static void
9300 ironlake_get_initial_plane_config(struct intel_crtc *crtc,
9301                                   struct intel_initial_plane_config *plane_config)
9302 {
9303         struct drm_device *dev = crtc->base.dev;
9304         struct drm_i915_private *dev_priv = dev->dev_private;
9305         u32 val, base, offset;
9306         int pipe = crtc->pipe;
9307         int fourcc, pixel_format;
9308         unsigned int aligned_height;
9309         struct drm_framebuffer *fb;
9310         struct intel_framebuffer *intel_fb;
9311
9312         val = I915_READ(DSPCNTR(pipe));
9313         if (!(val & DISPLAY_PLANE_ENABLE))
9314                 return;
9315
9316         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
9317         if (!intel_fb) {
9318                 DRM_DEBUG_KMS("failed to alloc fb\n");
9319                 return;
9320         }
9321
9322         fb = &intel_fb->base;
9323
9324         if (INTEL_INFO(dev)->gen >= 4) {
9325                 if (val & DISPPLANE_TILED) {
9326                         plane_config->tiling = I915_TILING_X;
9327                         fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
9328                 }
9329         }
9330
9331         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
9332         fourcc = i9xx_format_to_fourcc(pixel_format);
9333         fb->pixel_format = fourcc;
9334         fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
9335
9336         base = I915_READ(DSPSURF(pipe)) & 0xfffff000;
9337         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
9338                 offset = I915_READ(DSPOFFSET(pipe));
9339         } else {
9340                 if (plane_config->tiling)
9341                         offset = I915_READ(DSPTILEOFF(pipe));
9342                 else
9343                         offset = I915_READ(DSPLINOFF(pipe));
9344         }
9345         plane_config->base = base;
9346
9347         val = I915_READ(PIPESRC(pipe));
9348         fb->width = ((val >> 16) & 0xfff) + 1;
9349         fb->height = ((val >> 0) & 0xfff) + 1;
9350
9351         val = I915_READ(DSPSTRIDE(pipe));
9352         fb->pitches[0] = val & 0xffffffc0;
9353
9354         aligned_height = intel_fb_align_height(dev, fb->height,
9355                                                fb->pixel_format,
9356                                                fb->modifier[0]);
9357
9358         plane_config->size = fb->pitches[0] * aligned_height;
9359
9360         DRM_DEBUG_KMS("pipe %c with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9361                       pipe_name(pipe), fb->width, fb->height,
9362                       fb->bits_per_pixel, base, fb->pitches[0],
9363                       plane_config->size);
9364
9365         plane_config->fb = intel_fb;
9366 }
9367
9368 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
9369                                      struct intel_crtc_state *pipe_config)
9370 {
9371         struct drm_device *dev = crtc->base.dev;
9372         struct drm_i915_private *dev_priv = dev->dev_private;
9373         enum intel_display_power_domain power_domain;
9374         uint32_t tmp;
9375         bool ret;
9376
9377         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
9378         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
9379                 return false;
9380
9381         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9382         pipe_config->shared_dpll = NULL;
9383
9384         ret = false;
9385         tmp = I915_READ(PIPECONF(crtc->pipe));
9386         if (!(tmp & PIPECONF_ENABLE))
9387                 goto out;
9388
9389         switch (tmp & PIPECONF_BPC_MASK) {
9390         case PIPECONF_6BPC:
9391                 pipe_config->pipe_bpp = 18;
9392                 break;
9393         case PIPECONF_8BPC:
9394                 pipe_config->pipe_bpp = 24;
9395                 break;
9396         case PIPECONF_10BPC:
9397                 pipe_config->pipe_bpp = 30;
9398                 break;
9399         case PIPECONF_12BPC:
9400                 pipe_config->pipe_bpp = 36;
9401                 break;
9402         default:
9403                 break;
9404         }
9405
9406         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
9407                 pipe_config->limited_color_range = true;
9408
9409         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
9410                 struct intel_shared_dpll *pll;
9411                 enum intel_dpll_id pll_id;
9412
9413                 pipe_config->has_pch_encoder = true;
9414
9415                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
9416                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9417                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9418
9419                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9420
9421                 if (HAS_PCH_IBX(dev_priv)) {
9422                         /*
9423                          * The pipe->pch transcoder and pch transcoder->pll
9424                          * mapping is fixed.
9425                          */
9426                         pll_id = (enum intel_dpll_id) crtc->pipe;
9427                 } else {
9428                         tmp = I915_READ(PCH_DPLL_SEL);
9429                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
9430                                 pll_id = DPLL_ID_PCH_PLL_B;
9431                         else
9432                                 pll_id= DPLL_ID_PCH_PLL_A;
9433                 }
9434
9435                 pipe_config->shared_dpll =
9436                         intel_get_shared_dpll_by_id(dev_priv, pll_id);
9437                 pll = pipe_config->shared_dpll;
9438
9439                 WARN_ON(!pll->funcs.get_hw_state(dev_priv, pll,
9440                                                  &pipe_config->dpll_hw_state));
9441
9442                 tmp = pipe_config->dpll_hw_state.dpll;
9443                 pipe_config->pixel_multiplier =
9444                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
9445                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
9446
9447                 ironlake_pch_clock_get(crtc, pipe_config);
9448         } else {
9449                 pipe_config->pixel_multiplier = 1;
9450         }
9451
9452         intel_get_pipe_timings(crtc, pipe_config);
9453         intel_get_pipe_src_size(crtc, pipe_config);
9454
9455         ironlake_get_pfit_config(crtc, pipe_config);
9456
9457         ret = true;
9458
9459 out:
9460         intel_display_power_put(dev_priv, power_domain);
9461
9462         return ret;
9463 }
9464
9465 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
9466 {
9467         struct drm_device *dev = dev_priv->dev;
9468         struct intel_crtc *crtc;
9469
9470         for_each_intel_crtc(dev, crtc)
9471                 I915_STATE_WARN(crtc->active, "CRTC for pipe %c enabled\n",
9472                      pipe_name(crtc->pipe));
9473
9474         I915_STATE_WARN(I915_READ(HSW_PWR_WELL_DRIVER), "Power well on\n");
9475         I915_STATE_WARN(I915_READ(SPLL_CTL) & SPLL_PLL_ENABLE, "SPLL enabled\n");
9476         I915_STATE_WARN(I915_READ(WRPLL_CTL(0)) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n");
9477         I915_STATE_WARN(I915_READ(WRPLL_CTL(1)) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n");
9478         I915_STATE_WARN(I915_READ(PCH_PP_STATUS) & PP_ON, "Panel power on\n");
9479         I915_STATE_WARN(I915_READ(BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
9480              "CPU PWM1 enabled\n");
9481         if (IS_HASWELL(dev))
9482                 I915_STATE_WARN(I915_READ(HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
9483                      "CPU PWM2 enabled\n");
9484         I915_STATE_WARN(I915_READ(BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
9485              "PCH PWM1 enabled\n");
9486         I915_STATE_WARN(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
9487              "Utility pin enabled\n");
9488         I915_STATE_WARN(I915_READ(PCH_GTC_CTL) & PCH_GTC_ENABLE, "PCH GTC enabled\n");
9489
9490         /*
9491          * In theory we can still leave IRQs enabled, as long as only the HPD
9492          * interrupts remain enabled. We used to check for that, but since it's
9493          * gen-specific and since we only disable LCPLL after we fully disable
9494          * the interrupts, the check below should be enough.
9495          */
9496         I915_STATE_WARN(intel_irqs_enabled(dev_priv), "IRQs enabled\n");
9497 }
9498
9499 static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv)
9500 {
9501         struct drm_device *dev = dev_priv->dev;
9502
9503         if (IS_HASWELL(dev))
9504                 return I915_READ(D_COMP_HSW);
9505         else
9506                 return I915_READ(D_COMP_BDW);
9507 }
9508
9509 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val)
9510 {
9511         struct drm_device *dev = dev_priv->dev;
9512
9513         if (IS_HASWELL(dev)) {
9514                 mutex_lock(&dev_priv->rps.hw_lock);
9515                 if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP,
9516                                             val))
9517                         DRM_ERROR("Failed to write to D_COMP\n");
9518                 mutex_unlock(&dev_priv->rps.hw_lock);
9519         } else {
9520                 I915_WRITE(D_COMP_BDW, val);
9521                 POSTING_READ(D_COMP_BDW);
9522         }
9523 }
9524
9525 /*
9526  * This function implements pieces of two sequences from BSpec:
9527  * - Sequence for display software to disable LCPLL
9528  * - Sequence for display software to allow package C8+
9529  * The steps implemented here are just the steps that actually touch the LCPLL
9530  * register. Callers should take care of disabling all the display engine
9531  * functions, doing the mode unset, fixing interrupts, etc.
9532  */
9533 static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
9534                               bool switch_to_fclk, bool allow_power_down)
9535 {
9536         uint32_t val;
9537
9538         assert_can_disable_lcpll(dev_priv);
9539
9540         val = I915_READ(LCPLL_CTL);
9541
9542         if (switch_to_fclk) {
9543                 val |= LCPLL_CD_SOURCE_FCLK;
9544                 I915_WRITE(LCPLL_CTL, val);
9545
9546                 if (wait_for_us(I915_READ(LCPLL_CTL) &
9547                                 LCPLL_CD_SOURCE_FCLK_DONE, 1))
9548                         DRM_ERROR("Switching to FCLK failed\n");
9549
9550                 val = I915_READ(LCPLL_CTL);
9551         }
9552
9553         val |= LCPLL_PLL_DISABLE;
9554         I915_WRITE(LCPLL_CTL, val);
9555         POSTING_READ(LCPLL_CTL);
9556
9557         if (wait_for((I915_READ(LCPLL_CTL) & LCPLL_PLL_LOCK) == 0, 1))
9558                 DRM_ERROR("LCPLL still locked\n");
9559
9560         val = hsw_read_dcomp(dev_priv);
9561         val |= D_COMP_COMP_DISABLE;
9562         hsw_write_dcomp(dev_priv, val);
9563         ndelay(100);
9564
9565         if (wait_for((hsw_read_dcomp(dev_priv) & D_COMP_RCOMP_IN_PROGRESS) == 0,
9566                      1))
9567                 DRM_ERROR("D_COMP RCOMP still in progress\n");
9568
9569         if (allow_power_down) {
9570                 val = I915_READ(LCPLL_CTL);
9571                 val |= LCPLL_POWER_DOWN_ALLOW;
9572                 I915_WRITE(LCPLL_CTL, val);
9573                 POSTING_READ(LCPLL_CTL);
9574         }
9575 }
9576
9577 /*
9578  * Fully restores LCPLL, disallowing power down and switching back to LCPLL
9579  * source.
9580  */
9581 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
9582 {
9583         uint32_t val;
9584
9585         val = I915_READ(LCPLL_CTL);
9586
9587         if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
9588                     LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
9589                 return;
9590
9591         /*
9592          * Make sure we're not on PC8 state before disabling PC8, otherwise
9593          * we'll hang the machine. To prevent PC8 state, just enable force_wake.
9594          */
9595         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
9596
9597         if (val & LCPLL_POWER_DOWN_ALLOW) {
9598                 val &= ~LCPLL_POWER_DOWN_ALLOW;
9599                 I915_WRITE(LCPLL_CTL, val);
9600                 POSTING_READ(LCPLL_CTL);
9601         }
9602
9603         val = hsw_read_dcomp(dev_priv);
9604         val |= D_COMP_COMP_FORCE;
9605         val &= ~D_COMP_COMP_DISABLE;
9606         hsw_write_dcomp(dev_priv, val);
9607
9608         val = I915_READ(LCPLL_CTL);
9609         val &= ~LCPLL_PLL_DISABLE;
9610         I915_WRITE(LCPLL_CTL, val);
9611
9612         if (wait_for(I915_READ(LCPLL_CTL) & LCPLL_PLL_LOCK, 5))
9613                 DRM_ERROR("LCPLL not locked yet\n");
9614
9615         if (val & LCPLL_CD_SOURCE_FCLK) {
9616                 val = I915_READ(LCPLL_CTL);
9617                 val &= ~LCPLL_CD_SOURCE_FCLK;
9618                 I915_WRITE(LCPLL_CTL, val);
9619
9620                 if (wait_for_us((I915_READ(LCPLL_CTL) &
9621                                  LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
9622                         DRM_ERROR("Switching back to LCPLL failed\n");
9623         }
9624
9625         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
9626         intel_update_cdclk(dev_priv->dev);
9627 }
9628
9629 /*
9630  * Package states C8 and deeper are really deep PC states that can only be
9631  * reached when all the devices on the system allow it, so even if the graphics
9632  * device allows PC8+, it doesn't mean the system will actually get to these
9633  * states. Our driver only allows PC8+ when going into runtime PM.
9634  *
9635  * The requirements for PC8+ are that all the outputs are disabled, the power
9636  * well is disabled and most interrupts are disabled, and these are also
9637  * requirements for runtime PM. When these conditions are met, we manually do
9638  * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
9639  * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
9640  * hang the machine.
9641  *
9642  * When we really reach PC8 or deeper states (not just when we allow it) we lose
9643  * the state of some registers, so when we come back from PC8+ we need to
9644  * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
9645  * need to take care of the registers kept by RC6. Notice that this happens even
9646  * if we don't put the device in PCI D3 state (which is what currently happens
9647  * because of the runtime PM support).
9648  *
9649  * For more, read "Display Sequences for Package C8" on the hardware
9650  * documentation.
9651  */
9652 void hsw_enable_pc8(struct drm_i915_private *dev_priv)
9653 {
9654         struct drm_device *dev = dev_priv->dev;
9655         uint32_t val;
9656
9657         DRM_DEBUG_KMS("Enabling package C8+\n");
9658
9659         if (HAS_PCH_LPT_LP(dev)) {
9660                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9661                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
9662                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9663         }
9664
9665         lpt_disable_clkout_dp(dev);
9666         hsw_disable_lcpll(dev_priv, true, true);
9667 }
9668
9669 void hsw_disable_pc8(struct drm_i915_private *dev_priv)
9670 {
9671         struct drm_device *dev = dev_priv->dev;
9672         uint32_t val;
9673
9674         DRM_DEBUG_KMS("Disabling package C8+\n");
9675
9676         hsw_restore_lcpll(dev_priv);
9677         lpt_init_pch_refclk(dev);
9678
9679         if (HAS_PCH_LPT_LP(dev)) {
9680                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9681                 val |= PCH_LP_PARTITION_LEVEL_DISABLE;
9682                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9683         }
9684 }
9685
9686 static void bxt_modeset_commit_cdclk(struct drm_atomic_state *old_state)
9687 {
9688         struct drm_device *dev = old_state->dev;
9689         struct intel_atomic_state *old_intel_state =
9690                 to_intel_atomic_state(old_state);
9691         unsigned int req_cdclk = old_intel_state->dev_cdclk;
9692
9693         bxt_set_cdclk(to_i915(dev), req_cdclk);
9694 }
9695
9696 /* compute the max rate for new configuration */
9697 static int ilk_max_pixel_rate(struct drm_atomic_state *state)
9698 {
9699         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
9700         struct drm_i915_private *dev_priv = state->dev->dev_private;
9701         struct drm_crtc *crtc;
9702         struct drm_crtc_state *cstate;
9703         struct intel_crtc_state *crtc_state;
9704         unsigned max_pixel_rate = 0, i;
9705         enum pipe pipe;
9706
9707         memcpy(intel_state->min_pixclk, dev_priv->min_pixclk,
9708                sizeof(intel_state->min_pixclk));
9709
9710         for_each_crtc_in_state(state, crtc, cstate, i) {
9711                 int pixel_rate;
9712
9713                 crtc_state = to_intel_crtc_state(cstate);
9714                 if (!crtc_state->base.enable) {
9715                         intel_state->min_pixclk[i] = 0;
9716                         continue;
9717                 }
9718
9719                 pixel_rate = ilk_pipe_pixel_rate(crtc_state);
9720
9721                 /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
9722                 if (IS_BROADWELL(dev_priv) && crtc_state->ips_enabled)
9723                         pixel_rate = DIV_ROUND_UP(pixel_rate * 100, 95);
9724
9725                 intel_state->min_pixclk[i] = pixel_rate;
9726         }
9727
9728         for_each_pipe(dev_priv, pipe)
9729                 max_pixel_rate = max(intel_state->min_pixclk[pipe], max_pixel_rate);
9730
9731         return max_pixel_rate;
9732 }
9733
9734 static void broadwell_set_cdclk(struct drm_device *dev, int cdclk)
9735 {
9736         struct drm_i915_private *dev_priv = dev->dev_private;
9737         uint32_t val, data;
9738         int ret;
9739
9740         if (WARN((I915_READ(LCPLL_CTL) &
9741                   (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
9742                    LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
9743                    LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
9744                    LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
9745                  "trying to change cdclk frequency with cdclk not enabled\n"))
9746                 return;
9747
9748         mutex_lock(&dev_priv->rps.hw_lock);
9749         ret = sandybridge_pcode_write(dev_priv,
9750                                       BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
9751         mutex_unlock(&dev_priv->rps.hw_lock);
9752         if (ret) {
9753                 DRM_ERROR("failed to inform pcode about cdclk change\n");
9754                 return;
9755         }
9756
9757         val = I915_READ(LCPLL_CTL);
9758         val |= LCPLL_CD_SOURCE_FCLK;
9759         I915_WRITE(LCPLL_CTL, val);
9760
9761         if (wait_for_us(I915_READ(LCPLL_CTL) &
9762                         LCPLL_CD_SOURCE_FCLK_DONE, 1))
9763                 DRM_ERROR("Switching to FCLK failed\n");
9764
9765         val = I915_READ(LCPLL_CTL);
9766         val &= ~LCPLL_CLK_FREQ_MASK;
9767
9768         switch (cdclk) {
9769         case 450000:
9770                 val |= LCPLL_CLK_FREQ_450;
9771                 data = 0;
9772                 break;
9773         case 540000:
9774                 val |= LCPLL_CLK_FREQ_54O_BDW;
9775                 data = 1;
9776                 break;
9777         case 337500:
9778                 val |= LCPLL_CLK_FREQ_337_5_BDW;
9779                 data = 2;
9780                 break;
9781         case 675000:
9782                 val |= LCPLL_CLK_FREQ_675_BDW;
9783                 data = 3;
9784                 break;
9785         default:
9786                 WARN(1, "invalid cdclk frequency\n");
9787                 return;
9788         }
9789
9790         I915_WRITE(LCPLL_CTL, val);
9791
9792         val = I915_READ(LCPLL_CTL);
9793         val &= ~LCPLL_CD_SOURCE_FCLK;
9794         I915_WRITE(LCPLL_CTL, val);
9795
9796         if (wait_for_us((I915_READ(LCPLL_CTL) &
9797                         LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
9798                 DRM_ERROR("Switching back to LCPLL failed\n");
9799
9800         mutex_lock(&dev_priv->rps.hw_lock);
9801         sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ, data);
9802         mutex_unlock(&dev_priv->rps.hw_lock);
9803
9804         I915_WRITE(CDCLK_FREQ, DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
9805
9806         intel_update_cdclk(dev);
9807
9808         WARN(cdclk != dev_priv->cdclk_freq,
9809              "cdclk requested %d kHz but got %d kHz\n",
9810              cdclk, dev_priv->cdclk_freq);
9811 }
9812
9813 static int broadwell_calc_cdclk(int max_pixclk)
9814 {
9815         if (max_pixclk > 540000)
9816                 return 675000;
9817         else if (max_pixclk > 450000)
9818                 return 540000;
9819         else if (max_pixclk > 337500)
9820                 return 450000;
9821         else
9822                 return 337500;
9823 }
9824
9825 static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state)
9826 {
9827         struct drm_i915_private *dev_priv = to_i915(state->dev);
9828         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
9829         int max_pixclk = ilk_max_pixel_rate(state);
9830         int cdclk;
9831
9832         /*
9833          * FIXME should also account for plane ratio
9834          * once 64bpp pixel formats are supported.
9835          */
9836         cdclk = broadwell_calc_cdclk(max_pixclk);
9837
9838         if (cdclk > dev_priv->max_cdclk_freq) {
9839                 DRM_DEBUG_KMS("requested cdclk (%d kHz) exceeds max (%d kHz)\n",
9840                               cdclk, dev_priv->max_cdclk_freq);
9841                 return -EINVAL;
9842         }
9843
9844         intel_state->cdclk = intel_state->dev_cdclk = cdclk;
9845         if (!intel_state->active_crtcs)
9846                 intel_state->dev_cdclk = broadwell_calc_cdclk(0);
9847
9848         return 0;
9849 }
9850
9851 static void broadwell_modeset_commit_cdclk(struct drm_atomic_state *old_state)
9852 {
9853         struct drm_device *dev = old_state->dev;
9854         struct intel_atomic_state *old_intel_state =
9855                 to_intel_atomic_state(old_state);
9856         unsigned req_cdclk = old_intel_state->dev_cdclk;
9857
9858         broadwell_set_cdclk(dev, req_cdclk);
9859 }
9860
9861 static int skl_modeset_calc_cdclk(struct drm_atomic_state *state)
9862 {
9863         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
9864         struct drm_i915_private *dev_priv = to_i915(state->dev);
9865         const int max_pixclk = ilk_max_pixel_rate(state);
9866         int vco = intel_state->cdclk_pll_vco;
9867         int cdclk;
9868
9869         /*
9870          * FIXME should also account for plane ratio
9871          * once 64bpp pixel formats are supported.
9872          */
9873         cdclk = skl_calc_cdclk(max_pixclk, vco);
9874
9875         /*
9876          * FIXME move the cdclk caclulation to
9877          * compute_config() so we can fail gracegully.
9878          */
9879         if (cdclk > dev_priv->max_cdclk_freq) {
9880                 DRM_ERROR("requested cdclk (%d kHz) exceeds max (%d kHz)\n",
9881                           cdclk, dev_priv->max_cdclk_freq);
9882                 cdclk = dev_priv->max_cdclk_freq;
9883         }
9884
9885         intel_state->cdclk = intel_state->dev_cdclk = cdclk;
9886         if (!intel_state->active_crtcs)
9887                 intel_state->dev_cdclk = skl_calc_cdclk(0, vco);
9888
9889         return 0;
9890 }
9891
9892 static void skl_modeset_commit_cdclk(struct drm_atomic_state *old_state)
9893 {
9894         struct drm_i915_private *dev_priv = to_i915(old_state->dev);
9895         struct intel_atomic_state *intel_state = to_intel_atomic_state(old_state);
9896         unsigned int req_cdclk = intel_state->dev_cdclk;
9897         unsigned int req_vco = intel_state->cdclk_pll_vco;
9898
9899         skl_set_cdclk(dev_priv, req_cdclk, req_vco);
9900 }
9901
9902 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
9903                                       struct intel_crtc_state *crtc_state)
9904 {
9905         struct intel_encoder *intel_encoder =
9906                 intel_ddi_get_crtc_new_encoder(crtc_state);
9907
9908         if (intel_encoder->type != INTEL_OUTPUT_DSI) {
9909                 if (!intel_ddi_pll_select(crtc, crtc_state))
9910                         return -EINVAL;
9911         }
9912
9913         crtc->lowfreq_avail = false;
9914
9915         return 0;
9916 }
9917
9918 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
9919                                 enum port port,
9920                                 struct intel_crtc_state *pipe_config)
9921 {
9922         enum intel_dpll_id id;
9923
9924         switch (port) {
9925         case PORT_A:
9926                 pipe_config->ddi_pll_sel = SKL_DPLL0;
9927                 id = DPLL_ID_SKL_DPLL0;
9928                 break;
9929         case PORT_B:
9930                 pipe_config->ddi_pll_sel = SKL_DPLL1;
9931                 id = DPLL_ID_SKL_DPLL1;
9932                 break;
9933         case PORT_C:
9934                 pipe_config->ddi_pll_sel = SKL_DPLL2;
9935                 id = DPLL_ID_SKL_DPLL2;
9936                 break;
9937         default:
9938                 DRM_ERROR("Incorrect port type\n");
9939                 return;
9940         }
9941
9942         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9943 }
9944
9945 static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
9946                                 enum port port,
9947                                 struct intel_crtc_state *pipe_config)
9948 {
9949         enum intel_dpll_id id;
9950         u32 temp;
9951
9952         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
9953         pipe_config->ddi_pll_sel = temp >> (port * 3 + 1);
9954
9955         switch (pipe_config->ddi_pll_sel) {
9956         case SKL_DPLL0:
9957                 id = DPLL_ID_SKL_DPLL0;
9958                 break;
9959         case SKL_DPLL1:
9960                 id = DPLL_ID_SKL_DPLL1;
9961                 break;
9962         case SKL_DPLL2:
9963                 id = DPLL_ID_SKL_DPLL2;
9964                 break;
9965         case SKL_DPLL3:
9966                 id = DPLL_ID_SKL_DPLL3;
9967                 break;
9968         default:
9969                 MISSING_CASE(pipe_config->ddi_pll_sel);
9970                 return;
9971         }
9972
9973         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
9974 }
9975
9976 static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
9977                                 enum port port,
9978                                 struct intel_crtc_state *pipe_config)
9979 {
9980         enum intel_dpll_id id;
9981
9982         pipe_config->ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
9983
9984         switch (pipe_config->ddi_pll_sel) {
9985         case PORT_CLK_SEL_WRPLL1:
9986                 id = DPLL_ID_WRPLL1;
9987                 break;
9988         case PORT_CLK_SEL_WRPLL2:
9989                 id = DPLL_ID_WRPLL2;
9990                 break;
9991         case PORT_CLK_SEL_SPLL:
9992                 id = DPLL_ID_SPLL;
9993                 break;
9994         case PORT_CLK_SEL_LCPLL_810:
9995                 id = DPLL_ID_LCPLL_810;
9996                 break;
9997         case PORT_CLK_SEL_LCPLL_1350:
9998                 id = DPLL_ID_LCPLL_1350;
9999                 break;
10000         case PORT_CLK_SEL_LCPLL_2700:
10001                 id = DPLL_ID_LCPLL_2700;
10002                 break;
10003         default:
10004                 MISSING_CASE(pipe_config->ddi_pll_sel);
10005                 /* fall through */
10006         case PORT_CLK_SEL_NONE:
10007                 return;
10008         }
10009
10010         pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id);
10011 }
10012
10013 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
10014                                      struct intel_crtc_state *pipe_config,
10015                                      unsigned long *power_domain_mask)
10016 {
10017         struct drm_device *dev = crtc->base.dev;
10018         struct drm_i915_private *dev_priv = dev->dev_private;
10019         enum intel_display_power_domain power_domain;
10020         u32 tmp;
10021
10022         /*
10023          * The pipe->transcoder mapping is fixed with the exception of the eDP
10024          * transcoder handled below.
10025          */
10026         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
10027
10028         /*
10029          * XXX: Do intel_display_power_get_if_enabled before reading this (for
10030          * consistency and less surprising code; it's in always on power).
10031          */
10032         tmp = I915_READ(TRANS_DDI_FUNC_CTL(TRANSCODER_EDP));
10033         if (tmp & TRANS_DDI_FUNC_ENABLE) {
10034                 enum pipe trans_edp_pipe;
10035                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
10036                 default:
10037                         WARN(1, "unknown pipe linked to edp transcoder\n");
10038                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
10039                 case TRANS_DDI_EDP_INPUT_A_ON:
10040                         trans_edp_pipe = PIPE_A;
10041                         break;
10042                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
10043                         trans_edp_pipe = PIPE_B;
10044                         break;
10045                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
10046                         trans_edp_pipe = PIPE_C;
10047                         break;
10048                 }
10049
10050                 if (trans_edp_pipe == crtc->pipe)
10051                         pipe_config->cpu_transcoder = TRANSCODER_EDP;
10052         }
10053
10054         power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
10055         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
10056                 return false;
10057         *power_domain_mask |= BIT(power_domain);
10058
10059         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
10060
10061         return tmp & PIPECONF_ENABLE;
10062 }
10063
10064 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
10065                                          struct intel_crtc_state *pipe_config,
10066                                          unsigned long *power_domain_mask)
10067 {
10068         struct drm_device *dev = crtc->base.dev;
10069         struct drm_i915_private *dev_priv = dev->dev_private;
10070         enum intel_display_power_domain power_domain;
10071         enum port port;
10072         enum transcoder cpu_transcoder;
10073         u32 tmp;
10074
10075         pipe_config->has_dsi_encoder = false;
10076
10077         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
10078                 if (port == PORT_A)
10079                         cpu_transcoder = TRANSCODER_DSI_A;
10080                 else
10081                         cpu_transcoder = TRANSCODER_DSI_C;
10082
10083                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
10084                 if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
10085                         continue;
10086                 *power_domain_mask |= BIT(power_domain);
10087
10088                 /*
10089                  * The PLL needs to be enabled with a valid divider
10090                  * configuration, otherwise accessing DSI registers will hang
10091                  * the machine. See BSpec North Display Engine
10092                  * registers/MIPI[BXT]. We can break out here early, since we
10093                  * need the same DSI PLL to be enabled for both DSI ports.
10094                  */
10095                 if (!intel_dsi_pll_is_enabled(dev_priv))
10096                         break;
10097
10098                 /* XXX: this works for video mode only */
10099                 tmp = I915_READ(BXT_MIPI_PORT_CTRL(port));
10100                 if (!(tmp & DPI_ENABLE))
10101                         continue;
10102
10103                 tmp = I915_READ(MIPI_CTRL(port));
10104                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
10105                         continue;
10106
10107                 pipe_config->cpu_transcoder = cpu_transcoder;
10108                 pipe_config->has_dsi_encoder = true;
10109                 break;
10110         }
10111
10112         return pipe_config->has_dsi_encoder;
10113 }
10114
10115 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
10116                                        struct intel_crtc_state *pipe_config)
10117 {
10118         struct drm_device *dev = crtc->base.dev;
10119         struct drm_i915_private *dev_priv = dev->dev_private;
10120         struct intel_shared_dpll *pll;
10121         enum port port;
10122         uint32_t tmp;
10123
10124         tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
10125
10126         port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
10127
10128         if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
10129                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
10130         else if (IS_BROXTON(dev))
10131                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
10132         else
10133                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
10134
10135         pll = pipe_config->shared_dpll;
10136         if (pll) {
10137                 WARN_ON(!pll->funcs.get_hw_state(dev_priv, pll,
10138                                                  &pipe_config->dpll_hw_state));
10139         }
10140
10141         /*
10142          * Haswell has only FDI/PCH transcoder A. It is which is connected to
10143          * DDI E. So just check whether this pipe is wired to DDI E and whether
10144          * the PCH transcoder is on.
10145          */
10146         if (INTEL_INFO(dev)->gen < 9 &&
10147             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
10148                 pipe_config->has_pch_encoder = true;
10149
10150                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
10151                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
10152                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
10153
10154                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
10155         }
10156 }
10157
10158 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
10159                                     struct intel_crtc_state *pipe_config)
10160 {
10161         struct drm_device *dev = crtc->base.dev;
10162         struct drm_i915_private *dev_priv = dev->dev_private;
10163         enum intel_display_power_domain power_domain;
10164         unsigned long power_domain_mask;
10165         bool active;
10166
10167         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
10168         if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
10169                 return false;
10170         power_domain_mask = BIT(power_domain);
10171
10172         pipe_config->shared_dpll = NULL;
10173
10174         active = hsw_get_transcoder_state(crtc, pipe_config, &power_domain_mask);
10175
10176         if (IS_BROXTON(dev_priv)) {
10177                 bxt_get_dsi_transcoder_state(crtc, pipe_config,
10178                                              &power_domain_mask);
10179                 WARN_ON(active && pipe_config->has_dsi_encoder);
10180                 if (pipe_config->has_dsi_encoder)
10181                         active = true;
10182         }
10183
10184         if (!active)
10185                 goto out;
10186
10187         if (!pipe_config->has_dsi_encoder) {
10188                 haswell_get_ddi_port_state(crtc, pipe_config);
10189                 intel_get_pipe_timings(crtc, pipe_config);
10190         }
10191
10192         intel_get_pipe_src_size(crtc, pipe_config);
10193
10194         pipe_config->gamma_mode =
10195                 I915_READ(GAMMA_MODE(crtc->pipe)) & GAMMA_MODE_MODE_MASK;
10196
10197         if (INTEL_INFO(dev)->gen >= 9) {
10198                 skl_init_scalers(dev, crtc, pipe_config);
10199         }
10200
10201         if (INTEL_INFO(dev)->gen >= 9) {
10202                 pipe_config->scaler_state.scaler_id = -1;
10203                 pipe_config->scaler_state.scaler_users &= ~(1 << SKL_CRTC_INDEX);
10204         }
10205
10206         power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
10207         if (intel_display_power_get_if_enabled(dev_priv, power_domain)) {
10208                 power_domain_mask |= BIT(power_domain);
10209                 if (INTEL_INFO(dev)->gen >= 9)
10210                         skylake_get_pfit_config(crtc, pipe_config);
10211                 else
10212                         ironlake_get_pfit_config(crtc, pipe_config);
10213         }
10214
10215         if (IS_HASWELL(dev))
10216                 pipe_config->ips_enabled = hsw_crtc_supports_ips(crtc) &&
10217                         (I915_READ(IPS_CTL) & IPS_ENABLE);
10218
10219         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
10220             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
10221                 pipe_config->pixel_multiplier =
10222                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
10223         } else {
10224                 pipe_config->pixel_multiplier = 1;
10225         }
10226
10227 out:
10228         for_each_power_domain(power_domain, power_domain_mask)
10229                 intel_display_power_put(dev_priv, power_domain);
10230
10231         return active;
10232 }
10233
10234 static void i845_update_cursor(struct drm_crtc *crtc, u32 base,
10235                                const struct intel_plane_state *plane_state)
10236 {
10237         struct drm_device *dev = crtc->dev;
10238         struct drm_i915_private *dev_priv = dev->dev_private;
10239         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10240         uint32_t cntl = 0, size = 0;
10241
10242         if (plane_state && plane_state->visible) {
10243                 unsigned int width = plane_state->base.crtc_w;
10244                 unsigned int height = plane_state->base.crtc_h;
10245                 unsigned int stride = roundup_pow_of_two(width) * 4;
10246
10247                 switch (stride) {
10248                 default:
10249                         WARN_ONCE(1, "Invalid cursor width/stride, width=%u, stride=%u\n",
10250                                   width, stride);
10251                         stride = 256;
10252                         /* fallthrough */
10253                 case 256:
10254                 case 512:
10255                 case 1024:
10256                 case 2048:
10257                         break;
10258                 }
10259
10260                 cntl |= CURSOR_ENABLE |
10261                         CURSOR_GAMMA_ENABLE |
10262                         CURSOR_FORMAT_ARGB |
10263                         CURSOR_STRIDE(stride);
10264
10265                 size = (height << 12) | width;
10266         }
10267
10268         if (intel_crtc->cursor_cntl != 0 &&
10269             (intel_crtc->cursor_base != base ||
10270              intel_crtc->cursor_size != size ||
10271              intel_crtc->cursor_cntl != cntl)) {
10272                 /* On these chipsets we can only modify the base/size/stride
10273                  * whilst the cursor is disabled.
10274                  */
10275                 I915_WRITE(CURCNTR(PIPE_A), 0);
10276                 POSTING_READ(CURCNTR(PIPE_A));
10277                 intel_crtc->cursor_cntl = 0;
10278         }
10279
10280         if (intel_crtc->cursor_base != base) {
10281                 I915_WRITE(CURBASE(PIPE_A), base);
10282                 intel_crtc->cursor_base = base;
10283         }
10284
10285         if (intel_crtc->cursor_size != size) {
10286                 I915_WRITE(CURSIZE, size);
10287                 intel_crtc->cursor_size = size;
10288         }
10289
10290         if (intel_crtc->cursor_cntl != cntl) {
10291                 I915_WRITE(CURCNTR(PIPE_A), cntl);
10292                 POSTING_READ(CURCNTR(PIPE_A));
10293                 intel_crtc->cursor_cntl = cntl;
10294         }
10295 }
10296
10297 static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base,
10298                                const struct intel_plane_state *plane_state)
10299 {
10300         struct drm_device *dev = crtc->dev;
10301         struct drm_i915_private *dev_priv = dev->dev_private;
10302         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10303         int pipe = intel_crtc->pipe;
10304         uint32_t cntl = 0;
10305
10306         if (plane_state && plane_state->visible) {
10307                 cntl = MCURSOR_GAMMA_ENABLE;
10308                 switch (plane_state->base.crtc_w) {
10309                         case 64:
10310                                 cntl |= CURSOR_MODE_64_ARGB_AX;
10311                                 break;
10312                         case 128:
10313                                 cntl |= CURSOR_MODE_128_ARGB_AX;
10314                                 break;
10315                         case 256:
10316                                 cntl |= CURSOR_MODE_256_ARGB_AX;
10317                                 break;
10318                         default:
10319                                 MISSING_CASE(plane_state->base.crtc_w);
10320                                 return;
10321                 }
10322                 cntl |= pipe << 28; /* Connect to correct pipe */
10323
10324                 if (HAS_DDI(dev))
10325                         cntl |= CURSOR_PIPE_CSC_ENABLE;
10326
10327                 if (plane_state->base.rotation == BIT(DRM_ROTATE_180))
10328                         cntl |= CURSOR_ROTATE_180;
10329         }
10330
10331         if (intel_crtc->cursor_cntl != cntl) {
10332                 I915_WRITE(CURCNTR(pipe), cntl);
10333                 POSTING_READ(CURCNTR(pipe));
10334                 intel_crtc->cursor_cntl = cntl;
10335         }
10336
10337         /* and commit changes on next vblank */
10338         I915_WRITE(CURBASE(pipe), base);
10339         POSTING_READ(CURBASE(pipe));
10340
10341         intel_crtc->cursor_base = base;
10342 }
10343
10344 /* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */
10345 static void intel_crtc_update_cursor(struct drm_crtc *crtc,
10346                                      const struct intel_plane_state *plane_state)
10347 {
10348         struct drm_device *dev = crtc->dev;
10349         struct drm_i915_private *dev_priv = dev->dev_private;
10350         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10351         int pipe = intel_crtc->pipe;
10352         u32 base = intel_crtc->cursor_addr;
10353         u32 pos = 0;
10354
10355         if (plane_state) {
10356                 int x = plane_state->base.crtc_x;
10357                 int y = plane_state->base.crtc_y;
10358
10359                 if (x < 0) {
10360                         pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
10361                         x = -x;
10362                 }
10363                 pos |= x << CURSOR_X_SHIFT;
10364
10365                 if (y < 0) {
10366                         pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
10367                         y = -y;
10368                 }
10369                 pos |= y << CURSOR_Y_SHIFT;
10370
10371                 /* ILK+ do this automagically */
10372                 if (HAS_GMCH_DISPLAY(dev) &&
10373                     plane_state->base.rotation == BIT(DRM_ROTATE_180)) {
10374                         base += (plane_state->base.crtc_h *
10375                                  plane_state->base.crtc_w - 1) * 4;
10376                 }
10377         }
10378
10379         I915_WRITE(CURPOS(pipe), pos);
10380
10381         if (IS_845G(dev) || IS_I865G(dev))
10382                 i845_update_cursor(crtc, base, plane_state);
10383         else
10384                 i9xx_update_cursor(crtc, base, plane_state);
10385 }
10386
10387 static bool cursor_size_ok(struct drm_device *dev,
10388                            uint32_t width, uint32_t height)
10389 {
10390         if (width == 0 || height == 0)
10391                 return false;
10392
10393         /*
10394          * 845g/865g are special in that they are only limited by
10395          * the width of their cursors, the height is arbitrary up to
10396          * the precision of the register. Everything else requires
10397          * square cursors, limited to a few power-of-two sizes.
10398          */
10399         if (IS_845G(dev) || IS_I865G(dev)) {
10400                 if ((width & 63) != 0)
10401                         return false;
10402
10403                 if (width > (IS_845G(dev) ? 64 : 512))
10404                         return false;
10405
10406                 if (height > 1023)
10407                         return false;
10408         } else {
10409                 switch (width | height) {
10410                 case 256:
10411                 case 128:
10412                         if (IS_GEN2(dev))
10413                                 return false;
10414                 case 64:
10415                         break;
10416                 default:
10417                         return false;
10418                 }
10419         }
10420
10421         return true;
10422 }
10423
10424 /* VESA 640x480x72Hz mode to set on the pipe */
10425 static struct drm_display_mode load_detect_mode = {
10426         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
10427                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
10428 };
10429
10430 struct drm_framebuffer *
10431 __intel_framebuffer_create(struct drm_device *dev,
10432                            struct drm_mode_fb_cmd2 *mode_cmd,
10433                            struct drm_i915_gem_object *obj)
10434 {
10435         struct intel_framebuffer *intel_fb;
10436         int ret;
10437
10438         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
10439         if (!intel_fb)
10440                 return ERR_PTR(-ENOMEM);
10441
10442         ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj);
10443         if (ret)
10444                 goto err;
10445
10446         return &intel_fb->base;
10447
10448 err:
10449         kfree(intel_fb);
10450         return ERR_PTR(ret);
10451 }
10452
10453 static struct drm_framebuffer *
10454 intel_framebuffer_create(struct drm_device *dev,
10455                          struct drm_mode_fb_cmd2 *mode_cmd,
10456                          struct drm_i915_gem_object *obj)
10457 {
10458         struct drm_framebuffer *fb;
10459         int ret;
10460
10461         ret = i915_mutex_lock_interruptible(dev);
10462         if (ret)
10463                 return ERR_PTR(ret);
10464         fb = __intel_framebuffer_create(dev, mode_cmd, obj);
10465         mutex_unlock(&dev->struct_mutex);
10466
10467         return fb;
10468 }
10469
10470 static u32
10471 intel_framebuffer_pitch_for_width(int width, int bpp)
10472 {
10473         u32 pitch = DIV_ROUND_UP(width * bpp, 8);
10474         return ALIGN(pitch, 64);
10475 }
10476
10477 static u32
10478 intel_framebuffer_size_for_mode(struct drm_display_mode *mode, int bpp)
10479 {
10480         u32 pitch = intel_framebuffer_pitch_for_width(mode->hdisplay, bpp);
10481         return PAGE_ALIGN(pitch * mode->vdisplay);
10482 }
10483
10484 static struct drm_framebuffer *
10485 intel_framebuffer_create_for_mode(struct drm_device *dev,
10486                                   struct drm_display_mode *mode,
10487                                   int depth, int bpp)
10488 {
10489         struct drm_framebuffer *fb;
10490         struct drm_i915_gem_object *obj;
10491         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
10492
10493         obj = i915_gem_object_create(dev,
10494                                     intel_framebuffer_size_for_mode(mode, bpp));
10495         if (IS_ERR(obj))
10496                 return ERR_CAST(obj);
10497
10498         mode_cmd.width = mode->hdisplay;
10499         mode_cmd.height = mode->vdisplay;
10500         mode_cmd.pitches[0] = intel_framebuffer_pitch_for_width(mode_cmd.width,
10501                                                                 bpp);
10502         mode_cmd.pixel_format = drm_mode_legacy_fb_format(bpp, depth);
10503
10504         fb = intel_framebuffer_create(dev, &mode_cmd, obj);
10505         if (IS_ERR(fb))
10506                 drm_gem_object_unreference_unlocked(&obj->base);
10507
10508         return fb;
10509 }
10510
10511 static struct drm_framebuffer *
10512 mode_fits_in_fbdev(struct drm_device *dev,
10513                    struct drm_display_mode *mode)
10514 {
10515 #ifdef CONFIG_DRM_FBDEV_EMULATION
10516         struct drm_i915_private *dev_priv = dev->dev_private;
10517         struct drm_i915_gem_object *obj;
10518         struct drm_framebuffer *fb;
10519
10520         if (!dev_priv->fbdev)
10521                 return NULL;
10522
10523         if (!dev_priv->fbdev->fb)
10524                 return NULL;
10525
10526         obj = dev_priv->fbdev->fb->obj;
10527         BUG_ON(!obj);
10528
10529         fb = &dev_priv->fbdev->fb->base;
10530         if (fb->pitches[0] < intel_framebuffer_pitch_for_width(mode->hdisplay,
10531                                                                fb->bits_per_pixel))
10532                 return NULL;
10533
10534         if (obj->base.size < mode->vdisplay * fb->pitches[0])
10535                 return NULL;
10536
10537         drm_framebuffer_reference(fb);
10538         return fb;
10539 #else
10540         return NULL;
10541 #endif
10542 }
10543
10544 static int intel_modeset_setup_plane_state(struct drm_atomic_state *state,
10545                                            struct drm_crtc *crtc,
10546                                            struct drm_display_mode *mode,
10547                                            struct drm_framebuffer *fb,
10548                                            int x, int y)
10549 {
10550         struct drm_plane_state *plane_state;
10551         int hdisplay, vdisplay;
10552         int ret;
10553
10554         plane_state = drm_atomic_get_plane_state(state, crtc->primary);
10555         if (IS_ERR(plane_state))
10556                 return PTR_ERR(plane_state);
10557
10558         if (mode)
10559                 drm_crtc_get_hv_timing(mode, &hdisplay, &vdisplay);
10560         else
10561                 hdisplay = vdisplay = 0;
10562
10563         ret = drm_atomic_set_crtc_for_plane(plane_state, fb ? crtc : NULL);
10564         if (ret)
10565                 return ret;
10566         drm_atomic_set_fb_for_plane(plane_state, fb);
10567         plane_state->crtc_x = 0;
10568         plane_state->crtc_y = 0;
10569         plane_state->crtc_w = hdisplay;
10570         plane_state->crtc_h = vdisplay;
10571         plane_state->src_x = x << 16;
10572         plane_state->src_y = y << 16;
10573         plane_state->src_w = hdisplay << 16;
10574         plane_state->src_h = vdisplay << 16;
10575
10576         return 0;
10577 }
10578
10579 bool intel_get_load_detect_pipe(struct drm_connector *connector,
10580                                 struct drm_display_mode *mode,
10581                                 struct intel_load_detect_pipe *old,
10582                                 struct drm_modeset_acquire_ctx *ctx)
10583 {
10584         struct intel_crtc *intel_crtc;
10585         struct intel_encoder *intel_encoder =
10586                 intel_attached_encoder(connector);
10587         struct drm_crtc *possible_crtc;
10588         struct drm_encoder *encoder = &intel_encoder->base;
10589         struct drm_crtc *crtc = NULL;
10590         struct drm_device *dev = encoder->dev;
10591         struct drm_framebuffer *fb;
10592         struct drm_mode_config *config = &dev->mode_config;
10593         struct drm_atomic_state *state = NULL, *restore_state = NULL;
10594         struct drm_connector_state *connector_state;
10595         struct intel_crtc_state *crtc_state;
10596         int ret, i = -1;
10597
10598         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10599                       connector->base.id, connector->name,
10600                       encoder->base.id, encoder->name);
10601
10602         old->restore_state = NULL;
10603
10604 retry:
10605         ret = drm_modeset_lock(&config->connection_mutex, ctx);
10606         if (ret)
10607                 goto fail;
10608
10609         /*
10610          * Algorithm gets a little messy:
10611          *
10612          *   - if the connector already has an assigned crtc, use it (but make
10613          *     sure it's on first)
10614          *
10615          *   - try to find the first unused crtc that can drive this connector,
10616          *     and use that if we find one
10617          */
10618
10619         /* See if we already have a CRTC for this connector */
10620         if (connector->state->crtc) {
10621                 crtc = connector->state->crtc;
10622
10623                 ret = drm_modeset_lock(&crtc->mutex, ctx);
10624                 if (ret)
10625                         goto fail;
10626
10627                 /* Make sure the crtc and connector are running */
10628                 goto found;
10629         }
10630
10631         /* Find an unused one (if possible) */
10632         for_each_crtc(dev, possible_crtc) {
10633                 i++;
10634                 if (!(encoder->possible_crtcs & (1 << i)))
10635                         continue;
10636
10637                 ret = drm_modeset_lock(&possible_crtc->mutex, ctx);
10638                 if (ret)
10639                         goto fail;
10640
10641                 if (possible_crtc->state->enable) {
10642                         drm_modeset_unlock(&possible_crtc->mutex);
10643                         continue;
10644                 }
10645
10646                 crtc = possible_crtc;
10647                 break;
10648         }
10649
10650         /*
10651          * If we didn't find an unused CRTC, don't use any.
10652          */
10653         if (!crtc) {
10654                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
10655                 goto fail;
10656         }
10657
10658 found:
10659         intel_crtc = to_intel_crtc(crtc);
10660
10661         ret = drm_modeset_lock(&crtc->primary->mutex, ctx);
10662         if (ret)
10663                 goto fail;
10664
10665         state = drm_atomic_state_alloc(dev);
10666         restore_state = drm_atomic_state_alloc(dev);
10667         if (!state || !restore_state) {
10668                 ret = -ENOMEM;
10669                 goto fail;
10670         }
10671
10672         state->acquire_ctx = ctx;
10673         restore_state->acquire_ctx = ctx;
10674
10675         connector_state = drm_atomic_get_connector_state(state, connector);
10676         if (IS_ERR(connector_state)) {
10677                 ret = PTR_ERR(connector_state);
10678                 goto fail;
10679         }
10680
10681         ret = drm_atomic_set_crtc_for_connector(connector_state, crtc);
10682         if (ret)
10683                 goto fail;
10684
10685         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
10686         if (IS_ERR(crtc_state)) {
10687                 ret = PTR_ERR(crtc_state);
10688                 goto fail;
10689         }
10690
10691         crtc_state->base.active = crtc_state->base.enable = true;
10692
10693         if (!mode)
10694                 mode = &load_detect_mode;
10695
10696         /* We need a framebuffer large enough to accommodate all accesses
10697          * that the plane may generate whilst we perform load detection.
10698          * We can not rely on the fbcon either being present (we get called
10699          * during its initialisation to detect all boot displays, or it may
10700          * not even exist) or that it is large enough to satisfy the
10701          * requested mode.
10702          */
10703         fb = mode_fits_in_fbdev(dev, mode);
10704         if (fb == NULL) {
10705                 DRM_DEBUG_KMS("creating tmp fb for load-detection\n");
10706                 fb = intel_framebuffer_create_for_mode(dev, mode, 24, 32);
10707         } else
10708                 DRM_DEBUG_KMS("reusing fbdev for load-detection framebuffer\n");
10709         if (IS_ERR(fb)) {
10710                 DRM_DEBUG_KMS("failed to allocate framebuffer for load-detection\n");
10711                 goto fail;
10712         }
10713
10714         ret = intel_modeset_setup_plane_state(state, crtc, mode, fb, 0, 0);
10715         if (ret)
10716                 goto fail;
10717
10718         drm_framebuffer_unreference(fb);
10719
10720         ret = drm_atomic_set_mode_for_crtc(&crtc_state->base, mode);
10721         if (ret)
10722                 goto fail;
10723
10724         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
10725         if (!ret)
10726                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
10727         if (!ret)
10728                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_plane_state(restore_state, crtc->primary));
10729         if (ret) {
10730                 DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
10731                 goto fail;
10732         }
10733
10734         ret = drm_atomic_commit(state);
10735         if (ret) {
10736                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
10737                 goto fail;
10738         }
10739
10740         old->restore_state = restore_state;
10741
10742         /* let the connector get through one full cycle before testing */
10743         intel_wait_for_vblank(dev, intel_crtc->pipe);
10744         return true;
10745
10746 fail:
10747         drm_atomic_state_free(state);
10748         drm_atomic_state_free(restore_state);
10749         restore_state = state = NULL;
10750
10751         if (ret == -EDEADLK) {
10752                 drm_modeset_backoff(ctx);
10753                 goto retry;
10754         }
10755
10756         return false;
10757 }
10758
10759 void intel_release_load_detect_pipe(struct drm_connector *connector,
10760                                     struct intel_load_detect_pipe *old,
10761                                     struct drm_modeset_acquire_ctx *ctx)
10762 {
10763         struct intel_encoder *intel_encoder =
10764                 intel_attached_encoder(connector);
10765         struct drm_encoder *encoder = &intel_encoder->base;
10766         struct drm_atomic_state *state = old->restore_state;
10767         int ret;
10768
10769         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10770                       connector->base.id, connector->name,
10771                       encoder->base.id, encoder->name);
10772
10773         if (!state)
10774                 return;
10775
10776         ret = drm_atomic_commit(state);
10777         if (ret) {
10778                 DRM_DEBUG_KMS("Couldn't release load detect pipe: %i\n", ret);
10779                 drm_atomic_state_free(state);
10780         }
10781 }
10782
10783 static int i9xx_pll_refclk(struct drm_device *dev,
10784                            const struct intel_crtc_state *pipe_config)
10785 {
10786         struct drm_i915_private *dev_priv = dev->dev_private;
10787         u32 dpll = pipe_config->dpll_hw_state.dpll;
10788
10789         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
10790                 return dev_priv->vbt.lvds_ssc_freq;
10791         else if (HAS_PCH_SPLIT(dev))
10792                 return 120000;
10793         else if (!IS_GEN2(dev))
10794                 return 96000;
10795         else
10796                 return 48000;
10797 }
10798
10799 /* Returns the clock of the currently programmed mode of the given pipe. */
10800 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
10801                                 struct intel_crtc_state *pipe_config)
10802 {
10803         struct drm_device *dev = crtc->base.dev;
10804         struct drm_i915_private *dev_priv = dev->dev_private;
10805         int pipe = pipe_config->cpu_transcoder;
10806         u32 dpll = pipe_config->dpll_hw_state.dpll;
10807         u32 fp;
10808         struct dpll clock;
10809         int port_clock;
10810         int refclk = i9xx_pll_refclk(dev, pipe_config);
10811
10812         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
10813                 fp = pipe_config->dpll_hw_state.fp0;
10814         else
10815                 fp = pipe_config->dpll_hw_state.fp1;
10816
10817         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
10818         if (IS_PINEVIEW(dev)) {
10819                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
10820                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
10821         } else {
10822                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
10823                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
10824         }
10825
10826         if (!IS_GEN2(dev)) {
10827                 if (IS_PINEVIEW(dev))
10828                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
10829                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
10830                 else
10831                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
10832                                DPLL_FPA01_P1_POST_DIV_SHIFT);
10833
10834                 switch (dpll & DPLL_MODE_MASK) {
10835                 case DPLLB_MODE_DAC_SERIAL:
10836                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
10837                                 5 : 10;
10838                         break;
10839                 case DPLLB_MODE_LVDS:
10840                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
10841                                 7 : 14;
10842                         break;
10843                 default:
10844                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
10845                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
10846                         return;
10847                 }
10848
10849                 if (IS_PINEVIEW(dev))
10850                         port_clock = pnv_calc_dpll_params(refclk, &clock);
10851                 else
10852                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
10853         } else {
10854                 u32 lvds = IS_I830(dev) ? 0 : I915_READ(LVDS);
10855                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
10856
10857                 if (is_lvds) {
10858                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
10859                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
10860
10861                         if (lvds & LVDS_CLKB_POWER_UP)
10862                                 clock.p2 = 7;
10863                         else
10864                                 clock.p2 = 14;
10865                 } else {
10866                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
10867                                 clock.p1 = 2;
10868                         else {
10869                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
10870                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
10871                         }
10872                         if (dpll & PLL_P2_DIVIDE_BY_4)
10873                                 clock.p2 = 4;
10874                         else
10875                                 clock.p2 = 2;
10876                 }
10877
10878                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
10879         }
10880
10881         /*
10882          * This value includes pixel_multiplier. We will use
10883          * port_clock to compute adjusted_mode.crtc_clock in the
10884          * encoder's get_config() function.
10885          */
10886         pipe_config->port_clock = port_clock;
10887 }
10888
10889 int intel_dotclock_calculate(int link_freq,
10890                              const struct intel_link_m_n *m_n)
10891 {
10892         /*
10893          * The calculation for the data clock is:
10894          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
10895          * But we want to avoid losing precison if possible, so:
10896          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
10897          *
10898          * and the link clock is simpler:
10899          * link_clock = (m * link_clock) / n
10900          */
10901
10902         if (!m_n->link_n)
10903                 return 0;
10904
10905         return div_u64((u64)m_n->link_m * link_freq, m_n->link_n);
10906 }
10907
10908 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
10909                                    struct intel_crtc_state *pipe_config)
10910 {
10911         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
10912
10913         /* read out port_clock from the DPLL */
10914         i9xx_crtc_clock_get(crtc, pipe_config);
10915
10916         /*
10917          * In case there is an active pipe without active ports,
10918          * we may need some idea for the dotclock anyway.
10919          * Calculate one based on the FDI configuration.
10920          */
10921         pipe_config->base.adjusted_mode.crtc_clock =
10922                 intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
10923                                          &pipe_config->fdi_m_n);
10924 }
10925
10926 /** Returns the currently programmed mode of the given pipe. */
10927 struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
10928                                              struct drm_crtc *crtc)
10929 {
10930         struct drm_i915_private *dev_priv = dev->dev_private;
10931         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10932         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
10933         struct drm_display_mode *mode;
10934         struct intel_crtc_state *pipe_config;
10935         int htot = I915_READ(HTOTAL(cpu_transcoder));
10936         int hsync = I915_READ(HSYNC(cpu_transcoder));
10937         int vtot = I915_READ(VTOTAL(cpu_transcoder));
10938         int vsync = I915_READ(VSYNC(cpu_transcoder));
10939         enum pipe pipe = intel_crtc->pipe;
10940
10941         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
10942         if (!mode)
10943                 return NULL;
10944
10945         pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL);
10946         if (!pipe_config) {
10947                 kfree(mode);
10948                 return NULL;
10949         }
10950
10951         /*
10952          * Construct a pipe_config sufficient for getting the clock info
10953          * back out of crtc_clock_get.
10954          *
10955          * Note, if LVDS ever uses a non-1 pixel multiplier, we'll need
10956          * to use a real value here instead.
10957          */
10958         pipe_config->cpu_transcoder = (enum transcoder) pipe;
10959         pipe_config->pixel_multiplier = 1;
10960         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(pipe));
10961         pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(pipe));
10962         pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(pipe));
10963         i9xx_crtc_clock_get(intel_crtc, pipe_config);
10964
10965         mode->clock = pipe_config->port_clock / pipe_config->pixel_multiplier;
10966         mode->hdisplay = (htot & 0xffff) + 1;
10967         mode->htotal = ((htot & 0xffff0000) >> 16) + 1;
10968         mode->hsync_start = (hsync & 0xffff) + 1;
10969         mode->hsync_end = ((hsync & 0xffff0000) >> 16) + 1;
10970         mode->vdisplay = (vtot & 0xffff) + 1;
10971         mode->vtotal = ((vtot & 0xffff0000) >> 16) + 1;
10972         mode->vsync_start = (vsync & 0xffff) + 1;
10973         mode->vsync_end = ((vsync & 0xffff0000) >> 16) + 1;
10974
10975         drm_mode_set_name(mode);
10976
10977         kfree(pipe_config);
10978
10979         return mode;
10980 }
10981
10982 void intel_mark_busy(struct drm_i915_private *dev_priv)
10983 {
10984         if (dev_priv->mm.busy)
10985                 return;
10986
10987         intel_runtime_pm_get(dev_priv);
10988         i915_update_gfx_val(dev_priv);
10989         if (INTEL_GEN(dev_priv) >= 6)
10990                 gen6_rps_busy(dev_priv);
10991         dev_priv->mm.busy = true;
10992 }
10993
10994 void intel_mark_idle(struct drm_i915_private *dev_priv)
10995 {
10996         if (!dev_priv->mm.busy)
10997                 return;
10998
10999         dev_priv->mm.busy = false;
11000
11001         if (INTEL_GEN(dev_priv) >= 6)
11002                 gen6_rps_idle(dev_priv);
11003
11004         intel_runtime_pm_put(dev_priv);
11005 }
11006
11007 static void intel_crtc_destroy(struct drm_crtc *crtc)
11008 {
11009         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11010         struct drm_device *dev = crtc->dev;
11011         struct intel_flip_work *work;
11012
11013         spin_lock_irq(&dev->event_lock);
11014         work = intel_crtc->flip_work;
11015         intel_crtc->flip_work = NULL;
11016         spin_unlock_irq(&dev->event_lock);
11017
11018         if (work) {
11019                 cancel_work_sync(&work->mmio_work);
11020                 cancel_work_sync(&work->unpin_work);
11021                 kfree(work);
11022         }
11023
11024         drm_crtc_cleanup(crtc);
11025
11026         kfree(intel_crtc);
11027 }
11028
11029 static void intel_unpin_work_fn(struct work_struct *__work)
11030 {
11031         struct intel_flip_work *work =
11032                 container_of(__work, struct intel_flip_work, unpin_work);
11033         struct intel_crtc *crtc = to_intel_crtc(work->crtc);
11034         struct drm_device *dev = crtc->base.dev;
11035         struct drm_plane *primary = crtc->base.primary;
11036
11037         if (is_mmio_work(work))
11038                 flush_work(&work->mmio_work);
11039
11040         mutex_lock(&dev->struct_mutex);
11041         intel_unpin_fb_obj(work->old_fb, primary->state->rotation);
11042         drm_gem_object_unreference(&work->pending_flip_obj->base);
11043
11044         if (work->flip_queued_req)
11045                 i915_gem_request_assign(&work->flip_queued_req, NULL);
11046         mutex_unlock(&dev->struct_mutex);
11047
11048         intel_frontbuffer_flip_complete(dev, to_intel_plane(primary)->frontbuffer_bit);
11049         intel_fbc_post_update(crtc);
11050         drm_framebuffer_unreference(work->old_fb);
11051
11052         BUG_ON(atomic_read(&crtc->unpin_work_count) == 0);
11053         atomic_dec(&crtc->unpin_work_count);
11054
11055         kfree(work);
11056 }
11057
11058 /* Is 'a' after or equal to 'b'? */
11059 static bool g4x_flip_count_after_eq(u32 a, u32 b)
11060 {
11061         return !((a - b) & 0x80000000);
11062 }
11063
11064 static bool __pageflip_finished_cs(struct intel_crtc *crtc,
11065                                    struct intel_flip_work *work)
11066 {
11067         struct drm_device *dev = crtc->base.dev;
11068         struct drm_i915_private *dev_priv = dev->dev_private;
11069         unsigned reset_counter;
11070
11071         reset_counter = i915_reset_counter(&dev_priv->gpu_error);
11072         if (crtc->reset_counter != reset_counter)
11073                 return true;
11074
11075         /*
11076          * The relevant registers doen't exist on pre-ctg.
11077          * As the flip done interrupt doesn't trigger for mmio
11078          * flips on gmch platforms, a flip count check isn't
11079          * really needed there. But since ctg has the registers,
11080          * include it in the check anyway.
11081          */
11082         if (INTEL_INFO(dev)->gen < 5 && !IS_G4X(dev))
11083                 return true;
11084
11085         /*
11086          * BDW signals flip done immediately if the plane
11087          * is disabled, even if the plane enable is already
11088          * armed to occur at the next vblank :(
11089          */
11090
11091         /*
11092          * A DSPSURFLIVE check isn't enough in case the mmio and CS flips
11093          * used the same base address. In that case the mmio flip might
11094          * have completed, but the CS hasn't even executed the flip yet.
11095          *
11096          * A flip count check isn't enough as the CS might have updated
11097          * the base address just after start of vblank, but before we
11098          * managed to process the interrupt. This means we'd complete the
11099          * CS flip too soon.
11100          *
11101          * Combining both checks should get us a good enough result. It may
11102          * still happen that the CS flip has been executed, but has not
11103          * yet actually completed. But in case the base address is the same
11104          * anyway, we don't really care.
11105          */
11106         return (I915_READ(DSPSURFLIVE(crtc->plane)) & ~0xfff) ==
11107                 crtc->flip_work->gtt_offset &&
11108                 g4x_flip_count_after_eq(I915_READ(PIPE_FLIPCOUNT_G4X(crtc->pipe)),
11109                                     crtc->flip_work->flip_count);
11110 }
11111
11112 static bool
11113 __pageflip_finished_mmio(struct intel_crtc *crtc,
11114                                struct intel_flip_work *work)
11115 {
11116         /*
11117          * MMIO work completes when vblank is different from
11118          * flip_queued_vblank.
11119          *
11120          * Reset counter value doesn't matter, this is handled by
11121          * i915_wait_request finishing early, so no need to handle
11122          * reset here.
11123          */
11124         return intel_crtc_get_vblank_counter(crtc) != work->flip_queued_vblank;
11125 }
11126
11127
11128 static bool pageflip_finished(struct intel_crtc *crtc,
11129                               struct intel_flip_work *work)
11130 {
11131         if (!atomic_read(&work->pending))
11132                 return false;
11133
11134         smp_rmb();
11135
11136         if (is_mmio_work(work))
11137                 return __pageflip_finished_mmio(crtc, work);
11138         else
11139                 return __pageflip_finished_cs(crtc, work);
11140 }
11141
11142 void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe)
11143 {
11144         struct drm_device *dev = dev_priv->dev;
11145         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
11146         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11147         struct intel_flip_work *work;
11148         unsigned long flags;
11149
11150         /* Ignore early vblank irqs */
11151         if (!crtc)
11152                 return;
11153
11154         /*
11155          * This is called both by irq handlers and the reset code (to complete
11156          * lost pageflips) so needs the full irqsave spinlocks.
11157          */
11158         spin_lock_irqsave(&dev->event_lock, flags);
11159         work = intel_crtc->flip_work;
11160
11161         if (work != NULL &&
11162             !is_mmio_work(work) &&
11163             pageflip_finished(intel_crtc, work))
11164                 page_flip_completed(intel_crtc);
11165
11166         spin_unlock_irqrestore(&dev->event_lock, flags);
11167 }
11168
11169 void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe)
11170 {
11171         struct drm_device *dev = dev_priv->dev;
11172         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
11173         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11174         struct intel_flip_work *work;
11175         unsigned long flags;
11176
11177         /* Ignore early vblank irqs */
11178         if (!crtc)
11179                 return;
11180
11181         /*
11182          * This is called both by irq handlers and the reset code (to complete
11183          * lost pageflips) so needs the full irqsave spinlocks.
11184          */
11185         spin_lock_irqsave(&dev->event_lock, flags);
11186         work = intel_crtc->flip_work;
11187
11188         if (work != NULL &&
11189             is_mmio_work(work) &&
11190             pageflip_finished(intel_crtc, work))
11191                 page_flip_completed(intel_crtc);
11192
11193         spin_unlock_irqrestore(&dev->event_lock, flags);
11194 }
11195
11196 static inline void intel_mark_page_flip_active(struct intel_crtc *crtc,
11197                                                struct intel_flip_work *work)
11198 {
11199         work->flip_queued_vblank = intel_crtc_get_vblank_counter(crtc);
11200
11201         /* Ensure that the work item is consistent when activating it ... */
11202         smp_mb__before_atomic();
11203         atomic_set(&work->pending, 1);
11204 }
11205
11206 static int intel_gen2_queue_flip(struct drm_device *dev,
11207                                  struct drm_crtc *crtc,
11208                                  struct drm_framebuffer *fb,
11209                                  struct drm_i915_gem_object *obj,
11210                                  struct drm_i915_gem_request *req,
11211                                  uint32_t flags)
11212 {
11213         struct intel_engine_cs *engine = req->engine;
11214         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11215         u32 flip_mask;
11216         int ret;
11217
11218         ret = intel_ring_begin(req, 6);
11219         if (ret)
11220                 return ret;
11221
11222         /* Can't queue multiple flips, so wait for the previous
11223          * one to finish before executing the next.
11224          */
11225         if (intel_crtc->plane)
11226                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
11227         else
11228                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
11229         intel_ring_emit(engine, MI_WAIT_FOR_EVENT | flip_mask);
11230         intel_ring_emit(engine, MI_NOOP);
11231         intel_ring_emit(engine, MI_DISPLAY_FLIP |
11232                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
11233         intel_ring_emit(engine, fb->pitches[0]);
11234         intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset);
11235         intel_ring_emit(engine, 0); /* aux display base address, unused */
11236
11237         return 0;
11238 }
11239
11240 static int intel_gen3_queue_flip(struct drm_device *dev,
11241                                  struct drm_crtc *crtc,
11242                                  struct drm_framebuffer *fb,
11243                                  struct drm_i915_gem_object *obj,
11244                                  struct drm_i915_gem_request *req,
11245                                  uint32_t flags)
11246 {
11247         struct intel_engine_cs *engine = req->engine;
11248         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11249         u32 flip_mask;
11250         int ret;
11251
11252         ret = intel_ring_begin(req, 6);
11253         if (ret)
11254                 return ret;
11255
11256         if (intel_crtc->plane)
11257                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
11258         else
11259                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
11260         intel_ring_emit(engine, MI_WAIT_FOR_EVENT | flip_mask);
11261         intel_ring_emit(engine, MI_NOOP);
11262         intel_ring_emit(engine, MI_DISPLAY_FLIP_I915 |
11263                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
11264         intel_ring_emit(engine, fb->pitches[0]);
11265         intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset);
11266         intel_ring_emit(engine, MI_NOOP);
11267
11268         return 0;
11269 }
11270
11271 static int intel_gen4_queue_flip(struct drm_device *dev,
11272                                  struct drm_crtc *crtc,
11273                                  struct drm_framebuffer *fb,
11274                                  struct drm_i915_gem_object *obj,
11275                                  struct drm_i915_gem_request *req,
11276                                  uint32_t flags)
11277 {
11278         struct intel_engine_cs *engine = req->engine;
11279         struct drm_i915_private *dev_priv = dev->dev_private;
11280         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11281         uint32_t pf, pipesrc;
11282         int ret;
11283
11284         ret = intel_ring_begin(req, 4);
11285         if (ret)
11286                 return ret;
11287
11288         /* i965+ uses the linear or tiled offsets from the
11289          * Display Registers (which do not change across a page-flip)
11290          * so we need only reprogram the base address.
11291          */
11292         intel_ring_emit(engine, MI_DISPLAY_FLIP |
11293                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
11294         intel_ring_emit(engine, fb->pitches[0]);
11295         intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset |
11296                         obj->tiling_mode);
11297
11298         /* XXX Enabling the panel-fitter across page-flip is so far
11299          * untested on non-native modes, so ignore it for now.
11300          * pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
11301          */
11302         pf = 0;
11303         pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
11304         intel_ring_emit(engine, pf | pipesrc);
11305
11306         return 0;
11307 }
11308
11309 static int intel_gen6_queue_flip(struct drm_device *dev,
11310                                  struct drm_crtc *crtc,
11311                                  struct drm_framebuffer *fb,
11312                                  struct drm_i915_gem_object *obj,
11313                                  struct drm_i915_gem_request *req,
11314                                  uint32_t flags)
11315 {
11316         struct intel_engine_cs *engine = req->engine;
11317         struct drm_i915_private *dev_priv = dev->dev_private;
11318         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11319         uint32_t pf, pipesrc;
11320         int ret;
11321
11322         ret = intel_ring_begin(req, 4);
11323         if (ret)
11324                 return ret;
11325
11326         intel_ring_emit(engine, MI_DISPLAY_FLIP |
11327                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
11328         intel_ring_emit(engine, fb->pitches[0] | obj->tiling_mode);
11329         intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset);
11330
11331         /* Contrary to the suggestions in the documentation,
11332          * "Enable Panel Fitter" does not seem to be required when page
11333          * flipping with a non-native mode, and worse causes a normal
11334          * modeset to fail.
11335          * pf = I915_READ(PF_CTL(intel_crtc->pipe)) & PF_ENABLE;
11336          */
11337         pf = 0;
11338         pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
11339         intel_ring_emit(engine, pf | pipesrc);
11340
11341         return 0;
11342 }
11343
11344 static int intel_gen7_queue_flip(struct drm_device *dev,
11345                                  struct drm_crtc *crtc,
11346                                  struct drm_framebuffer *fb,
11347                                  struct drm_i915_gem_object *obj,
11348                                  struct drm_i915_gem_request *req,
11349                                  uint32_t flags)
11350 {
11351         struct intel_engine_cs *engine = req->engine;
11352         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11353         uint32_t plane_bit = 0;
11354         int len, ret;
11355
11356         switch (intel_crtc->plane) {
11357         case PLANE_A:
11358                 plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_A;
11359                 break;
11360         case PLANE_B:
11361                 plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_B;
11362                 break;
11363         case PLANE_C:
11364                 plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_C;
11365                 break;
11366         default:
11367                 WARN_ONCE(1, "unknown plane in flip command\n");
11368                 return -ENODEV;
11369         }
11370
11371         len = 4;
11372         if (engine->id == RCS) {
11373                 len += 6;
11374                 /*
11375                  * On Gen 8, SRM is now taking an extra dword to accommodate
11376                  * 48bits addresses, and we need a NOOP for the batch size to
11377                  * stay even.
11378                  */
11379                 if (IS_GEN8(dev))
11380                         len += 2;
11381         }
11382
11383         /*
11384          * BSpec MI_DISPLAY_FLIP for IVB:
11385          * "The full packet must be contained within the same cache line."
11386          *
11387          * Currently the LRI+SRM+MI_DISPLAY_FLIP all fit within the same
11388          * cacheline, if we ever start emitting more commands before
11389          * the MI_DISPLAY_FLIP we may need to first emit everything else,
11390          * then do the cacheline alignment, and finally emit the
11391          * MI_DISPLAY_FLIP.
11392          */
11393         ret = intel_ring_cacheline_align(req);
11394         if (ret)
11395                 return ret;
11396
11397         ret = intel_ring_begin(req, len);
11398         if (ret)
11399                 return ret;
11400
11401         /* Unmask the flip-done completion message. Note that the bspec says that
11402          * we should do this for both the BCS and RCS, and that we must not unmask
11403          * more than one flip event at any time (or ensure that one flip message
11404          * can be sent by waiting for flip-done prior to queueing new flips).
11405          * Experimentation says that BCS works despite DERRMR masking all
11406          * flip-done completion events and that unmasking all planes at once
11407          * for the RCS also doesn't appear to drop events. Setting the DERRMR
11408          * to zero does lead to lockups within MI_DISPLAY_FLIP.
11409          */
11410         if (engine->id == RCS) {
11411                 intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1));
11412                 intel_ring_emit_reg(engine, DERRMR);
11413                 intel_ring_emit(engine, ~(DERRMR_PIPEA_PRI_FLIP_DONE |
11414                                           DERRMR_PIPEB_PRI_FLIP_DONE |
11415                                           DERRMR_PIPEC_PRI_FLIP_DONE));
11416                 if (IS_GEN8(dev))
11417                         intel_ring_emit(engine, MI_STORE_REGISTER_MEM_GEN8 |
11418                                               MI_SRM_LRM_GLOBAL_GTT);
11419                 else
11420                         intel_ring_emit(engine, MI_STORE_REGISTER_MEM |
11421                                               MI_SRM_LRM_GLOBAL_GTT);
11422                 intel_ring_emit_reg(engine, DERRMR);
11423                 intel_ring_emit(engine, engine->scratch.gtt_offset + 256);
11424                 if (IS_GEN8(dev)) {
11425                         intel_ring_emit(engine, 0);
11426                         intel_ring_emit(engine, MI_NOOP);
11427                 }
11428         }
11429
11430         intel_ring_emit(engine, MI_DISPLAY_FLIP_I915 | plane_bit);
11431         intel_ring_emit(engine, (fb->pitches[0] | obj->tiling_mode));
11432         intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset);
11433         intel_ring_emit(engine, (MI_NOOP));
11434
11435         return 0;
11436 }
11437
11438 static bool use_mmio_flip(struct intel_engine_cs *engine,
11439                           struct drm_i915_gem_object *obj)
11440 {
11441         struct reservation_object *resv;
11442
11443         /*
11444          * This is not being used for older platforms, because
11445          * non-availability of flip done interrupt forces us to use
11446          * CS flips. Older platforms derive flip done using some clever
11447          * tricks involving the flip_pending status bits and vblank irqs.
11448          * So using MMIO flips there would disrupt this mechanism.
11449          */
11450
11451         if (engine == NULL)
11452                 return true;
11453
11454         if (INTEL_GEN(engine->i915) < 5)
11455                 return false;
11456
11457         if (i915.use_mmio_flip < 0)
11458                 return false;
11459         else if (i915.use_mmio_flip > 0)
11460                 return true;
11461         else if (i915.enable_execlists)
11462                 return true;
11463
11464         resv = i915_gem_object_get_dmabuf_resv(obj);
11465         if (resv && !reservation_object_test_signaled_rcu(resv, false))
11466                 return true;
11467
11468         return engine != i915_gem_request_get_engine(obj->last_write_req);
11469 }
11470
11471 static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
11472                              unsigned int rotation,
11473                              struct intel_flip_work *work)
11474 {
11475         struct drm_device *dev = intel_crtc->base.dev;
11476         struct drm_i915_private *dev_priv = dev->dev_private;
11477         struct drm_framebuffer *fb = intel_crtc->base.primary->fb;
11478         const enum pipe pipe = intel_crtc->pipe;
11479         u32 ctl, stride, tile_height;
11480
11481         ctl = I915_READ(PLANE_CTL(pipe, 0));
11482         ctl &= ~PLANE_CTL_TILED_MASK;
11483         switch (fb->modifier[0]) {
11484         case DRM_FORMAT_MOD_NONE:
11485                 break;
11486         case I915_FORMAT_MOD_X_TILED:
11487                 ctl |= PLANE_CTL_TILED_X;
11488                 break;
11489         case I915_FORMAT_MOD_Y_TILED:
11490                 ctl |= PLANE_CTL_TILED_Y;
11491                 break;
11492         case I915_FORMAT_MOD_Yf_TILED:
11493                 ctl |= PLANE_CTL_TILED_YF;
11494                 break;
11495         default:
11496                 MISSING_CASE(fb->modifier[0]);
11497         }
11498
11499         /*
11500          * The stride is either expressed as a multiple of 64 bytes chunks for
11501          * linear buffers or in number of tiles for tiled buffers.
11502          */
11503         if (intel_rotation_90_or_270(rotation)) {
11504                 /* stride = Surface height in tiles */
11505                 tile_height = intel_tile_height(dev_priv, fb->modifier[0], 0);
11506                 stride = DIV_ROUND_UP(fb->height, tile_height);
11507         } else {
11508                 stride = fb->pitches[0] /
11509                         intel_fb_stride_alignment(dev_priv, fb->modifier[0],
11510                                                   fb->pixel_format);
11511         }
11512
11513         /*
11514          * Both PLANE_CTL and PLANE_STRIDE are not updated on vblank but on
11515          * PLANE_SURF updates, the update is then guaranteed to be atomic.
11516          */
11517         I915_WRITE(PLANE_CTL(pipe, 0), ctl);
11518         I915_WRITE(PLANE_STRIDE(pipe, 0), stride);
11519
11520         I915_WRITE(PLANE_SURF(pipe, 0), work->gtt_offset);
11521         POSTING_READ(PLANE_SURF(pipe, 0));
11522 }
11523
11524 static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc,
11525                              struct intel_flip_work *work)
11526 {
11527         struct drm_device *dev = intel_crtc->base.dev;
11528         struct drm_i915_private *dev_priv = dev->dev_private;
11529         struct intel_framebuffer *intel_fb =
11530                 to_intel_framebuffer(intel_crtc->base.primary->fb);
11531         struct drm_i915_gem_object *obj = intel_fb->obj;
11532         i915_reg_t reg = DSPCNTR(intel_crtc->plane);
11533         u32 dspcntr;
11534
11535         dspcntr = I915_READ(reg);
11536
11537         if (obj->tiling_mode != I915_TILING_NONE)
11538                 dspcntr |= DISPPLANE_TILED;
11539         else
11540                 dspcntr &= ~DISPPLANE_TILED;
11541
11542         I915_WRITE(reg, dspcntr);
11543
11544         I915_WRITE(DSPSURF(intel_crtc->plane), work->gtt_offset);
11545         POSTING_READ(DSPSURF(intel_crtc->plane));
11546 }
11547
11548 static void intel_mmio_flip_work_func(struct work_struct *w)
11549 {
11550         struct intel_flip_work *work =
11551                 container_of(w, struct intel_flip_work, mmio_work);
11552         struct intel_crtc *crtc = to_intel_crtc(work->crtc);
11553         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
11554         struct intel_framebuffer *intel_fb =
11555                 to_intel_framebuffer(crtc->base.primary->fb);
11556         struct drm_i915_gem_object *obj = intel_fb->obj;
11557         struct reservation_object *resv;
11558
11559         if (work->flip_queued_req)
11560                 WARN_ON(__i915_wait_request(work->flip_queued_req,
11561                                             false, NULL,
11562                                             &dev_priv->rps.mmioflips));
11563
11564         /* For framebuffer backed by dmabuf, wait for fence */
11565         resv = i915_gem_object_get_dmabuf_resv(obj);
11566         if (resv)
11567                 WARN_ON(reservation_object_wait_timeout_rcu(resv, false, false,
11568                                                             MAX_SCHEDULE_TIMEOUT) < 0);
11569
11570         intel_pipe_update_start(crtc);
11571
11572         if (INTEL_GEN(dev_priv) >= 9)
11573                 skl_do_mmio_flip(crtc, work->rotation, work);
11574         else
11575                 /* use_mmio_flip() retricts MMIO flips to ilk+ */
11576                 ilk_do_mmio_flip(crtc, work);
11577
11578         intel_pipe_update_end(crtc, work);
11579 }
11580
11581 static int intel_default_queue_flip(struct drm_device *dev,
11582                                     struct drm_crtc *crtc,
11583                                     struct drm_framebuffer *fb,
11584                                     struct drm_i915_gem_object *obj,
11585                                     struct drm_i915_gem_request *req,
11586                                     uint32_t flags)
11587 {
11588         return -ENODEV;
11589 }
11590
11591 static bool __pageflip_stall_check_cs(struct drm_i915_private *dev_priv,
11592                                       struct intel_crtc *intel_crtc,
11593                                       struct intel_flip_work *work)
11594 {
11595         u32 addr, vblank;
11596
11597         if (!atomic_read(&work->pending))
11598                 return false;
11599
11600         smp_rmb();
11601
11602         vblank = intel_crtc_get_vblank_counter(intel_crtc);
11603         if (work->flip_ready_vblank == 0) {
11604                 if (work->flip_queued_req &&
11605                     !i915_gem_request_completed(work->flip_queued_req, true))
11606                         return false;
11607
11608                 work->flip_ready_vblank = vblank;
11609         }
11610
11611         if (vblank - work->flip_ready_vblank < 3)
11612                 return false;
11613
11614         /* Potential stall - if we see that the flip has happened,
11615          * assume a missed interrupt. */
11616         if (INTEL_GEN(dev_priv) >= 4)
11617                 addr = I915_HI_DISPBASE(I915_READ(DSPSURF(intel_crtc->plane)));
11618         else
11619                 addr = I915_READ(DSPADDR(intel_crtc->plane));
11620
11621         /* There is a potential issue here with a false positive after a flip
11622          * to the same address. We could address this by checking for a
11623          * non-incrementing frame counter.
11624          */
11625         return addr == work->gtt_offset;
11626 }
11627
11628 void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe)
11629 {
11630         struct drm_device *dev = dev_priv->dev;
11631         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
11632         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11633         struct intel_flip_work *work;
11634
11635         WARN_ON(!in_interrupt());
11636
11637         if (crtc == NULL)
11638                 return;
11639
11640         spin_lock(&dev->event_lock);
11641         work = intel_crtc->flip_work;
11642
11643         if (work != NULL && !is_mmio_work(work) &&
11644             __pageflip_stall_check_cs(dev_priv, intel_crtc, work)) {
11645                 WARN_ONCE(1,
11646                           "Kicking stuck page flip: queued at %d, now %d\n",
11647                         work->flip_queued_vblank, intel_crtc_get_vblank_counter(intel_crtc));
11648                 page_flip_completed(intel_crtc);
11649                 work = NULL;
11650         }
11651
11652         if (work != NULL && !is_mmio_work(work) &&
11653             intel_crtc_get_vblank_counter(intel_crtc) - work->flip_queued_vblank > 1)
11654                 intel_queue_rps_boost_for_request(work->flip_queued_req);
11655         spin_unlock(&dev->event_lock);
11656 }
11657
11658 static int intel_crtc_page_flip(struct drm_crtc *crtc,
11659                                 struct drm_framebuffer *fb,
11660                                 struct drm_pending_vblank_event *event,
11661                                 uint32_t page_flip_flags)
11662 {
11663         struct drm_device *dev = crtc->dev;
11664         struct drm_i915_private *dev_priv = dev->dev_private;
11665         struct drm_framebuffer *old_fb = crtc->primary->fb;
11666         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
11667         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11668         struct drm_plane *primary = crtc->primary;
11669         enum pipe pipe = intel_crtc->pipe;
11670         struct intel_flip_work *work;
11671         struct intel_engine_cs *engine;
11672         bool mmio_flip;
11673         struct drm_i915_gem_request *request = NULL;
11674         int ret;
11675
11676         /*
11677          * drm_mode_page_flip_ioctl() should already catch this, but double
11678          * check to be safe.  In the future we may enable pageflipping from
11679          * a disabled primary plane.
11680          */
11681         if (WARN_ON(intel_fb_obj(old_fb) == NULL))
11682                 return -EBUSY;
11683
11684         /* Can't change pixel format via MI display flips. */
11685         if (fb->pixel_format != crtc->primary->fb->pixel_format)
11686                 return -EINVAL;
11687
11688         /*
11689          * TILEOFF/LINOFF registers can't be changed via MI display flips.
11690          * Note that pitch changes could also affect these register.
11691          */
11692         if (INTEL_INFO(dev)->gen > 3 &&
11693             (fb->offsets[0] != crtc->primary->fb->offsets[0] ||
11694              fb->pitches[0] != crtc->primary->fb->pitches[0]))
11695                 return -EINVAL;
11696
11697         if (i915_terminally_wedged(&dev_priv->gpu_error))
11698                 goto out_hang;
11699
11700         work = kzalloc(sizeof(*work), GFP_KERNEL);
11701         if (work == NULL)
11702                 return -ENOMEM;
11703
11704         work->event = event;
11705         work->crtc = crtc;
11706         work->old_fb = old_fb;
11707         INIT_WORK(&work->unpin_work, intel_unpin_work_fn);
11708
11709         ret = drm_crtc_vblank_get(crtc);
11710         if (ret)
11711                 goto free_work;
11712
11713         /* We borrow the event spin lock for protecting flip_work */
11714         spin_lock_irq(&dev->event_lock);
11715         if (intel_crtc->flip_work) {
11716                 /* Before declaring the flip queue wedged, check if
11717                  * the hardware completed the operation behind our backs.
11718                  */
11719                 if (pageflip_finished(intel_crtc, intel_crtc->flip_work)) {
11720                         DRM_DEBUG_DRIVER("flip queue: previous flip completed, continuing\n");
11721                         page_flip_completed(intel_crtc);
11722                 } else {
11723                         DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
11724                         spin_unlock_irq(&dev->event_lock);
11725
11726                         drm_crtc_vblank_put(crtc);
11727                         kfree(work);
11728                         return -EBUSY;
11729                 }
11730         }
11731         intel_crtc->flip_work = work;
11732         spin_unlock_irq(&dev->event_lock);
11733
11734         if (atomic_read(&intel_crtc->unpin_work_count) >= 2)
11735                 flush_workqueue(dev_priv->wq);
11736
11737         /* Reference the objects for the scheduled work. */
11738         drm_framebuffer_reference(work->old_fb);
11739         drm_gem_object_reference(&obj->base);
11740
11741         crtc->primary->fb = fb;
11742         update_state_fb(crtc->primary);
11743
11744         intel_fbc_pre_update(intel_crtc, intel_crtc->config,
11745                              to_intel_plane_state(primary->state));
11746
11747         work->pending_flip_obj = obj;
11748
11749         ret = i915_mutex_lock_interruptible(dev);
11750         if (ret)
11751                 goto cleanup;
11752
11753         intel_crtc->reset_counter = i915_reset_counter(&dev_priv->gpu_error);
11754         if (__i915_reset_in_progress_or_wedged(intel_crtc->reset_counter)) {
11755                 ret = -EIO;
11756                 goto cleanup;
11757         }
11758
11759         atomic_inc(&intel_crtc->unpin_work_count);
11760
11761         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
11762                 work->flip_count = I915_READ(PIPE_FLIPCOUNT_G4X(pipe)) + 1;
11763
11764         if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
11765                 engine = &dev_priv->engine[BCS];
11766                 if (obj->tiling_mode != intel_fb_obj(work->old_fb)->tiling_mode)
11767                         /* vlv: DISPLAY_FLIP fails to change tiling */
11768                         engine = NULL;
11769         } else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
11770                 engine = &dev_priv->engine[BCS];
11771         } else if (INTEL_INFO(dev)->gen >= 7) {
11772                 engine = i915_gem_request_get_engine(obj->last_write_req);
11773                 if (engine == NULL || engine->id != RCS)
11774                         engine = &dev_priv->engine[BCS];
11775         } else {
11776                 engine = &dev_priv->engine[RCS];
11777         }
11778
11779         mmio_flip = use_mmio_flip(engine, obj);
11780
11781         /* When using CS flips, we want to emit semaphores between rings.
11782          * However, when using mmio flips we will create a task to do the
11783          * synchronisation, so all we want here is to pin the framebuffer
11784          * into the display plane and skip any waits.
11785          */
11786         if (!mmio_flip) {
11787                 ret = i915_gem_object_sync(obj, engine, &request);
11788                 if (!ret && !request) {
11789                         request = i915_gem_request_alloc(engine, NULL);
11790                         ret = PTR_ERR_OR_ZERO(request);
11791                 }
11792
11793                 if (ret)
11794                         goto cleanup_pending;
11795         }
11796
11797         ret = intel_pin_and_fence_fb_obj(fb, primary->state->rotation);
11798         if (ret)
11799                 goto cleanup_pending;
11800
11801         work->gtt_offset = intel_plane_obj_offset(to_intel_plane(primary),
11802                                                   obj, 0);
11803         work->gtt_offset += intel_crtc->dspaddr_offset;
11804         work->rotation = crtc->primary->state->rotation;
11805
11806         if (mmio_flip) {
11807                 INIT_WORK(&work->mmio_work, intel_mmio_flip_work_func);
11808
11809                 i915_gem_request_assign(&work->flip_queued_req,
11810                                         obj->last_write_req);
11811
11812                 schedule_work(&work->mmio_work);
11813         } else {
11814                 i915_gem_request_assign(&work->flip_queued_req, request);
11815                 ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, request,
11816                                                    page_flip_flags);
11817                 if (ret)
11818                         goto cleanup_unpin;
11819
11820                 intel_mark_page_flip_active(intel_crtc, work);
11821
11822                 i915_add_request_no_flush(request);
11823         }
11824
11825         i915_gem_track_fb(intel_fb_obj(old_fb), obj,
11826                           to_intel_plane(primary)->frontbuffer_bit);
11827         mutex_unlock(&dev->struct_mutex);
11828
11829         intel_frontbuffer_flip_prepare(dev,
11830                                        to_intel_plane(primary)->frontbuffer_bit);
11831
11832         trace_i915_flip_request(intel_crtc->plane, obj);
11833
11834         return 0;
11835
11836 cleanup_unpin:
11837         intel_unpin_fb_obj(fb, crtc->primary->state->rotation);
11838 cleanup_pending:
11839         if (!IS_ERR_OR_NULL(request))
11840                 i915_add_request_no_flush(request);
11841         atomic_dec(&intel_crtc->unpin_work_count);
11842         mutex_unlock(&dev->struct_mutex);
11843 cleanup:
11844         crtc->primary->fb = old_fb;
11845         update_state_fb(crtc->primary);
11846
11847         drm_gem_object_unreference_unlocked(&obj->base);
11848         drm_framebuffer_unreference(work->old_fb);
11849
11850         spin_lock_irq(&dev->event_lock);
11851         intel_crtc->flip_work = NULL;
11852         spin_unlock_irq(&dev->event_lock);
11853
11854         drm_crtc_vblank_put(crtc);
11855 free_work:
11856         kfree(work);
11857
11858         if (ret == -EIO) {
11859                 struct drm_atomic_state *state;
11860                 struct drm_plane_state *plane_state;
11861
11862 out_hang:
11863                 state = drm_atomic_state_alloc(dev);
11864                 if (!state)
11865                         return -ENOMEM;
11866                 state->acquire_ctx = drm_modeset_legacy_acquire_ctx(crtc);
11867
11868 retry:
11869                 plane_state = drm_atomic_get_plane_state(state, primary);
11870                 ret = PTR_ERR_OR_ZERO(plane_state);
11871                 if (!ret) {
11872                         drm_atomic_set_fb_for_plane(plane_state, fb);
11873
11874                         ret = drm_atomic_set_crtc_for_plane(plane_state, crtc);
11875                         if (!ret)
11876                                 ret = drm_atomic_commit(state);
11877                 }
11878
11879                 if (ret == -EDEADLK) {
11880                         drm_modeset_backoff(state->acquire_ctx);
11881                         drm_atomic_state_clear(state);
11882                         goto retry;
11883                 }
11884
11885                 if (ret)
11886                         drm_atomic_state_free(state);
11887
11888                 if (ret == 0 && event) {
11889                         spin_lock_irq(&dev->event_lock);
11890                         drm_crtc_send_vblank_event(crtc, event);
11891                         spin_unlock_irq(&dev->event_lock);
11892                 }
11893         }
11894         return ret;
11895 }
11896
11897
11898 /**
11899  * intel_wm_need_update - Check whether watermarks need updating
11900  * @plane: drm plane
11901  * @state: new plane state
11902  *
11903  * Check current plane state versus the new one to determine whether
11904  * watermarks need to be recalculated.
11905  *
11906  * Returns true or false.
11907  */
11908 static bool intel_wm_need_update(struct drm_plane *plane,
11909                                  struct drm_plane_state *state)
11910 {
11911         struct intel_plane_state *new = to_intel_plane_state(state);
11912         struct intel_plane_state *cur = to_intel_plane_state(plane->state);
11913
11914         /* Update watermarks on tiling or size changes. */
11915         if (new->visible != cur->visible)
11916                 return true;
11917
11918         if (!cur->base.fb || !new->base.fb)
11919                 return false;
11920
11921         if (cur->base.fb->modifier[0] != new->base.fb->modifier[0] ||
11922             cur->base.rotation != new->base.rotation ||
11923             drm_rect_width(&new->src) != drm_rect_width(&cur->src) ||
11924             drm_rect_height(&new->src) != drm_rect_height(&cur->src) ||
11925             drm_rect_width(&new->dst) != drm_rect_width(&cur->dst) ||
11926             drm_rect_height(&new->dst) != drm_rect_height(&cur->dst))
11927                 return true;
11928
11929         return false;
11930 }
11931
11932 static bool needs_scaling(struct intel_plane_state *state)
11933 {
11934         int src_w = drm_rect_width(&state->src) >> 16;
11935         int src_h = drm_rect_height(&state->src) >> 16;
11936         int dst_w = drm_rect_width(&state->dst);
11937         int dst_h = drm_rect_height(&state->dst);
11938
11939         return (src_w != dst_w || src_h != dst_h);
11940 }
11941
11942 int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
11943                                     struct drm_plane_state *plane_state)
11944 {
11945         struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc_state);
11946         struct drm_crtc *crtc = crtc_state->crtc;
11947         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11948         struct drm_plane *plane = plane_state->plane;
11949         struct drm_device *dev = crtc->dev;
11950         struct drm_i915_private *dev_priv = to_i915(dev);
11951         struct intel_plane_state *old_plane_state =
11952                 to_intel_plane_state(plane->state);
11953         bool mode_changed = needs_modeset(crtc_state);
11954         bool was_crtc_enabled = crtc->state->active;
11955         bool is_crtc_enabled = crtc_state->active;
11956         bool turn_off, turn_on, visible, was_visible;
11957         struct drm_framebuffer *fb = plane_state->fb;
11958         int ret;
11959
11960         if (crtc_state && INTEL_INFO(dev)->gen >= 9 &&
11961             plane->type != DRM_PLANE_TYPE_CURSOR) {
11962                 ret = skl_update_scaler_plane(
11963                         to_intel_crtc_state(crtc_state),
11964                         to_intel_plane_state(plane_state));
11965                 if (ret)
11966                         return ret;
11967         }
11968
11969         was_visible = old_plane_state->visible;
11970         visible = to_intel_plane_state(plane_state)->visible;
11971
11972         if (!was_crtc_enabled && WARN_ON(was_visible))
11973                 was_visible = false;
11974
11975         /*
11976          * Visibility is calculated as if the crtc was on, but
11977          * after scaler setup everything depends on it being off
11978          * when the crtc isn't active.
11979          *
11980          * FIXME this is wrong for watermarks. Watermarks should also
11981          * be computed as if the pipe would be active. Perhaps move
11982          * per-plane wm computation to the .check_plane() hook, and
11983          * only combine the results from all planes in the current place?
11984          */
11985         if (!is_crtc_enabled)
11986                 to_intel_plane_state(plane_state)->visible = visible = false;
11987
11988         if (!was_visible && !visible)
11989                 return 0;
11990
11991         if (fb != old_plane_state->base.fb)
11992                 pipe_config->fb_changed = true;
11993
11994         turn_off = was_visible && (!visible || mode_changed);
11995         turn_on = visible && (!was_visible || mode_changed);
11996
11997         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] has [PLANE:%d:%s] with fb %i\n",
11998                          intel_crtc->base.base.id,
11999                          intel_crtc->base.name,
12000                          plane->base.id, plane->name,
12001                          fb ? fb->base.id : -1);
12002
12003         DRM_DEBUG_ATOMIC("[PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
12004                          plane->base.id, plane->name,
12005                          was_visible, visible,
12006                          turn_off, turn_on, mode_changed);
12007
12008         if (turn_on) {
12009                 pipe_config->update_wm_pre = true;
12010
12011                 /* must disable cxsr around plane enable/disable */
12012                 if (plane->type != DRM_PLANE_TYPE_CURSOR)
12013                         pipe_config->disable_cxsr = true;
12014         } else if (turn_off) {
12015                 pipe_config->update_wm_post = true;
12016
12017                 /* must disable cxsr around plane enable/disable */
12018                 if (plane->type != DRM_PLANE_TYPE_CURSOR)
12019                         pipe_config->disable_cxsr = true;
12020         } else if (intel_wm_need_update(plane, plane_state)) {
12021                 /* FIXME bollocks */
12022                 pipe_config->update_wm_pre = true;
12023                 pipe_config->update_wm_post = true;
12024         }
12025
12026         /* Pre-gen9 platforms need two-step watermark updates */
12027         if ((pipe_config->update_wm_pre || pipe_config->update_wm_post) &&
12028             INTEL_INFO(dev)->gen < 9 && dev_priv->display.optimize_watermarks)
12029                 to_intel_crtc_state(crtc_state)->wm.need_postvbl_update = true;
12030
12031         if (visible || was_visible)
12032                 pipe_config->fb_bits |= to_intel_plane(plane)->frontbuffer_bit;
12033
12034         /*
12035          * WaCxSRDisabledForSpriteScaling:ivb
12036          *
12037          * cstate->update_wm was already set above, so this flag will
12038          * take effect when we commit and program watermarks.
12039          */
12040         if (plane->type == DRM_PLANE_TYPE_OVERLAY && IS_IVYBRIDGE(dev) &&
12041             needs_scaling(to_intel_plane_state(plane_state)) &&
12042             !needs_scaling(old_plane_state))
12043                 pipe_config->disable_lp_wm = true;
12044
12045         return 0;
12046 }
12047
12048 static bool encoders_cloneable(const struct intel_encoder *a,
12049                                const struct intel_encoder *b)
12050 {
12051         /* masks could be asymmetric, so check both ways */
12052         return a == b || (a->cloneable & (1 << b->type) &&
12053                           b->cloneable & (1 << a->type));
12054 }
12055
12056 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
12057                                          struct intel_crtc *crtc,
12058                                          struct intel_encoder *encoder)
12059 {
12060         struct intel_encoder *source_encoder;
12061         struct drm_connector *connector;
12062         struct drm_connector_state *connector_state;
12063         int i;
12064
12065         for_each_connector_in_state(state, connector, connector_state, i) {
12066                 if (connector_state->crtc != &crtc->base)
12067                         continue;
12068
12069                 source_encoder =
12070                         to_intel_encoder(connector_state->best_encoder);
12071                 if (!encoders_cloneable(encoder, source_encoder))
12072                         return false;
12073         }
12074
12075         return true;
12076 }
12077
12078 static bool check_encoder_cloning(struct drm_atomic_state *state,
12079                                   struct intel_crtc *crtc)
12080 {
12081         struct intel_encoder *encoder;
12082         struct drm_connector *connector;
12083         struct drm_connector_state *connector_state;
12084         int i;
12085
12086         for_each_connector_in_state(state, connector, connector_state, i) {
12087                 if (connector_state->crtc != &crtc->base)
12088                         continue;
12089
12090                 encoder = to_intel_encoder(connector_state->best_encoder);
12091                 if (!check_single_encoder_cloning(state, crtc, encoder))
12092                         return false;
12093         }
12094
12095         return true;
12096 }
12097
12098 static int intel_crtc_atomic_check(struct drm_crtc *crtc,
12099                                    struct drm_crtc_state *crtc_state)
12100 {
12101         struct drm_device *dev = crtc->dev;
12102         struct drm_i915_private *dev_priv = dev->dev_private;
12103         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12104         struct intel_crtc_state *pipe_config =
12105                 to_intel_crtc_state(crtc_state);
12106         struct drm_atomic_state *state = crtc_state->state;
12107         int ret;
12108         bool mode_changed = needs_modeset(crtc_state);
12109
12110         if (mode_changed && !check_encoder_cloning(state, intel_crtc)) {
12111                 DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
12112                 return -EINVAL;
12113         }
12114
12115         if (mode_changed && !crtc_state->active)
12116                 pipe_config->update_wm_post = true;
12117
12118         if (mode_changed && crtc_state->enable &&
12119             dev_priv->display.crtc_compute_clock &&
12120             !WARN_ON(pipe_config->shared_dpll)) {
12121                 ret = dev_priv->display.crtc_compute_clock(intel_crtc,
12122                                                            pipe_config);
12123                 if (ret)
12124                         return ret;
12125         }
12126
12127         if (crtc_state->color_mgmt_changed) {
12128                 ret = intel_color_check(crtc, crtc_state);
12129                 if (ret)
12130                         return ret;
12131         }
12132
12133         ret = 0;
12134         if (dev_priv->display.compute_pipe_wm) {
12135                 ret = dev_priv->display.compute_pipe_wm(pipe_config);
12136                 if (ret) {
12137                         DRM_DEBUG_KMS("Target pipe watermarks are invalid\n");
12138                         return ret;
12139                 }
12140         }
12141
12142         if (dev_priv->display.compute_intermediate_wm &&
12143             !to_intel_atomic_state(state)->skip_intermediate_wm) {
12144                 if (WARN_ON(!dev_priv->display.compute_pipe_wm))
12145                         return 0;
12146
12147                 /*
12148                  * Calculate 'intermediate' watermarks that satisfy both the
12149                  * old state and the new state.  We can program these
12150                  * immediately.
12151                  */
12152                 ret = dev_priv->display.compute_intermediate_wm(crtc->dev,
12153                                                                 intel_crtc,
12154                                                                 pipe_config);
12155                 if (ret) {
12156                         DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n");
12157                         return ret;
12158                 }
12159         } else if (dev_priv->display.compute_intermediate_wm) {
12160                 if (HAS_PCH_SPLIT(dev_priv) && INTEL_GEN(dev_priv) < 9)
12161                         pipe_config->wm.ilk.intermediate = pipe_config->wm.ilk.optimal;
12162         }
12163
12164         if (INTEL_INFO(dev)->gen >= 9) {
12165                 if (mode_changed)
12166                         ret = skl_update_scaler_crtc(pipe_config);
12167
12168                 if (!ret)
12169                         ret = intel_atomic_setup_scalers(dev, intel_crtc,
12170                                                          pipe_config);
12171         }
12172
12173         return ret;
12174 }
12175
12176 static const struct drm_crtc_helper_funcs intel_helper_funcs = {
12177         .mode_set_base_atomic = intel_pipe_set_base_atomic,
12178         .atomic_begin = intel_begin_crtc_commit,
12179         .atomic_flush = intel_finish_crtc_commit,
12180         .atomic_check = intel_crtc_atomic_check,
12181 };
12182
12183 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
12184 {
12185         struct intel_connector *connector;
12186
12187         for_each_intel_connector(dev, connector) {
12188                 if (connector->base.state->crtc)
12189                         drm_connector_unreference(&connector->base);
12190
12191                 if (connector->base.encoder) {
12192                         connector->base.state->best_encoder =
12193                                 connector->base.encoder;
12194                         connector->base.state->crtc =
12195                                 connector->base.encoder->crtc;
12196
12197                         drm_connector_reference(&connector->base);
12198                 } else {
12199                         connector->base.state->best_encoder = NULL;
12200                         connector->base.state->crtc = NULL;
12201                 }
12202         }
12203 }
12204
12205 static void
12206 connected_sink_compute_bpp(struct intel_connector *connector,
12207                            struct intel_crtc_state *pipe_config)
12208 {
12209         int bpp = pipe_config->pipe_bpp;
12210
12211         DRM_DEBUG_KMS("[CONNECTOR:%d:%s] checking for sink bpp constrains\n",
12212                 connector->base.base.id,
12213                 connector->base.name);
12214
12215         /* Don't use an invalid EDID bpc value */
12216         if (connector->base.display_info.bpc &&
12217             connector->base.display_info.bpc * 3 < bpp) {
12218                 DRM_DEBUG_KMS("clamping display bpp (was %d) to EDID reported max of %d\n",
12219                               bpp, connector->base.display_info.bpc*3);
12220                 pipe_config->pipe_bpp = connector->base.display_info.bpc*3;
12221         }
12222
12223         /* Clamp bpp to default limit on screens without EDID 1.4 */
12224         if (connector->base.display_info.bpc == 0) {
12225                 int type = connector->base.connector_type;
12226                 int clamp_bpp = 24;
12227
12228                 /* Fall back to 18 bpp when DP sink capability is unknown. */
12229                 if (type == DRM_MODE_CONNECTOR_DisplayPort ||
12230                     type == DRM_MODE_CONNECTOR_eDP)
12231                         clamp_bpp = 18;
12232
12233                 if (bpp > clamp_bpp) {
12234                         DRM_DEBUG_KMS("clamping display bpp (was %d) to default limit of %d\n",
12235                                       bpp, clamp_bpp);
12236                         pipe_config->pipe_bpp = clamp_bpp;
12237                 }
12238         }
12239 }
12240
12241 static int
12242 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
12243                           struct intel_crtc_state *pipe_config)
12244 {
12245         struct drm_device *dev = crtc->base.dev;
12246         struct drm_atomic_state *state;
12247         struct drm_connector *connector;
12248         struct drm_connector_state *connector_state;
12249         int bpp, i;
12250
12251         if ((IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)))
12252                 bpp = 10*3;
12253         else if (INTEL_INFO(dev)->gen >= 5)
12254                 bpp = 12*3;
12255         else
12256                 bpp = 8*3;
12257
12258
12259         pipe_config->pipe_bpp = bpp;
12260
12261         state = pipe_config->base.state;
12262
12263         /* Clamp display bpp to EDID value */
12264         for_each_connector_in_state(state, connector, connector_state, i) {
12265                 if (connector_state->crtc != &crtc->base)
12266                         continue;
12267
12268                 connected_sink_compute_bpp(to_intel_connector(connector),
12269                                            pipe_config);
12270         }
12271
12272         return bpp;
12273 }
12274
12275 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
12276 {
12277         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
12278                         "type: 0x%x flags: 0x%x\n",
12279                 mode->crtc_clock,
12280                 mode->crtc_hdisplay, mode->crtc_hsync_start,
12281                 mode->crtc_hsync_end, mode->crtc_htotal,
12282                 mode->crtc_vdisplay, mode->crtc_vsync_start,
12283                 mode->crtc_vsync_end, mode->crtc_vtotal, mode->type, mode->flags);
12284 }
12285
12286 static void intel_dump_pipe_config(struct intel_crtc *crtc,
12287                                    struct intel_crtc_state *pipe_config,
12288                                    const char *context)
12289 {
12290         struct drm_device *dev = crtc->base.dev;
12291         struct drm_plane *plane;
12292         struct intel_plane *intel_plane;
12293         struct intel_plane_state *state;
12294         struct drm_framebuffer *fb;
12295
12296         DRM_DEBUG_KMS("[CRTC:%d:%s]%s config %p for pipe %c\n",
12297                       crtc->base.base.id, crtc->base.name,
12298                       context, pipe_config, pipe_name(crtc->pipe));
12299
12300         DRM_DEBUG_KMS("cpu_transcoder: %s\n", transcoder_name(pipe_config->cpu_transcoder));
12301         DRM_DEBUG_KMS("pipe bpp: %i, dithering: %i\n",
12302                       pipe_config->pipe_bpp, pipe_config->dither);
12303         DRM_DEBUG_KMS("fdi/pch: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
12304                       pipe_config->has_pch_encoder,
12305                       pipe_config->fdi_lanes,
12306                       pipe_config->fdi_m_n.gmch_m, pipe_config->fdi_m_n.gmch_n,
12307                       pipe_config->fdi_m_n.link_m, pipe_config->fdi_m_n.link_n,
12308                       pipe_config->fdi_m_n.tu);
12309         DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
12310                       pipe_config->has_dp_encoder,
12311                       pipe_config->lane_count,
12312                       pipe_config->dp_m_n.gmch_m, pipe_config->dp_m_n.gmch_n,
12313                       pipe_config->dp_m_n.link_m, pipe_config->dp_m_n.link_n,
12314                       pipe_config->dp_m_n.tu);
12315
12316         DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n",
12317                       pipe_config->has_dp_encoder,
12318                       pipe_config->lane_count,
12319                       pipe_config->dp_m2_n2.gmch_m,
12320                       pipe_config->dp_m2_n2.gmch_n,
12321                       pipe_config->dp_m2_n2.link_m,
12322                       pipe_config->dp_m2_n2.link_n,
12323                       pipe_config->dp_m2_n2.tu);
12324
12325         DRM_DEBUG_KMS("audio: %i, infoframes: %i\n",
12326                       pipe_config->has_audio,
12327                       pipe_config->has_infoframe);
12328
12329         DRM_DEBUG_KMS("requested mode:\n");
12330         drm_mode_debug_printmodeline(&pipe_config->base.mode);
12331         DRM_DEBUG_KMS("adjusted mode:\n");
12332         drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode);
12333         intel_dump_crtc_timings(&pipe_config->base.adjusted_mode);
12334         DRM_DEBUG_KMS("port clock: %d\n", pipe_config->port_clock);
12335         DRM_DEBUG_KMS("pipe src size: %dx%d\n",
12336                       pipe_config->pipe_src_w, pipe_config->pipe_src_h);
12337         DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
12338                       crtc->num_scalers,
12339                       pipe_config->scaler_state.scaler_users,
12340                       pipe_config->scaler_state.scaler_id);
12341         DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
12342                       pipe_config->gmch_pfit.control,
12343                       pipe_config->gmch_pfit.pgm_ratios,
12344                       pipe_config->gmch_pfit.lvds_border_bits);
12345         DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n",
12346                       pipe_config->pch_pfit.pos,
12347                       pipe_config->pch_pfit.size,
12348                       pipe_config->pch_pfit.enabled ? "enabled" : "disabled");
12349         DRM_DEBUG_KMS("ips: %i\n", pipe_config->ips_enabled);
12350         DRM_DEBUG_KMS("double wide: %i\n", pipe_config->double_wide);
12351
12352         if (IS_BROXTON(dev)) {
12353                 DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: ebb0: 0x%x, ebb4: 0x%x,"
12354                               "pll0: 0x%x, pll1: 0x%x, pll2: 0x%x, pll3: 0x%x, "
12355                               "pll6: 0x%x, pll8: 0x%x, pll9: 0x%x, pll10: 0x%x, pcsdw12: 0x%x\n",
12356                               pipe_config->ddi_pll_sel,
12357                               pipe_config->dpll_hw_state.ebb0,
12358                               pipe_config->dpll_hw_state.ebb4,
12359                               pipe_config->dpll_hw_state.pll0,
12360                               pipe_config->dpll_hw_state.pll1,
12361                               pipe_config->dpll_hw_state.pll2,
12362                               pipe_config->dpll_hw_state.pll3,
12363                               pipe_config->dpll_hw_state.pll6,
12364                               pipe_config->dpll_hw_state.pll8,
12365                               pipe_config->dpll_hw_state.pll9,
12366                               pipe_config->dpll_hw_state.pll10,
12367                               pipe_config->dpll_hw_state.pcsdw12);
12368         } else if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
12369                 DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: "
12370                               "ctrl1: 0x%x, cfgcr1: 0x%x, cfgcr2: 0x%x\n",
12371                               pipe_config->ddi_pll_sel,
12372                               pipe_config->dpll_hw_state.ctrl1,
12373                               pipe_config->dpll_hw_state.cfgcr1,
12374                               pipe_config->dpll_hw_state.cfgcr2);
12375         } else if (HAS_DDI(dev)) {
12376                 DRM_DEBUG_KMS("ddi_pll_sel: 0x%x; dpll_hw_state: wrpll: 0x%x spll: 0x%x\n",
12377                               pipe_config->ddi_pll_sel,
12378                               pipe_config->dpll_hw_state.wrpll,
12379                               pipe_config->dpll_hw_state.spll);
12380         } else {
12381                 DRM_DEBUG_KMS("dpll_hw_state: dpll: 0x%x, dpll_md: 0x%x, "
12382                               "fp0: 0x%x, fp1: 0x%x\n",
12383                               pipe_config->dpll_hw_state.dpll,
12384                               pipe_config->dpll_hw_state.dpll_md,
12385                               pipe_config->dpll_hw_state.fp0,
12386                               pipe_config->dpll_hw_state.fp1);
12387         }
12388
12389         DRM_DEBUG_KMS("planes on this crtc\n");
12390         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
12391                 intel_plane = to_intel_plane(plane);
12392                 if (intel_plane->pipe != crtc->pipe)
12393                         continue;
12394
12395                 state = to_intel_plane_state(plane->state);
12396                 fb = state->base.fb;
12397                 if (!fb) {
12398                         DRM_DEBUG_KMS("[PLANE:%d:%s] disabled, scaler_id = %d\n",
12399                                       plane->base.id, plane->name, state->scaler_id);
12400                         continue;
12401                 }
12402
12403                 DRM_DEBUG_KMS("[PLANE:%d:%s] enabled",
12404                               plane->base.id, plane->name);
12405                 DRM_DEBUG_KMS("\tFB:%d, fb = %ux%u format = %s",
12406                               fb->base.id, fb->width, fb->height,
12407                               drm_get_format_name(fb->pixel_format));
12408                 DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n",
12409                               state->scaler_id,
12410                               state->src.x1 >> 16, state->src.y1 >> 16,
12411                               drm_rect_width(&state->src) >> 16,
12412                               drm_rect_height(&state->src) >> 16,
12413                               state->dst.x1, state->dst.y1,
12414                               drm_rect_width(&state->dst),
12415                               drm_rect_height(&state->dst));
12416         }
12417 }
12418
12419 static bool check_digital_port_conflicts(struct drm_atomic_state *state)
12420 {
12421         struct drm_device *dev = state->dev;
12422         struct drm_connector *connector;
12423         unsigned int used_ports = 0;
12424
12425         /*
12426          * Walk the connector list instead of the encoder
12427          * list to detect the problem on ddi platforms
12428          * where there's just one encoder per digital port.
12429          */
12430         drm_for_each_connector(connector, dev) {
12431                 struct drm_connector_state *connector_state;
12432                 struct intel_encoder *encoder;
12433
12434                 connector_state = drm_atomic_get_existing_connector_state(state, connector);
12435                 if (!connector_state)
12436                         connector_state = connector->state;
12437
12438                 if (!connector_state->best_encoder)
12439                         continue;
12440
12441                 encoder = to_intel_encoder(connector_state->best_encoder);
12442
12443                 WARN_ON(!connector_state->crtc);
12444
12445                 switch (encoder->type) {
12446                         unsigned int port_mask;
12447                 case INTEL_OUTPUT_UNKNOWN:
12448                         if (WARN_ON(!HAS_DDI(dev)))
12449                                 break;
12450                 case INTEL_OUTPUT_DISPLAYPORT:
12451                 case INTEL_OUTPUT_HDMI:
12452                 case INTEL_OUTPUT_EDP:
12453                         port_mask = 1 << enc_to_dig_port(&encoder->base)->port;
12454
12455                         /* the same port mustn't appear more than once */
12456                         if (used_ports & port_mask)
12457                                 return false;
12458
12459                         used_ports |= port_mask;
12460                 default:
12461                         break;
12462                 }
12463         }
12464
12465         return true;
12466 }
12467
12468 static void
12469 clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
12470 {
12471         struct drm_crtc_state tmp_state;
12472         struct intel_crtc_scaler_state scaler_state;
12473         struct intel_dpll_hw_state dpll_hw_state;
12474         struct intel_shared_dpll *shared_dpll;
12475         uint32_t ddi_pll_sel;
12476         bool force_thru;
12477
12478         /* FIXME: before the switch to atomic started, a new pipe_config was
12479          * kzalloc'd. Code that depends on any field being zero should be
12480          * fixed, so that the crtc_state can be safely duplicated. For now,
12481          * only fields that are know to not cause problems are preserved. */
12482
12483         tmp_state = crtc_state->base;
12484         scaler_state = crtc_state->scaler_state;
12485         shared_dpll = crtc_state->shared_dpll;
12486         dpll_hw_state = crtc_state->dpll_hw_state;
12487         ddi_pll_sel = crtc_state->ddi_pll_sel;
12488         force_thru = crtc_state->pch_pfit.force_thru;
12489
12490         memset(crtc_state, 0, sizeof *crtc_state);
12491
12492         crtc_state->base = tmp_state;
12493         crtc_state->scaler_state = scaler_state;
12494         crtc_state->shared_dpll = shared_dpll;
12495         crtc_state->dpll_hw_state = dpll_hw_state;
12496         crtc_state->ddi_pll_sel = ddi_pll_sel;
12497         crtc_state->pch_pfit.force_thru = force_thru;
12498 }
12499
12500 static int
12501 intel_modeset_pipe_config(struct drm_crtc *crtc,
12502                           struct intel_crtc_state *pipe_config)
12503 {
12504         struct drm_atomic_state *state = pipe_config->base.state;
12505         struct intel_encoder *encoder;
12506         struct drm_connector *connector;
12507         struct drm_connector_state *connector_state;
12508         int base_bpp, ret = -EINVAL;
12509         int i;
12510         bool retry = true;
12511
12512         clear_intel_crtc_state(pipe_config);
12513
12514         pipe_config->cpu_transcoder =
12515                 (enum transcoder) to_intel_crtc(crtc)->pipe;
12516
12517         /*
12518          * Sanitize sync polarity flags based on requested ones. If neither
12519          * positive or negative polarity is requested, treat this as meaning
12520          * negative polarity.
12521          */
12522         if (!(pipe_config->base.adjusted_mode.flags &
12523               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
12524                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
12525
12526         if (!(pipe_config->base.adjusted_mode.flags &
12527               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
12528                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
12529
12530         base_bpp = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
12531                                              pipe_config);
12532         if (base_bpp < 0)
12533                 goto fail;
12534
12535         /*
12536          * Determine the real pipe dimensions. Note that stereo modes can
12537          * increase the actual pipe size due to the frame doubling and
12538          * insertion of additional space for blanks between the frame. This
12539          * is stored in the crtc timings. We use the requested mode to do this
12540          * computation to clearly distinguish it from the adjusted mode, which
12541          * can be changed by the connectors in the below retry loop.
12542          */
12543         drm_crtc_get_hv_timing(&pipe_config->base.mode,
12544                                &pipe_config->pipe_src_w,
12545                                &pipe_config->pipe_src_h);
12546
12547 encoder_retry:
12548         /* Ensure the port clock defaults are reset when retrying. */
12549         pipe_config->port_clock = 0;
12550         pipe_config->pixel_multiplier = 1;
12551
12552         /* Fill in default crtc timings, allow encoders to overwrite them. */
12553         drm_mode_set_crtcinfo(&pipe_config->base.adjusted_mode,
12554                               CRTC_STEREO_DOUBLE);
12555
12556         /* Pass our mode to the connectors and the CRTC to give them a chance to
12557          * adjust it according to limitations or connector properties, and also
12558          * a chance to reject the mode entirely.
12559          */
12560         for_each_connector_in_state(state, connector, connector_state, i) {
12561                 if (connector_state->crtc != crtc)
12562                         continue;
12563
12564                 encoder = to_intel_encoder(connector_state->best_encoder);
12565
12566                 if (!(encoder->compute_config(encoder, pipe_config))) {
12567                         DRM_DEBUG_KMS("Encoder config failure\n");
12568                         goto fail;
12569                 }
12570         }
12571
12572         /* Set default port clock if not overwritten by the encoder. Needs to be
12573          * done afterwards in case the encoder adjusts the mode. */
12574         if (!pipe_config->port_clock)
12575                 pipe_config->port_clock = pipe_config->base.adjusted_mode.crtc_clock
12576                         * pipe_config->pixel_multiplier;
12577
12578         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
12579         if (ret < 0) {
12580                 DRM_DEBUG_KMS("CRTC fixup failed\n");
12581                 goto fail;
12582         }
12583
12584         if (ret == RETRY) {
12585                 if (WARN(!retry, "loop in pipe configuration computation\n")) {
12586                         ret = -EINVAL;
12587                         goto fail;
12588                 }
12589
12590                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
12591                 retry = false;
12592                 goto encoder_retry;
12593         }
12594
12595         /* Dithering seems to not pass-through bits correctly when it should, so
12596          * only enable it on 6bpc panels. */
12597         pipe_config->dither = pipe_config->pipe_bpp == 6*3;
12598         DRM_DEBUG_KMS("hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
12599                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
12600
12601 fail:
12602         return ret;
12603 }
12604
12605 static void
12606 intel_modeset_update_crtc_state(struct drm_atomic_state *state)
12607 {
12608         struct drm_crtc *crtc;
12609         struct drm_crtc_state *crtc_state;
12610         int i;
12611
12612         /* Double check state. */
12613         for_each_crtc_in_state(state, crtc, crtc_state, i) {
12614                 to_intel_crtc(crtc)->config = to_intel_crtc_state(crtc->state);
12615
12616                 /* Update hwmode for vblank functions */
12617                 if (crtc->state->active)
12618                         crtc->hwmode = crtc->state->adjusted_mode;
12619                 else
12620                         crtc->hwmode.crtc_clock = 0;
12621
12622                 /*
12623                  * Update legacy state to satisfy fbc code. This can
12624                  * be removed when fbc uses the atomic state.
12625                  */
12626                 if (drm_atomic_get_existing_plane_state(state, crtc->primary)) {
12627                         struct drm_plane_state *plane_state = crtc->primary->state;
12628
12629                         crtc->primary->fb = plane_state->fb;
12630                         crtc->x = plane_state->src_x >> 16;
12631                         crtc->y = plane_state->src_y >> 16;
12632                 }
12633         }
12634 }
12635
12636 static bool intel_fuzzy_clock_check(int clock1, int clock2)
12637 {
12638         int diff;
12639
12640         if (clock1 == clock2)
12641                 return true;
12642
12643         if (!clock1 || !clock2)
12644                 return false;
12645
12646         diff = abs(clock1 - clock2);
12647
12648         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
12649                 return true;
12650
12651         return false;
12652 }
12653
12654 #define for_each_intel_crtc_masked(dev, mask, intel_crtc) \
12655         list_for_each_entry((intel_crtc), \
12656                             &(dev)->mode_config.crtc_list, \
12657                             base.head) \
12658                 for_each_if (mask & (1 <<(intel_crtc)->pipe))
12659
12660 static bool
12661 intel_compare_m_n(unsigned int m, unsigned int n,
12662                   unsigned int m2, unsigned int n2,
12663                   bool exact)
12664 {
12665         if (m == m2 && n == n2)
12666                 return true;
12667
12668         if (exact || !m || !n || !m2 || !n2)
12669                 return false;
12670
12671         BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
12672
12673         if (n > n2) {
12674                 while (n > n2) {
12675                         m2 <<= 1;
12676                         n2 <<= 1;
12677                 }
12678         } else if (n < n2) {
12679                 while (n < n2) {
12680                         m <<= 1;
12681                         n <<= 1;
12682                 }
12683         }
12684
12685         if (n != n2)
12686                 return false;
12687
12688         return intel_fuzzy_clock_check(m, m2);
12689 }
12690
12691 static bool
12692 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
12693                        struct intel_link_m_n *m2_n2,
12694                        bool adjust)
12695 {
12696         if (m_n->tu == m2_n2->tu &&
12697             intel_compare_m_n(m_n->gmch_m, m_n->gmch_n,
12698                               m2_n2->gmch_m, m2_n2->gmch_n, !adjust) &&
12699             intel_compare_m_n(m_n->link_m, m_n->link_n,
12700                               m2_n2->link_m, m2_n2->link_n, !adjust)) {
12701                 if (adjust)
12702                         *m2_n2 = *m_n;
12703
12704                 return true;
12705         }
12706
12707         return false;
12708 }
12709
12710 static bool
12711 intel_pipe_config_compare(struct drm_device *dev,
12712                           struct intel_crtc_state *current_config,
12713                           struct intel_crtc_state *pipe_config,
12714                           bool adjust)
12715 {
12716         bool ret = true;
12717
12718 #define INTEL_ERR_OR_DBG_KMS(fmt, ...) \
12719         do { \
12720                 if (!adjust) \
12721                         DRM_ERROR(fmt, ##__VA_ARGS__); \
12722                 else \
12723                         DRM_DEBUG_KMS(fmt, ##__VA_ARGS__); \
12724         } while (0)
12725
12726 #define PIPE_CONF_CHECK_X(name) \
12727         if (current_config->name != pipe_config->name) { \
12728                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12729                           "(expected 0x%08x, found 0x%08x)\n", \
12730                           current_config->name, \
12731                           pipe_config->name); \
12732                 ret = false; \
12733         }
12734
12735 #define PIPE_CONF_CHECK_I(name) \
12736         if (current_config->name != pipe_config->name) { \
12737                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12738                           "(expected %i, found %i)\n", \
12739                           current_config->name, \
12740                           pipe_config->name); \
12741                 ret = false; \
12742         }
12743
12744 #define PIPE_CONF_CHECK_P(name) \
12745         if (current_config->name != pipe_config->name) { \
12746                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12747                           "(expected %p, found %p)\n", \
12748                           current_config->name, \
12749                           pipe_config->name); \
12750                 ret = false; \
12751         }
12752
12753 #define PIPE_CONF_CHECK_M_N(name) \
12754         if (!intel_compare_link_m_n(&current_config->name, \
12755                                     &pipe_config->name,\
12756                                     adjust)) { \
12757                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12758                           "(expected tu %i gmch %i/%i link %i/%i, " \
12759                           "found tu %i, gmch %i/%i link %i/%i)\n", \
12760                           current_config->name.tu, \
12761                           current_config->name.gmch_m, \
12762                           current_config->name.gmch_n, \
12763                           current_config->name.link_m, \
12764                           current_config->name.link_n, \
12765                           pipe_config->name.tu, \
12766                           pipe_config->name.gmch_m, \
12767                           pipe_config->name.gmch_n, \
12768                           pipe_config->name.link_m, \
12769                           pipe_config->name.link_n); \
12770                 ret = false; \
12771         }
12772
12773 /* This is required for BDW+ where there is only one set of registers for
12774  * switching between high and low RR.
12775  * This macro can be used whenever a comparison has to be made between one
12776  * hw state and multiple sw state variables.
12777  */
12778 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) \
12779         if (!intel_compare_link_m_n(&current_config->name, \
12780                                     &pipe_config->name, adjust) && \
12781             !intel_compare_link_m_n(&current_config->alt_name, \
12782                                     &pipe_config->name, adjust)) { \
12783                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12784                           "(expected tu %i gmch %i/%i link %i/%i, " \
12785                           "or tu %i gmch %i/%i link %i/%i, " \
12786                           "found tu %i, gmch %i/%i link %i/%i)\n", \
12787                           current_config->name.tu, \
12788                           current_config->name.gmch_m, \
12789                           current_config->name.gmch_n, \
12790                           current_config->name.link_m, \
12791                           current_config->name.link_n, \
12792                           current_config->alt_name.tu, \
12793                           current_config->alt_name.gmch_m, \
12794                           current_config->alt_name.gmch_n, \
12795                           current_config->alt_name.link_m, \
12796                           current_config->alt_name.link_n, \
12797                           pipe_config->name.tu, \
12798                           pipe_config->name.gmch_m, \
12799                           pipe_config->name.gmch_n, \
12800                           pipe_config->name.link_m, \
12801                           pipe_config->name.link_n); \
12802                 ret = false; \
12803         }
12804
12805 #define PIPE_CONF_CHECK_FLAGS(name, mask)       \
12806         if ((current_config->name ^ pipe_config->name) & (mask)) { \
12807                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name "(" #mask ") " \
12808                           "(expected %i, found %i)\n", \
12809                           current_config->name & (mask), \
12810                           pipe_config->name & (mask)); \
12811                 ret = false; \
12812         }
12813
12814 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) \
12815         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
12816                 INTEL_ERR_OR_DBG_KMS("mismatch in " #name " " \
12817                           "(expected %i, found %i)\n", \
12818                           current_config->name, \
12819                           pipe_config->name); \
12820                 ret = false; \
12821         }
12822
12823 #define PIPE_CONF_QUIRK(quirk)  \
12824         ((current_config->quirks | pipe_config->quirks) & (quirk))
12825
12826         PIPE_CONF_CHECK_I(cpu_transcoder);
12827
12828         PIPE_CONF_CHECK_I(has_pch_encoder);
12829         PIPE_CONF_CHECK_I(fdi_lanes);
12830         PIPE_CONF_CHECK_M_N(fdi_m_n);
12831
12832         PIPE_CONF_CHECK_I(has_dp_encoder);
12833         PIPE_CONF_CHECK_I(lane_count);
12834         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
12835
12836         if (INTEL_INFO(dev)->gen < 8) {
12837                 PIPE_CONF_CHECK_M_N(dp_m_n);
12838
12839                 if (current_config->has_drrs)
12840                         PIPE_CONF_CHECK_M_N(dp_m2_n2);
12841         } else
12842                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
12843
12844         PIPE_CONF_CHECK_I(has_dsi_encoder);
12845
12846         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
12847         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
12848         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start);
12849         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_end);
12850         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_start);
12851         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_end);
12852
12853         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vdisplay);
12854         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vtotal);
12855         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_start);
12856         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_end);
12857         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_start);
12858         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end);
12859
12860         PIPE_CONF_CHECK_I(pixel_multiplier);
12861         PIPE_CONF_CHECK_I(has_hdmi_sink);
12862         if ((INTEL_INFO(dev)->gen < 8 && !IS_HASWELL(dev)) ||
12863             IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
12864                 PIPE_CONF_CHECK_I(limited_color_range);
12865         PIPE_CONF_CHECK_I(has_infoframe);
12866
12867         PIPE_CONF_CHECK_I(has_audio);
12868
12869         PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12870                               DRM_MODE_FLAG_INTERLACE);
12871
12872         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
12873                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12874                                       DRM_MODE_FLAG_PHSYNC);
12875                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12876                                       DRM_MODE_FLAG_NHSYNC);
12877                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12878                                       DRM_MODE_FLAG_PVSYNC);
12879                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12880                                       DRM_MODE_FLAG_NVSYNC);
12881         }
12882
12883         PIPE_CONF_CHECK_X(gmch_pfit.control);
12884         /* pfit ratios are autocomputed by the hw on gen4+ */
12885         if (INTEL_INFO(dev)->gen < 4)
12886                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
12887         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
12888
12889         if (!adjust) {
12890                 PIPE_CONF_CHECK_I(pipe_src_w);
12891                 PIPE_CONF_CHECK_I(pipe_src_h);
12892
12893                 PIPE_CONF_CHECK_I(pch_pfit.enabled);
12894                 if (current_config->pch_pfit.enabled) {
12895                         PIPE_CONF_CHECK_X(pch_pfit.pos);
12896                         PIPE_CONF_CHECK_X(pch_pfit.size);
12897                 }
12898
12899                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
12900         }
12901
12902         /* BDW+ don't expose a synchronous way to read the state */
12903         if (IS_HASWELL(dev))
12904                 PIPE_CONF_CHECK_I(ips_enabled);
12905
12906         PIPE_CONF_CHECK_I(double_wide);
12907
12908         PIPE_CONF_CHECK_X(ddi_pll_sel);
12909
12910         PIPE_CONF_CHECK_P(shared_dpll);
12911         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
12912         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
12913         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
12914         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
12915         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
12916         PIPE_CONF_CHECK_X(dpll_hw_state.spll);
12917         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
12918         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
12919         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
12920
12921         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
12922         PIPE_CONF_CHECK_X(dsi_pll.div);
12923
12924         if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5)
12925                 PIPE_CONF_CHECK_I(pipe_bpp);
12926
12927         PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
12928         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
12929
12930 #undef PIPE_CONF_CHECK_X
12931 #undef PIPE_CONF_CHECK_I
12932 #undef PIPE_CONF_CHECK_P
12933 #undef PIPE_CONF_CHECK_FLAGS
12934 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
12935 #undef PIPE_CONF_QUIRK
12936 #undef INTEL_ERR_OR_DBG_KMS
12937
12938         return ret;
12939 }
12940
12941 static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
12942                                            const struct intel_crtc_state *pipe_config)
12943 {
12944         if (pipe_config->has_pch_encoder) {
12945                 int fdi_dotclock = intel_dotclock_calculate(intel_fdi_link_freq(dev_priv, pipe_config),
12946                                                             &pipe_config->fdi_m_n);
12947                 int dotclock = pipe_config->base.adjusted_mode.crtc_clock;
12948
12949                 /*
12950                  * FDI already provided one idea for the dotclock.
12951                  * Yell if the encoder disagrees.
12952                  */
12953                 WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock),
12954                      "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
12955                      fdi_dotclock, dotclock);
12956         }
12957 }
12958
12959 static void verify_wm_state(struct drm_crtc *crtc,
12960                             struct drm_crtc_state *new_state)
12961 {
12962         struct drm_device *dev = crtc->dev;
12963         struct drm_i915_private *dev_priv = dev->dev_private;
12964         struct skl_ddb_allocation hw_ddb, *sw_ddb;
12965         struct skl_ddb_entry *hw_entry, *sw_entry;
12966         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
12967         const enum pipe pipe = intel_crtc->pipe;
12968         int plane;
12969
12970         if (INTEL_INFO(dev)->gen < 9 || !new_state->active)
12971                 return;
12972
12973         skl_ddb_get_hw_state(dev_priv, &hw_ddb);
12974         sw_ddb = &dev_priv->wm.skl_hw.ddb;
12975
12976         /* planes */
12977         for_each_plane(dev_priv, pipe, plane) {
12978                 hw_entry = &hw_ddb.plane[pipe][plane];
12979                 sw_entry = &sw_ddb->plane[pipe][plane];
12980
12981                 if (skl_ddb_entry_equal(hw_entry, sw_entry))
12982                         continue;
12983
12984                 DRM_ERROR("mismatch in DDB state pipe %c plane %d "
12985                           "(expected (%u,%u), found (%u,%u))\n",
12986                           pipe_name(pipe), plane + 1,
12987                           sw_entry->start, sw_entry->end,
12988                           hw_entry->start, hw_entry->end);
12989         }
12990
12991         /* cursor */
12992         hw_entry = &hw_ddb.plane[pipe][PLANE_CURSOR];
12993         sw_entry = &sw_ddb->plane[pipe][PLANE_CURSOR];
12994
12995         if (!skl_ddb_entry_equal(hw_entry, sw_entry)) {
12996                 DRM_ERROR("mismatch in DDB state pipe %c cursor "
12997                           "(expected (%u,%u), found (%u,%u))\n",
12998                           pipe_name(pipe),
12999                           sw_entry->start, sw_entry->end,
13000                           hw_entry->start, hw_entry->end);
13001         }
13002 }
13003
13004 static void
13005 verify_connector_state(struct drm_device *dev, struct drm_crtc *crtc)
13006 {
13007         struct drm_connector *connector;
13008
13009         drm_for_each_connector(connector, dev) {
13010                 struct drm_encoder *encoder = connector->encoder;
13011                 struct drm_connector_state *state = connector->state;
13012
13013                 if (state->crtc != crtc)
13014                         continue;
13015
13016                 intel_connector_verify_state(to_intel_connector(connector));
13017
13018                 I915_STATE_WARN(state->best_encoder != encoder,
13019                      "connector's atomic encoder doesn't match legacy encoder\n");
13020         }
13021 }
13022
13023 static void
13024 verify_encoder_state(struct drm_device *dev)
13025 {
13026         struct intel_encoder *encoder;
13027         struct intel_connector *connector;
13028
13029         for_each_intel_encoder(dev, encoder) {
13030                 bool enabled = false;
13031                 enum pipe pipe;
13032
13033                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
13034                               encoder->base.base.id,
13035                               encoder->base.name);
13036
13037                 for_each_intel_connector(dev, connector) {
13038                         if (connector->base.state->best_encoder != &encoder->base)
13039                                 continue;
13040                         enabled = true;
13041
13042                         I915_STATE_WARN(connector->base.state->crtc !=
13043                                         encoder->base.crtc,
13044                              "connector's crtc doesn't match encoder crtc\n");
13045                 }
13046
13047                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
13048                      "encoder's enabled state mismatch "
13049                      "(expected %i, found %i)\n",
13050                      !!encoder->base.crtc, enabled);
13051
13052                 if (!encoder->base.crtc) {
13053                         bool active;
13054
13055                         active = encoder->get_hw_state(encoder, &pipe);
13056                         I915_STATE_WARN(active,
13057                              "encoder detached but still enabled on pipe %c.\n",
13058                              pipe_name(pipe));
13059                 }
13060         }
13061 }
13062
13063 static void
13064 verify_crtc_state(struct drm_crtc *crtc,
13065                   struct drm_crtc_state *old_crtc_state,
13066                   struct drm_crtc_state *new_crtc_state)
13067 {
13068         struct drm_device *dev = crtc->dev;
13069         struct drm_i915_private *dev_priv = dev->dev_private;
13070         struct intel_encoder *encoder;
13071         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13072         struct intel_crtc_state *pipe_config, *sw_config;
13073         struct drm_atomic_state *old_state;
13074         bool active;
13075
13076         old_state = old_crtc_state->state;
13077         __drm_atomic_helper_crtc_destroy_state(old_crtc_state);
13078         pipe_config = to_intel_crtc_state(old_crtc_state);
13079         memset(pipe_config, 0, sizeof(*pipe_config));
13080         pipe_config->base.crtc = crtc;
13081         pipe_config->base.state = old_state;
13082
13083         DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
13084
13085         active = dev_priv->display.get_pipe_config(intel_crtc, pipe_config);
13086
13087         /* hw state is inconsistent with the pipe quirk */
13088         if ((intel_crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
13089             (intel_crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
13090                 active = new_crtc_state->active;
13091
13092         I915_STATE_WARN(new_crtc_state->active != active,
13093              "crtc active state doesn't match with hw state "
13094              "(expected %i, found %i)\n", new_crtc_state->active, active);
13095
13096         I915_STATE_WARN(intel_crtc->active != new_crtc_state->active,
13097              "transitional active state does not match atomic hw state "
13098              "(expected %i, found %i)\n", new_crtc_state->active, intel_crtc->active);
13099
13100         for_each_encoder_on_crtc(dev, crtc, encoder) {
13101                 enum pipe pipe;
13102
13103                 active = encoder->get_hw_state(encoder, &pipe);
13104                 I915_STATE_WARN(active != new_crtc_state->active,
13105                         "[ENCODER:%i] active %i with crtc active %i\n",
13106                         encoder->base.base.id, active, new_crtc_state->active);
13107
13108                 I915_STATE_WARN(active && intel_crtc->pipe != pipe,
13109                                 "Encoder connected to wrong pipe %c\n",
13110                                 pipe_name(pipe));
13111
13112                 if (active)
13113                         encoder->get_config(encoder, pipe_config);
13114         }
13115
13116         if (!new_crtc_state->active)
13117                 return;
13118
13119         intel_pipe_config_sanity_check(dev_priv, pipe_config);
13120
13121         sw_config = to_intel_crtc_state(crtc->state);
13122         if (!intel_pipe_config_compare(dev, sw_config,
13123                                        pipe_config, false)) {
13124                 I915_STATE_WARN(1, "pipe state doesn't match!\n");
13125                 intel_dump_pipe_config(intel_crtc, pipe_config,
13126                                        "[hw state]");
13127                 intel_dump_pipe_config(intel_crtc, sw_config,
13128                                        "[sw state]");
13129         }
13130 }
13131
13132 static void
13133 verify_single_dpll_state(struct drm_i915_private *dev_priv,
13134                          struct intel_shared_dpll *pll,
13135                          struct drm_crtc *crtc,
13136                          struct drm_crtc_state *new_state)
13137 {
13138         struct intel_dpll_hw_state dpll_hw_state;
13139         unsigned crtc_mask;
13140         bool active;
13141
13142         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
13143
13144         DRM_DEBUG_KMS("%s\n", pll->name);
13145
13146         active = pll->funcs.get_hw_state(dev_priv, pll, &dpll_hw_state);
13147
13148         if (!(pll->flags & INTEL_DPLL_ALWAYS_ON)) {
13149                 I915_STATE_WARN(!pll->on && pll->active_mask,
13150                      "pll in active use but not on in sw tracking\n");
13151                 I915_STATE_WARN(pll->on && !pll->active_mask,
13152                      "pll is on but not used by any active crtc\n");
13153                 I915_STATE_WARN(pll->on != active,
13154                      "pll on state mismatch (expected %i, found %i)\n",
13155                      pll->on, active);
13156         }
13157
13158         if (!crtc) {
13159                 I915_STATE_WARN(pll->active_mask & ~pll->config.crtc_mask,
13160                                 "more active pll users than references: %x vs %x\n",
13161                                 pll->active_mask, pll->config.crtc_mask);
13162
13163                 return;
13164         }
13165
13166         crtc_mask = 1 << drm_crtc_index(crtc);
13167
13168         if (new_state->active)
13169                 I915_STATE_WARN(!(pll->active_mask & crtc_mask),
13170                                 "pll active mismatch (expected pipe %c in active mask 0x%02x)\n",
13171                                 pipe_name(drm_crtc_index(crtc)), pll->active_mask);
13172         else
13173                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13174                                 "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n",
13175                                 pipe_name(drm_crtc_index(crtc)), pll->active_mask);
13176
13177         I915_STATE_WARN(!(pll->config.crtc_mask & crtc_mask),
13178                         "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n",
13179                         crtc_mask, pll->config.crtc_mask);
13180
13181         I915_STATE_WARN(pll->on && memcmp(&pll->config.hw_state,
13182                                           &dpll_hw_state,
13183                                           sizeof(dpll_hw_state)),
13184                         "pll hw state mismatch\n");
13185 }
13186
13187 static void
13188 verify_shared_dpll_state(struct drm_device *dev, struct drm_crtc *crtc,
13189                          struct drm_crtc_state *old_crtc_state,
13190                          struct drm_crtc_state *new_crtc_state)
13191 {
13192         struct drm_i915_private *dev_priv = dev->dev_private;
13193         struct intel_crtc_state *old_state = to_intel_crtc_state(old_crtc_state);
13194         struct intel_crtc_state *new_state = to_intel_crtc_state(new_crtc_state);
13195
13196         if (new_state->shared_dpll)
13197                 verify_single_dpll_state(dev_priv, new_state->shared_dpll, crtc, new_crtc_state);
13198
13199         if (old_state->shared_dpll &&
13200             old_state->shared_dpll != new_state->shared_dpll) {
13201                 unsigned crtc_mask = 1 << drm_crtc_index(crtc);
13202                 struct intel_shared_dpll *pll = old_state->shared_dpll;
13203
13204                 I915_STATE_WARN(pll->active_mask & crtc_mask,
13205                                 "pll active mismatch (didn't expect pipe %c in active mask)\n",
13206                                 pipe_name(drm_crtc_index(crtc)));
13207                 I915_STATE_WARN(pll->config.crtc_mask & crtc_mask,
13208                                 "pll enabled crtcs mismatch (found %x in enabled mask)\n",
13209                                 pipe_name(drm_crtc_index(crtc)));
13210         }
13211 }
13212
13213 static void
13214 intel_modeset_verify_crtc(struct drm_crtc *crtc,
13215                          struct drm_crtc_state *old_state,
13216                          struct drm_crtc_state *new_state)
13217 {
13218         if (!needs_modeset(new_state) &&
13219             !to_intel_crtc_state(new_state)->update_pipe)
13220                 return;
13221
13222         verify_wm_state(crtc, new_state);
13223         verify_connector_state(crtc->dev, crtc);
13224         verify_crtc_state(crtc, old_state, new_state);
13225         verify_shared_dpll_state(crtc->dev, crtc, old_state, new_state);
13226 }
13227
13228 static void
13229 verify_disabled_dpll_state(struct drm_device *dev)
13230 {
13231         struct drm_i915_private *dev_priv = dev->dev_private;
13232         int i;
13233
13234         for (i = 0; i < dev_priv->num_shared_dpll; i++)
13235                 verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL);
13236 }
13237
13238 static void
13239 intel_modeset_verify_disabled(struct drm_device *dev)
13240 {
13241         verify_encoder_state(dev);
13242         verify_connector_state(dev, NULL);
13243         verify_disabled_dpll_state(dev);
13244 }
13245
13246 static void update_scanline_offset(struct intel_crtc *crtc)
13247 {
13248         struct drm_device *dev = crtc->base.dev;
13249
13250         /*
13251          * The scanline counter increments at the leading edge of hsync.
13252          *
13253          * On most platforms it starts counting from vtotal-1 on the
13254          * first active line. That means the scanline counter value is
13255          * always one less than what we would expect. Ie. just after
13256          * start of vblank, which also occurs at start of hsync (on the
13257          * last active line), the scanline counter will read vblank_start-1.
13258          *
13259          * On gen2 the scanline counter starts counting from 1 instead
13260          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
13261          * to keep the value positive), instead of adding one.
13262          *
13263          * On HSW+ the behaviour of the scanline counter depends on the output
13264          * type. For DP ports it behaves like most other platforms, but on HDMI
13265          * there's an extra 1 line difference. So we need to add two instead of
13266          * one to the value.
13267          */
13268         if (IS_GEN2(dev)) {
13269                 const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
13270                 int vtotal;
13271
13272                 vtotal = adjusted_mode->crtc_vtotal;
13273                 if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
13274                         vtotal /= 2;
13275
13276                 crtc->scanline_offset = vtotal - 1;
13277         } else if (HAS_DDI(dev) &&
13278                    intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI)) {
13279                 crtc->scanline_offset = 2;
13280         } else
13281                 crtc->scanline_offset = 1;
13282 }
13283
13284 static void intel_modeset_clear_plls(struct drm_atomic_state *state)
13285 {
13286         struct drm_device *dev = state->dev;
13287         struct drm_i915_private *dev_priv = to_i915(dev);
13288         struct intel_shared_dpll_config *shared_dpll = NULL;
13289         struct drm_crtc *crtc;
13290         struct drm_crtc_state *crtc_state;
13291         int i;
13292
13293         if (!dev_priv->display.crtc_compute_clock)
13294                 return;
13295
13296         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13297                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13298                 struct intel_shared_dpll *old_dpll =
13299                         to_intel_crtc_state(crtc->state)->shared_dpll;
13300
13301                 if (!needs_modeset(crtc_state))
13302                         continue;
13303
13304                 to_intel_crtc_state(crtc_state)->shared_dpll = NULL;
13305
13306                 if (!old_dpll)
13307                         continue;
13308
13309                 if (!shared_dpll)
13310                         shared_dpll = intel_atomic_get_shared_dpll_state(state);
13311
13312                 intel_shared_dpll_config_put(shared_dpll, old_dpll, intel_crtc);
13313         }
13314 }
13315
13316 /*
13317  * This implements the workaround described in the "notes" section of the mode
13318  * set sequence documentation. When going from no pipes or single pipe to
13319  * multiple pipes, and planes are enabled after the pipe, we need to wait at
13320  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
13321  */
13322 static int haswell_mode_set_planes_workaround(struct drm_atomic_state *state)
13323 {
13324         struct drm_crtc_state *crtc_state;
13325         struct intel_crtc *intel_crtc;
13326         struct drm_crtc *crtc;
13327         struct intel_crtc_state *first_crtc_state = NULL;
13328         struct intel_crtc_state *other_crtc_state = NULL;
13329         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
13330         int i;
13331
13332         /* look at all crtc's that are going to be enabled in during modeset */
13333         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13334                 intel_crtc = to_intel_crtc(crtc);
13335
13336                 if (!crtc_state->active || !needs_modeset(crtc_state))
13337                         continue;
13338
13339                 if (first_crtc_state) {
13340                         other_crtc_state = to_intel_crtc_state(crtc_state);
13341                         break;
13342                 } else {
13343                         first_crtc_state = to_intel_crtc_state(crtc_state);
13344                         first_pipe = intel_crtc->pipe;
13345                 }
13346         }
13347
13348         /* No workaround needed? */
13349         if (!first_crtc_state)
13350                 return 0;
13351
13352         /* w/a possibly needed, check how many crtc's are already enabled. */
13353         for_each_intel_crtc(state->dev, intel_crtc) {
13354                 struct intel_crtc_state *pipe_config;
13355
13356                 pipe_config = intel_atomic_get_crtc_state(state, intel_crtc);
13357                 if (IS_ERR(pipe_config))
13358                         return PTR_ERR(pipe_config);
13359
13360                 pipe_config->hsw_workaround_pipe = INVALID_PIPE;
13361
13362                 if (!pipe_config->base.active ||
13363                     needs_modeset(&pipe_config->base))
13364                         continue;
13365
13366                 /* 2 or more enabled crtcs means no need for w/a */
13367                 if (enabled_pipe != INVALID_PIPE)
13368                         return 0;
13369
13370                 enabled_pipe = intel_crtc->pipe;
13371         }
13372
13373         if (enabled_pipe != INVALID_PIPE)
13374                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
13375         else if (other_crtc_state)
13376                 other_crtc_state->hsw_workaround_pipe = first_pipe;
13377
13378         return 0;
13379 }
13380
13381 static int intel_modeset_all_pipes(struct drm_atomic_state *state)
13382 {
13383         struct drm_crtc *crtc;
13384         struct drm_crtc_state *crtc_state;
13385         int ret = 0;
13386
13387         /* add all active pipes to the state */
13388         for_each_crtc(state->dev, crtc) {
13389                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
13390                 if (IS_ERR(crtc_state))
13391                         return PTR_ERR(crtc_state);
13392
13393                 if (!crtc_state->active || needs_modeset(crtc_state))
13394                         continue;
13395
13396                 crtc_state->mode_changed = true;
13397
13398                 ret = drm_atomic_add_affected_connectors(state, crtc);
13399                 if (ret)
13400                         break;
13401
13402                 ret = drm_atomic_add_affected_planes(state, crtc);
13403                 if (ret)
13404                         break;
13405         }
13406
13407         return ret;
13408 }
13409
13410 static int intel_modeset_checks(struct drm_atomic_state *state)
13411 {
13412         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
13413         struct drm_i915_private *dev_priv = state->dev->dev_private;
13414         struct drm_crtc *crtc;
13415         struct drm_crtc_state *crtc_state;
13416         int ret = 0, i;
13417
13418         if (!check_digital_port_conflicts(state)) {
13419                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
13420                 return -EINVAL;
13421         }
13422
13423         intel_state->modeset = true;
13424         intel_state->active_crtcs = dev_priv->active_crtcs;
13425
13426         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13427                 if (crtc_state->active)
13428                         intel_state->active_crtcs |= 1 << i;
13429                 else
13430                         intel_state->active_crtcs &= ~(1 << i);
13431
13432                 if (crtc_state->active != crtc->state->active)
13433                         intel_state->active_pipe_changes |= drm_crtc_mask(crtc);
13434         }
13435
13436         /*
13437          * See if the config requires any additional preparation, e.g.
13438          * to adjust global state with pipes off.  We need to do this
13439          * here so we can get the modeset_pipe updated config for the new
13440          * mode set on this crtc.  For other crtcs we need to use the
13441          * adjusted_mode bits in the crtc directly.
13442          */
13443         if (dev_priv->display.modeset_calc_cdclk) {
13444                 if (!intel_state->cdclk_pll_vco)
13445                         intel_state->cdclk_pll_vco = dev_priv->cdclk_pll.vco;
13446                 if (!intel_state->cdclk_pll_vco)
13447                         intel_state->cdclk_pll_vco = dev_priv->skl_preferred_vco_freq;
13448
13449                 ret = dev_priv->display.modeset_calc_cdclk(state);
13450                 if (ret < 0)
13451                         return ret;
13452
13453                 if (intel_state->dev_cdclk != dev_priv->cdclk_freq ||
13454                     intel_state->cdclk_pll_vco != dev_priv->cdclk_pll.vco)
13455                         ret = intel_modeset_all_pipes(state);
13456
13457                 if (ret < 0)
13458                         return ret;
13459
13460                 DRM_DEBUG_KMS("New cdclk calculated to be atomic %u, actual %u\n",
13461                               intel_state->cdclk, intel_state->dev_cdclk);
13462         } else
13463                 to_intel_atomic_state(state)->cdclk = dev_priv->atomic_cdclk_freq;
13464
13465         intel_modeset_clear_plls(state);
13466
13467         if (IS_HASWELL(dev_priv))
13468                 return haswell_mode_set_planes_workaround(state);
13469
13470         return 0;
13471 }
13472
13473 /*
13474  * Handle calculation of various watermark data at the end of the atomic check
13475  * phase.  The code here should be run after the per-crtc and per-plane 'check'
13476  * handlers to ensure that all derived state has been updated.
13477  */
13478 static int calc_watermark_data(struct drm_atomic_state *state)
13479 {
13480         struct drm_device *dev = state->dev;
13481         struct drm_i915_private *dev_priv = to_i915(dev);
13482
13483         /* Is there platform-specific watermark information to calculate? */
13484         if (dev_priv->display.compute_global_watermarks)
13485                 return dev_priv->display.compute_global_watermarks(state);
13486
13487         return 0;
13488 }
13489
13490 /**
13491  * intel_atomic_check - validate state object
13492  * @dev: drm device
13493  * @state: state to validate
13494  */
13495 static int intel_atomic_check(struct drm_device *dev,
13496                               struct drm_atomic_state *state)
13497 {
13498         struct drm_i915_private *dev_priv = to_i915(dev);
13499         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
13500         struct drm_crtc *crtc;
13501         struct drm_crtc_state *crtc_state;
13502         int ret, i;
13503         bool any_ms = false;
13504
13505         ret = drm_atomic_helper_check_modeset(dev, state);
13506         if (ret)
13507                 return ret;
13508
13509         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13510                 struct intel_crtc_state *pipe_config =
13511                         to_intel_crtc_state(crtc_state);
13512
13513                 /* Catch I915_MODE_FLAG_INHERITED */
13514                 if (crtc_state->mode.private_flags != crtc->state->mode.private_flags)
13515                         crtc_state->mode_changed = true;
13516
13517                 if (!needs_modeset(crtc_state))
13518                         continue;
13519
13520                 if (!crtc_state->enable) {
13521                         any_ms = true;
13522                         continue;
13523                 }
13524
13525                 /* FIXME: For only active_changed we shouldn't need to do any
13526                  * state recomputation at all. */
13527
13528                 ret = drm_atomic_add_affected_connectors(state, crtc);
13529                 if (ret)
13530                         return ret;
13531
13532                 ret = intel_modeset_pipe_config(crtc, pipe_config);
13533                 if (ret) {
13534                         intel_dump_pipe_config(to_intel_crtc(crtc),
13535                                                pipe_config, "[failed]");
13536                         return ret;
13537                 }
13538
13539                 if (i915.fastboot &&
13540                     intel_pipe_config_compare(dev,
13541                                         to_intel_crtc_state(crtc->state),
13542                                         pipe_config, true)) {
13543                         crtc_state->mode_changed = false;
13544                         to_intel_crtc_state(crtc_state)->update_pipe = true;
13545                 }
13546
13547                 if (needs_modeset(crtc_state))
13548                         any_ms = true;
13549
13550                 ret = drm_atomic_add_affected_planes(state, crtc);
13551                 if (ret)
13552                         return ret;
13553
13554                 intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config,
13555                                        needs_modeset(crtc_state) ?
13556                                        "[modeset]" : "[fastset]");
13557         }
13558
13559         if (any_ms) {
13560                 ret = intel_modeset_checks(state);
13561
13562                 if (ret)
13563                         return ret;
13564         } else
13565                 intel_state->cdclk = dev_priv->cdclk_freq;
13566
13567         ret = drm_atomic_helper_check_planes(dev, state);
13568         if (ret)
13569                 return ret;
13570
13571         intel_fbc_choose_crtc(dev_priv, state);
13572         return calc_watermark_data(state);
13573 }
13574
13575 static int intel_atomic_prepare_commit(struct drm_device *dev,
13576                                        struct drm_atomic_state *state,
13577                                        bool nonblock)
13578 {
13579         struct drm_i915_private *dev_priv = dev->dev_private;
13580         struct drm_plane_state *plane_state;
13581         struct drm_crtc_state *crtc_state;
13582         struct drm_plane *plane;
13583         struct drm_crtc *crtc;
13584         int i, ret;
13585
13586         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13587                 if (state->legacy_cursor_update)
13588                         continue;
13589
13590                 ret = intel_crtc_wait_for_pending_flips(crtc);
13591                 if (ret)
13592                         return ret;
13593
13594                 if (atomic_read(&to_intel_crtc(crtc)->unpin_work_count) >= 2)
13595                         flush_workqueue(dev_priv->wq);
13596         }
13597
13598         ret = mutex_lock_interruptible(&dev->struct_mutex);
13599         if (ret)
13600                 return ret;
13601
13602         ret = drm_atomic_helper_prepare_planes(dev, state);
13603         mutex_unlock(&dev->struct_mutex);
13604
13605         if (!ret && !nonblock) {
13606                 for_each_plane_in_state(state, plane, plane_state, i) {
13607                         struct intel_plane_state *intel_plane_state =
13608                                 to_intel_plane_state(plane_state);
13609
13610                         if (!intel_plane_state->wait_req)
13611                                 continue;
13612
13613                         ret = __i915_wait_request(intel_plane_state->wait_req,
13614                                                   true, NULL, NULL);
13615                         if (ret) {
13616                                 /* Any hang should be swallowed by the wait */
13617                                 WARN_ON(ret == -EIO);
13618                                 mutex_lock(&dev->struct_mutex);
13619                                 drm_atomic_helper_cleanup_planes(dev, state);
13620                                 mutex_unlock(&dev->struct_mutex);
13621                                 break;
13622                         }
13623                 }
13624         }
13625
13626         return ret;
13627 }
13628
13629 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
13630 {
13631         struct drm_device *dev = crtc->base.dev;
13632
13633         if (!dev->max_vblank_count)
13634                 return drm_accurate_vblank_count(&crtc->base);
13635
13636         return dev->driver->get_vblank_counter(dev, crtc->pipe);
13637 }
13638
13639 static void intel_atomic_wait_for_vblanks(struct drm_device *dev,
13640                                           struct drm_i915_private *dev_priv,
13641                                           unsigned crtc_mask)
13642 {
13643         unsigned last_vblank_count[I915_MAX_PIPES];
13644         enum pipe pipe;
13645         int ret;
13646
13647         if (!crtc_mask)
13648                 return;
13649
13650         for_each_pipe(dev_priv, pipe) {
13651                 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
13652
13653                 if (!((1 << pipe) & crtc_mask))
13654                         continue;
13655
13656                 ret = drm_crtc_vblank_get(crtc);
13657                 if (WARN_ON(ret != 0)) {
13658                         crtc_mask &= ~(1 << pipe);
13659                         continue;
13660                 }
13661
13662                 last_vblank_count[pipe] = drm_crtc_vblank_count(crtc);
13663         }
13664
13665         for_each_pipe(dev_priv, pipe) {
13666                 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
13667                 long lret;
13668
13669                 if (!((1 << pipe) & crtc_mask))
13670                         continue;
13671
13672                 lret = wait_event_timeout(dev->vblank[pipe].queue,
13673                                 last_vblank_count[pipe] !=
13674                                         drm_crtc_vblank_count(crtc),
13675                                 msecs_to_jiffies(50));
13676
13677                 WARN(!lret, "pipe %c vblank wait timed out\n", pipe_name(pipe));
13678
13679                 drm_crtc_vblank_put(crtc);
13680         }
13681 }
13682
13683 static bool needs_vblank_wait(struct intel_crtc_state *crtc_state)
13684 {
13685         /* fb updated, need to unpin old fb */
13686         if (crtc_state->fb_changed)
13687                 return true;
13688
13689         /* wm changes, need vblank before final wm's */
13690         if (crtc_state->update_wm_post)
13691                 return true;
13692
13693         /*
13694          * cxsr is re-enabled after vblank.
13695          * This is already handled by crtc_state->update_wm_post,
13696          * but added for clarity.
13697          */
13698         if (crtc_state->disable_cxsr)
13699                 return true;
13700
13701         return false;
13702 }
13703
13704 static void intel_atomic_commit_tail(struct drm_atomic_state *state)
13705 {
13706         struct drm_device *dev = state->dev;
13707         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
13708         struct drm_i915_private *dev_priv = dev->dev_private;
13709         struct drm_crtc_state *old_crtc_state;
13710         struct drm_crtc *crtc;
13711         struct intel_crtc_state *intel_cstate;
13712         struct drm_plane *plane;
13713         struct drm_plane_state *plane_state;
13714         bool hw_check = intel_state->modeset;
13715         unsigned long put_domains[I915_MAX_PIPES] = {};
13716         unsigned crtc_vblank_mask = 0;
13717         int i, ret;
13718
13719         for_each_plane_in_state(state, plane, plane_state, i) {
13720                 struct intel_plane_state *intel_plane_state =
13721                         to_intel_plane_state(plane_state);
13722
13723                 if (!intel_plane_state->wait_req)
13724                         continue;
13725
13726                 ret = __i915_wait_request(intel_plane_state->wait_req,
13727                                           true, NULL, NULL);
13728                 /* EIO should be eaten, and we can't get interrupted in the
13729                  * worker, and blocking commits have waited already. */
13730                 WARN_ON(ret);
13731         }
13732
13733         drm_atomic_helper_wait_for_dependencies(state);
13734
13735         if (intel_state->modeset) {
13736                 memcpy(dev_priv->min_pixclk, intel_state->min_pixclk,
13737                        sizeof(intel_state->min_pixclk));
13738                 dev_priv->active_crtcs = intel_state->active_crtcs;
13739                 dev_priv->atomic_cdclk_freq = intel_state->cdclk;
13740
13741                 intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
13742         }
13743
13744         for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
13745                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13746
13747                 if (needs_modeset(crtc->state) ||
13748                     to_intel_crtc_state(crtc->state)->update_pipe) {
13749                         hw_check = true;
13750
13751                         put_domains[to_intel_crtc(crtc)->pipe] =
13752                                 modeset_get_crtc_power_domains(crtc,
13753                                         to_intel_crtc_state(crtc->state));
13754                 }
13755
13756                 if (!needs_modeset(crtc->state))
13757                         continue;
13758
13759                 intel_pre_plane_update(to_intel_crtc_state(old_crtc_state));
13760
13761                 if (old_crtc_state->active) {
13762                         intel_crtc_disable_planes(crtc, old_crtc_state->plane_mask);
13763                         dev_priv->display.crtc_disable(crtc);
13764                         intel_crtc->active = false;
13765                         intel_fbc_disable(intel_crtc);
13766                         intel_disable_shared_dpll(intel_crtc);
13767
13768                         /*
13769                          * Underruns don't always raise
13770                          * interrupts, so check manually.
13771                          */
13772                         intel_check_cpu_fifo_underruns(dev_priv);
13773                         intel_check_pch_fifo_underruns(dev_priv);
13774
13775                         if (!crtc->state->active)
13776                                 intel_update_watermarks(crtc);
13777                 }
13778         }
13779
13780         /* Only after disabling all output pipelines that will be changed can we
13781          * update the the output configuration. */
13782         intel_modeset_update_crtc_state(state);
13783
13784         if (intel_state->modeset) {
13785                 drm_atomic_helper_update_legacy_modeset_state(state->dev, state);
13786
13787                 if (dev_priv->display.modeset_commit_cdclk &&
13788                     (intel_state->dev_cdclk != dev_priv->cdclk_freq ||
13789                      intel_state->cdclk_pll_vco != dev_priv->cdclk_pll.vco))
13790                         dev_priv->display.modeset_commit_cdclk(state);
13791
13792                 intel_modeset_verify_disabled(dev);
13793         }
13794
13795         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
13796         for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
13797                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13798                 bool modeset = needs_modeset(crtc->state);
13799                 struct intel_crtc_state *pipe_config =
13800                         to_intel_crtc_state(crtc->state);
13801
13802                 if (modeset && crtc->state->active) {
13803                         update_scanline_offset(to_intel_crtc(crtc));
13804                         dev_priv->display.crtc_enable(crtc);
13805                 }
13806
13807                 /* Complete events for now disable pipes here. */
13808                 if (modeset && !crtc->state->active && crtc->state->event) {
13809                         spin_lock_irq(&dev->event_lock);
13810                         drm_crtc_send_vblank_event(crtc, crtc->state->event);
13811                         spin_unlock_irq(&dev->event_lock);
13812
13813                         crtc->state->event = NULL;
13814                 }
13815
13816                 if (!modeset)
13817                         intel_pre_plane_update(to_intel_crtc_state(old_crtc_state));
13818
13819                 if (crtc->state->active &&
13820                     drm_atomic_get_existing_plane_state(state, crtc->primary))
13821                         intel_fbc_enable(intel_crtc, pipe_config, to_intel_plane_state(crtc->primary->state));
13822
13823                 if (crtc->state->active)
13824                         drm_atomic_helper_commit_planes_on_crtc(old_crtc_state);
13825
13826                 if (pipe_config->base.active && needs_vblank_wait(pipe_config))
13827                         crtc_vblank_mask |= 1 << i;
13828         }
13829
13830         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
13831          * already, but still need the state for the delayed optimization. To
13832          * fix this:
13833          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
13834          * - schedule that vblank worker _before_ calling hw_done
13835          * - at the start of commit_tail, cancel it _synchrously
13836          * - switch over to the vblank wait helper in the core after that since
13837          *   we don't need out special handling any more.
13838          */
13839         if (!state->legacy_cursor_update)
13840                 intel_atomic_wait_for_vblanks(dev, dev_priv, crtc_vblank_mask);
13841
13842         /*
13843          * Now that the vblank has passed, we can go ahead and program the
13844          * optimal watermarks on platforms that need two-step watermark
13845          * programming.
13846          *
13847          * TODO: Move this (and other cleanup) to an async worker eventually.
13848          */
13849         for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
13850                 intel_cstate = to_intel_crtc_state(crtc->state);
13851
13852                 if (dev_priv->display.optimize_watermarks)
13853                         dev_priv->display.optimize_watermarks(intel_cstate);
13854         }
13855
13856         for_each_crtc_in_state(state, crtc, old_crtc_state, i) {
13857                 intel_post_plane_update(to_intel_crtc_state(old_crtc_state));
13858
13859                 if (put_domains[i])
13860                         modeset_put_power_domains(dev_priv, put_domains[i]);
13861
13862                 intel_modeset_verify_crtc(crtc, old_crtc_state, crtc->state);
13863         }
13864
13865         drm_atomic_helper_commit_hw_done(state);
13866
13867         if (intel_state->modeset)
13868                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET);
13869
13870         mutex_lock(&dev->struct_mutex);
13871         drm_atomic_helper_cleanup_planes(dev, state);
13872         mutex_unlock(&dev->struct_mutex);
13873
13874         drm_atomic_helper_commit_cleanup_done(state);
13875
13876         drm_atomic_state_free(state);
13877
13878         /* As one of the primary mmio accessors, KMS has a high likelihood
13879          * of triggering bugs in unclaimed access. After we finish
13880          * modesetting, see if an error has been flagged, and if so
13881          * enable debugging for the next modeset - and hope we catch
13882          * the culprit.
13883          *
13884          * XXX note that we assume display power is on at this point.
13885          * This might hold true now but we need to add pm helper to check
13886          * unclaimed only when the hardware is on, as atomic commits
13887          * can happen also when the device is completely off.
13888          */
13889         intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
13890 }
13891
13892 static void intel_atomic_commit_work(struct work_struct *work)
13893 {
13894         struct drm_atomic_state *state = container_of(work,
13895                                                       struct drm_atomic_state,
13896                                                       commit_work);
13897         intel_atomic_commit_tail(state);
13898 }
13899
13900 static void intel_atomic_track_fbs(struct drm_atomic_state *state)
13901 {
13902         struct drm_plane_state *old_plane_state;
13903         struct drm_plane *plane;
13904         struct drm_i915_gem_object *obj, *old_obj;
13905         struct intel_plane *intel_plane;
13906         int i;
13907
13908         mutex_lock(&state->dev->struct_mutex);
13909         for_each_plane_in_state(state, plane, old_plane_state, i) {
13910                 obj = intel_fb_obj(plane->state->fb);
13911                 old_obj = intel_fb_obj(old_plane_state->fb);
13912                 intel_plane = to_intel_plane(plane);
13913
13914                 i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit);
13915         }
13916         mutex_unlock(&state->dev->struct_mutex);
13917 }
13918
13919 /**
13920  * intel_atomic_commit - commit validated state object
13921  * @dev: DRM device
13922  * @state: the top-level driver state object
13923  * @nonblock: nonblocking commit
13924  *
13925  * This function commits a top-level state object that has been validated
13926  * with drm_atomic_helper_check().
13927  *
13928  * FIXME:  Atomic modeset support for i915 is not yet complete.  At the moment
13929  * nonblocking commits are only safe for pure plane updates. Everything else
13930  * should work though.
13931  *
13932  * RETURNS
13933  * Zero for success or -errno.
13934  */
13935 static int intel_atomic_commit(struct drm_device *dev,
13936                                struct drm_atomic_state *state,
13937                                bool nonblock)
13938 {
13939         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
13940         struct drm_i915_private *dev_priv = dev->dev_private;
13941         int ret = 0;
13942
13943         if (intel_state->modeset && nonblock) {
13944                 DRM_DEBUG_KMS("nonblocking commit for modeset not yet implemented.\n");
13945                 return -EINVAL;
13946         }
13947
13948         ret = drm_atomic_helper_setup_commit(state, nonblock);
13949         if (ret)
13950                 return ret;
13951
13952         INIT_WORK(&state->commit_work, intel_atomic_commit_work);
13953
13954         ret = intel_atomic_prepare_commit(dev, state, nonblock);
13955         if (ret) {
13956                 DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
13957                 return ret;
13958         }
13959
13960         drm_atomic_helper_swap_state(state, true);
13961         dev_priv->wm.distrust_bios_wm = false;
13962         dev_priv->wm.skl_results = intel_state->wm_results;
13963         intel_shared_dpll_commit(state);
13964         intel_atomic_track_fbs(state);
13965
13966         if (nonblock)
13967                 queue_work(system_unbound_wq, &state->commit_work);
13968         else
13969                 intel_atomic_commit_tail(state);
13970
13971         return 0;
13972 }
13973
13974 void intel_crtc_restore_mode(struct drm_crtc *crtc)
13975 {
13976         struct drm_device *dev = crtc->dev;
13977         struct drm_atomic_state *state;
13978         struct drm_crtc_state *crtc_state;
13979         int ret;
13980
13981         state = drm_atomic_state_alloc(dev);
13982         if (!state) {
13983                 DRM_DEBUG_KMS("[CRTC:%d:%s] crtc restore failed, out of memory",
13984                               crtc->base.id, crtc->name);
13985                 return;
13986         }
13987
13988         state->acquire_ctx = drm_modeset_legacy_acquire_ctx(crtc);
13989
13990 retry:
13991         crtc_state = drm_atomic_get_crtc_state(state, crtc);
13992         ret = PTR_ERR_OR_ZERO(crtc_state);
13993         if (!ret) {
13994                 if (!crtc_state->active)
13995                         goto out;
13996
13997                 crtc_state->mode_changed = true;
13998                 ret = drm_atomic_commit(state);
13999         }
14000
14001         if (ret == -EDEADLK) {
14002                 drm_atomic_state_clear(state);
14003                 drm_modeset_backoff(state->acquire_ctx);
14004                 goto retry;
14005         }
14006
14007         if (ret)
14008 out:
14009                 drm_atomic_state_free(state);
14010 }
14011
14012 #undef for_each_intel_crtc_masked
14013
14014 static const struct drm_crtc_funcs intel_crtc_funcs = {
14015         .gamma_set = drm_atomic_helper_legacy_gamma_set,
14016         .set_config = drm_atomic_helper_set_config,
14017         .set_property = drm_atomic_helper_crtc_set_property,
14018         .destroy = intel_crtc_destroy,
14019         .page_flip = intel_crtc_page_flip,
14020         .atomic_duplicate_state = intel_crtc_duplicate_state,
14021         .atomic_destroy_state = intel_crtc_destroy_state,
14022 };
14023
14024 /**
14025  * intel_prepare_plane_fb - Prepare fb for usage on plane
14026  * @plane: drm plane to prepare for
14027  * @fb: framebuffer to prepare for presentation
14028  *
14029  * Prepares a framebuffer for usage on a display plane.  Generally this
14030  * involves pinning the underlying object and updating the frontbuffer tracking
14031  * bits.  Some older platforms need special physical address handling for
14032  * cursor planes.
14033  *
14034  * Must be called with struct_mutex held.
14035  *
14036  * Returns 0 on success, negative error code on failure.
14037  */
14038 int
14039 intel_prepare_plane_fb(struct drm_plane *plane,
14040                        const struct drm_plane_state *new_state)
14041 {
14042         struct drm_device *dev = plane->dev;
14043         struct drm_framebuffer *fb = new_state->fb;
14044         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14045         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
14046         struct reservation_object *resv;
14047         int ret = 0;
14048
14049         if (!obj && !old_obj)
14050                 return 0;
14051
14052         if (old_obj) {
14053                 struct drm_crtc_state *crtc_state =
14054                         drm_atomic_get_existing_crtc_state(new_state->state, plane->state->crtc);
14055
14056                 /* Big Hammer, we also need to ensure that any pending
14057                  * MI_WAIT_FOR_EVENT inside a user batch buffer on the
14058                  * current scanout is retired before unpinning the old
14059                  * framebuffer. Note that we rely on userspace rendering
14060                  * into the buffer attached to the pipe they are waiting
14061                  * on. If not, userspace generates a GPU hang with IPEHR
14062                  * point to the MI_WAIT_FOR_EVENT.
14063                  *
14064                  * This should only fail upon a hung GPU, in which case we
14065                  * can safely continue.
14066                  */
14067                 if (needs_modeset(crtc_state))
14068                         ret = i915_gem_object_wait_rendering(old_obj, true);
14069                 if (ret) {
14070                         /* GPU hangs should have been swallowed by the wait */
14071                         WARN_ON(ret == -EIO);
14072                         return ret;
14073                 }
14074         }
14075
14076         if (!obj)
14077                 return 0;
14078
14079         /* For framebuffer backed by dmabuf, wait for fence */
14080         resv = i915_gem_object_get_dmabuf_resv(obj);
14081         if (resv) {
14082                 long lret;
14083
14084                 lret = reservation_object_wait_timeout_rcu(resv, false, true,
14085                                                            MAX_SCHEDULE_TIMEOUT);
14086                 if (lret == -ERESTARTSYS)
14087                         return lret;
14088
14089                 WARN(lret < 0, "waiting returns %li\n", lret);
14090         }
14091
14092         if (plane->type == DRM_PLANE_TYPE_CURSOR &&
14093             INTEL_INFO(dev)->cursor_needs_physical) {
14094                 int align = IS_I830(dev) ? 16 * 1024 : 256;
14095                 ret = i915_gem_object_attach_phys(obj, align);
14096                 if (ret)
14097                         DRM_DEBUG_KMS("failed to attach phys object\n");
14098         } else {
14099                 ret = intel_pin_and_fence_fb_obj(fb, new_state->rotation);
14100         }
14101
14102         if (ret == 0) {
14103                 struct intel_plane_state *plane_state =
14104                         to_intel_plane_state(new_state);
14105
14106                 i915_gem_request_assign(&plane_state->wait_req,
14107                                         obj->last_write_req);
14108         }
14109
14110         return ret;
14111 }
14112
14113 /**
14114  * intel_cleanup_plane_fb - Cleans up an fb after plane use
14115  * @plane: drm plane to clean up for
14116  * @fb: old framebuffer that was on plane
14117  *
14118  * Cleans up a framebuffer that has just been removed from a plane.
14119  *
14120  * Must be called with struct_mutex held.
14121  */
14122 void
14123 intel_cleanup_plane_fb(struct drm_plane *plane,
14124                        const struct drm_plane_state *old_state)
14125 {
14126         struct drm_device *dev = plane->dev;
14127         struct intel_plane_state *old_intel_state;
14128         struct drm_i915_gem_object *old_obj = intel_fb_obj(old_state->fb);
14129         struct drm_i915_gem_object *obj = intel_fb_obj(plane->state->fb);
14130
14131         old_intel_state = to_intel_plane_state(old_state);
14132
14133         if (!obj && !old_obj)
14134                 return;
14135
14136         if (old_obj && (plane->type != DRM_PLANE_TYPE_CURSOR ||
14137             !INTEL_INFO(dev)->cursor_needs_physical))
14138                 intel_unpin_fb_obj(old_state->fb, old_state->rotation);
14139
14140         i915_gem_request_assign(&old_intel_state->wait_req, NULL);
14141 }
14142
14143 int
14144 skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state)
14145 {
14146         int max_scale;
14147         struct drm_device *dev;
14148         struct drm_i915_private *dev_priv;
14149         int crtc_clock, cdclk;
14150
14151         if (!intel_crtc || !crtc_state->base.enable)
14152                 return DRM_PLANE_HELPER_NO_SCALING;
14153
14154         dev = intel_crtc->base.dev;
14155         dev_priv = dev->dev_private;
14156         crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
14157         cdclk = to_intel_atomic_state(crtc_state->base.state)->cdclk;
14158
14159         if (WARN_ON_ONCE(!crtc_clock || cdclk < crtc_clock))
14160                 return DRM_PLANE_HELPER_NO_SCALING;
14161
14162         /*
14163          * skl max scale is lower of:
14164          *    close to 3 but not 3, -1 is for that purpose
14165          *            or
14166          *    cdclk/crtc_clock
14167          */
14168         max_scale = min((1 << 16) * 3 - 1, (1 << 8) * ((cdclk << 8) / crtc_clock));
14169
14170         return max_scale;
14171 }
14172
14173 static int
14174 intel_check_primary_plane(struct drm_plane *plane,
14175                           struct intel_crtc_state *crtc_state,
14176                           struct intel_plane_state *state)
14177 {
14178         struct drm_crtc *crtc = state->base.crtc;
14179         struct drm_framebuffer *fb = state->base.fb;
14180         int min_scale = DRM_PLANE_HELPER_NO_SCALING;
14181         int max_scale = DRM_PLANE_HELPER_NO_SCALING;
14182         bool can_position = false;
14183
14184         if (INTEL_INFO(plane->dev)->gen >= 9) {
14185                 /* use scaler when colorkey is not required */
14186                 if (state->ckey.flags == I915_SET_COLORKEY_NONE) {
14187                         min_scale = 1;
14188                         max_scale = skl_max_scale(to_intel_crtc(crtc), crtc_state);
14189                 }
14190                 can_position = true;
14191         }
14192
14193         return drm_plane_helper_check_update(plane, crtc, fb, &state->src,
14194                                              &state->dst, &state->clip,
14195                                              state->base.rotation,
14196                                              min_scale, max_scale,
14197                                              can_position, true,
14198                                              &state->visible);
14199 }
14200
14201 static void intel_begin_crtc_commit(struct drm_crtc *crtc,
14202                                     struct drm_crtc_state *old_crtc_state)
14203 {
14204         struct drm_device *dev = crtc->dev;
14205         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
14206         struct intel_crtc_state *old_intel_state =
14207                 to_intel_crtc_state(old_crtc_state);
14208         bool modeset = needs_modeset(crtc->state);
14209
14210         /* Perform vblank evasion around commit operation */
14211         intel_pipe_update_start(intel_crtc);
14212
14213         if (modeset)
14214                 return;
14215
14216         if (crtc->state->color_mgmt_changed || to_intel_crtc_state(crtc->state)->update_pipe) {
14217                 intel_color_set_csc(crtc->state);
14218                 intel_color_load_luts(crtc->state);
14219         }
14220
14221         if (to_intel_crtc_state(crtc->state)->update_pipe)
14222                 intel_update_pipe_config(intel_crtc, old_intel_state);
14223         else if (INTEL_INFO(dev)->gen >= 9)
14224                 skl_detach_scalers(intel_crtc);
14225 }
14226
14227 static void intel_finish_crtc_commit(struct drm_crtc *crtc,
14228                                      struct drm_crtc_state *old_crtc_state)
14229 {
14230         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
14231
14232         intel_pipe_update_end(intel_crtc, NULL);
14233 }
14234
14235 /**
14236  * intel_plane_destroy - destroy a plane
14237  * @plane: plane to destroy
14238  *
14239  * Common destruction function for all types of planes (primary, cursor,
14240  * sprite).
14241  */
14242 void intel_plane_destroy(struct drm_plane *plane)
14243 {
14244         if (!plane)
14245                 return;
14246
14247         drm_plane_cleanup(plane);
14248         kfree(to_intel_plane(plane));
14249 }
14250
14251 const struct drm_plane_funcs intel_plane_funcs = {
14252         .update_plane = drm_atomic_helper_update_plane,
14253         .disable_plane = drm_atomic_helper_disable_plane,
14254         .destroy = intel_plane_destroy,
14255         .set_property = drm_atomic_helper_plane_set_property,
14256         .atomic_get_property = intel_plane_atomic_get_property,
14257         .atomic_set_property = intel_plane_atomic_set_property,
14258         .atomic_duplicate_state = intel_plane_duplicate_state,
14259         .atomic_destroy_state = intel_plane_destroy_state,
14260
14261 };
14262
14263 static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
14264                                                     int pipe)
14265 {
14266         struct intel_plane *primary = NULL;
14267         struct intel_plane_state *state = NULL;
14268         const uint32_t *intel_primary_formats;
14269         unsigned int num_formats;
14270         int ret;
14271
14272         primary = kzalloc(sizeof(*primary), GFP_KERNEL);
14273         if (!primary)
14274                 goto fail;
14275
14276         state = intel_create_plane_state(&primary->base);
14277         if (!state)
14278                 goto fail;
14279         primary->base.state = &state->base;
14280
14281         primary->can_scale = false;
14282         primary->max_downscale = 1;
14283         if (INTEL_INFO(dev)->gen >= 9) {
14284                 primary->can_scale = true;
14285                 state->scaler_id = -1;
14286         }
14287         primary->pipe = pipe;
14288         primary->plane = pipe;
14289         primary->frontbuffer_bit = INTEL_FRONTBUFFER_PRIMARY(pipe);
14290         primary->check_plane = intel_check_primary_plane;
14291         if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4)
14292                 primary->plane = !pipe;
14293
14294         if (INTEL_INFO(dev)->gen >= 9) {
14295                 intel_primary_formats = skl_primary_formats;
14296                 num_formats = ARRAY_SIZE(skl_primary_formats);
14297
14298                 primary->update_plane = skylake_update_primary_plane;
14299                 primary->disable_plane = skylake_disable_primary_plane;
14300         } else if (HAS_PCH_SPLIT(dev)) {
14301                 intel_primary_formats = i965_primary_formats;
14302                 num_formats = ARRAY_SIZE(i965_primary_formats);
14303
14304                 primary->update_plane = ironlake_update_primary_plane;
14305                 primary->disable_plane = i9xx_disable_primary_plane;
14306         } else if (INTEL_INFO(dev)->gen >= 4) {
14307                 intel_primary_formats = i965_primary_formats;
14308                 num_formats = ARRAY_SIZE(i965_primary_formats);
14309
14310                 primary->update_plane = i9xx_update_primary_plane;
14311                 primary->disable_plane = i9xx_disable_primary_plane;
14312         } else {
14313                 intel_primary_formats = i8xx_primary_formats;
14314                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
14315
14316                 primary->update_plane = i9xx_update_primary_plane;
14317                 primary->disable_plane = i9xx_disable_primary_plane;
14318         }
14319
14320         if (INTEL_INFO(dev)->gen >= 9)
14321                 ret = drm_universal_plane_init(dev, &primary->base, 0,
14322                                                &intel_plane_funcs,
14323                                                intel_primary_formats, num_formats,
14324                                                DRM_PLANE_TYPE_PRIMARY,
14325                                                "plane 1%c", pipe_name(pipe));
14326         else if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
14327                 ret = drm_universal_plane_init(dev, &primary->base, 0,
14328                                                &intel_plane_funcs,
14329                                                intel_primary_formats, num_formats,
14330                                                DRM_PLANE_TYPE_PRIMARY,
14331                                                "primary %c", pipe_name(pipe));
14332         else
14333                 ret = drm_universal_plane_init(dev, &primary->base, 0,
14334                                                &intel_plane_funcs,
14335                                                intel_primary_formats, num_formats,
14336                                                DRM_PLANE_TYPE_PRIMARY,
14337                                                "plane %c", plane_name(primary->plane));
14338         if (ret)
14339                 goto fail;
14340
14341         if (INTEL_INFO(dev)->gen >= 4)
14342                 intel_create_rotation_property(dev, primary);
14343
14344         drm_plane_helper_add(&primary->base, &intel_plane_helper_funcs);
14345
14346         return &primary->base;
14347
14348 fail:
14349         kfree(state);
14350         kfree(primary);
14351
14352         return NULL;
14353 }
14354
14355 void intel_create_rotation_property(struct drm_device *dev, struct intel_plane *plane)
14356 {
14357         if (!dev->mode_config.rotation_property) {
14358                 unsigned long flags = BIT(DRM_ROTATE_0) |
14359                         BIT(DRM_ROTATE_180);
14360
14361                 if (INTEL_INFO(dev)->gen >= 9)
14362                         flags |= BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_270);
14363
14364                 dev->mode_config.rotation_property =
14365                         drm_mode_create_rotation_property(dev, flags);
14366         }
14367         if (dev->mode_config.rotation_property)
14368                 drm_object_attach_property(&plane->base.base,
14369                                 dev->mode_config.rotation_property,
14370                                 plane->base.state->rotation);
14371 }
14372
14373 static int
14374 intel_check_cursor_plane(struct drm_plane *plane,
14375                          struct intel_crtc_state *crtc_state,
14376                          struct intel_plane_state *state)
14377 {
14378         struct drm_crtc *crtc = crtc_state->base.crtc;
14379         struct drm_framebuffer *fb = state->base.fb;
14380         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
14381         enum pipe pipe = to_intel_plane(plane)->pipe;
14382         unsigned stride;
14383         int ret;
14384
14385         ret = drm_plane_helper_check_update(plane, crtc, fb, &state->src,
14386                                             &state->dst, &state->clip,
14387                                             state->base.rotation,
14388                                             DRM_PLANE_HELPER_NO_SCALING,
14389                                             DRM_PLANE_HELPER_NO_SCALING,
14390                                             true, true, &state->visible);
14391         if (ret)
14392                 return ret;
14393
14394         /* if we want to turn off the cursor ignore width and height */
14395         if (!obj)
14396                 return 0;
14397
14398         /* Check for which cursor types we support */
14399         if (!cursor_size_ok(plane->dev, state->base.crtc_w, state->base.crtc_h)) {
14400                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
14401                           state->base.crtc_w, state->base.crtc_h);
14402                 return -EINVAL;
14403         }
14404
14405         stride = roundup_pow_of_two(state->base.crtc_w) * 4;
14406         if (obj->base.size < stride * state->base.crtc_h) {
14407                 DRM_DEBUG_KMS("buffer is too small\n");
14408                 return -ENOMEM;
14409         }
14410
14411         if (fb->modifier[0] != DRM_FORMAT_MOD_NONE) {
14412                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
14413                 return -EINVAL;
14414         }
14415
14416         /*
14417          * There's something wrong with the cursor on CHV pipe C.
14418          * If it straddles the left edge of the screen then
14419          * moving it away from the edge or disabling it often
14420          * results in a pipe underrun, and often that can lead to
14421          * dead pipe (constant underrun reported, and it scans
14422          * out just a solid color). To recover from that, the
14423          * display power well must be turned off and on again.
14424          * Refuse the put the cursor into that compromised position.
14425          */
14426         if (IS_CHERRYVIEW(plane->dev) && pipe == PIPE_C &&
14427             state->visible && state->base.crtc_x < 0) {
14428                 DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
14429                 return -EINVAL;
14430         }
14431
14432         return 0;
14433 }
14434
14435 static void
14436 intel_disable_cursor_plane(struct drm_plane *plane,
14437                            struct drm_crtc *crtc)
14438 {
14439         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
14440
14441         intel_crtc->cursor_addr = 0;
14442         intel_crtc_update_cursor(crtc, NULL);
14443 }
14444
14445 static void
14446 intel_update_cursor_plane(struct drm_plane *plane,
14447                           const struct intel_crtc_state *crtc_state,
14448                           const struct intel_plane_state *state)
14449 {
14450         struct drm_crtc *crtc = crtc_state->base.crtc;
14451         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
14452         struct drm_device *dev = plane->dev;
14453         struct drm_i915_gem_object *obj = intel_fb_obj(state->base.fb);
14454         uint32_t addr;
14455
14456         if (!obj)
14457                 addr = 0;
14458         else if (!INTEL_INFO(dev)->cursor_needs_physical)
14459                 addr = i915_gem_obj_ggtt_offset(obj);
14460         else
14461                 addr = obj->phys_handle->busaddr;
14462
14463         intel_crtc->cursor_addr = addr;
14464         intel_crtc_update_cursor(crtc, state);
14465 }
14466
14467 static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
14468                                                    int pipe)
14469 {
14470         struct intel_plane *cursor = NULL;
14471         struct intel_plane_state *state = NULL;
14472         int ret;
14473
14474         cursor = kzalloc(sizeof(*cursor), GFP_KERNEL);
14475         if (!cursor)
14476                 goto fail;
14477
14478         state = intel_create_plane_state(&cursor->base);
14479         if (!state)
14480                 goto fail;
14481         cursor->base.state = &state->base;
14482
14483         cursor->can_scale = false;
14484         cursor->max_downscale = 1;
14485         cursor->pipe = pipe;
14486         cursor->plane = pipe;
14487         cursor->frontbuffer_bit = INTEL_FRONTBUFFER_CURSOR(pipe);
14488         cursor->check_plane = intel_check_cursor_plane;
14489         cursor->update_plane = intel_update_cursor_plane;
14490         cursor->disable_plane = intel_disable_cursor_plane;
14491
14492         ret = drm_universal_plane_init(dev, &cursor->base, 0,
14493                                        &intel_plane_funcs,
14494                                        intel_cursor_formats,
14495                                        ARRAY_SIZE(intel_cursor_formats),
14496                                        DRM_PLANE_TYPE_CURSOR,
14497                                        "cursor %c", pipe_name(pipe));
14498         if (ret)
14499                 goto fail;
14500
14501         if (INTEL_INFO(dev)->gen >= 4) {
14502                 if (!dev->mode_config.rotation_property)
14503                         dev->mode_config.rotation_property =
14504                                 drm_mode_create_rotation_property(dev,
14505                                                         BIT(DRM_ROTATE_0) |
14506                                                         BIT(DRM_ROTATE_180));
14507                 if (dev->mode_config.rotation_property)
14508                         drm_object_attach_property(&cursor->base.base,
14509                                 dev->mode_config.rotation_property,
14510                                 state->base.rotation);
14511         }
14512
14513         if (INTEL_INFO(dev)->gen >=9)
14514                 state->scaler_id = -1;
14515
14516         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
14517
14518         return &cursor->base;
14519
14520 fail:
14521         kfree(state);
14522         kfree(cursor);
14523
14524         return NULL;
14525 }
14526
14527 static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc,
14528         struct intel_crtc_state *crtc_state)
14529 {
14530         int i;
14531         struct intel_scaler *intel_scaler;
14532         struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state;
14533
14534         for (i = 0; i < intel_crtc->num_scalers; i++) {
14535                 intel_scaler = &scaler_state->scalers[i];
14536                 intel_scaler->in_use = 0;
14537                 intel_scaler->mode = PS_SCALER_MODE_DYN;
14538         }
14539
14540         scaler_state->scaler_id = -1;
14541 }
14542
14543 static void intel_crtc_init(struct drm_device *dev, int pipe)
14544 {
14545         struct drm_i915_private *dev_priv = dev->dev_private;
14546         struct intel_crtc *intel_crtc;
14547         struct intel_crtc_state *crtc_state = NULL;
14548         struct drm_plane *primary = NULL;
14549         struct drm_plane *cursor = NULL;
14550         int ret;
14551
14552         intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL);
14553         if (intel_crtc == NULL)
14554                 return;
14555
14556         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
14557         if (!crtc_state)
14558                 goto fail;
14559         intel_crtc->config = crtc_state;
14560         intel_crtc->base.state = &crtc_state->base;
14561         crtc_state->base.crtc = &intel_crtc->base;
14562
14563         /* initialize shared scalers */
14564         if (INTEL_INFO(dev)->gen >= 9) {
14565                 if (pipe == PIPE_C)
14566                         intel_crtc->num_scalers = 1;
14567                 else
14568                         intel_crtc->num_scalers = SKL_NUM_SCALERS;
14569
14570                 skl_init_scalers(dev, intel_crtc, crtc_state);
14571         }
14572
14573         primary = intel_primary_plane_create(dev, pipe);
14574         if (!primary)
14575                 goto fail;
14576
14577         cursor = intel_cursor_plane_create(dev, pipe);
14578         if (!cursor)
14579                 goto fail;
14580
14581         ret = drm_crtc_init_with_planes(dev, &intel_crtc->base, primary,
14582                                         cursor, &intel_crtc_funcs,
14583                                         "pipe %c", pipe_name(pipe));
14584         if (ret)
14585                 goto fail;
14586
14587         /*
14588          * On gen2/3 only plane A can do fbc, but the panel fitter and lvds port
14589          * is hooked to pipe B. Hence we want plane A feeding pipe B.
14590          */
14591         intel_crtc->pipe = pipe;
14592         intel_crtc->plane = pipe;
14593         if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4) {
14594                 DRM_DEBUG_KMS("swapping pipes & planes for FBC\n");
14595                 intel_crtc->plane = !pipe;
14596         }
14597
14598         intel_crtc->cursor_base = ~0;
14599         intel_crtc->cursor_cntl = ~0;
14600         intel_crtc->cursor_size = ~0;
14601
14602         intel_crtc->wm.cxsr_allowed = true;
14603
14604         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
14605                dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL);
14606         dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base;
14607         dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base;
14608
14609         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
14610
14611         intel_color_init(&intel_crtc->base);
14612
14613         WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe);
14614         return;
14615
14616 fail:
14617         intel_plane_destroy(primary);
14618         intel_plane_destroy(cursor);
14619         kfree(crtc_state);
14620         kfree(intel_crtc);
14621 }
14622
14623 enum pipe intel_get_pipe_from_connector(struct intel_connector *connector)
14624 {
14625         struct drm_encoder *encoder = connector->base.encoder;
14626         struct drm_device *dev = connector->base.dev;
14627
14628         WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
14629
14630         if (!encoder || WARN_ON(!encoder->crtc))
14631                 return INVALID_PIPE;
14632
14633         return to_intel_crtc(encoder->crtc)->pipe;
14634 }
14635
14636 int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
14637                                 struct drm_file *file)
14638 {
14639         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
14640         struct drm_crtc *drmmode_crtc;
14641         struct intel_crtc *crtc;
14642
14643         drmmode_crtc = drm_crtc_find(dev, pipe_from_crtc_id->crtc_id);
14644         if (!drmmode_crtc)
14645                 return -ENOENT;
14646
14647         crtc = to_intel_crtc(drmmode_crtc);
14648         pipe_from_crtc_id->pipe = crtc->pipe;
14649
14650         return 0;
14651 }
14652
14653 static int intel_encoder_clones(struct intel_encoder *encoder)
14654 {
14655         struct drm_device *dev = encoder->base.dev;
14656         struct intel_encoder *source_encoder;
14657         int index_mask = 0;
14658         int entry = 0;
14659
14660         for_each_intel_encoder(dev, source_encoder) {
14661                 if (encoders_cloneable(encoder, source_encoder))
14662                         index_mask |= (1 << entry);
14663
14664                 entry++;
14665         }
14666
14667         return index_mask;
14668 }
14669
14670 static bool has_edp_a(struct drm_device *dev)
14671 {
14672         struct drm_i915_private *dev_priv = dev->dev_private;
14673
14674         if (!IS_MOBILE(dev))
14675                 return false;
14676
14677         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
14678                 return false;
14679
14680         if (IS_GEN5(dev) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
14681                 return false;
14682
14683         return true;
14684 }
14685
14686 static bool intel_crt_present(struct drm_device *dev)
14687 {
14688         struct drm_i915_private *dev_priv = dev->dev_private;
14689
14690         if (INTEL_INFO(dev)->gen >= 9)
14691                 return false;
14692
14693         if (IS_HSW_ULT(dev) || IS_BDW_ULT(dev))
14694                 return false;
14695
14696         if (IS_CHERRYVIEW(dev))
14697                 return false;
14698
14699         if (HAS_PCH_LPT_H(dev) && I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
14700                 return false;
14701
14702         /* DDI E can't be used if DDI A requires 4 lanes */
14703         if (HAS_DDI(dev) && I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
14704                 return false;
14705
14706         if (!dev_priv->vbt.int_crt_support)
14707                 return false;
14708
14709         return true;
14710 }
14711
14712 static void intel_setup_outputs(struct drm_device *dev)
14713 {
14714         struct drm_i915_private *dev_priv = dev->dev_private;
14715         struct intel_encoder *encoder;
14716         bool dpd_is_edp = false;
14717
14718         /*
14719          * intel_edp_init_connector() depends on this completing first, to
14720          * prevent the registeration of both eDP and LVDS and the incorrect
14721          * sharing of the PPS.
14722          */
14723         intel_lvds_init(dev);
14724
14725         if (intel_crt_present(dev))
14726                 intel_crt_init(dev);
14727
14728         if (IS_BROXTON(dev)) {
14729                 /*
14730                  * FIXME: Broxton doesn't support port detection via the
14731                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
14732                  * detect the ports.
14733                  */
14734                 intel_ddi_init(dev, PORT_A);
14735                 intel_ddi_init(dev, PORT_B);
14736                 intel_ddi_init(dev, PORT_C);
14737
14738                 intel_dsi_init(dev);
14739         } else if (HAS_DDI(dev)) {
14740                 int found;
14741
14742                 /*
14743                  * Haswell uses DDI functions to detect digital outputs.
14744                  * On SKL pre-D0 the strap isn't connected, so we assume
14745                  * it's there.
14746                  */
14747                 found = I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
14748                 /* WaIgnoreDDIAStrap: skl */
14749                 if (found || IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
14750                         intel_ddi_init(dev, PORT_A);
14751
14752                 /* DDI B, C and D detection is indicated by the SFUSE_STRAP
14753                  * register */
14754                 found = I915_READ(SFUSE_STRAP);
14755
14756                 if (found & SFUSE_STRAP_DDIB_DETECTED)
14757                         intel_ddi_init(dev, PORT_B);
14758                 if (found & SFUSE_STRAP_DDIC_DETECTED)
14759                         intel_ddi_init(dev, PORT_C);
14760                 if (found & SFUSE_STRAP_DDID_DETECTED)
14761                         intel_ddi_init(dev, PORT_D);
14762                 /*
14763                  * On SKL we don't have a way to detect DDI-E so we rely on VBT.
14764                  */
14765                 if ((IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) &&
14766                     (dev_priv->vbt.ddi_port_info[PORT_E].supports_dp ||
14767                      dev_priv->vbt.ddi_port_info[PORT_E].supports_dvi ||
14768                      dev_priv->vbt.ddi_port_info[PORT_E].supports_hdmi))
14769                         intel_ddi_init(dev, PORT_E);
14770
14771         } else if (HAS_PCH_SPLIT(dev)) {
14772                 int found;
14773                 dpd_is_edp = intel_dp_is_edp(dev, PORT_D);
14774
14775                 if (has_edp_a(dev))
14776                         intel_dp_init(dev, DP_A, PORT_A);
14777
14778                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
14779                         /* PCH SDVOB multiplex with HDMIB */
14780                         found = intel_sdvo_init(dev, PCH_SDVOB, PORT_B);
14781                         if (!found)
14782                                 intel_hdmi_init(dev, PCH_HDMIB, PORT_B);
14783                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
14784                                 intel_dp_init(dev, PCH_DP_B, PORT_B);
14785                 }
14786
14787                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
14788                         intel_hdmi_init(dev, PCH_HDMIC, PORT_C);
14789
14790                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
14791                         intel_hdmi_init(dev, PCH_HDMID, PORT_D);
14792
14793                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
14794                         intel_dp_init(dev, PCH_DP_C, PORT_C);
14795
14796                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
14797                         intel_dp_init(dev, PCH_DP_D, PORT_D);
14798         } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
14799                 bool has_edp, has_port;
14800
14801                 /*
14802                  * The DP_DETECTED bit is the latched state of the DDC
14803                  * SDA pin at boot. However since eDP doesn't require DDC
14804                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
14805                  * eDP ports may have been muxed to an alternate function.
14806                  * Thus we can't rely on the DP_DETECTED bit alone to detect
14807                  * eDP ports. Consult the VBT as well as DP_DETECTED to
14808                  * detect eDP ports.
14809                  *
14810                  * Sadly the straps seem to be missing sometimes even for HDMI
14811                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
14812                  * and VBT for the presence of the port. Additionally we can't
14813                  * trust the port type the VBT declares as we've seen at least
14814                  * HDMI ports that the VBT claim are DP or eDP.
14815                  */
14816                 has_edp = intel_dp_is_edp(dev, PORT_B);
14817                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
14818                 if (I915_READ(VLV_DP_B) & DP_DETECTED || has_port)
14819                         has_edp &= intel_dp_init(dev, VLV_DP_B, PORT_B);
14820                 if ((I915_READ(VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
14821                         intel_hdmi_init(dev, VLV_HDMIB, PORT_B);
14822
14823                 has_edp = intel_dp_is_edp(dev, PORT_C);
14824                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
14825                 if (I915_READ(VLV_DP_C) & DP_DETECTED || has_port)
14826                         has_edp &= intel_dp_init(dev, VLV_DP_C, PORT_C);
14827                 if ((I915_READ(VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
14828                         intel_hdmi_init(dev, VLV_HDMIC, PORT_C);
14829
14830                 if (IS_CHERRYVIEW(dev)) {
14831                         /*
14832                          * eDP not supported on port D,
14833                          * so no need to worry about it
14834                          */
14835                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
14836                         if (I915_READ(CHV_DP_D) & DP_DETECTED || has_port)
14837                                 intel_dp_init(dev, CHV_DP_D, PORT_D);
14838                         if (I915_READ(CHV_HDMID) & SDVO_DETECTED || has_port)
14839                                 intel_hdmi_init(dev, CHV_HDMID, PORT_D);
14840                 }
14841
14842                 intel_dsi_init(dev);
14843         } else if (!IS_GEN2(dev) && !IS_PINEVIEW(dev)) {
14844                 bool found = false;
14845
14846                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14847                         DRM_DEBUG_KMS("probing SDVOB\n");
14848                         found = intel_sdvo_init(dev, GEN3_SDVOB, PORT_B);
14849                         if (!found && IS_G4X(dev)) {
14850                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
14851                                 intel_hdmi_init(dev, GEN4_HDMIB, PORT_B);
14852                         }
14853
14854                         if (!found && IS_G4X(dev))
14855                                 intel_dp_init(dev, DP_B, PORT_B);
14856                 }
14857
14858                 /* Before G4X SDVOC doesn't have its own detect register */
14859
14860                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14861                         DRM_DEBUG_KMS("probing SDVOC\n");
14862                         found = intel_sdvo_init(dev, GEN3_SDVOC, PORT_C);
14863                 }
14864
14865                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
14866
14867                         if (IS_G4X(dev)) {
14868                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
14869                                 intel_hdmi_init(dev, GEN4_HDMIC, PORT_C);
14870                         }
14871                         if (IS_G4X(dev))
14872                                 intel_dp_init(dev, DP_C, PORT_C);
14873                 }
14874
14875                 if (IS_G4X(dev) &&
14876                     (I915_READ(DP_D) & DP_DETECTED))
14877                         intel_dp_init(dev, DP_D, PORT_D);
14878         } else if (IS_GEN2(dev))
14879                 intel_dvo_init(dev);
14880
14881         if (SUPPORTS_TV(dev))
14882                 intel_tv_init(dev);
14883
14884         intel_psr_init(dev);
14885
14886         for_each_intel_encoder(dev, encoder) {
14887                 encoder->base.possible_crtcs = encoder->crtc_mask;
14888                 encoder->base.possible_clones =
14889                         intel_encoder_clones(encoder);
14890         }
14891
14892         intel_init_pch_refclk(dev);
14893
14894         drm_helper_move_panel_connectors_to_head(dev);
14895 }
14896
14897 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
14898 {
14899         struct drm_device *dev = fb->dev;
14900         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14901
14902         drm_framebuffer_cleanup(fb);
14903         mutex_lock(&dev->struct_mutex);
14904         WARN_ON(!intel_fb->obj->framebuffer_references--);
14905         drm_gem_object_unreference(&intel_fb->obj->base);
14906         mutex_unlock(&dev->struct_mutex);
14907         kfree(intel_fb);
14908 }
14909
14910 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
14911                                                 struct drm_file *file,
14912                                                 unsigned int *handle)
14913 {
14914         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14915         struct drm_i915_gem_object *obj = intel_fb->obj;
14916
14917         if (obj->userptr.mm) {
14918                 DRM_DEBUG("attempting to use a userptr for a framebuffer, denied\n");
14919                 return -EINVAL;
14920         }
14921
14922         return drm_gem_handle_create(file, &obj->base, handle);
14923 }
14924
14925 static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
14926                                         struct drm_file *file,
14927                                         unsigned flags, unsigned color,
14928                                         struct drm_clip_rect *clips,
14929                                         unsigned num_clips)
14930 {
14931         struct drm_device *dev = fb->dev;
14932         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14933         struct drm_i915_gem_object *obj = intel_fb->obj;
14934
14935         mutex_lock(&dev->struct_mutex);
14936         intel_fb_obj_flush(obj, false, ORIGIN_DIRTYFB);
14937         mutex_unlock(&dev->struct_mutex);
14938
14939         return 0;
14940 }
14941
14942 static const struct drm_framebuffer_funcs intel_fb_funcs = {
14943         .destroy = intel_user_framebuffer_destroy,
14944         .create_handle = intel_user_framebuffer_create_handle,
14945         .dirty = intel_user_framebuffer_dirty,
14946 };
14947
14948 static
14949 u32 intel_fb_pitch_limit(struct drm_device *dev, uint64_t fb_modifier,
14950                          uint32_t pixel_format)
14951 {
14952         u32 gen = INTEL_INFO(dev)->gen;
14953
14954         if (gen >= 9) {
14955                 int cpp = drm_format_plane_cpp(pixel_format, 0);
14956
14957                 /* "The stride in bytes must not exceed the of the size of 8K
14958                  *  pixels and 32K bytes."
14959                  */
14960                 return min(8192 * cpp, 32768);
14961         } else if (gen >= 5 && !IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
14962                 return 32*1024;
14963         } else if (gen >= 4) {
14964                 if (fb_modifier == I915_FORMAT_MOD_X_TILED)
14965                         return 16*1024;
14966                 else
14967                         return 32*1024;
14968         } else if (gen >= 3) {
14969                 if (fb_modifier == I915_FORMAT_MOD_X_TILED)
14970                         return 8*1024;
14971                 else
14972                         return 16*1024;
14973         } else {
14974                 /* XXX DSPC is limited to 4k tiled */
14975                 return 8*1024;
14976         }
14977 }
14978
14979 static int intel_framebuffer_init(struct drm_device *dev,
14980                                   struct intel_framebuffer *intel_fb,
14981                                   struct drm_mode_fb_cmd2 *mode_cmd,
14982                                   struct drm_i915_gem_object *obj)
14983 {
14984         struct drm_i915_private *dev_priv = to_i915(dev);
14985         unsigned int aligned_height;
14986         int ret;
14987         u32 pitch_limit, stride_alignment;
14988
14989         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
14990
14991         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
14992                 /* Enforce that fb modifier and tiling mode match, but only for
14993                  * X-tiled. This is needed for FBC. */
14994                 if (!!(obj->tiling_mode == I915_TILING_X) !=
14995                     !!(mode_cmd->modifier[0] == I915_FORMAT_MOD_X_TILED)) {
14996                         DRM_DEBUG("tiling_mode doesn't match fb modifier\n");
14997                         return -EINVAL;
14998                 }
14999         } else {
15000                 if (obj->tiling_mode == I915_TILING_X)
15001                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
15002                 else if (obj->tiling_mode == I915_TILING_Y) {
15003                         DRM_DEBUG("No Y tiling for legacy addfb\n");
15004                         return -EINVAL;
15005                 }
15006         }
15007
15008         /* Passed in modifier sanity checking. */
15009         switch (mode_cmd->modifier[0]) {
15010         case I915_FORMAT_MOD_Y_TILED:
15011         case I915_FORMAT_MOD_Yf_TILED:
15012                 if (INTEL_INFO(dev)->gen < 9) {
15013                         DRM_DEBUG("Unsupported tiling 0x%llx!\n",
15014                                   mode_cmd->modifier[0]);
15015                         return -EINVAL;
15016                 }
15017         case DRM_FORMAT_MOD_NONE:
15018         case I915_FORMAT_MOD_X_TILED:
15019                 break;
15020         default:
15021                 DRM_DEBUG("Unsupported fb modifier 0x%llx!\n",
15022                           mode_cmd->modifier[0]);
15023                 return -EINVAL;
15024         }
15025
15026         stride_alignment = intel_fb_stride_alignment(dev_priv,
15027                                                      mode_cmd->modifier[0],
15028                                                      mode_cmd->pixel_format);
15029         if (mode_cmd->pitches[0] & (stride_alignment - 1)) {
15030                 DRM_DEBUG("pitch (%d) must be at least %u byte aligned\n",
15031                           mode_cmd->pitches[0], stride_alignment);
15032                 return -EINVAL;
15033         }
15034
15035         pitch_limit = intel_fb_pitch_limit(dev, mode_cmd->modifier[0],
15036                                            mode_cmd->pixel_format);
15037         if (mode_cmd->pitches[0] > pitch_limit) {
15038                 DRM_DEBUG("%s pitch (%u) must be at less than %d\n",
15039                           mode_cmd->modifier[0] != DRM_FORMAT_MOD_NONE ?
15040                           "tiled" : "linear",
15041                           mode_cmd->pitches[0], pitch_limit);
15042                 return -EINVAL;
15043         }
15044
15045         if (mode_cmd->modifier[0] == I915_FORMAT_MOD_X_TILED &&
15046             mode_cmd->pitches[0] != obj->stride) {
15047                 DRM_DEBUG("pitch (%d) must match tiling stride (%d)\n",
15048                           mode_cmd->pitches[0], obj->stride);
15049                 return -EINVAL;
15050         }
15051
15052         /* Reject formats not supported by any plane early. */
15053         switch (mode_cmd->pixel_format) {
15054         case DRM_FORMAT_C8:
15055         case DRM_FORMAT_RGB565:
15056         case DRM_FORMAT_XRGB8888:
15057         case DRM_FORMAT_ARGB8888:
15058                 break;
15059         case DRM_FORMAT_XRGB1555:
15060                 if (INTEL_INFO(dev)->gen > 3) {
15061                         DRM_DEBUG("unsupported pixel format: %s\n",
15062                                   drm_get_format_name(mode_cmd->pixel_format));
15063                         return -EINVAL;
15064                 }
15065                 break;
15066         case DRM_FORMAT_ABGR8888:
15067                 if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
15068                     INTEL_INFO(dev)->gen < 9) {
15069                         DRM_DEBUG("unsupported pixel format: %s\n",
15070                                   drm_get_format_name(mode_cmd->pixel_format));
15071                         return -EINVAL;
15072                 }
15073                 break;
15074         case DRM_FORMAT_XBGR8888:
15075         case DRM_FORMAT_XRGB2101010:
15076         case DRM_FORMAT_XBGR2101010:
15077                 if (INTEL_INFO(dev)->gen < 4) {
15078                         DRM_DEBUG("unsupported pixel format: %s\n",
15079                                   drm_get_format_name(mode_cmd->pixel_format));
15080                         return -EINVAL;
15081                 }
15082                 break;
15083         case DRM_FORMAT_ABGR2101010:
15084                 if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
15085                         DRM_DEBUG("unsupported pixel format: %s\n",
15086                                   drm_get_format_name(mode_cmd->pixel_format));
15087                         return -EINVAL;
15088                 }
15089                 break;
15090         case DRM_FORMAT_YUYV:
15091         case DRM_FORMAT_UYVY:
15092         case DRM_FORMAT_YVYU:
15093         case DRM_FORMAT_VYUY:
15094                 if (INTEL_INFO(dev)->gen < 5) {
15095                         DRM_DEBUG("unsupported pixel format: %s\n",
15096                                   drm_get_format_name(mode_cmd->pixel_format));
15097                         return -EINVAL;
15098                 }
15099                 break;
15100         default:
15101                 DRM_DEBUG("unsupported pixel format: %s\n",
15102                           drm_get_format_name(mode_cmd->pixel_format));
15103                 return -EINVAL;
15104         }
15105
15106         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
15107         if (mode_cmd->offsets[0] != 0)
15108                 return -EINVAL;
15109
15110         aligned_height = intel_fb_align_height(dev, mode_cmd->height,
15111                                                mode_cmd->pixel_format,
15112                                                mode_cmd->modifier[0]);
15113         /* FIXME drm helper for size checks (especially planar formats)? */
15114         if (obj->base.size < aligned_height * mode_cmd->pitches[0])
15115                 return -EINVAL;
15116
15117         drm_helper_mode_fill_fb_struct(&intel_fb->base, mode_cmd);
15118         intel_fb->obj = obj;
15119
15120         intel_fill_fb_info(dev_priv, &intel_fb->base);
15121
15122         ret = drm_framebuffer_init(dev, &intel_fb->base, &intel_fb_funcs);
15123         if (ret) {
15124                 DRM_ERROR("framebuffer init failed %d\n", ret);
15125                 return ret;
15126         }
15127
15128         intel_fb->obj->framebuffer_references++;
15129
15130         return 0;
15131 }
15132
15133 static struct drm_framebuffer *
15134 intel_user_framebuffer_create(struct drm_device *dev,
15135                               struct drm_file *filp,
15136                               const struct drm_mode_fb_cmd2 *user_mode_cmd)
15137 {
15138         struct drm_framebuffer *fb;
15139         struct drm_i915_gem_object *obj;
15140         struct drm_mode_fb_cmd2 mode_cmd = *user_mode_cmd;
15141
15142         obj = to_intel_bo(drm_gem_object_lookup(filp, mode_cmd.handles[0]));
15143         if (&obj->base == NULL)
15144                 return ERR_PTR(-ENOENT);
15145
15146         fb = intel_framebuffer_create(dev, &mode_cmd, obj);
15147         if (IS_ERR(fb))
15148                 drm_gem_object_unreference_unlocked(&obj->base);
15149
15150         return fb;
15151 }
15152
15153 #ifndef CONFIG_DRM_FBDEV_EMULATION
15154 static inline void intel_fbdev_output_poll_changed(struct drm_device *dev)
15155 {
15156 }
15157 #endif
15158
15159 static const struct drm_mode_config_funcs intel_mode_funcs = {
15160         .fb_create = intel_user_framebuffer_create,
15161         .output_poll_changed = intel_fbdev_output_poll_changed,
15162         .atomic_check = intel_atomic_check,
15163         .atomic_commit = intel_atomic_commit,
15164         .atomic_state_alloc = intel_atomic_state_alloc,
15165         .atomic_state_clear = intel_atomic_state_clear,
15166 };
15167
15168 /**
15169  * intel_init_display_hooks - initialize the display modesetting hooks
15170  * @dev_priv: device private
15171  */
15172 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
15173 {
15174         if (INTEL_INFO(dev_priv)->gen >= 9) {
15175                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
15176                 dev_priv->display.get_initial_plane_config =
15177                         skylake_get_initial_plane_config;
15178                 dev_priv->display.crtc_compute_clock =
15179                         haswell_crtc_compute_clock;
15180                 dev_priv->display.crtc_enable = haswell_crtc_enable;
15181                 dev_priv->display.crtc_disable = haswell_crtc_disable;
15182         } else if (HAS_DDI(dev_priv)) {
15183                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
15184                 dev_priv->display.get_initial_plane_config =
15185                         ironlake_get_initial_plane_config;
15186                 dev_priv->display.crtc_compute_clock =
15187                         haswell_crtc_compute_clock;
15188                 dev_priv->display.crtc_enable = haswell_crtc_enable;
15189                 dev_priv->display.crtc_disable = haswell_crtc_disable;
15190         } else if (HAS_PCH_SPLIT(dev_priv)) {
15191                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
15192                 dev_priv->display.get_initial_plane_config =
15193                         ironlake_get_initial_plane_config;
15194                 dev_priv->display.crtc_compute_clock =
15195                         ironlake_crtc_compute_clock;
15196                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
15197                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
15198         } else if (IS_CHERRYVIEW(dev_priv)) {
15199                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15200                 dev_priv->display.get_initial_plane_config =
15201                         i9xx_get_initial_plane_config;
15202                 dev_priv->display.crtc_compute_clock = chv_crtc_compute_clock;
15203                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
15204                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15205         } else if (IS_VALLEYVIEW(dev_priv)) {
15206                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15207                 dev_priv->display.get_initial_plane_config =
15208                         i9xx_get_initial_plane_config;
15209                 dev_priv->display.crtc_compute_clock = vlv_crtc_compute_clock;
15210                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
15211                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15212         } else if (IS_G4X(dev_priv)) {
15213                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15214                 dev_priv->display.get_initial_plane_config =
15215                         i9xx_get_initial_plane_config;
15216                 dev_priv->display.crtc_compute_clock = g4x_crtc_compute_clock;
15217                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15218                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15219         } else if (IS_PINEVIEW(dev_priv)) {
15220                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15221                 dev_priv->display.get_initial_plane_config =
15222                         i9xx_get_initial_plane_config;
15223                 dev_priv->display.crtc_compute_clock = pnv_crtc_compute_clock;
15224                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15225                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15226         } else if (!IS_GEN2(dev_priv)) {
15227                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15228                 dev_priv->display.get_initial_plane_config =
15229                         i9xx_get_initial_plane_config;
15230                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
15231                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15232                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15233         } else {
15234                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
15235                 dev_priv->display.get_initial_plane_config =
15236                         i9xx_get_initial_plane_config;
15237                 dev_priv->display.crtc_compute_clock = i8xx_crtc_compute_clock;
15238                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
15239                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
15240         }
15241
15242         /* Returns the core display clock speed */
15243         if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
15244                 dev_priv->display.get_display_clock_speed =
15245                         skylake_get_display_clock_speed;
15246         else if (IS_BROXTON(dev_priv))
15247                 dev_priv->display.get_display_clock_speed =
15248                         broxton_get_display_clock_speed;
15249         else if (IS_BROADWELL(dev_priv))
15250                 dev_priv->display.get_display_clock_speed =
15251                         broadwell_get_display_clock_speed;
15252         else if (IS_HASWELL(dev_priv))
15253                 dev_priv->display.get_display_clock_speed =
15254                         haswell_get_display_clock_speed;
15255         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
15256                 dev_priv->display.get_display_clock_speed =
15257                         valleyview_get_display_clock_speed;
15258         else if (IS_GEN5(dev_priv))
15259                 dev_priv->display.get_display_clock_speed =
15260                         ilk_get_display_clock_speed;
15261         else if (IS_I945G(dev_priv) || IS_BROADWATER(dev_priv) ||
15262                  IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv))
15263                 dev_priv->display.get_display_clock_speed =
15264                         i945_get_display_clock_speed;
15265         else if (IS_GM45(dev_priv))
15266                 dev_priv->display.get_display_clock_speed =
15267                         gm45_get_display_clock_speed;
15268         else if (IS_CRESTLINE(dev_priv))
15269                 dev_priv->display.get_display_clock_speed =
15270                         i965gm_get_display_clock_speed;
15271         else if (IS_PINEVIEW(dev_priv))
15272                 dev_priv->display.get_display_clock_speed =
15273                         pnv_get_display_clock_speed;
15274         else if (IS_G33(dev_priv) || IS_G4X(dev_priv))
15275                 dev_priv->display.get_display_clock_speed =
15276                         g33_get_display_clock_speed;
15277         else if (IS_I915G(dev_priv))
15278                 dev_priv->display.get_display_clock_speed =
15279                         i915_get_display_clock_speed;
15280         else if (IS_I945GM(dev_priv) || IS_845G(dev_priv))
15281                 dev_priv->display.get_display_clock_speed =
15282                         i9xx_misc_get_display_clock_speed;
15283         else if (IS_I915GM(dev_priv))
15284                 dev_priv->display.get_display_clock_speed =
15285                         i915gm_get_display_clock_speed;
15286         else if (IS_I865G(dev_priv))
15287                 dev_priv->display.get_display_clock_speed =
15288                         i865_get_display_clock_speed;
15289         else if (IS_I85X(dev_priv))
15290                 dev_priv->display.get_display_clock_speed =
15291                         i85x_get_display_clock_speed;
15292         else { /* 830 */
15293                 WARN(!IS_I830(dev_priv), "Unknown platform. Assuming 133 MHz CDCLK\n");
15294                 dev_priv->display.get_display_clock_speed =
15295                         i830_get_display_clock_speed;
15296         }
15297
15298         if (IS_GEN5(dev_priv)) {
15299                 dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
15300         } else if (IS_GEN6(dev_priv)) {
15301                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
15302         } else if (IS_IVYBRIDGE(dev_priv)) {
15303                 /* FIXME: detect B0+ stepping and use auto training */
15304                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
15305         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
15306                 dev_priv->display.fdi_link_train = hsw_fdi_link_train;
15307         }
15308
15309         if (IS_BROADWELL(dev_priv)) {
15310                 dev_priv->display.modeset_commit_cdclk =
15311                         broadwell_modeset_commit_cdclk;
15312                 dev_priv->display.modeset_calc_cdclk =
15313                         broadwell_modeset_calc_cdclk;
15314         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
15315                 dev_priv->display.modeset_commit_cdclk =
15316                         valleyview_modeset_commit_cdclk;
15317                 dev_priv->display.modeset_calc_cdclk =
15318                         valleyview_modeset_calc_cdclk;
15319         } else if (IS_BROXTON(dev_priv)) {
15320                 dev_priv->display.modeset_commit_cdclk =
15321                         bxt_modeset_commit_cdclk;
15322                 dev_priv->display.modeset_calc_cdclk =
15323                         bxt_modeset_calc_cdclk;
15324         } else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
15325                 dev_priv->display.modeset_commit_cdclk =
15326                         skl_modeset_commit_cdclk;
15327                 dev_priv->display.modeset_calc_cdclk =
15328                         skl_modeset_calc_cdclk;
15329         }
15330
15331         switch (INTEL_INFO(dev_priv)->gen) {
15332         case 2:
15333                 dev_priv->display.queue_flip = intel_gen2_queue_flip;
15334                 break;
15335
15336         case 3:
15337                 dev_priv->display.queue_flip = intel_gen3_queue_flip;
15338                 break;
15339
15340         case 4:
15341         case 5:
15342                 dev_priv->display.queue_flip = intel_gen4_queue_flip;
15343                 break;
15344
15345         case 6:
15346                 dev_priv->display.queue_flip = intel_gen6_queue_flip;
15347                 break;
15348         case 7:
15349         case 8: /* FIXME(BDW): Check that the gen8 RCS flip works. */
15350                 dev_priv->display.queue_flip = intel_gen7_queue_flip;
15351                 break;
15352         case 9:
15353                 /* Drop through - unsupported since execlist only. */
15354         default:
15355                 /* Default just returns -ENODEV to indicate unsupported */
15356                 dev_priv->display.queue_flip = intel_default_queue_flip;
15357         }
15358 }
15359
15360 /*
15361  * Some BIOSes insist on assuming the GPU's pipe A is enabled at suspend,
15362  * resume, or other times.  This quirk makes sure that's the case for
15363  * affected systems.
15364  */
15365 static void quirk_pipea_force(struct drm_device *dev)
15366 {
15367         struct drm_i915_private *dev_priv = dev->dev_private;
15368
15369         dev_priv->quirks |= QUIRK_PIPEA_FORCE;
15370         DRM_INFO("applying pipe a force quirk\n");
15371 }
15372
15373 static void quirk_pipeb_force(struct drm_device *dev)
15374 {
15375         struct drm_i915_private *dev_priv = dev->dev_private;
15376
15377         dev_priv->quirks |= QUIRK_PIPEB_FORCE;
15378         DRM_INFO("applying pipe b force quirk\n");
15379 }
15380
15381 /*
15382  * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
15383  */
15384 static void quirk_ssc_force_disable(struct drm_device *dev)
15385 {
15386         struct drm_i915_private *dev_priv = dev->dev_private;
15387         dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE;
15388         DRM_INFO("applying lvds SSC disable quirk\n");
15389 }
15390
15391 /*
15392  * A machine (e.g. Acer Aspire 5734Z) may need to invert the panel backlight
15393  * brightness value
15394  */
15395 static void quirk_invert_brightness(struct drm_device *dev)
15396 {
15397         struct drm_i915_private *dev_priv = dev->dev_private;
15398         dev_priv->quirks |= QUIRK_INVERT_BRIGHTNESS;
15399         DRM_INFO("applying inverted panel brightness quirk\n");
15400 }
15401
15402 /* Some VBT's incorrectly indicate no backlight is present */
15403 static void quirk_backlight_present(struct drm_device *dev)
15404 {
15405         struct drm_i915_private *dev_priv = dev->dev_private;
15406         dev_priv->quirks |= QUIRK_BACKLIGHT_PRESENT;
15407         DRM_INFO("applying backlight present quirk\n");
15408 }
15409
15410 struct intel_quirk {
15411         int device;
15412         int subsystem_vendor;
15413         int subsystem_device;
15414         void (*hook)(struct drm_device *dev);
15415 };
15416
15417 /* For systems that don't have a meaningful PCI subdevice/subvendor ID */
15418 struct intel_dmi_quirk {
15419         void (*hook)(struct drm_device *dev);
15420         const struct dmi_system_id (*dmi_id_list)[];
15421 };
15422
15423 static int intel_dmi_reverse_brightness(const struct dmi_system_id *id)
15424 {
15425         DRM_INFO("Backlight polarity reversed on %s\n", id->ident);
15426         return 1;
15427 }
15428
15429 static const struct intel_dmi_quirk intel_dmi_quirks[] = {
15430         {
15431                 .dmi_id_list = &(const struct dmi_system_id[]) {
15432                         {
15433                                 .callback = intel_dmi_reverse_brightness,
15434                                 .ident = "NCR Corporation",
15435                                 .matches = {DMI_MATCH(DMI_SYS_VENDOR, "NCR Corporation"),
15436                                             DMI_MATCH(DMI_PRODUCT_NAME, ""),
15437                                 },
15438                         },
15439                         { }  /* terminating entry */
15440                 },
15441                 .hook = quirk_invert_brightness,
15442         },
15443 };
15444
15445 static struct intel_quirk intel_quirks[] = {
15446         /* Toshiba Protege R-205, S-209 needs pipe A force quirk */
15447         { 0x2592, 0x1179, 0x0001, quirk_pipea_force },
15448
15449         /* ThinkPad T60 needs pipe A force quirk (bug #16494) */
15450         { 0x2782, 0x17aa, 0x201a, quirk_pipea_force },
15451
15452         /* 830 needs to leave pipe A & dpll A up */
15453         { 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
15454
15455         /* 830 needs to leave pipe B & dpll B up */
15456         { 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipeb_force },
15457
15458         /* Lenovo U160 cannot use SSC on LVDS */
15459         { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
15460
15461         /* Sony Vaio Y cannot use SSC on LVDS */
15462         { 0x0046, 0x104d, 0x9076, quirk_ssc_force_disable },
15463
15464         /* Acer Aspire 5734Z must invert backlight brightness */
15465         { 0x2a42, 0x1025, 0x0459, quirk_invert_brightness },
15466
15467         /* Acer/eMachines G725 */
15468         { 0x2a42, 0x1025, 0x0210, quirk_invert_brightness },
15469
15470         /* Acer/eMachines e725 */
15471         { 0x2a42, 0x1025, 0x0212, quirk_invert_brightness },
15472
15473         /* Acer/Packard Bell NCL20 */
15474         { 0x2a42, 0x1025, 0x034b, quirk_invert_brightness },
15475
15476         /* Acer Aspire 4736Z */
15477         { 0x2a42, 0x1025, 0x0260, quirk_invert_brightness },
15478
15479         /* Acer Aspire 5336 */
15480         { 0x2a42, 0x1025, 0x048a, quirk_invert_brightness },
15481
15482         /* Acer C720 and C720P Chromebooks (Celeron 2955U) have backlights */
15483         { 0x0a06, 0x1025, 0x0a11, quirk_backlight_present },
15484
15485         /* Acer C720 Chromebook (Core i3 4005U) */
15486         { 0x0a16, 0x1025, 0x0a11, quirk_backlight_present },
15487
15488         /* Apple Macbook 2,1 (Core 2 T7400) */
15489         { 0x27a2, 0x8086, 0x7270, quirk_backlight_present },
15490
15491         /* Apple Macbook 4,1 */
15492         { 0x2a02, 0x106b, 0x00a1, quirk_backlight_present },
15493
15494         /* Toshiba CB35 Chromebook (Celeron 2955U) */
15495         { 0x0a06, 0x1179, 0x0a88, quirk_backlight_present },
15496
15497         /* HP Chromebook 14 (Celeron 2955U) */
15498         { 0x0a06, 0x103c, 0x21ed, quirk_backlight_present },
15499
15500         /* Dell Chromebook 11 */
15501         { 0x0a06, 0x1028, 0x0a35, quirk_backlight_present },
15502
15503         /* Dell Chromebook 11 (2015 version) */
15504         { 0x0a16, 0x1028, 0x0a35, quirk_backlight_present },
15505 };
15506
15507 static void intel_init_quirks(struct drm_device *dev)
15508 {
15509         struct pci_dev *d = dev->pdev;
15510         int i;
15511
15512         for (i = 0; i < ARRAY_SIZE(intel_quirks); i++) {
15513                 struct intel_quirk *q = &intel_quirks[i];
15514
15515                 if (d->device == q->device &&
15516                     (d->subsystem_vendor == q->subsystem_vendor ||
15517                      q->subsystem_vendor == PCI_ANY_ID) &&
15518                     (d->subsystem_device == q->subsystem_device ||
15519                      q->subsystem_device == PCI_ANY_ID))
15520                         q->hook(dev);
15521         }
15522         for (i = 0; i < ARRAY_SIZE(intel_dmi_quirks); i++) {
15523                 if (dmi_check_system(*intel_dmi_quirks[i].dmi_id_list) != 0)
15524                         intel_dmi_quirks[i].hook(dev);
15525         }
15526 }
15527
15528 /* Disable the VGA plane that we never use */
15529 static void i915_disable_vga(struct drm_device *dev)
15530 {
15531         struct drm_i915_private *dev_priv = dev->dev_private;
15532         u8 sr1;
15533         i915_reg_t vga_reg = i915_vgacntrl_reg(dev);
15534
15535         /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
15536         vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
15537         outb(SR01, VGA_SR_INDEX);
15538         sr1 = inb(VGA_SR_DATA);
15539         outb(sr1 | 1<<5, VGA_SR_DATA);
15540         vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
15541         udelay(300);
15542
15543         I915_WRITE(vga_reg, VGA_DISP_DISABLE);
15544         POSTING_READ(vga_reg);
15545 }
15546
15547 void intel_modeset_init_hw(struct drm_device *dev)
15548 {
15549         struct drm_i915_private *dev_priv = dev->dev_private;
15550
15551         intel_update_cdclk(dev);
15552
15553         dev_priv->atomic_cdclk_freq = dev_priv->cdclk_freq;
15554
15555         intel_init_clock_gating(dev);
15556         intel_enable_gt_powersave(dev_priv);
15557 }
15558
15559 /*
15560  * Calculate what we think the watermarks should be for the state we've read
15561  * out of the hardware and then immediately program those watermarks so that
15562  * we ensure the hardware settings match our internal state.
15563  *
15564  * We can calculate what we think WM's should be by creating a duplicate of the
15565  * current state (which was constructed during hardware readout) and running it
15566  * through the atomic check code to calculate new watermark values in the
15567  * state object.
15568  */
15569 static void sanitize_watermarks(struct drm_device *dev)
15570 {
15571         struct drm_i915_private *dev_priv = to_i915(dev);
15572         struct drm_atomic_state *state;
15573         struct drm_crtc *crtc;
15574         struct drm_crtc_state *cstate;
15575         struct drm_modeset_acquire_ctx ctx;
15576         int ret;
15577         int i;
15578
15579         /* Only supported on platforms that use atomic watermark design */
15580         if (!dev_priv->display.optimize_watermarks)
15581                 return;
15582
15583         /*
15584          * We need to hold connection_mutex before calling duplicate_state so
15585          * that the connector loop is protected.
15586          */
15587         drm_modeset_acquire_init(&ctx, 0);
15588 retry:
15589         ret = drm_modeset_lock_all_ctx(dev, &ctx);
15590         if (ret == -EDEADLK) {
15591                 drm_modeset_backoff(&ctx);
15592                 goto retry;
15593         } else if (WARN_ON(ret)) {
15594                 goto fail;
15595         }
15596
15597         state = drm_atomic_helper_duplicate_state(dev, &ctx);
15598         if (WARN_ON(IS_ERR(state)))
15599                 goto fail;
15600
15601         /*
15602          * Hardware readout is the only time we don't want to calculate
15603          * intermediate watermarks (since we don't trust the current
15604          * watermarks).
15605          */
15606         to_intel_atomic_state(state)->skip_intermediate_wm = true;
15607
15608         ret = intel_atomic_check(dev, state);
15609         if (ret) {
15610                 /*
15611                  * If we fail here, it means that the hardware appears to be
15612                  * programmed in a way that shouldn't be possible, given our
15613                  * understanding of watermark requirements.  This might mean a
15614                  * mistake in the hardware readout code or a mistake in the
15615                  * watermark calculations for a given platform.  Raise a WARN
15616                  * so that this is noticeable.
15617                  *
15618                  * If this actually happens, we'll have to just leave the
15619                  * BIOS-programmed watermarks untouched and hope for the best.
15620                  */
15621                 WARN(true, "Could not determine valid watermarks for inherited state\n");
15622                 goto fail;
15623         }
15624
15625         /* Write calculated watermark values back */
15626         for_each_crtc_in_state(state, crtc, cstate, i) {
15627                 struct intel_crtc_state *cs = to_intel_crtc_state(cstate);
15628
15629                 cs->wm.need_postvbl_update = true;
15630                 dev_priv->display.optimize_watermarks(cs);
15631         }
15632
15633         drm_atomic_state_free(state);
15634 fail:
15635         drm_modeset_drop_locks(&ctx);
15636         drm_modeset_acquire_fini(&ctx);
15637 }
15638
15639 void intel_modeset_init(struct drm_device *dev)
15640 {
15641         struct drm_i915_private *dev_priv = to_i915(dev);
15642         struct i915_ggtt *ggtt = &dev_priv->ggtt;
15643         int sprite, ret;
15644         enum pipe pipe;
15645         struct intel_crtc *crtc;
15646
15647         drm_mode_config_init(dev);
15648
15649         dev->mode_config.min_width = 0;
15650         dev->mode_config.min_height = 0;
15651
15652         dev->mode_config.preferred_depth = 24;
15653         dev->mode_config.prefer_shadow = 1;
15654
15655         dev->mode_config.allow_fb_modifiers = true;
15656
15657         dev->mode_config.funcs = &intel_mode_funcs;
15658
15659         intel_init_quirks(dev);
15660
15661         intel_init_pm(dev);
15662
15663         if (INTEL_INFO(dev)->num_pipes == 0)
15664                 return;
15665
15666         /*
15667          * There may be no VBT; and if the BIOS enabled SSC we can
15668          * just keep using it to avoid unnecessary flicker.  Whereas if the
15669          * BIOS isn't using it, don't assume it will work even if the VBT
15670          * indicates as much.
15671          */
15672         if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) {
15673                 bool bios_lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) &
15674                                             DREF_SSC1_ENABLE);
15675
15676                 if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
15677                         DRM_DEBUG_KMS("SSC %sabled by BIOS, overriding VBT which says %sabled\n",
15678                                      bios_lvds_use_ssc ? "en" : "dis",
15679                                      dev_priv->vbt.lvds_use_ssc ? "en" : "dis");
15680                         dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
15681                 }
15682         }
15683
15684         if (IS_GEN2(dev)) {
15685                 dev->mode_config.max_width = 2048;
15686                 dev->mode_config.max_height = 2048;
15687         } else if (IS_GEN3(dev)) {
15688                 dev->mode_config.max_width = 4096;
15689                 dev->mode_config.max_height = 4096;
15690         } else {
15691                 dev->mode_config.max_width = 8192;
15692                 dev->mode_config.max_height = 8192;
15693         }
15694
15695         if (IS_845G(dev) || IS_I865G(dev)) {
15696                 dev->mode_config.cursor_width = IS_845G(dev) ? 64 : 512;
15697                 dev->mode_config.cursor_height = 1023;
15698         } else if (IS_GEN2(dev)) {
15699                 dev->mode_config.cursor_width = GEN2_CURSOR_WIDTH;
15700                 dev->mode_config.cursor_height = GEN2_CURSOR_HEIGHT;
15701         } else {
15702                 dev->mode_config.cursor_width = MAX_CURSOR_WIDTH;
15703                 dev->mode_config.cursor_height = MAX_CURSOR_HEIGHT;
15704         }
15705
15706         dev->mode_config.fb_base = ggtt->mappable_base;
15707
15708         DRM_DEBUG_KMS("%d display pipe%s available.\n",
15709                       INTEL_INFO(dev)->num_pipes,
15710                       INTEL_INFO(dev)->num_pipes > 1 ? "s" : "");
15711
15712         for_each_pipe(dev_priv, pipe) {
15713                 intel_crtc_init(dev, pipe);
15714                 for_each_sprite(dev_priv, pipe, sprite) {
15715                         ret = intel_plane_init(dev, pipe, sprite);
15716                         if (ret)
15717                                 DRM_DEBUG_KMS("pipe %c sprite %c init failed: %d\n",
15718                                               pipe_name(pipe), sprite_name(pipe, sprite), ret);
15719                 }
15720         }
15721
15722         intel_update_czclk(dev_priv);
15723         intel_update_cdclk(dev);
15724
15725         intel_shared_dpll_init(dev);
15726
15727         if (dev_priv->max_cdclk_freq == 0)
15728                 intel_update_max_cdclk(dev);
15729
15730         /* Just disable it once at startup */
15731         i915_disable_vga(dev);
15732         intel_setup_outputs(dev);
15733
15734         drm_modeset_lock_all(dev);
15735         intel_modeset_setup_hw_state(dev);
15736         drm_modeset_unlock_all(dev);
15737
15738         for_each_intel_crtc(dev, crtc) {
15739                 struct intel_initial_plane_config plane_config = {};
15740
15741                 if (!crtc->active)
15742                         continue;
15743
15744                 /*
15745                  * Note that reserving the BIOS fb up front prevents us
15746                  * from stuffing other stolen allocations like the ring
15747                  * on top.  This prevents some ugliness at boot time, and
15748                  * can even allow for smooth boot transitions if the BIOS
15749                  * fb is large enough for the active pipe configuration.
15750                  */
15751                 dev_priv->display.get_initial_plane_config(crtc,
15752                                                            &plane_config);
15753
15754                 /*
15755                  * If the fb is shared between multiple heads, we'll
15756                  * just get the first one.
15757                  */
15758                 intel_find_initial_plane_obj(crtc, &plane_config);
15759         }
15760
15761         /*
15762          * Make sure hardware watermarks really match the state we read out.
15763          * Note that we need to do this after reconstructing the BIOS fb's
15764          * since the watermark calculation done here will use pstate->fb.
15765          */
15766         sanitize_watermarks(dev);
15767 }
15768
15769 static void intel_enable_pipe_a(struct drm_device *dev)
15770 {
15771         struct intel_connector *connector;
15772         struct drm_connector *crt = NULL;
15773         struct intel_load_detect_pipe load_detect_temp;
15774         struct drm_modeset_acquire_ctx *ctx = dev->mode_config.acquire_ctx;
15775
15776         /* We can't just switch on the pipe A, we need to set things up with a
15777          * proper mode and output configuration. As a gross hack, enable pipe A
15778          * by enabling the load detect pipe once. */
15779         for_each_intel_connector(dev, connector) {
15780                 if (connector->encoder->type == INTEL_OUTPUT_ANALOG) {
15781                         crt = &connector->base;
15782                         break;
15783                 }
15784         }
15785
15786         if (!crt)
15787                 return;
15788
15789         if (intel_get_load_detect_pipe(crt, NULL, &load_detect_temp, ctx))
15790                 intel_release_load_detect_pipe(crt, &load_detect_temp, ctx);
15791 }
15792
15793 static bool
15794 intel_check_plane_mapping(struct intel_crtc *crtc)
15795 {
15796         struct drm_device *dev = crtc->base.dev;
15797         struct drm_i915_private *dev_priv = dev->dev_private;
15798         u32 val;
15799
15800         if (INTEL_INFO(dev)->num_pipes == 1)
15801                 return true;
15802
15803         val = I915_READ(DSPCNTR(!crtc->plane));
15804
15805         if ((val & DISPLAY_PLANE_ENABLE) &&
15806             (!!(val & DISPPLANE_SEL_PIPE_MASK) == crtc->pipe))
15807                 return false;
15808
15809         return true;
15810 }
15811
15812 static bool intel_crtc_has_encoders(struct intel_crtc *crtc)
15813 {
15814         struct drm_device *dev = crtc->base.dev;
15815         struct intel_encoder *encoder;
15816
15817         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
15818                 return true;
15819
15820         return false;
15821 }
15822
15823 static bool intel_encoder_has_connectors(struct intel_encoder *encoder)
15824 {
15825         struct drm_device *dev = encoder->base.dev;
15826         struct intel_connector *connector;
15827
15828         for_each_connector_on_encoder(dev, &encoder->base, connector)
15829                 return true;
15830
15831         return false;
15832 }
15833
15834 static void intel_sanitize_crtc(struct intel_crtc *crtc)
15835 {
15836         struct drm_device *dev = crtc->base.dev;
15837         struct drm_i915_private *dev_priv = dev->dev_private;
15838         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
15839
15840         /* Clear any frame start delays used for debugging left by the BIOS */
15841         if (!transcoder_is_dsi(cpu_transcoder)) {
15842                 i915_reg_t reg = PIPECONF(cpu_transcoder);
15843
15844                 I915_WRITE(reg,
15845                            I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
15846         }
15847
15848         /* restore vblank interrupts to correct state */
15849         drm_crtc_vblank_reset(&crtc->base);
15850         if (crtc->active) {
15851                 struct intel_plane *plane;
15852
15853                 drm_crtc_vblank_on(&crtc->base);
15854
15855                 /* Disable everything but the primary plane */
15856                 for_each_intel_plane_on_crtc(dev, crtc, plane) {
15857                         if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
15858                                 continue;
15859
15860                         plane->disable_plane(&plane->base, &crtc->base);
15861                 }
15862         }
15863
15864         /* We need to sanitize the plane -> pipe mapping first because this will
15865          * disable the crtc (and hence change the state) if it is wrong. Note
15866          * that gen4+ has a fixed plane -> pipe mapping.  */
15867         if (INTEL_INFO(dev)->gen < 4 && !intel_check_plane_mapping(crtc)) {
15868                 bool plane;
15869
15870                 DRM_DEBUG_KMS("[CRTC:%d:%s] wrong plane connection detected!\n",
15871                               crtc->base.base.id, crtc->base.name);
15872
15873                 /* Pipe has the wrong plane attached and the plane is active.
15874                  * Temporarily change the plane mapping and disable everything
15875                  * ...  */
15876                 plane = crtc->plane;
15877                 to_intel_plane_state(crtc->base.primary->state)->visible = true;
15878                 crtc->plane = !plane;
15879                 intel_crtc_disable_noatomic(&crtc->base);
15880                 crtc->plane = plane;
15881         }
15882
15883         if (dev_priv->quirks & QUIRK_PIPEA_FORCE &&
15884             crtc->pipe == PIPE_A && !crtc->active) {
15885                 /* BIOS forgot to enable pipe A, this mostly happens after
15886                  * resume. Force-enable the pipe to fix this, the update_dpms
15887                  * call below we restore the pipe to the right state, but leave
15888                  * the required bits on. */
15889                 intel_enable_pipe_a(dev);
15890         }
15891
15892         /* Adjust the state of the output pipe according to whether we
15893          * have active connectors/encoders. */
15894         if (crtc->active && !intel_crtc_has_encoders(crtc))
15895                 intel_crtc_disable_noatomic(&crtc->base);
15896
15897         if (crtc->active || HAS_GMCH_DISPLAY(dev)) {
15898                 /*
15899                  * We start out with underrun reporting disabled to avoid races.
15900                  * For correct bookkeeping mark this on active crtcs.
15901                  *
15902                  * Also on gmch platforms we dont have any hardware bits to
15903                  * disable the underrun reporting. Which means we need to start
15904                  * out with underrun reporting disabled also on inactive pipes,
15905                  * since otherwise we'll complain about the garbage we read when
15906                  * e.g. coming up after runtime pm.
15907                  *
15908                  * No protection against concurrent access is required - at
15909                  * worst a fifo underrun happens which also sets this to false.
15910                  */
15911                 crtc->cpu_fifo_underrun_disabled = true;
15912                 crtc->pch_fifo_underrun_disabled = true;
15913         }
15914 }
15915
15916 static void intel_sanitize_encoder(struct intel_encoder *encoder)
15917 {
15918         struct intel_connector *connector;
15919         struct drm_device *dev = encoder->base.dev;
15920
15921         /* We need to check both for a crtc link (meaning that the
15922          * encoder is active and trying to read from a pipe) and the
15923          * pipe itself being active. */
15924         bool has_active_crtc = encoder->base.crtc &&
15925                 to_intel_crtc(encoder->base.crtc)->active;
15926
15927         if (intel_encoder_has_connectors(encoder) && !has_active_crtc) {
15928                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
15929                               encoder->base.base.id,
15930                               encoder->base.name);
15931
15932                 /* Connector is active, but has no active pipe. This is
15933                  * fallout from our resume register restoring. Disable
15934                  * the encoder manually again. */
15935                 if (encoder->base.crtc) {
15936                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
15937                                       encoder->base.base.id,
15938                                       encoder->base.name);
15939                         encoder->disable(encoder);
15940                         if (encoder->post_disable)
15941                                 encoder->post_disable(encoder);
15942                 }
15943                 encoder->base.crtc = NULL;
15944
15945                 /* Inconsistent output/port/pipe state happens presumably due to
15946                  * a bug in one of the get_hw_state functions. Or someplace else
15947                  * in our code, like the register restore mess on resume. Clamp
15948                  * things to off as a safer default. */
15949                 for_each_intel_connector(dev, connector) {
15950                         if (connector->encoder != encoder)
15951                                 continue;
15952                         connector->base.dpms = DRM_MODE_DPMS_OFF;
15953                         connector->base.encoder = NULL;
15954                 }
15955         }
15956         /* Enabled encoders without active connectors will be fixed in
15957          * the crtc fixup. */
15958 }
15959
15960 void i915_redisable_vga_power_on(struct drm_device *dev)
15961 {
15962         struct drm_i915_private *dev_priv = dev->dev_private;
15963         i915_reg_t vga_reg = i915_vgacntrl_reg(dev);
15964
15965         if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
15966                 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
15967                 i915_disable_vga(dev);
15968         }
15969 }
15970
15971 void i915_redisable_vga(struct drm_device *dev)
15972 {
15973         struct drm_i915_private *dev_priv = dev->dev_private;
15974
15975         /* This function can be called both from intel_modeset_setup_hw_state or
15976          * at a very early point in our resume sequence, where the power well
15977          * structures are not yet restored. Since this function is at a very
15978          * paranoid "someone might have enabled VGA while we were not looking"
15979          * level, just check if the power well is enabled instead of trying to
15980          * follow the "don't touch the power well if we don't need it" policy
15981          * the rest of the driver uses. */
15982         if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_VGA))
15983                 return;
15984
15985         i915_redisable_vga_power_on(dev);
15986
15987         intel_display_power_put(dev_priv, POWER_DOMAIN_VGA);
15988 }
15989
15990 static bool primary_get_hw_state(struct intel_plane *plane)
15991 {
15992         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
15993
15994         return I915_READ(DSPCNTR(plane->plane)) & DISPLAY_PLANE_ENABLE;
15995 }
15996
15997 /* FIXME read out full plane state for all planes */
15998 static void readout_plane_state(struct intel_crtc *crtc)
15999 {
16000         struct drm_plane *primary = crtc->base.primary;
16001         struct intel_plane_state *plane_state =
16002                 to_intel_plane_state(primary->state);
16003
16004         plane_state->visible = crtc->active &&
16005                 primary_get_hw_state(to_intel_plane(primary));
16006
16007         if (plane_state->visible)
16008                 crtc->base.state->plane_mask |= 1 << drm_plane_index(primary);
16009 }
16010
16011 static void intel_modeset_readout_hw_state(struct drm_device *dev)
16012 {
16013         struct drm_i915_private *dev_priv = dev->dev_private;
16014         enum pipe pipe;
16015         struct intel_crtc *crtc;
16016         struct intel_encoder *encoder;
16017         struct intel_connector *connector;
16018         int i;
16019
16020         dev_priv->active_crtcs = 0;
16021
16022         for_each_intel_crtc(dev, crtc) {
16023                 struct intel_crtc_state *crtc_state = crtc->config;
16024                 int pixclk = 0;
16025
16026                 __drm_atomic_helper_crtc_destroy_state(&crtc_state->base);
16027                 memset(crtc_state, 0, sizeof(*crtc_state));
16028                 crtc_state->base.crtc = &crtc->base;
16029
16030                 crtc_state->base.active = crtc_state->base.enable =
16031                         dev_priv->display.get_pipe_config(crtc, crtc_state);
16032
16033                 crtc->base.enabled = crtc_state->base.enable;
16034                 crtc->active = crtc_state->base.active;
16035
16036                 if (crtc_state->base.active) {
16037                         dev_priv->active_crtcs |= 1 << crtc->pipe;
16038
16039                         if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
16040                                 pixclk = ilk_pipe_pixel_rate(crtc_state);
16041                         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
16042                                 pixclk = crtc_state->base.adjusted_mode.crtc_clock;
16043                         else
16044                                 WARN_ON(dev_priv->display.modeset_calc_cdclk);
16045
16046                         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
16047                         if (IS_BROADWELL(dev_priv) && crtc_state->ips_enabled)
16048                                 pixclk = DIV_ROUND_UP(pixclk * 100, 95);
16049                 }
16050
16051                 dev_priv->min_pixclk[crtc->pipe] = pixclk;
16052
16053                 readout_plane_state(crtc);
16054
16055                 DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
16056                               crtc->base.base.id, crtc->base.name,
16057                               crtc->active ? "enabled" : "disabled");
16058         }
16059
16060         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
16061                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
16062
16063                 pll->on = pll->funcs.get_hw_state(dev_priv, pll,
16064                                                   &pll->config.hw_state);
16065                 pll->config.crtc_mask = 0;
16066                 for_each_intel_crtc(dev, crtc) {
16067                         if (crtc->active && crtc->config->shared_dpll == pll)
16068                                 pll->config.crtc_mask |= 1 << crtc->pipe;
16069                 }
16070                 pll->active_mask = pll->config.crtc_mask;
16071
16072                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
16073                               pll->name, pll->config.crtc_mask, pll->on);
16074         }
16075
16076         for_each_intel_encoder(dev, encoder) {
16077                 pipe = 0;
16078
16079                 if (encoder->get_hw_state(encoder, &pipe)) {
16080                         crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
16081                         encoder->base.crtc = &crtc->base;
16082                         encoder->get_config(encoder, crtc->config);
16083                 } else {
16084                         encoder->base.crtc = NULL;
16085                 }
16086
16087                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
16088                               encoder->base.base.id,
16089                               encoder->base.name,
16090                               encoder->base.crtc ? "enabled" : "disabled",
16091                               pipe_name(pipe));
16092         }
16093
16094         for_each_intel_connector(dev, connector) {
16095                 if (connector->get_hw_state(connector)) {
16096                         connector->base.dpms = DRM_MODE_DPMS_ON;
16097
16098                         encoder = connector->encoder;
16099                         connector->base.encoder = &encoder->base;
16100
16101                         if (encoder->base.crtc &&
16102                             encoder->base.crtc->state->active) {
16103                                 /*
16104                                  * This has to be done during hardware readout
16105                                  * because anything calling .crtc_disable may
16106                                  * rely on the connector_mask being accurate.
16107                                  */
16108                                 encoder->base.crtc->state->connector_mask |=
16109                                         1 << drm_connector_index(&connector->base);
16110                                 encoder->base.crtc->state->encoder_mask |=
16111                                         1 << drm_encoder_index(&encoder->base);
16112                         }
16113
16114                 } else {
16115                         connector->base.dpms = DRM_MODE_DPMS_OFF;
16116                         connector->base.encoder = NULL;
16117                 }
16118                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
16119                               connector->base.base.id,
16120                               connector->base.name,
16121                               connector->base.encoder ? "enabled" : "disabled");
16122         }
16123
16124         for_each_intel_crtc(dev, crtc) {
16125                 crtc->base.hwmode = crtc->config->base.adjusted_mode;
16126
16127                 memset(&crtc->base.mode, 0, sizeof(crtc->base.mode));
16128                 if (crtc->base.state->active) {
16129                         intel_mode_from_pipe_config(&crtc->base.mode, crtc->config);
16130                         intel_mode_from_pipe_config(&crtc->base.state->adjusted_mode, crtc->config);
16131                         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->base.state, &crtc->base.mode));
16132
16133                         /*
16134                          * The initial mode needs to be set in order to keep
16135                          * the atomic core happy. It wants a valid mode if the
16136                          * crtc's enabled, so we do the above call.
16137                          *
16138                          * At this point some state updated by the connectors
16139                          * in their ->detect() callback has not run yet, so
16140                          * no recalculation can be done yet.
16141                          *
16142                          * Even if we could do a recalculation and modeset
16143                          * right now it would cause a double modeset if
16144                          * fbdev or userspace chooses a different initial mode.
16145                          *
16146                          * If that happens, someone indicated they wanted a
16147                          * mode change, which means it's safe to do a full
16148                          * recalculation.
16149                          */
16150                         crtc->base.state->mode.private_flags = I915_MODE_FLAG_INHERITED;
16151
16152                         drm_calc_timestamping_constants(&crtc->base, &crtc->base.hwmode);
16153                         update_scanline_offset(crtc);
16154                 }
16155
16156                 intel_pipe_config_sanity_check(dev_priv, crtc->config);
16157         }
16158 }
16159
16160 /* Scan out the current hw modeset state,
16161  * and sanitizes it to the current state
16162  */
16163 static void
16164 intel_modeset_setup_hw_state(struct drm_device *dev)
16165 {
16166         struct drm_i915_private *dev_priv = dev->dev_private;
16167         enum pipe pipe;
16168         struct intel_crtc *crtc;
16169         struct intel_encoder *encoder;
16170         int i;
16171
16172         intel_modeset_readout_hw_state(dev);
16173
16174         /* HW state is read out, now we need to sanitize this mess. */
16175         for_each_intel_encoder(dev, encoder) {
16176                 intel_sanitize_encoder(encoder);
16177         }
16178
16179         for_each_pipe(dev_priv, pipe) {
16180                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
16181                 intel_sanitize_crtc(crtc);
16182                 intel_dump_pipe_config(crtc, crtc->config,
16183                                        "[setup_hw_state]");
16184         }
16185
16186         intel_modeset_update_connector_atomic_state(dev);
16187
16188         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
16189                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
16190
16191                 if (!pll->on || pll->active_mask)
16192                         continue;
16193
16194                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n", pll->name);
16195
16196                 pll->funcs.disable(dev_priv, pll);
16197                 pll->on = false;
16198         }
16199
16200         if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
16201                 vlv_wm_get_hw_state(dev);
16202         else if (IS_GEN9(dev))
16203                 skl_wm_get_hw_state(dev);
16204         else if (HAS_PCH_SPLIT(dev))
16205                 ilk_wm_get_hw_state(dev);
16206
16207         for_each_intel_crtc(dev, crtc) {
16208                 unsigned long put_domains;
16209
16210                 put_domains = modeset_get_crtc_power_domains(&crtc->base, crtc->config);
16211                 if (WARN_ON(put_domains))
16212                         modeset_put_power_domains(dev_priv, put_domains);
16213         }
16214         intel_display_set_init_power(dev_priv, false);
16215
16216         intel_fbc_init_pipe_state(dev_priv);
16217 }
16218
16219 void intel_display_resume(struct drm_device *dev)
16220 {
16221         struct drm_i915_private *dev_priv = to_i915(dev);
16222         struct drm_atomic_state *state = dev_priv->modeset_restore_state;
16223         struct drm_modeset_acquire_ctx ctx;
16224         int ret;
16225         bool setup = false;
16226
16227         dev_priv->modeset_restore_state = NULL;
16228
16229         /*
16230          * This is a cludge because with real atomic modeset mode_config.mutex
16231          * won't be taken. Unfortunately some probed state like
16232          * audio_codec_enable is still protected by mode_config.mutex, so lock
16233          * it here for now.
16234          */
16235         mutex_lock(&dev->mode_config.mutex);
16236         drm_modeset_acquire_init(&ctx, 0);
16237
16238 retry:
16239         ret = drm_modeset_lock_all_ctx(dev, &ctx);
16240
16241         if (ret == 0 && !setup) {
16242                 setup = true;
16243
16244                 intel_modeset_setup_hw_state(dev);
16245                 i915_redisable_vga(dev);
16246         }
16247
16248         if (ret == 0 && state) {
16249                 struct drm_crtc_state *crtc_state;
16250                 struct drm_crtc *crtc;
16251                 int i;
16252
16253                 state->acquire_ctx = &ctx;
16254
16255                 /* ignore any reset values/BIOS leftovers in the WM registers */
16256                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
16257
16258                 for_each_crtc_in_state(state, crtc, crtc_state, i) {
16259                         /*
16260                          * Force recalculation even if we restore
16261                          * current state. With fast modeset this may not result
16262                          * in a modeset when the state is compatible.
16263                          */
16264                         crtc_state->mode_changed = true;
16265                 }
16266
16267                 ret = drm_atomic_commit(state);
16268         }
16269
16270         if (ret == -EDEADLK) {
16271                 drm_modeset_backoff(&ctx);
16272                 goto retry;
16273         }
16274
16275         drm_modeset_drop_locks(&ctx);
16276         drm_modeset_acquire_fini(&ctx);
16277         mutex_unlock(&dev->mode_config.mutex);
16278
16279         if (ret) {
16280                 DRM_ERROR("Restoring old state failed with %i\n", ret);
16281                 drm_atomic_state_free(state);
16282         }
16283 }
16284
16285 void intel_modeset_gem_init(struct drm_device *dev)
16286 {
16287         struct drm_i915_private *dev_priv = to_i915(dev);
16288         struct drm_crtc *c;
16289         struct drm_i915_gem_object *obj;
16290         int ret;
16291
16292         intel_init_gt_powersave(dev_priv);
16293
16294         intel_modeset_init_hw(dev);
16295
16296         intel_setup_overlay(dev_priv);
16297
16298         /*
16299          * Make sure any fbs we allocated at startup are properly
16300          * pinned & fenced.  When we do the allocation it's too early
16301          * for this.
16302          */
16303         for_each_crtc(dev, c) {
16304                 obj = intel_fb_obj(c->primary->fb);
16305                 if (obj == NULL)
16306                         continue;
16307
16308                 mutex_lock(&dev->struct_mutex);
16309                 ret = intel_pin_and_fence_fb_obj(c->primary->fb,
16310                                                  c->primary->state->rotation);
16311                 mutex_unlock(&dev->struct_mutex);
16312                 if (ret) {
16313                         DRM_ERROR("failed to pin boot fb on pipe %d\n",
16314                                   to_intel_crtc(c)->pipe);
16315                         drm_framebuffer_unreference(c->primary->fb);
16316                         c->primary->fb = NULL;
16317                         c->primary->crtc = c->primary->state->crtc = NULL;
16318                         update_state_fb(c->primary);
16319                         c->state->plane_mask &= ~(1 << drm_plane_index(c->primary));
16320                 }
16321         }
16322 }
16323
16324 int intel_connector_register(struct drm_connector *connector)
16325 {
16326         struct intel_connector *intel_connector = to_intel_connector(connector);
16327         int ret;
16328
16329         ret = intel_backlight_device_register(intel_connector);
16330         if (ret)
16331                 goto err;
16332
16333         return 0;
16334
16335 err:
16336         return ret;
16337 }
16338
16339 void intel_connector_unregister(struct drm_connector *connector)
16340 {
16341         struct intel_connector *intel_connector = to_intel_connector(connector);
16342
16343         intel_backlight_device_unregister(intel_connector);
16344         intel_panel_destroy_backlight(connector);
16345 }
16346
16347 void intel_modeset_cleanup(struct drm_device *dev)
16348 {
16349         struct drm_i915_private *dev_priv = dev->dev_private;
16350
16351         intel_disable_gt_powersave(dev_priv);
16352
16353         /*
16354          * Interrupts and polling as the first thing to avoid creating havoc.
16355          * Too much stuff here (turning of connectors, ...) would
16356          * experience fancy races otherwise.
16357          */
16358         intel_irq_uninstall(dev_priv);
16359
16360         /*
16361          * Due to the hpd irq storm handling the hotplug work can re-arm the
16362          * poll handlers. Hence disable polling after hpd handling is shut down.
16363          */
16364         drm_kms_helper_poll_fini(dev);
16365
16366         intel_unregister_dsm_handler();
16367
16368         intel_fbc_global_disable(dev_priv);
16369
16370         /* flush any delayed tasks or pending work */
16371         flush_scheduled_work();
16372
16373         drm_mode_config_cleanup(dev);
16374
16375         intel_cleanup_overlay(dev_priv);
16376
16377         intel_cleanup_gt_powersave(dev_priv);
16378
16379         intel_teardown_gmbus(dev);
16380 }
16381
16382 void intel_connector_attach_encoder(struct intel_connector *connector,
16383                                     struct intel_encoder *encoder)
16384 {
16385         connector->encoder = encoder;
16386         drm_mode_connector_attach_encoder(&connector->base,
16387                                           &encoder->base);
16388 }
16389
16390 /*
16391  * set vga decode state - true == enable VGA decode
16392  */
16393 int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
16394 {
16395         struct drm_i915_private *dev_priv = dev->dev_private;
16396         unsigned reg = INTEL_INFO(dev)->gen >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
16397         u16 gmch_ctrl;
16398
16399         if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
16400                 DRM_ERROR("failed to read control word\n");
16401                 return -EIO;
16402         }
16403
16404         if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state)
16405                 return 0;
16406
16407         if (state)
16408                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
16409         else
16410                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
16411
16412         if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) {
16413                 DRM_ERROR("failed to write control word\n");
16414                 return -EIO;
16415         }
16416
16417         return 0;
16418 }
16419
16420 struct intel_display_error_state {
16421
16422         u32 power_well_driver;
16423
16424         int num_transcoders;
16425
16426         struct intel_cursor_error_state {
16427                 u32 control;
16428                 u32 position;
16429                 u32 base;
16430                 u32 size;
16431         } cursor[I915_MAX_PIPES];
16432
16433         struct intel_pipe_error_state {
16434                 bool power_domain_on;
16435                 u32 source;
16436                 u32 stat;
16437         } pipe[I915_MAX_PIPES];
16438
16439         struct intel_plane_error_state {
16440                 u32 control;
16441                 u32 stride;
16442                 u32 size;
16443                 u32 pos;
16444                 u32 addr;
16445                 u32 surface;
16446                 u32 tile_offset;
16447         } plane[I915_MAX_PIPES];
16448
16449         struct intel_transcoder_error_state {
16450                 bool power_domain_on;
16451                 enum transcoder cpu_transcoder;
16452
16453                 u32 conf;
16454
16455                 u32 htotal;
16456                 u32 hblank;
16457                 u32 hsync;
16458                 u32 vtotal;
16459                 u32 vblank;
16460                 u32 vsync;
16461         } transcoder[4];
16462 };
16463
16464 struct intel_display_error_state *
16465 intel_display_capture_error_state(struct drm_i915_private *dev_priv)
16466 {
16467         struct intel_display_error_state *error;
16468         int transcoders[] = {
16469                 TRANSCODER_A,
16470                 TRANSCODER_B,
16471                 TRANSCODER_C,
16472                 TRANSCODER_EDP,
16473         };
16474         int i;
16475
16476         if (INTEL_INFO(dev_priv)->num_pipes == 0)
16477                 return NULL;
16478
16479         error = kzalloc(sizeof(*error), GFP_ATOMIC);
16480         if (error == NULL)
16481                 return NULL;
16482
16483         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
16484                 error->power_well_driver = I915_READ(HSW_PWR_WELL_DRIVER);
16485
16486         for_each_pipe(dev_priv, i) {
16487                 error->pipe[i].power_domain_on =
16488                         __intel_display_power_is_enabled(dev_priv,
16489                                                          POWER_DOMAIN_PIPE(i));
16490                 if (!error->pipe[i].power_domain_on)
16491                         continue;
16492
16493                 error->cursor[i].control = I915_READ(CURCNTR(i));
16494                 error->cursor[i].position = I915_READ(CURPOS(i));
16495                 error->cursor[i].base = I915_READ(CURBASE(i));
16496
16497                 error->plane[i].control = I915_READ(DSPCNTR(i));
16498                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
16499                 if (INTEL_GEN(dev_priv) <= 3) {
16500                         error->plane[i].size = I915_READ(DSPSIZE(i));
16501                         error->plane[i].pos = I915_READ(DSPPOS(i));
16502                 }
16503                 if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
16504                         error->plane[i].addr = I915_READ(DSPADDR(i));
16505                 if (INTEL_GEN(dev_priv) >= 4) {
16506                         error->plane[i].surface = I915_READ(DSPSURF(i));
16507                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
16508                 }
16509
16510                 error->pipe[i].source = I915_READ(PIPESRC(i));
16511
16512                 if (HAS_GMCH_DISPLAY(dev_priv))
16513                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
16514         }
16515
16516         /* Note: this does not include DSI transcoders. */
16517         error->num_transcoders = INTEL_INFO(dev_priv)->num_pipes;
16518         if (HAS_DDI(dev_priv))
16519                 error->num_transcoders++; /* Account for eDP. */
16520
16521         for (i = 0; i < error->num_transcoders; i++) {
16522                 enum transcoder cpu_transcoder = transcoders[i];
16523
16524                 error->transcoder[i].power_domain_on =
16525                         __intel_display_power_is_enabled(dev_priv,
16526                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
16527                 if (!error->transcoder[i].power_domain_on)
16528                         continue;
16529
16530                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
16531
16532                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
16533                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
16534                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
16535                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
16536                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
16537                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
16538                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
16539         }
16540
16541         return error;
16542 }
16543
16544 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
16545
16546 void
16547 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
16548                                 struct drm_device *dev,
16549                                 struct intel_display_error_state *error)
16550 {
16551         struct drm_i915_private *dev_priv = dev->dev_private;
16552         int i;
16553
16554         if (!error)
16555                 return;
16556
16557         err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev)->num_pipes);
16558         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
16559                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
16560                            error->power_well_driver);
16561         for_each_pipe(dev_priv, i) {
16562                 err_printf(m, "Pipe [%d]:\n", i);
16563                 err_printf(m, "  Power: %s\n",
16564                            onoff(error->pipe[i].power_domain_on));
16565                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
16566                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
16567
16568                 err_printf(m, "Plane [%d]:\n", i);
16569                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
16570                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
16571                 if (INTEL_INFO(dev)->gen <= 3) {
16572                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
16573                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
16574                 }
16575                 if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
16576                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
16577                 if (INTEL_INFO(dev)->gen >= 4) {
16578                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
16579                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
16580                 }
16581
16582                 err_printf(m, "Cursor [%d]:\n", i);
16583                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
16584                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
16585                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
16586         }
16587
16588         for (i = 0; i < error->num_transcoders; i++) {
16589                 err_printf(m, "CPU transcoder: %s\n",
16590                            transcoder_name(error->transcoder[i].cpu_transcoder));
16591                 err_printf(m, "  Power: %s\n",
16592                            onoff(error->transcoder[i].power_domain_on));
16593                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
16594                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
16595                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
16596                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
16597                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
16598                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
16599                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
16600         }
16601 }