drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include <linux/log2.h>
  31 #include <drm/drmP.h>
  32 #include "i915_drv.h"
  33 #include <drm/i915_drm.h>
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36
  37 /* Rough estimate of the typical request size, performing a flush,
  38  * set-context and then emitting the batch.
  39  */
  40 #define LEGACY_REQUEST_SIZE 200
  41
  42 int __intel_ring_space(int head, int tail, int size)
  43 {
  44         int space = head - tail;
  45         if (space <= 0)
  46                 space += size;
  47         return space - I915_RING_FREE_SPACE;
  48 }
  49
  50 void intel_ring_update_space(struct intel_ringbuffer *ringbuf)
  51 {
  52         if (ringbuf->last_retired_head != -1) {
  53                 ringbuf->head = ringbuf->last_retired_head;
  54                 ringbuf->last_retired_head = -1;
  55         }
  56
  57         ringbuf->space = __intel_ring_space(ringbuf->head & HEAD_ADDR,
  58                                             ringbuf->tail, ringbuf->size);
  59 }
  60
  61 bool intel_engine_stopped(struct intel_engine_cs *engine)
  62 {
  63         struct drm_i915_private *dev_priv = engine->i915;
  64         return dev_priv->gpu_error.stop_rings & intel_engine_flag(engine);
  65 }
  66
  67 static void __intel_ring_advance(struct intel_engine_cs *engine)
  68 {
  69         struct intel_ringbuffer *ringbuf = engine->buffer;
  70         ringbuf->tail &= ringbuf->size - 1;
  71         if (intel_engine_stopped(engine))
  72                 return;
  73         engine->write_tail(engine, ringbuf->tail);
  74 }
  75
  76 static int
  77 gen2_render_ring_flush(struct drm_i915_gem_request *req,
  78                        u32      invalidate_domains,
  79                        u32      flush_domains)
  80 {
  81         struct intel_engine_cs *engine = req->engine;
  82         u32 cmd;
  83         int ret;
  84
  85         cmd = MI_FLUSH;
  86         if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
  87                 cmd |= MI_NO_WRITE_FLUSH;
  88
  89         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  90                 cmd |= MI_READ_FLUSH;
  91
  92         ret = intel_ring_begin(req, 2);
  93         if (ret)
  94                 return ret;
  95
  96         intel_ring_emit(engine, cmd);
  97         intel_ring_emit(engine, MI_NOOP);
  98         intel_ring_advance(engine);
  99
 100         return 0;
 101 }
 102
 103 static int
 104 gen4_render_ring_flush(struct drm_i915_gem_request *req,
 105                        u32      invalidate_domains,
 106                        u32      flush_domains)
 107 {
 108         struct intel_engine_cs *engine = req->engine;
 109         u32 cmd;
 110         int ret;
 111
 112         /*
 113          * read/write caches:
 114          *
 115          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
 116          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
 117          * also flushed at 2d versus 3d pipeline switches.
 118          *
 119          * read-only caches:
 120          *
 121          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
 122          * MI_READ_FLUSH is set, and is always flushed on 965.
 123          *
 124          * I915_GEM_DOMAIN_COMMAND may not exist?
 125          *
 126          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
 127          * invalidated when MI_EXE_FLUSH is set.
 128          *
 129          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
 130          * invalidated with every MI_FLUSH.
 131          *
 132          * TLBs:
 133          *
 134          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
 135          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
 136          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
 137          * are flushed at any MI_FLUSH.
 138          */
 139
 140         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
 141         if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
 142                 cmd &= ~MI_NO_WRITE_FLUSH;
 143         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 144                 cmd |= MI_EXE_FLUSH;
 145
 146         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
 147             (IS_G4X(req->i915) || IS_GEN5(req->i915)))
 148                 cmd |= MI_INVALIDATE_ISP;
 149
 150         ret = intel_ring_begin(req, 2);
 151         if (ret)
 152                 return ret;
 153
 154         intel_ring_emit(engine, cmd);
 155         intel_ring_emit(engine, MI_NOOP);
 156         intel_ring_advance(engine);
 157
 158         return 0;
 159 }
 160
 161 /**
 162  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 163  * implementing two workarounds on gen6.  From section 1.4.7.1
 164  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 165  *
 166  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 167  * produced by non-pipelined state commands), software needs to first
 168  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 169  * 0.
 170  *
 171  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 172  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 173  *
 174  * And the workaround for these two requires this workaround first:
 175  *
 176  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 177  * BEFORE the pipe-control with a post-sync op and no write-cache
 178  * flushes.
 179  *
 180  * And this last workaround is tricky because of the requirements on
 181  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 182  * volume 2 part 1:
 183  *
 184  *     "1 of the following must also be set:
 185  *      - Render Target Cache Flush Enable ([12] of DW1)
 186  *      - Depth Cache Flush Enable ([0] of DW1)
 187  *      - Stall at Pixel Scoreboard ([1] of DW1)
 188  *      - Depth Stall ([13] of DW1)
 189  *      - Post-Sync Operation ([13] of DW1)
 190  *      - Notify Enable ([8] of DW1)"
 191  *
 192  * The cache flushes require the workaround flush that triggered this
 193  * one, so we can't use it.  Depth stall would trigger the same.
 194  * Post-sync nonzero is what triggered this second workaround, so we
 195  * can't use that one either.  Notify enable is IRQs, which aren't
 196  * really our business.  That leaves only stall at scoreboard.
 197  */
 198 static int
 199 intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req)
 200 {
 201         struct intel_engine_cs *engine = req->engine;
 202         u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 203         int ret;
 204
 205         ret = intel_ring_begin(req, 6);
 206         if (ret)
 207                 return ret;
 208
 209         intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(5));
 210         intel_ring_emit(engine, PIPE_CONTROL_CS_STALL |
 211                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
 212         intel_ring_emit(engine, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 213         intel_ring_emit(engine, 0); /* low dword */
 214         intel_ring_emit(engine, 0); /* high dword */
 215         intel_ring_emit(engine, MI_NOOP);
 216         intel_ring_advance(engine);
 217
 218         ret = intel_ring_begin(req, 6);
 219         if (ret)
 220                 return ret;
 221
 222         intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(5));
 223         intel_ring_emit(engine, PIPE_CONTROL_QW_WRITE);
 224         intel_ring_emit(engine, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 225         intel_ring_emit(engine, 0);
 226         intel_ring_emit(engine, 0);
 227         intel_ring_emit(engine, MI_NOOP);
 228         intel_ring_advance(engine);
 229
 230         return 0;
 231 }
 232
 233 static int
 234 gen6_render_ring_flush(struct drm_i915_gem_request *req,
 235                        u32 invalidate_domains, u32 flush_domains)
 236 {
 237         struct intel_engine_cs *engine = req->engine;
 238         u32 flags = 0;
 239         u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 240         int ret;
 241
 242         /* Force SNB workarounds for PIPE_CONTROL flushes */
 243         ret = intel_emit_post_sync_nonzero_flush(req);
 244         if (ret)
 245                 return ret;
 246
 247         /* Just flush everything.  Experiments have shown that reducing the
 248          * number of bits based on the write domains has little performance
 249          * impact.
 250          */
 251         if (flush_domains) {
 252                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 253                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 254                 /*
 255                  * Ensure that any following seqno writes only happen
 256                  * when the render cache is indeed flushed.
 257                  */
 258                 flags |= PIPE_CONTROL_CS_STALL;
 259         }
 260         if (invalidate_domains) {
 261                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 262                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 263                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 264                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 265                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 266                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 267                 /*
 268                  * TLB invalidate requires a post-sync write.
 269                  */
 270                 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
 271         }
 272
 273         ret = intel_ring_begin(req, 4);
 274         if (ret)
 275                 return ret;
 276
 277         intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(4));
 278         intel_ring_emit(engine, flags);
 279         intel_ring_emit(engine, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
 280         intel_ring_emit(engine, 0);
 281         intel_ring_advance(engine);
 282
 283         return 0;
 284 }
 285
 286 static int
 287 gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req)
 288 {
 289         struct intel_engine_cs *engine = req->engine;
 290         int ret;
 291
 292         ret = intel_ring_begin(req, 4);
 293         if (ret)
 294                 return ret;
 295
 296         intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(4));
 297         intel_ring_emit(engine, PIPE_CONTROL_CS_STALL |
 298                               PIPE_CONTROL_STALL_AT_SCOREBOARD);
 299         intel_ring_emit(engine, 0);
 300         intel_ring_emit(engine, 0);
 301         intel_ring_advance(engine);
 302
 303         return 0;
 304 }
 305
 306 static int
 307 gen7_render_ring_flush(struct drm_i915_gem_request *req,
 308                        u32 invalidate_domains, u32 flush_domains)
 309 {
 310         struct intel_engine_cs *engine = req->engine;
 311         u32 flags = 0;
 312         u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 313         int ret;
 314
 315         /*
 316          * Ensure that any following seqno writes only happen when the render
 317          * cache is indeed flushed.
 318          *
 319          * Workaround: 4th PIPE_CONTROL command (except the ones with only
 320          * read-cache invalidate bits set) must have the CS_STALL bit set. We
 321          * don't try to be clever and just set it unconditionally.
 322          */
 323         flags |= PIPE_CONTROL_CS_STALL;
 324
 325         /* Just flush everything.  Experiments have shown that reducing the
 326          * number of bits based on the write domains has little performance
 327          * impact.
 328          */
 329         if (flush_domains) {
 330                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 331                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 332                 flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
 333                 flags |= PIPE_CONTROL_FLUSH_ENABLE;
 334         }
 335         if (invalidate_domains) {
 336                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 337                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 338                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 339                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 340                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 341                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 342                 flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR;
 343                 /*
 344                  * TLB invalidate requires a post-sync write.
 345                  */
 346                 flags |= PIPE_CONTROL_QW_WRITE;
 347                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
 348
 349                 flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD;
 350
 351                 /* Workaround: we must issue a pipe_control with CS-stall bit
 352                  * set before a pipe_control command that has the state cache
 353                  * invalidate bit set. */
 354                 gen7_render_ring_cs_stall_wa(req);
 355         }
 356
 357         ret = intel_ring_begin(req, 4);
 358         if (ret)
 359                 return ret;
 360
 361         intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(4));
 362         intel_ring_emit(engine, flags);
 363         intel_ring_emit(engine, scratch_addr);
 364         intel_ring_emit(engine, 0);
 365         intel_ring_advance(engine);
 366
 367         return 0;
 368 }
 369
 370 static int
 371 gen8_emit_pipe_control(struct drm_i915_gem_request *req,
 372                        u32 flags, u32 scratch_addr)
 373 {
 374         struct intel_engine_cs *engine = req->engine;
 375         int ret;
 376
 377         ret = intel_ring_begin(req, 6);
 378         if (ret)
 379                 return ret;
 380
 381         intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(6));
 382         intel_ring_emit(engine, flags);
 383         intel_ring_emit(engine, scratch_addr);
 384         intel_ring_emit(engine, 0);
 385         intel_ring_emit(engine, 0);
 386         intel_ring_emit(engine, 0);
 387         intel_ring_advance(engine);
 388
 389         return 0;
 390 }
 391
 392 static int
 393 gen8_render_ring_flush(struct drm_i915_gem_request *req,
 394                        u32 invalidate_domains, u32 flush_domains)
 395 {
 396         u32 flags = 0;
 397         u32 scratch_addr = req->engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 398         int ret;
 399
 400         flags |= PIPE_CONTROL_CS_STALL;
 401
 402         if (flush_domains) {
 403                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 404                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 405                 flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
 406                 flags |= PIPE_CONTROL_FLUSH_ENABLE;
 407         }
 408         if (invalidate_domains) {
 409                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 410                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 411                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 412                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 413                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 414                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 415                 flags |= PIPE_CONTROL_QW_WRITE;
 416                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
 417
 418                 /* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
 419                 ret = gen8_emit_pipe_control(req,
 420                                              PIPE_CONTROL_CS_STALL |
 421                                              PIPE_CONTROL_STALL_AT_SCOREBOARD,
 422                                              0);
 423                 if (ret)
 424                         return ret;
 425         }
 426
 427         return gen8_emit_pipe_control(req, flags, scratch_addr);
 428 }
 429
 430 static void ring_write_tail(struct intel_engine_cs *engine,
 431                             u32 value)
 432 {
 433         struct drm_i915_private *dev_priv = engine->i915;
 434         I915_WRITE_TAIL(engine, value);
 435 }
 436
 437 u64 intel_ring_get_active_head(struct intel_engine_cs *engine)
 438 {
 439         struct drm_i915_private *dev_priv = engine->i915;
 440         u64 acthd;
 441
 442         if (INTEL_GEN(dev_priv) >= 8)
 443                 acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
 444                                          RING_ACTHD_UDW(engine->mmio_base));
 445         else if (INTEL_GEN(dev_priv) >= 4)
 446                 acthd = I915_READ(RING_ACTHD(engine->mmio_base));
 447         else
 448                 acthd = I915_READ(ACTHD);
 449
 450         return acthd;
 451 }
 452
 453 static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
 454 {
 455         struct drm_i915_private *dev_priv = engine->i915;
 456         u32 addr;
 457
 458         addr = dev_priv->status_page_dmah->busaddr;
 459         if (INTEL_GEN(dev_priv) >= 4)
 460                 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
 461         I915_WRITE(HWS_PGA, addr);
 462 }
 463
 464 static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
 465 {
 466         struct drm_i915_private *dev_priv = engine->i915;
 467         i915_reg_t mmio;
 468
 469         /* The ring status page addresses are no longer next to the rest of
 470          * the ring registers as of gen7.
 471          */
 472         if (IS_GEN7(dev_priv)) {
 473                 switch (engine->id) {
 474                 case RCS:
 475                         mmio = RENDER_HWS_PGA_GEN7;
 476                         break;
 477                 case BCS:
 478                         mmio = BLT_HWS_PGA_GEN7;
 479                         break;
 480                 /*
 481                  * VCS2 actually doesn't exist on Gen7. Only shut up
 482                  * gcc switch check warning
 483                  */
 484                 case VCS2:
 485                 case VCS:
 486                         mmio = BSD_HWS_PGA_GEN7;
 487                         break;
 488                 case VECS:
 489                         mmio = VEBOX_HWS_PGA_GEN7;
 490                         break;
 491                 }
 492         } else if (IS_GEN6(dev_priv)) {
 493                 mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
 494         } else {
 495                 /* XXX: gen8 returns to sanity */
 496                 mmio = RING_HWS_PGA(engine->mmio_base);
 497         }
 498
 499         I915_WRITE(mmio, (u32)engine->status_page.gfx_addr);
 500         POSTING_READ(mmio);
 501
 502         /*
 503          * Flush the TLB for this page
 504          *
 505          * FIXME: These two bits have disappeared on gen8, so a question
 506          * arises: do we still need this and if so how should we go about
 507          * invalidating the TLB?
 508          */
 509         if (IS_GEN(dev_priv, 6, 7)) {
 510                 i915_reg_t reg = RING_INSTPM(engine->mmio_base);
 511
 512                 /* ring should be idle before issuing a sync flush*/
 513                 WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
 514
 515                 I915_WRITE(reg,
 516                            _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
 517                                               INSTPM_SYNC_FLUSH));
 518                 if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
 519                              1000))
 520                         DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
 521                                   engine->name);
 522         }
 523 }
 524
 525 static bool stop_ring(struct intel_engine_cs *engine)
 526 {
 527         struct drm_i915_private *dev_priv = engine->i915;
 528
 529         if (!IS_GEN2(dev_priv)) {
 530                 I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
 531                 if (wait_for((I915_READ_MODE(engine) & MODE_IDLE) != 0, 1000)) {
 532                         DRM_ERROR("%s : timed out trying to stop ring\n",
 533                                   engine->name);
 534                         /* Sometimes we observe that the idle flag is not
 535                          * set even though the ring is empty. So double
 536                          * check before giving up.
 537                          */
 538                         if (I915_READ_HEAD(engine) != I915_READ_TAIL(engine))
 539                                 return false;
 540                 }
 541         }
 542
 543         I915_WRITE_CTL(engine, 0);
 544         I915_WRITE_HEAD(engine, 0);
 545         engine->write_tail(engine, 0);
 546
 547         if (!IS_GEN2(dev_priv)) {
 548                 (void)I915_READ_CTL(engine);
 549                 I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING));
 550         }
 551
 552         return (I915_READ_HEAD(engine) & HEAD_ADDR) == 0;
 553 }
 554
 555 void intel_engine_init_hangcheck(struct intel_engine_cs *engine)
 556 {
 557         memset(&engine->hangcheck, 0, sizeof(engine->hangcheck));
 558 }
 559
 560 static int init_ring_common(struct intel_engine_cs *engine)
 561 {
 562         struct drm_i915_private *dev_priv = engine->i915;
 563         struct intel_ringbuffer *ringbuf = engine->buffer;
 564         struct drm_i915_gem_object *obj = ringbuf->obj;
 565         int ret = 0;
 566
 567         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 568
 569         if (!stop_ring(engine)) {
 570                 /* G45 ring initialization often fails to reset head to zero */
 571                 DRM_DEBUG_KMS("%s head not reset to zero "
 572                               "ctl %08x head %08x tail %08x start %08x\n",
 573                               engine->name,
 574                               I915_READ_CTL(engine),
 575                               I915_READ_HEAD(engine),
 576                               I915_READ_TAIL(engine),
 577                               I915_READ_START(engine));
 578
 579                 if (!stop_ring(engine)) {
 580                         DRM_ERROR("failed to set %s head to zero "
 581                                   "ctl %08x head %08x tail %08x start %08x\n",
 582                                   engine->name,
 583                                   I915_READ_CTL(engine),
 584                                   I915_READ_HEAD(engine),
 585                                   I915_READ_TAIL(engine),
 586                                   I915_READ_START(engine));
 587                         ret = -EIO;
 588                         goto out;
 589                 }
 590         }
 591
 592         if (I915_NEED_GFX_HWS(dev_priv))
 593                 intel_ring_setup_status_page(engine);
 594         else
 595                 ring_setup_phys_status_page(engine);
 596
 597         /* Enforce ordering by reading HEAD register back */
 598         I915_READ_HEAD(engine);
 599
 600         /* Initialize the ring. This must happen _after_ we've cleared the ring
 601          * registers with the above sequence (the readback of the HEAD registers
 602          * also enforces ordering), otherwise the hw might lose the new ring
 603          * register values. */
 604         I915_WRITE_START(engine, i915_gem_obj_ggtt_offset(obj));
 605
 606         /* WaClearRingBufHeadRegAtInit:ctg,elk */
 607         if (I915_READ_HEAD(engine))
 608                 DRM_DEBUG("%s initialization failed [head=%08x], fudging\n",
 609                           engine->name, I915_READ_HEAD(engine));
 610         I915_WRITE_HEAD(engine, 0);
 611         (void)I915_READ_HEAD(engine);
 612
 613         I915_WRITE_CTL(engine,
 614                         ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES)
 615                         | RING_VALID);
 616
 617         /* If the head is still not zero, the ring is dead */
 618         if (wait_for((I915_READ_CTL(engine) & RING_VALID) != 0 &&
 619                      I915_READ_START(engine) == i915_gem_obj_ggtt_offset(obj) &&
 620                      (I915_READ_HEAD(engine) & HEAD_ADDR) == 0, 50)) {
 621                 DRM_ERROR("%s initialization failed "
 622                           "ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n",
 623                           engine->name,
 624                           I915_READ_CTL(engine),
 625                           I915_READ_CTL(engine) & RING_VALID,
 626                           I915_READ_HEAD(engine), I915_READ_TAIL(engine),
 627                           I915_READ_START(engine),
 628                           (unsigned long)i915_gem_obj_ggtt_offset(obj));
 629                 ret = -EIO;
 630                 goto out;
 631         }
 632
 633         ringbuf->last_retired_head = -1;
 634         ringbuf->head = I915_READ_HEAD(engine);
 635         ringbuf->tail = I915_READ_TAIL(engine) & TAIL_ADDR;
 636         intel_ring_update_space(ringbuf);
 637
 638         intel_engine_init_hangcheck(engine);
 639
 640 out:
 641         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 642
 643         return ret;
 644 }
 645
 646 void
 647 intel_fini_pipe_control(struct intel_engine_cs *engine)
 648 {
 649         if (engine->scratch.obj == NULL)
 650                 return;
 651
 652         if (INTEL_GEN(engine->i915) >= 5) {
 653                 kunmap(sg_page(engine->scratch.obj->pages->sgl));
 654                 i915_gem_object_ggtt_unpin(engine->scratch.obj);
 655         }
 656
 657         drm_gem_object_unreference(&engine->scratch.obj->base);
 658         engine->scratch.obj = NULL;
 659 }
 660
 661 int
 662 intel_init_pipe_control(struct intel_engine_cs *engine)
 663 {
 664         int ret;
 665
 666         WARN_ON(engine->scratch.obj);
 667
 668         engine->scratch.obj = i915_gem_object_create(engine->i915->dev, 4096);
 669         if (IS_ERR(engine->scratch.obj)) {
 670                 DRM_ERROR("Failed to allocate seqno page\n");
 671                 ret = PTR_ERR(engine->scratch.obj);
 672                 engine->scratch.obj = NULL;
 673                 goto err;
 674         }
 675
 676         ret = i915_gem_object_set_cache_level(engine->scratch.obj,
 677                                               I915_CACHE_LLC);
 678         if (ret)
 679                 goto err_unref;
 680
 681         ret = i915_gem_obj_ggtt_pin(engine->scratch.obj, 4096, 0);
 682         if (ret)
 683                 goto err_unref;
 684
 685         engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(engine->scratch.obj);
 686         engine->scratch.cpu_page = kmap(sg_page(engine->scratch.obj->pages->sgl));
 687         if (engine->scratch.cpu_page == NULL) {
 688                 ret = -ENOMEM;
 689                 goto err_unpin;
 690         }
 691
 692         DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
 693                          engine->name, engine->scratch.gtt_offset);
 694         return 0;
 695
 696 err_unpin:
 697         i915_gem_object_ggtt_unpin(engine->scratch.obj);
 698 err_unref:
 699         drm_gem_object_unreference(&engine->scratch.obj->base);
 700 err:
 701         return ret;
 702 }
 703
 704 static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
 705 {
 706         struct intel_engine_cs *engine = req->engine;
 707         struct i915_workarounds *w = &req->i915->workarounds;
 708         int ret, i;
 709
 710         if (w->count == 0)
 711                 return 0;
 712
 713         engine->gpu_caches_dirty = true;
 714         ret = intel_ring_flush_all_caches(req);
 715         if (ret)
 716                 return ret;
 717
 718         ret = intel_ring_begin(req, (w->count * 2 + 2));
 719         if (ret)
 720                 return ret;
 721
 722         intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(w->count));
 723         for (i = 0; i < w->count; i++) {
 724                 intel_ring_emit_reg(engine, w->reg[i].addr);
 725                 intel_ring_emit(engine, w->reg[i].value);
 726         }
 727         intel_ring_emit(engine, MI_NOOP);
 728
 729         intel_ring_advance(engine);
 730
 731         engine->gpu_caches_dirty = true;
 732         ret = intel_ring_flush_all_caches(req);
 733         if (ret)
 734                 return ret;
 735
 736         DRM_DEBUG_DRIVER("Number of Workarounds emitted: %d\n", w->count);
 737
 738         return 0;
 739 }
 740
 741 static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)
 742 {
 743         int ret;
 744
 745         ret = intel_ring_workarounds_emit(req);
 746         if (ret != 0)
 747                 return ret;
 748
 749         ret = i915_gem_render_state_init(req);
 750         if (ret)
 751                 return ret;
 752
 753         return 0;
 754 }
 755
 756 static int wa_add(struct drm_i915_private *dev_priv,
 757                   i915_reg_t addr,
 758                   const u32 mask, const u32 val)
 759 {
 760         const u32 idx = dev_priv->workarounds.count;
 761
 762         if (WARN_ON(idx >= I915_MAX_WA_REGS))
 763                 return -ENOSPC;
 764
 765         dev_priv->workarounds.reg[idx].addr = addr;
 766         dev_priv->workarounds.reg[idx].value = val;
 767         dev_priv->workarounds.reg[idx].mask = mask;
 768
 769         dev_priv->workarounds.count++;
 770
 771         return 0;
 772 }
 773
 774 #define WA_REG(addr, mask, val) do { \
 775                 const int r = wa_add(dev_priv, (addr), (mask), (val)); \
 776                 if (r) \
 777                         return r; \
 778         } while (0)
 779
 780 #define WA_SET_BIT_MASKED(addr, mask) \
 781         WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
 782
 783 #define WA_CLR_BIT_MASKED(addr, mask) \
 784         WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
 785
 786 #define WA_SET_FIELD_MASKED(addr, mask, value) \
 787         WA_REG(addr, mask, _MASKED_FIELD(mask, value))
 788
 789 #define WA_SET_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) | (mask))
 790 #define WA_CLR_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) & ~(mask))
 791
 792 #define WA_WRITE(addr, val) WA_REG(addr, 0xffffffff, val)
 793
 794 static int wa_ring_whitelist_reg(struct intel_engine_cs *engine,
 795                                  i915_reg_t reg)
 796 {
 797         struct drm_i915_private *dev_priv = engine->i915;
 798         struct i915_workarounds *wa = &dev_priv->workarounds;
 799         const uint32_t index = wa->hw_whitelist_count[engine->id];
 800
 801         if (WARN_ON(index >= RING_MAX_NONPRIV_SLOTS))
 802                 return -EINVAL;
 803
 804         WA_WRITE(RING_FORCE_TO_NONPRIV(engine->mmio_base, index),
 805                  i915_mmio_reg_offset(reg));
 806         wa->hw_whitelist_count[engine->id]++;
 807
 808         return 0;
 809 }
 810
 811 static int gen8_init_workarounds(struct intel_engine_cs *engine)
 812 {
 813         struct drm_i915_private *dev_priv = engine->i915;
 814
 815         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 816
 817         /* WaDisableAsyncFlipPerfMode:bdw,chv */
 818         WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
 819
 820         /* WaDisablePartialInstShootdown:bdw,chv */
 821         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 822                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 823
 824         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 825          * workaround for for a possible hang in the unlikely event a TLB
 826          * invalidation occurs during a PSD flush.
 827          */
 828         /* WaForceEnableNonCoherent:bdw,chv */
 829         /* WaHdcDisableFetchWhenMasked:bdw,chv */
 830         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 831                           HDC_DONOT_FETCH_MEM_WHEN_MASKED |
 832                           HDC_FORCE_NON_COHERENT);
 833
 834         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
 835          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
 836          *  polygons in the same 8x4 pixel/sample area to be processed without
 837          *  stalling waiting for the earlier ones to write to Hierarchical Z
 838          *  buffer."
 839          *
 840          * This optimization is off by default for BDW and CHV; turn it on.
 841          */
 842         WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 843
 844         /* Wa4x4STCOptimizationDisable:bdw,chv */
 845         WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 846
 847         /*
 848          * BSpec recommends 8x4 when MSAA is used,
 849          * however in practice 16x4 seems fastest.
 850          *
 851          * Note that PS/WM thread counts depend on the WIZ hashing
 852          * disable bit, which we don't touch here, but it's good
 853          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 854          */
 855         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 856                             GEN6_WIZ_HASHING_MASK,
 857                             GEN6_WIZ_HASHING_16x4);
 858
 859         return 0;
 860 }
 861
 862 static int bdw_init_workarounds(struct intel_engine_cs *engine)
 863 {
 864         struct drm_i915_private *dev_priv = engine->i915;
 865         int ret;
 866
 867         ret = gen8_init_workarounds(engine);
 868         if (ret)
 869                 return ret;
 870
 871         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 872         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 873
 874         /* WaDisableDopClockGating:bdw */
 875         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 876                           DOP_CLOCK_GATING_DISABLE);
 877
 878         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 879                           GEN8_SAMPLER_POWER_BYPASS_DIS);
 880
 881         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 882                           /* WaForceContextSaveRestoreNonCoherent:bdw */
 883                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 884                           /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
 885                           (IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 886
 887         return 0;
 888 }
 889
 890 static int chv_init_workarounds(struct intel_engine_cs *engine)
 891 {
 892         struct drm_i915_private *dev_priv = engine->i915;
 893         int ret;
 894
 895         ret = gen8_init_workarounds(engine);
 896         if (ret)
 897                 return ret;
 898
 899         /* WaDisableThreadStallDopClockGating:chv */
 900         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 901
 902         /* Improve HiZ throughput on CHV. */
 903         WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
 904
 905         return 0;
 906 }
 907
 908 static int gen9_init_workarounds(struct intel_engine_cs *engine)
 909 {
 910         struct drm_i915_private *dev_priv = engine->i915;
 911         int ret;
 912
 913         /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl */
 914         I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
 915                    GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
 916
 917         /* WaDisableKillLogic:bxt,skl,kbl */
 918         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
 919                    ECOCHK_DIS_TLB);
 920
 921         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl */
 922         /* WaDisablePartialInstShootdown:skl,bxt,kbl */
 923         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 924                           FLOW_CONTROL_ENABLE |
 925                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 926
 927         /* Syncing dependencies between camera and graphics:skl,bxt,kbl */
 928         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 929                           GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
 930
 931         /* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */
 932         if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
 933             IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
 934                 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 935                                   GEN9_DG_MIRROR_FIX_ENABLE);
 936
 937         /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
 938         if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
 939             IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
 940                 WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
 941                                   GEN9_RHWO_OPTIMIZATION_DISABLE);
 942                 /*
 943                  * WA also requires GEN9_SLICE_COMMON_ECO_CHICKEN0[14:14] to be set
 944                  * but we do that in per ctx batchbuffer as there is an issue
 945                  * with this register not getting restored on ctx restore
 946                  */
 947         }
 948
 949         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl */
 950         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl */
 951         WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 952                           GEN9_ENABLE_YV12_BUGFIX |
 953                           GEN9_ENABLE_GPGPU_PREEMPTION);
 954
 955         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl */
 956         /* WaDisablePartialResolveInVc:skl,bxt,kbl */
 957         WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE |
 958                                          GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
 959
 960         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl */
 961         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 962                           GEN9_CCS_TLB_PREFETCH_ENABLE);
 963
 964         /* WaDisableMaskBasedCammingInRCC:skl,bxt */
 965         if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, SKL_REVID_C0) ||
 966             IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
 967                 WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
 968                                   PIXEL_MASK_CAMMING_DISABLE);
 969
 970         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl */
 971         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 972                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 973                           HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
 974
 975         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
 976          * both tied to WaForceContextSaveRestoreNonCoherent
 977          * in some hsds for skl. We keep the tie for all gen9. The
 978          * documentation is a bit hazy and so we want to get common behaviour,
 979          * even though there is no clear evidence we would need both on kbl/bxt.
 980          * This area has been source of system hangs so we play it safe
 981          * and mimic the skl regardless of what bspec says.
 982          *
 983          * Use Force Non-Coherent whenever executing a 3D context. This
 984          * is a workaround for a possible hang in the unlikely event
 985          * a TLB invalidation occurs during a PSD flush.
 986          */
 987
 988         /* WaForceEnableNonCoherent:skl,bxt,kbl */
 989         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 990                           HDC_FORCE_NON_COHERENT);
 991
 992         /* WaDisableHDCInvalidation:skl,bxt,kbl */
 993         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
 994                    BDW_DISABLE_HDC_INVALIDATION);
 995
 996         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl */
 997         if (IS_SKYLAKE(dev_priv) ||
 998             IS_KABYLAKE(dev_priv) ||
 999             IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
1000                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
1001                                   GEN8_SAMPLER_POWER_BYPASS_DIS);
1002
1003         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl */
1004         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
1005
1006         /* WaOCLCoherentLineFlush:skl,bxt,kbl */
1007         I915_WRITE(GEN8_L3SQCREG4, (I915_READ(GEN8_L3SQCREG4) |
1008                                     GEN8_LQSC_FLUSH_COHERENT_LINES));
1009
1010         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt */
1011         ret = wa_ring_whitelist_reg(engine, GEN9_CTX_PREEMPT_REG);
1012         if (ret)
1013                 return ret;
1014
1015         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl */
1016         ret= wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
1017         if (ret)
1018                 return ret;
1019
1020         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl */
1021         ret = wa_ring_whitelist_reg(engine, GEN8_HDC_CHICKEN1);
1022         if (ret)
1023                 return ret;
1024
1025         return 0;
1026 }
1027
1028 static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
1029 {
1030         struct drm_i915_private *dev_priv = engine->i915;
1031         u8 vals[3] = { 0, 0, 0 };
1032         unsigned int i;
1033
1034         for (i = 0; i < 3; i++) {
1035                 u8 ss;
1036
1037                 /*
1038                  * Only consider slices where one, and only one, subslice has 7
1039                  * EUs
1040                  */
1041                 if (!is_power_of_2(dev_priv->info.subslice_7eu[i]))
1042                         continue;
1043
1044                 /*
1045                  * subslice_7eu[i] != 0 (because of the check above) and
1046                  * ss_max == 4 (maximum number of subslices possible per slice)
1047                  *
1048                  * ->    0 <= ss <= 3;
1049                  */
1050                 ss = ffs(dev_priv->info.subslice_7eu[i]) - 1;
1051                 vals[i] = 3 - ss;
1052         }
1053
1054         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
1055                 return 0;
1056
1057         /* Tune IZ hashing. See intel_device_info_runtime_init() */
1058         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
1059                             GEN9_IZ_HASHING_MASK(2) |
1060                             GEN9_IZ_HASHING_MASK(1) |
1061                             GEN9_IZ_HASHING_MASK(0),
1062                             GEN9_IZ_HASHING(2, vals[2]) |
1063                             GEN9_IZ_HASHING(1, vals[1]) |
1064                             GEN9_IZ_HASHING(0, vals[0]));
1065
1066         return 0;
1067 }
1068
1069 static int skl_init_workarounds(struct intel_engine_cs *engine)
1070 {
1071         struct drm_i915_private *dev_priv = engine->i915;
1072         int ret;
1073
1074         ret = gen9_init_workarounds(engine);
1075         if (ret)
1076                 return ret;
1077
1078         /*
1079          * Actual WA is to disable percontext preemption granularity control
1080          * until D0 which is the default case so this is equivalent to
1081          * !WaDisablePerCtxtPreemptionGranularityControl:skl
1082          */
1083         if (IS_SKL_REVID(dev_priv, SKL_REVID_E0, REVID_FOREVER)) {
1084                 I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
1085                            _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
1086         }
1087
1088         if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_D0)) {
1089                 /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
1090                 I915_WRITE(FF_SLICE_CS_CHICKEN2,
1091                            _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE));
1092         }
1093
1094         /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
1095          * involving this register should also be added to WA batch as required.
1096          */
1097         if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0))
1098                 /* WaDisableLSQCROPERFforOCL:skl */
1099                 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
1100                            GEN8_LQSC_RO_PERF_DIS);
1101
1102         /* WaEnableGapsTsvCreditFix:skl */
1103         if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, REVID_FOREVER)) {
1104                 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1105                                            GEN9_GAPS_TSV_CREDIT_DISABLE));
1106         }
1107
1108         /* WaDisablePowerCompilerClockGating:skl */
1109         if (IS_SKL_REVID(dev_priv, SKL_REVID_B0, SKL_REVID_B0))
1110                 WA_SET_BIT_MASKED(HIZ_CHICKEN,
1111                                   BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE);
1112
1113         /* WaBarrierPerformanceFixDisable:skl */
1114         if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, SKL_REVID_D0))
1115                 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1116                                   HDC_FENCE_DEST_SLM_DISABLE |
1117                                   HDC_BARRIER_PERFORMANCE_DISABLE);
1118
1119         /* WaDisableSbeCacheDispatchPortSharing:skl */
1120         if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0))
1121                 WA_SET_BIT_MASKED(
1122                         GEN7_HALF_SLICE_CHICKEN1,
1123                         GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1124
1125         /* WaDisableGafsUnitClkGating:skl */
1126         WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1127
1128         /* WaDisableLSQCROPERFforOCL:skl */
1129         ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1130         if (ret)
1131                 return ret;
1132
1133         return skl_tune_iz_hashing(engine);
1134 }
1135
1136 static int bxt_init_workarounds(struct intel_engine_cs *engine)
1137 {
1138         struct drm_i915_private *dev_priv = engine->i915;
1139         int ret;
1140
1141         ret = gen9_init_workarounds(engine);
1142         if (ret)
1143                 return ret;
1144
1145         /* WaStoreMultiplePTEenable:bxt */
1146         /* This is a requirement according to Hardware specification */
1147         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1148                 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
1149
1150         /* WaSetClckGatingDisableMedia:bxt */
1151         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1152                 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
1153                                             ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
1154         }
1155
1156         /* WaDisableThreadStallDopClockGating:bxt */
1157         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
1158                           STALL_DOP_GATING_DISABLE);
1159
1160         /* WaDisableSbeCacheDispatchPortSharing:bxt */
1161         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) {
1162                 WA_SET_BIT_MASKED(
1163                         GEN7_HALF_SLICE_CHICKEN1,
1164                         GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1165         }
1166
1167         /* WaDisableObjectLevelPreemptionForTrifanOrPolygon:bxt */
1168         /* WaDisableObjectLevelPreemptionForInstancedDraw:bxt */
1169         /* WaDisableObjectLevelPreemtionForInstanceId:bxt */
1170         /* WaDisableLSQCROPERFforOCL:bxt */
1171         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1172                 ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1);
1173                 if (ret)
1174                         return ret;
1175
1176                 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1177                 if (ret)
1178                         return ret;
1179         }
1180
1181         /* WaProgramL3SqcReg1DefaultForPerf:bxt */
1182         if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER))
1183                 I915_WRITE(GEN8_L3SQCREG1, L3_GENERAL_PRIO_CREDITS(62) |
1184                                            L3_HIGH_PRIO_CREDITS(2));
1185
1186         /* WaInsertDummyPushConstPs:bxt */
1187         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
1188                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1189                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1190
1191         return 0;
1192 }
1193
1194 static int kbl_init_workarounds(struct intel_engine_cs *engine)
1195 {
1196         struct drm_i915_private *dev_priv = engine->i915;
1197         int ret;
1198
1199         ret = gen9_init_workarounds(engine);
1200         if (ret)
1201                 return ret;
1202
1203         /* WaEnableGapsTsvCreditFix:kbl */
1204         I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1205                                    GEN9_GAPS_TSV_CREDIT_DISABLE));
1206
1207         /* WaDisableDynamicCreditSharing:kbl */
1208         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
1209                 WA_SET_BIT(GAMT_CHKN_BIT_REG,
1210                            GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
1211
1212         /* WaDisableFenceDestinationToSLM:kbl (pre-prod) */
1213         if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0))
1214                 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1215                                   HDC_FENCE_DEST_SLM_DISABLE);
1216
1217         /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
1218          * involving this register should also be added to WA batch as required.
1219          */
1220         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_E0))
1221                 /* WaDisableLSQCROPERFforOCL:kbl */
1222                 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
1223                            GEN8_LQSC_RO_PERF_DIS);
1224
1225         /* WaInsertDummyPushConstPs:kbl */
1226         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
1227                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1228                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1229
1230         /* WaDisableLSQCROPERFforOCL:kbl */
1231         ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1232         if (ret)
1233                 return ret;
1234
1235         return 0;
1236 }
1237
1238 int init_workarounds_ring(struct intel_engine_cs *engine)
1239 {
1240         struct drm_i915_private *dev_priv = engine->i915;
1241
1242         WARN_ON(engine->id != RCS);
1243
1244         dev_priv->workarounds.count = 0;
1245         dev_priv->workarounds.hw_whitelist_count[RCS] = 0;
1246
1247         if (IS_BROADWELL(dev_priv))
1248                 return bdw_init_workarounds(engine);
1249
1250         if (IS_CHERRYVIEW(dev_priv))
1251                 return chv_init_workarounds(engine);
1252
1253         if (IS_SKYLAKE(dev_priv))
1254                 return skl_init_workarounds(engine);
1255
1256         if (IS_BROXTON(dev_priv))
1257                 return bxt_init_workarounds(engine);
1258
1259         if (IS_KABYLAKE(dev_priv))
1260                 return kbl_init_workarounds(engine);
1261
1262         return 0;
1263 }
1264
1265 static int init_render_ring(struct intel_engine_cs *engine)
1266 {
1267         struct drm_i915_private *dev_priv = engine->i915;
1268         int ret = init_ring_common(engine);
1269         if (ret)
1270                 return ret;
1271
1272         /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
1273         if (IS_GEN(dev_priv, 4, 6))
1274                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
1275
1276         /* We need to disable the AsyncFlip performance optimisations in order
1277          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
1278          * programmed to '1' on all products.
1279          *
1280          * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1281          */
1282         if (IS_GEN(dev_priv, 6, 7))
1283                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
1284
1285         /* Required for the hardware to program scanline values for waiting */
1286         /* WaEnableFlushTlbInvalidationMode:snb */
1287         if (IS_GEN6(dev_priv))
1288                 I915_WRITE(GFX_MODE,
1289                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
1290
1291         /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1292         if (IS_GEN7(dev_priv))
1293                 I915_WRITE(GFX_MODE_GEN7,
1294                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
1295                            _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
1296
1297         if (IS_GEN6(dev_priv)) {
1298                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
1299                  * "If this bit is set, STCunit will have LRA as replacement
1300                  *  policy. [...] This bit must be reset.  LRA replacement
1301                  *  policy is not supported."
1302                  */
1303                 I915_WRITE(CACHE_MODE_0,
1304                            _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
1305         }
1306
1307         if (IS_GEN(dev_priv, 6, 7))
1308                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
1309
1310         if (HAS_L3_DPF(dev_priv))
1311                 I915_WRITE_IMR(engine, ~GT_PARITY_ERROR(dev_priv));
1312
1313         return init_workarounds_ring(engine);
1314 }
1315
1316 static void render_ring_cleanup(struct intel_engine_cs *engine)
1317 {
1318         struct drm_i915_private *dev_priv = engine->i915;
1319
1320         if (dev_priv->semaphore_obj) {
1321                 i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj);
1322                 drm_gem_object_unreference(&dev_priv->semaphore_obj->base);
1323                 dev_priv->semaphore_obj = NULL;
1324         }
1325
1326         intel_fini_pipe_control(engine);
1327 }
1328
1329 static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req,
1330                            unsigned int num_dwords)
1331 {
1332 #define MBOX_UPDATE_DWORDS 8
1333         struct intel_engine_cs *signaller = signaller_req->engine;
1334         struct drm_i915_private *dev_priv = signaller_req->i915;
1335         struct intel_engine_cs *waiter;
1336         enum intel_engine_id id;
1337         int ret, num_rings;
1338
1339         num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask);
1340         num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
1341 #undef MBOX_UPDATE_DWORDS
1342
1343         ret = intel_ring_begin(signaller_req, num_dwords);
1344         if (ret)
1345                 return ret;
1346
1347         for_each_engine_id(waiter, dev_priv, id) {
1348                 u32 seqno;
1349                 u64 gtt_offset = signaller->semaphore.signal_ggtt[id];
1350                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
1351                         continue;
1352
1353                 seqno = i915_gem_request_get_seqno(signaller_req);
1354                 intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
1355                 intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB |
1356                                            PIPE_CONTROL_QW_WRITE |
1357                                            PIPE_CONTROL_CS_STALL);
1358                 intel_ring_emit(signaller, lower_32_bits(gtt_offset));
1359                 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
1360                 intel_ring_emit(signaller, seqno);
1361                 intel_ring_emit(signaller, 0);
1362                 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
1363                                            MI_SEMAPHORE_TARGET(waiter->hw_id));
1364                 intel_ring_emit(signaller, 0);
1365         }
1366
1367         return 0;
1368 }
1369
1370 static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req,
1371                            unsigned int num_dwords)
1372 {
1373 #define MBOX_UPDATE_DWORDS 6
1374         struct intel_engine_cs *signaller = signaller_req->engine;
1375         struct drm_i915_private *dev_priv = signaller_req->i915;
1376         struct intel_engine_cs *waiter;
1377         enum intel_engine_id id;
1378         int ret, num_rings;
1379
1380         num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask);
1381         num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
1382 #undef MBOX_UPDATE_DWORDS
1383
1384         ret = intel_ring_begin(signaller_req, num_dwords);
1385         if (ret)
1386                 return ret;
1387
1388         for_each_engine_id(waiter, dev_priv, id) {
1389                 u32 seqno;
1390                 u64 gtt_offset = signaller->semaphore.signal_ggtt[id];
1391                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
1392                         continue;
1393
1394                 seqno = i915_gem_request_get_seqno(signaller_req);
1395                 intel_ring_emit(signaller, (MI_FLUSH_DW + 1) |
1396                                            MI_FLUSH_DW_OP_STOREDW);
1397                 intel_ring_emit(signaller, lower_32_bits(gtt_offset) |
1398                                            MI_FLUSH_DW_USE_GTT);
1399                 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
1400                 intel_ring_emit(signaller, seqno);
1401                 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
1402                                            MI_SEMAPHORE_TARGET(waiter->hw_id));
1403                 intel_ring_emit(signaller, 0);
1404         }
1405
1406         return 0;
1407 }
1408
1409 static int gen6_signal(struct drm_i915_gem_request *signaller_req,
1410                        unsigned int num_dwords)
1411 {
1412         struct intel_engine_cs *signaller = signaller_req->engine;
1413         struct drm_i915_private *dev_priv = signaller_req->i915;
1414         struct intel_engine_cs *useless;
1415         enum intel_engine_id id;
1416         int ret, num_rings;
1417
1418 #define MBOX_UPDATE_DWORDS 3
1419         num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask);
1420         num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2);
1421 #undef MBOX_UPDATE_DWORDS
1422
1423         ret = intel_ring_begin(signaller_req, num_dwords);
1424         if (ret)
1425                 return ret;
1426
1427         for_each_engine_id(useless, dev_priv, id) {
1428                 i915_reg_t mbox_reg = signaller->semaphore.mbox.signal[id];
1429
1430                 if (i915_mmio_reg_valid(mbox_reg)) {
1431                         u32 seqno = i915_gem_request_get_seqno(signaller_req);
1432
1433                         intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
1434                         intel_ring_emit_reg(signaller, mbox_reg);
1435                         intel_ring_emit(signaller, seqno);
1436                 }
1437         }
1438
1439         /* If num_dwords was rounded, make sure the tail pointer is correct */
1440         if (num_rings % 2 == 0)
1441                 intel_ring_emit(signaller, MI_NOOP);
1442
1443         return 0;
1444 }
1445
1446 /**
1447  * gen6_add_request - Update the semaphore mailbox registers
1448  *
1449  * @request - request to write to the ring
1450  *
1451  * Update the mailbox registers in the *other* rings with the current seqno.
1452  * This acts like a signal in the canonical semaphore.
1453  */
1454 static int
1455 gen6_add_request(struct drm_i915_gem_request *req)
1456 {
1457         struct intel_engine_cs *engine = req->engine;
1458         int ret;
1459
1460         if (engine->semaphore.signal)
1461                 ret = engine->semaphore.signal(req, 4);
1462         else
1463                 ret = intel_ring_begin(req, 4);
1464
1465         if (ret)
1466                 return ret;
1467
1468         intel_ring_emit(engine, MI_STORE_DWORD_INDEX);
1469         intel_ring_emit(engine,
1470                         I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1471         intel_ring_emit(engine, i915_gem_request_get_seqno(req));
1472         intel_ring_emit(engine, MI_USER_INTERRUPT);
1473         __intel_ring_advance(engine);
1474
1475         return 0;
1476 }
1477
1478 static int
1479 gen8_render_add_request(struct drm_i915_gem_request *req)
1480 {
1481         struct intel_engine_cs *engine = req->engine;
1482         int ret;
1483
1484         if (engine->semaphore.signal)
1485                 ret = engine->semaphore.signal(req, 8);
1486         else
1487                 ret = intel_ring_begin(req, 8);
1488         if (ret)
1489                 return ret;
1490
1491         intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(6));
1492         intel_ring_emit(engine, (PIPE_CONTROL_GLOBAL_GTT_IVB |
1493                                  PIPE_CONTROL_CS_STALL |
1494                                  PIPE_CONTROL_QW_WRITE));
1495         intel_ring_emit(engine, intel_hws_seqno_address(req->engine));
1496         intel_ring_emit(engine, 0);
1497         intel_ring_emit(engine, i915_gem_request_get_seqno(req));
1498         /* We're thrashing one dword of HWS. */
1499         intel_ring_emit(engine, 0);
1500         intel_ring_emit(engine, MI_USER_INTERRUPT);
1501         intel_ring_emit(engine, MI_NOOP);
1502         __intel_ring_advance(engine);
1503
1504         return 0;
1505 }
1506
1507 static inline bool i915_gem_has_seqno_wrapped(struct drm_i915_private *dev_priv,
1508                                               u32 seqno)
1509 {
1510         return dev_priv->last_seqno < seqno;
1511 }
1512
1513 /**
1514  * intel_ring_sync - sync the waiter to the signaller on seqno
1515  *
1516  * @waiter - ring that is waiting
1517  * @signaller - ring which has, or will signal
1518  * @seqno - seqno which the waiter will block on
1519  */
1520
1521 static int
1522 gen8_ring_sync(struct drm_i915_gem_request *waiter_req,
1523                struct intel_engine_cs *signaller,
1524                u32 seqno)
1525 {
1526         struct intel_engine_cs *waiter = waiter_req->engine;
1527         struct drm_i915_private *dev_priv = waiter_req->i915;
1528         struct i915_hw_ppgtt *ppgtt;
1529         int ret;
1530
1531         ret = intel_ring_begin(waiter_req, 4);
1532         if (ret)
1533                 return ret;
1534
1535         intel_ring_emit(waiter, MI_SEMAPHORE_WAIT |
1536                                 MI_SEMAPHORE_GLOBAL_GTT |
1537                                 MI_SEMAPHORE_SAD_GTE_SDD);
1538         intel_ring_emit(waiter, seqno);
1539         intel_ring_emit(waiter,
1540                         lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
1541         intel_ring_emit(waiter,
1542                         upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
1543         intel_ring_advance(waiter);
1544
1545         /* When the !RCS engines idle waiting upon a semaphore, they lose their
1546          * pagetables and we must reload them before executing the batch.
1547          * We do this on the i915_switch_context() following the wait and
1548          * before the dispatch.
1549          */
1550         ppgtt = waiter_req->ctx->ppgtt;
1551         if (ppgtt && waiter_req->engine->id != RCS)
1552                 ppgtt->pd_dirty_rings |= intel_engine_flag(waiter_req->engine);
1553         return 0;
1554 }
1555
1556 static int
1557 gen6_ring_sync(struct drm_i915_gem_request *waiter_req,
1558                struct intel_engine_cs *signaller,
1559                u32 seqno)
1560 {
1561         struct intel_engine_cs *waiter = waiter_req->engine;
1562         u32 dw1 = MI_SEMAPHORE_MBOX |
1563                   MI_SEMAPHORE_COMPARE |
1564                   MI_SEMAPHORE_REGISTER;
1565         u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id];
1566         int ret;
1567
1568         /* Throughout all of the GEM code, seqno passed implies our current
1569          * seqno is >= the last seqno executed. However for hardware the
1570          * comparison is strictly greater than.
1571          */
1572         seqno -= 1;
1573
1574         WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
1575
1576         ret = intel_ring_begin(waiter_req, 4);
1577         if (ret)
1578                 return ret;
1579
1580         /* If seqno wrap happened, omit the wait with no-ops */
1581         if (likely(!i915_gem_has_seqno_wrapped(waiter_req->i915, seqno))) {
1582                 intel_ring_emit(waiter, dw1 | wait_mbox);
1583                 intel_ring_emit(waiter, seqno);
1584                 intel_ring_emit(waiter, 0);
1585                 intel_ring_emit(waiter, MI_NOOP);
1586         } else {
1587                 intel_ring_emit(waiter, MI_NOOP);
1588                 intel_ring_emit(waiter, MI_NOOP);
1589                 intel_ring_emit(waiter, MI_NOOP);
1590                 intel_ring_emit(waiter, MI_NOOP);
1591         }
1592         intel_ring_advance(waiter);
1593
1594         return 0;
1595 }
1596
1597 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
1598 do {                                                                    \
1599         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
1600                  PIPE_CONTROL_DEPTH_STALL);                             \
1601         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
1602         intel_ring_emit(ring__, 0);                                                     \
1603         intel_ring_emit(ring__, 0);                                                     \
1604 } while (0)
1605
1606 static int
1607 pc_render_add_request(struct drm_i915_gem_request *req)
1608 {
1609         struct intel_engine_cs *engine = req->engine;
1610         u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
1611         int ret;
1612
1613         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
1614          * incoherent with writes to memory, i.e. completely fubar,
1615          * so we need to use PIPE_NOTIFY instead.
1616          *
1617          * However, we also need to workaround the qword write
1618          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
1619          * memory before requesting an interrupt.
1620          */
1621         ret = intel_ring_begin(req, 32);
1622         if (ret)
1623                 return ret;
1624
1625         intel_ring_emit(engine,
1626                         GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
1627                         PIPE_CONTROL_WRITE_FLUSH |
1628                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
1629         intel_ring_emit(engine,
1630                         engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
1631         intel_ring_emit(engine, i915_gem_request_get_seqno(req));
1632         intel_ring_emit(engine, 0);
1633         PIPE_CONTROL_FLUSH(engine, scratch_addr);
1634         scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */
1635         PIPE_CONTROL_FLUSH(engine, scratch_addr);
1636         scratch_addr += 2 * CACHELINE_BYTES;
1637         PIPE_CONTROL_FLUSH(engine, scratch_addr);
1638         scratch_addr += 2 * CACHELINE_BYTES;
1639         PIPE_CONTROL_FLUSH(engine, scratch_addr);
1640         scratch_addr += 2 * CACHELINE_BYTES;
1641         PIPE_CONTROL_FLUSH(engine, scratch_addr);
1642         scratch_addr += 2 * CACHELINE_BYTES;
1643         PIPE_CONTROL_FLUSH(engine, scratch_addr);
1644
1645         intel_ring_emit(engine,
1646                         GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
1647                         PIPE_CONTROL_WRITE_FLUSH |
1648                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
1649                         PIPE_CONTROL_NOTIFY);
1650         intel_ring_emit(engine,
1651                         engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
1652         intel_ring_emit(engine, i915_gem_request_get_seqno(req));
1653         intel_ring_emit(engine, 0);
1654         __intel_ring_advance(engine);
1655
1656         return 0;
1657 }
1658
1659 static void
1660 gen6_seqno_barrier(struct intel_engine_cs *engine)
1661 {
1662         struct drm_i915_private *dev_priv = engine->i915;
1663
1664         /* Workaround to force correct ordering between irq and seqno writes on
1665          * ivb (and maybe also on snb) by reading from a CS register (like
1666          * ACTHD) before reading the status page.
1667          *
1668          * Note that this effectively stalls the read by the time it takes to
1669          * do a memory transaction, which more or less ensures that the write
1670          * from the GPU has sufficient time to invalidate the CPU cacheline.
1671          * Alternatively we could delay the interrupt from the CS ring to give
1672          * the write time to land, but that would incur a delay after every
1673          * batch i.e. much more frequent than a delay when waiting for the
1674          * interrupt (with the same net latency).
1675          *
1676          * Also note that to prevent whole machine hangs on gen7, we have to
1677          * take the spinlock to guard against concurrent cacheline access.
1678          */
1679         spin_lock_irq(&dev_priv->uncore.lock);
1680         POSTING_READ_FW(RING_ACTHD(engine->mmio_base));
1681         spin_unlock_irq(&dev_priv->uncore.lock);
1682 }
1683
1684 static u32
1685 ring_get_seqno(struct intel_engine_cs *engine)
1686 {
1687         return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
1688 }
1689
1690 static void
1691 ring_set_seqno(struct intel_engine_cs *engine, u32 seqno)
1692 {
1693         intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
1694 }
1695
1696 static u32
1697 pc_render_get_seqno(struct intel_engine_cs *engine)
1698 {
1699         return engine->scratch.cpu_page[0];
1700 }
1701
1702 static void
1703 pc_render_set_seqno(struct intel_engine_cs *engine, u32 seqno)
1704 {
1705         engine->scratch.cpu_page[0] = seqno;
1706 }
1707
1708 static bool
1709 gen5_ring_get_irq(struct intel_engine_cs *engine)
1710 {
1711         struct drm_i915_private *dev_priv = engine->i915;
1712         unsigned long flags;
1713
1714         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1715                 return false;
1716
1717         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1718         if (engine->irq_refcount++ == 0)
1719                 gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
1720         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1721
1722         return true;
1723 }
1724
1725 static void
1726 gen5_ring_put_irq(struct intel_engine_cs *engine)
1727 {
1728         struct drm_i915_private *dev_priv = engine->i915;
1729         unsigned long flags;
1730
1731         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1732         if (--engine->irq_refcount == 0)
1733                 gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
1734         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1735 }
1736
1737 static bool
1738 i9xx_ring_get_irq(struct intel_engine_cs *engine)
1739 {
1740         struct drm_i915_private *dev_priv = engine->i915;
1741         unsigned long flags;
1742
1743         if (!intel_irqs_enabled(dev_priv))
1744                 return false;
1745
1746         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1747         if (engine->irq_refcount++ == 0) {
1748                 dev_priv->irq_mask &= ~engine->irq_enable_mask;
1749                 I915_WRITE(IMR, dev_priv->irq_mask);
1750                 POSTING_READ(IMR);
1751         }
1752         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1753
1754         return true;
1755 }
1756
1757 static void
1758 i9xx_ring_put_irq(struct intel_engine_cs *engine)
1759 {
1760         struct drm_i915_private *dev_priv = engine->i915;
1761         unsigned long flags;
1762
1763         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1764         if (--engine->irq_refcount == 0) {
1765                 dev_priv->irq_mask |= engine->irq_enable_mask;
1766                 I915_WRITE(IMR, dev_priv->irq_mask);
1767                 POSTING_READ(IMR);
1768         }
1769         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1770 }
1771
1772 static bool
1773 i8xx_ring_get_irq(struct intel_engine_cs *engine)
1774 {
1775         struct drm_i915_private *dev_priv = engine->i915;
1776         unsigned long flags;
1777
1778         if (!intel_irqs_enabled(dev_priv))
1779                 return false;
1780
1781         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1782         if (engine->irq_refcount++ == 0) {
1783                 dev_priv->irq_mask &= ~engine->irq_enable_mask;
1784                 I915_WRITE16(IMR, dev_priv->irq_mask);
1785                 POSTING_READ16(IMR);
1786         }
1787         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1788
1789         return true;
1790 }
1791
1792 static void
1793 i8xx_ring_put_irq(struct intel_engine_cs *engine)
1794 {
1795         struct drm_i915_private *dev_priv = engine->i915;
1796         unsigned long flags;
1797
1798         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1799         if (--engine->irq_refcount == 0) {
1800                 dev_priv->irq_mask |= engine->irq_enable_mask;
1801                 I915_WRITE16(IMR, dev_priv->irq_mask);
1802                 POSTING_READ16(IMR);
1803         }
1804         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1805 }
1806
1807 static int
1808 bsd_ring_flush(struct drm_i915_gem_request *req,
1809                u32     invalidate_domains,
1810                u32     flush_domains)
1811 {
1812         struct intel_engine_cs *engine = req->engine;
1813         int ret;
1814
1815         ret = intel_ring_begin(req, 2);
1816         if (ret)
1817                 return ret;
1818
1819         intel_ring_emit(engine, MI_FLUSH);
1820         intel_ring_emit(engine, MI_NOOP);
1821         intel_ring_advance(engine);
1822         return 0;
1823 }
1824
1825 static int
1826 i9xx_add_request(struct drm_i915_gem_request *req)
1827 {
1828         struct intel_engine_cs *engine = req->engine;
1829         int ret;
1830
1831         ret = intel_ring_begin(req, 4);
1832         if (ret)
1833                 return ret;
1834
1835         intel_ring_emit(engine, MI_STORE_DWORD_INDEX);
1836         intel_ring_emit(engine,
1837                         I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1838         intel_ring_emit(engine, i915_gem_request_get_seqno(req));
1839         intel_ring_emit(engine, MI_USER_INTERRUPT);
1840         __intel_ring_advance(engine);
1841
1842         return 0;
1843 }
1844
1845 static bool
1846 gen6_ring_get_irq(struct intel_engine_cs *engine)
1847 {
1848         struct drm_i915_private *dev_priv = engine->i915;
1849         unsigned long flags;
1850
1851         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1852                 return false;
1853
1854         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1855         if (engine->irq_refcount++ == 0) {
1856                 if (HAS_L3_DPF(dev_priv) && engine->id == RCS)
1857                         I915_WRITE_IMR(engine,
1858                                        ~(engine->irq_enable_mask |
1859                                          GT_PARITY_ERROR(dev_priv)));
1860                 else
1861                         I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1862                 gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
1863         }
1864         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1865
1866         return true;
1867 }
1868
1869 static void
1870 gen6_ring_put_irq(struct intel_engine_cs *engine)
1871 {
1872         struct drm_i915_private *dev_priv = engine->i915;
1873         unsigned long flags;
1874
1875         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1876         if (--engine->irq_refcount == 0) {
1877                 if (HAS_L3_DPF(dev_priv) && engine->id == RCS)
1878                         I915_WRITE_IMR(engine, ~GT_PARITY_ERROR(dev_priv));
1879                 else
1880                         I915_WRITE_IMR(engine, ~0);
1881                 gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
1882         }
1883         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1884 }
1885
1886 static bool
1887 hsw_vebox_get_irq(struct intel_engine_cs *engine)
1888 {
1889         struct drm_i915_private *dev_priv = engine->i915;
1890         unsigned long flags;
1891
1892         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1893                 return false;
1894
1895         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1896         if (engine->irq_refcount++ == 0) {
1897                 I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1898                 gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask);
1899         }
1900         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1901
1902         return true;
1903 }
1904
1905 static void
1906 hsw_vebox_put_irq(struct intel_engine_cs *engine)
1907 {
1908         struct drm_i915_private *dev_priv = engine->i915;
1909         unsigned long flags;
1910
1911         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1912         if (--engine->irq_refcount == 0) {
1913                 I915_WRITE_IMR(engine, ~0);
1914                 gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask);
1915         }
1916         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1917 }
1918
1919 static bool
1920 gen8_ring_get_irq(struct intel_engine_cs *engine)
1921 {
1922         struct drm_i915_private *dev_priv = engine->i915;
1923         unsigned long flags;
1924
1925         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1926                 return false;
1927
1928         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1929         if (engine->irq_refcount++ == 0) {
1930                 if (HAS_L3_DPF(dev_priv) && engine->id == RCS) {
1931                         I915_WRITE_IMR(engine,
1932                                        ~(engine->irq_enable_mask |
1933                                          GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
1934                 } else {
1935                         I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1936                 }
1937                 POSTING_READ(RING_IMR(engine->mmio_base));
1938         }
1939         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1940
1941         return true;
1942 }
1943
1944 static void
1945 gen8_ring_put_irq(struct intel_engine_cs *engine)
1946 {
1947         struct drm_i915_private *dev_priv = engine->i915;
1948         unsigned long flags;
1949
1950         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1951         if (--engine->irq_refcount == 0) {
1952                 if (HAS_L3_DPF(dev_priv) && engine->id == RCS) {
1953                         I915_WRITE_IMR(engine,
1954                                        ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
1955                 } else {
1956                         I915_WRITE_IMR(engine, ~0);
1957                 }
1958                 POSTING_READ(RING_IMR(engine->mmio_base));
1959         }
1960         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1961 }
1962
1963 static int
1964 i965_dispatch_execbuffer(struct drm_i915_gem_request *req,
1965                          u64 offset, u32 length,
1966                          unsigned dispatch_flags)
1967 {
1968         struct intel_engine_cs *engine = req->engine;
1969         int ret;
1970
1971         ret = intel_ring_begin(req, 2);
1972         if (ret)
1973                 return ret;
1974
1975         intel_ring_emit(engine,
1976                         MI_BATCH_BUFFER_START |
1977                         MI_BATCH_GTT |
1978                         (dispatch_flags & I915_DISPATCH_SECURE ?
1979                          0 : MI_BATCH_NON_SECURE_I965));
1980         intel_ring_emit(engine, offset);
1981         intel_ring_advance(engine);
1982
1983         return 0;
1984 }
1985
1986 /* Just userspace ABI convention to limit the wa batch bo to a resonable size */
1987 #define I830_BATCH_LIMIT (256*1024)
1988 #define I830_TLB_ENTRIES (2)
1989 #define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
1990 static int
1991 i830_dispatch_execbuffer(struct drm_i915_gem_request *req,
1992                          u64 offset, u32 len,
1993                          unsigned dispatch_flags)
1994 {
1995         struct intel_engine_cs *engine = req->engine;
1996         u32 cs_offset = engine->scratch.gtt_offset;
1997         int ret;
1998
1999         ret = intel_ring_begin(req, 6);
2000         if (ret)
2001                 return ret;
2002
2003         /* Evict the invalid PTE TLBs */
2004         intel_ring_emit(engine, COLOR_BLT_CMD | BLT_WRITE_RGBA);
2005         intel_ring_emit(engine, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096);
2006         intel_ring_emit(engine, I830_TLB_ENTRIES << 16 | 4); /* load each page */
2007         intel_ring_emit(engine, cs_offset);
2008         intel_ring_emit(engine, 0xdeadbeef);
2009         intel_ring_emit(engine, MI_NOOP);
2010         intel_ring_advance(engine);
2011
2012         if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) {
2013                 if (len > I830_BATCH_LIMIT)
2014                         return -ENOSPC;
2015
2016                 ret = intel_ring_begin(req, 6 + 2);
2017                 if (ret)
2018                         return ret;
2019
2020                 /* Blit the batch (which has now all relocs applied) to the
2021                  * stable batch scratch bo area (so that the CS never
2022                  * stumbles over its tlb invalidation bug) ...
2023                  */
2024                 intel_ring_emit(engine, SRC_COPY_BLT_CMD | BLT_WRITE_RGBA);
2025                 intel_ring_emit(engine,
2026                                 BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096);
2027                 intel_ring_emit(engine, DIV_ROUND_UP(len, 4096) << 16 | 4096);
2028                 intel_ring_emit(engine, cs_offset);
2029                 intel_ring_emit(engine, 4096);
2030                 intel_ring_emit(engine, offset);
2031
2032                 intel_ring_emit(engine, MI_FLUSH);
2033                 intel_ring_emit(engine, MI_NOOP);
2034                 intel_ring_advance(engine);
2035
2036                 /* ... and execute it. */
2037                 offset = cs_offset;
2038         }
2039
2040         ret = intel_ring_begin(req, 2);
2041         if (ret)
2042                 return ret;
2043
2044         intel_ring_emit(engine, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
2045         intel_ring_emit(engine, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
2046                                           0 : MI_BATCH_NON_SECURE));
2047         intel_ring_advance(engine);
2048
2049         return 0;
2050 }
2051
2052 static int
2053 i915_dispatch_execbuffer(struct drm_i915_gem_request *req,
2054                          u64 offset, u32 len,
2055                          unsigned dispatch_flags)
2056 {
2057         struct intel_engine_cs *engine = req->engine;
2058         int ret;
2059
2060         ret = intel_ring_begin(req, 2);
2061         if (ret)
2062                 return ret;
2063
2064         intel_ring_emit(engine, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
2065         intel_ring_emit(engine, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
2066                                           0 : MI_BATCH_NON_SECURE));
2067         intel_ring_advance(engine);
2068
2069         return 0;
2070 }
2071
2072 static void cleanup_phys_status_page(struct intel_engine_cs *engine)
2073 {
2074         struct drm_i915_private *dev_priv = engine->i915;
2075
2076         if (!dev_priv->status_page_dmah)
2077                 return;
2078
2079         drm_pci_free(dev_priv->dev, dev_priv->status_page_dmah);
2080         engine->status_page.page_addr = NULL;
2081 }
2082
2083 static void cleanup_status_page(struct intel_engine_cs *engine)
2084 {
2085         struct drm_i915_gem_object *obj;
2086
2087         obj = engine->status_page.obj;
2088         if (obj == NULL)
2089                 return;
2090
2091         kunmap(sg_page(obj->pages->sgl));
2092         i915_gem_object_ggtt_unpin(obj);
2093         drm_gem_object_unreference(&obj->base);
2094         engine->status_page.obj = NULL;
2095 }
2096
2097 static int init_status_page(struct intel_engine_cs *engine)
2098 {
2099         struct drm_i915_gem_object *obj = engine->status_page.obj;
2100
2101         if (obj == NULL) {
2102                 unsigned flags;
2103                 int ret;
2104
2105                 obj = i915_gem_object_create(engine->i915->dev, 4096);
2106                 if (IS_ERR(obj)) {
2107                         DRM_ERROR("Failed to allocate status page\n");
2108                         return PTR_ERR(obj);
2109                 }
2110
2111                 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2112                 if (ret)
2113                         goto err_unref;
2114
2115                 flags = 0;
2116                 if (!HAS_LLC(engine->i915))
2117                         /* On g33, we cannot place HWS above 256MiB, so
2118                          * restrict its pinning to the low mappable arena.
2119                          * Though this restriction is not documented for
2120                          * gen4, gen5, or byt, they also behave similarly
2121                          * and hang if the HWS is placed at the top of the
2122                          * GTT. To generalise, it appears that all !llc
2123                          * platforms have issues with us placing the HWS
2124                          * above the mappable region (even though we never
2125                          * actualy map it).
2126                          */
2127                         flags |= PIN_MAPPABLE;
2128                 ret = i915_gem_obj_ggtt_pin(obj, 4096, flags);
2129                 if (ret) {
2130 err_unref:
2131                         drm_gem_object_unreference(&obj->base);
2132                         return ret;
2133                 }
2134
2135                 engine->status_page.obj = obj;
2136         }
2137
2138         engine->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
2139         engine->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
2140         memset(engine->status_page.page_addr, 0, PAGE_SIZE);
2141
2142         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
2143                         engine->name, engine->status_page.gfx_addr);
2144
2145         return 0;
2146 }
2147
2148 static int init_phys_status_page(struct intel_engine_cs *engine)
2149 {
2150         struct drm_i915_private *dev_priv = engine->i915;
2151
2152         if (!dev_priv->status_page_dmah) {
2153                 dev_priv->status_page_dmah =
2154                         drm_pci_alloc(dev_priv->dev, PAGE_SIZE, PAGE_SIZE);
2155                 if (!dev_priv->status_page_dmah)
2156                         return -ENOMEM;
2157         }
2158
2159         engine->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
2160         memset(engine->status_page.page_addr, 0, PAGE_SIZE);
2161
2162         return 0;
2163 }
2164
2165 void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
2166 {
2167         GEM_BUG_ON(ringbuf->vma == NULL);
2168         GEM_BUG_ON(ringbuf->virtual_start == NULL);
2169
2170         if (HAS_LLC(ringbuf->obj->base.dev) && !ringbuf->obj->stolen)
2171                 i915_gem_object_unpin_map(ringbuf->obj);
2172         else
2173                 i915_vma_unpin_iomap(ringbuf->vma);
2174         ringbuf->virtual_start = NULL;
2175
2176         i915_gem_object_ggtt_unpin(ringbuf->obj);
2177         ringbuf->vma = NULL;
2178 }
2179
2180 int intel_pin_and_map_ringbuffer_obj(struct drm_i915_private *dev_priv,
2181                                      struct intel_ringbuffer *ringbuf)
2182 {
2183         struct drm_i915_gem_object *obj = ringbuf->obj;
2184         /* Ring wraparound at offset 0 sometimes hangs. No idea why. */
2185         unsigned flags = PIN_OFFSET_BIAS | 4096;
2186         void *addr;
2187         int ret;
2188
2189         if (HAS_LLC(dev_priv) && !obj->stolen) {
2190                 ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, flags);
2191                 if (ret)
2192                         return ret;
2193
2194                 ret = i915_gem_object_set_to_cpu_domain(obj, true);
2195                 if (ret)
2196                         goto err_unpin;
2197
2198                 addr = i915_gem_object_pin_map(obj);
2199                 if (IS_ERR(addr)) {
2200                         ret = PTR_ERR(addr);
2201                         goto err_unpin;
2202                 }
2203         } else {
2204                 ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE,
2205                                             flags | PIN_MAPPABLE);
2206                 if (ret)
2207                         return ret;
2208
2209                 ret = i915_gem_object_set_to_gtt_domain(obj, true);
2210                 if (ret)
2211                         goto err_unpin;
2212
2213                 /* Access through the GTT requires the device to be awake. */
2214                 assert_rpm_wakelock_held(dev_priv);
2215
2216                 addr = i915_vma_pin_iomap(i915_gem_obj_to_ggtt(obj));
2217                 if (IS_ERR(addr)) {
2218                         ret = PTR_ERR(addr);
2219                         goto err_unpin;
2220                 }
2221         }
2222
2223         ringbuf->virtual_start = addr;
2224         ringbuf->vma = i915_gem_obj_to_ggtt(obj);
2225         return 0;
2226
2227 err_unpin:
2228         i915_gem_object_ggtt_unpin(obj);
2229         return ret;
2230 }
2231
2232 static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
2233 {
2234         drm_gem_object_unreference(&ringbuf->obj->base);
2235         ringbuf->obj = NULL;
2236 }
2237
2238 static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
2239                                       struct intel_ringbuffer *ringbuf)
2240 {
2241         struct drm_i915_gem_object *obj;
2242
2243         obj = NULL;
2244         if (!HAS_LLC(dev))
2245                 obj = i915_gem_object_create_stolen(dev, ringbuf->size);
2246         if (obj == NULL)
2247                 obj = i915_gem_object_create(dev, ringbuf->size);
2248         if (IS_ERR(obj))
2249                 return PTR_ERR(obj);
2250
2251         /* mark ring buffers as read-only from GPU side by default */
2252         obj->gt_ro = 1;
2253
2254         ringbuf->obj = obj;
2255
2256         return 0;
2257 }
2258
2259 struct intel_ringbuffer *
2260 intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size)
2261 {
2262         struct intel_ringbuffer *ring;
2263         int ret;
2264
2265         ring = kzalloc(sizeof(*ring), GFP_KERNEL);
2266         if (ring == NULL) {
2267                 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n",
2268                                  engine->name);
2269                 return ERR_PTR(-ENOMEM);
2270         }
2271
2272         ring->engine = engine;
2273         list_add(&ring->link, &engine->buffers);
2274
2275         ring->size = size;
2276         /* Workaround an erratum on the i830 which causes a hang if
2277          * the TAIL pointer points to within the last 2 cachelines
2278          * of the buffer.
2279          */
2280         ring->effective_size = size;
2281         if (IS_I830(engine->i915) || IS_845G(engine->i915))
2282                 ring->effective_size -= 2 * CACHELINE_BYTES;
2283
2284         ring->last_retired_head = -1;
2285         intel_ring_update_space(ring);
2286
2287         ret = intel_alloc_ringbuffer_obj(engine->i915->dev, ring);
2288         if (ret) {
2289                 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s: %d\n",
2290                                  engine->name, ret);
2291                 list_del(&ring->link);
2292                 kfree(ring);
2293                 return ERR_PTR(ret);
2294         }
2295
2296         return ring;
2297 }
2298
2299 void
2300 intel_ringbuffer_free(struct intel_ringbuffer *ring)
2301 {
2302         intel_destroy_ringbuffer_obj(ring);
2303         list_del(&ring->link);
2304         kfree(ring);
2305 }
2306
2307 static int intel_init_ring_buffer(struct drm_device *dev,
2308                                   struct intel_engine_cs *engine)
2309 {
2310         struct drm_i915_private *dev_priv = to_i915(dev);
2311         struct intel_ringbuffer *ringbuf;
2312         int ret;
2313
2314         WARN_ON(engine->buffer);
2315
2316         engine->i915 = dev_priv;
2317         INIT_LIST_HEAD(&engine->active_list);
2318         INIT_LIST_HEAD(&engine->request_list);
2319         INIT_LIST_HEAD(&engine->execlist_queue);
2320         INIT_LIST_HEAD(&engine->buffers);
2321         i915_gem_batch_pool_init(dev, &engine->batch_pool);
2322         memset(engine->semaphore.sync_seqno, 0,
2323                sizeof(engine->semaphore.sync_seqno));
2324
2325         init_waitqueue_head(&engine->irq_queue);
2326
2327         ringbuf = intel_engine_create_ringbuffer(engine, 32 * PAGE_SIZE);
2328         if (IS_ERR(ringbuf)) {
2329                 ret = PTR_ERR(ringbuf);
2330                 goto error;
2331         }
2332         engine->buffer = ringbuf;
2333
2334         if (I915_NEED_GFX_HWS(dev_priv)) {
2335                 ret = init_status_page(engine);
2336                 if (ret)
2337                         goto error;
2338         } else {
2339                 WARN_ON(engine->id != RCS);
2340                 ret = init_phys_status_page(engine);
2341                 if (ret)
2342                         goto error;
2343         }
2344
2345         ret = intel_pin_and_map_ringbuffer_obj(dev_priv, ringbuf);
2346         if (ret) {
2347                 DRM_ERROR("Failed to pin and map ringbuffer %s: %d\n",
2348                                 engine->name, ret);
2349                 intel_destroy_ringbuffer_obj(ringbuf);
2350                 goto error;
2351         }
2352
2353         ret = i915_cmd_parser_init_ring(engine);
2354         if (ret)
2355                 goto error;
2356
2357         return 0;
2358
2359 error:
2360         intel_cleanup_engine(engine);
2361         return ret;
2362 }
2363
2364 void intel_cleanup_engine(struct intel_engine_cs *engine)
2365 {
2366         struct drm_i915_private *dev_priv;
2367
2368         if (!intel_engine_initialized(engine))
2369                 return;
2370
2371         dev_priv = engine->i915;
2372
2373         if (engine->buffer) {
2374                 intel_stop_engine(engine);
2375                 WARN_ON(!IS_GEN2(dev_priv) && (I915_READ_MODE(engine) & MODE_IDLE) == 0);
2376
2377                 intel_unpin_ringbuffer_obj(engine->buffer);
2378                 intel_ringbuffer_free(engine->buffer);
2379                 engine->buffer = NULL;
2380         }
2381
2382         if (engine->cleanup)
2383                 engine->cleanup(engine);
2384
2385         if (I915_NEED_GFX_HWS(dev_priv)) {
2386                 cleanup_status_page(engine);
2387         } else {
2388                 WARN_ON(engine->id != RCS);
2389                 cleanup_phys_status_page(engine);
2390         }
2391
2392         i915_cmd_parser_fini_ring(engine);
2393         i915_gem_batch_pool_fini(&engine->batch_pool);
2394         engine->i915 = NULL;
2395 }
2396
2397 int intel_engine_idle(struct intel_engine_cs *engine)
2398 {
2399         struct drm_i915_gem_request *req;
2400
2401         /* Wait upon the last request to be completed */
2402         if (list_empty(&engine->request_list))
2403                 return 0;
2404
2405         req = list_entry(engine->request_list.prev,
2406                          struct drm_i915_gem_request,
2407                          list);
2408
2409         /* Make sure we do not trigger any retires */
2410         return __i915_wait_request(req,
2411                                    req->i915->mm.interruptible,
2412                                    NULL, NULL);
2413 }
2414
2415 int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request)
2416 {
2417         int ret;
2418
2419         /* Flush enough space to reduce the likelihood of waiting after
2420          * we start building the request - in which case we will just
2421          * have to repeat work.
2422          */
2423         request->reserved_space += LEGACY_REQUEST_SIZE;
2424
2425         request->ringbuf = request->engine->buffer;
2426
2427         ret = intel_ring_begin(request, 0);
2428         if (ret)
2429                 return ret;
2430
2431         request->reserved_space -= LEGACY_REQUEST_SIZE;
2432         return 0;
2433 }
2434
2435 static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
2436 {
2437         struct intel_ringbuffer *ringbuf = req->ringbuf;
2438         struct intel_engine_cs *engine = req->engine;
2439         struct drm_i915_gem_request *target;
2440
2441         intel_ring_update_space(ringbuf);
2442         if (ringbuf->space >= bytes)
2443                 return 0;
2444
2445         /*
2446          * Space is reserved in the ringbuffer for finalising the request,
2447          * as that cannot be allowed to fail. During request finalisation,
2448          * reserved_space is set to 0 to stop the overallocation and the
2449          * assumption is that then we never need to wait (which has the
2450          * risk of failing with EINTR).
2451          *
2452          * See also i915_gem_request_alloc() and i915_add_request().
2453          */
2454         GEM_BUG_ON(!req->reserved_space);
2455
2456         list_for_each_entry(target, &engine->request_list, list) {
2457                 unsigned space;
2458
2459                 /*
2460                  * The request queue is per-engine, so can contain requests
2461                  * from multiple ringbuffers. Here, we must ignore any that
2462                  * aren't from the ringbuffer we're considering.
2463                  */
2464                 if (target->ringbuf != ringbuf)
2465                         continue;
2466
2467                 /* Would completion of this request free enough space? */
2468                 space = __intel_ring_space(target->postfix, ringbuf->tail,
2469                                            ringbuf->size);
2470                 if (space >= bytes)
2471                         break;
2472         }
2473
2474         if (WARN_ON(&target->list == &engine->request_list))
2475                 return -ENOSPC;
2476
2477         return i915_wait_request(target);
2478 }
2479
2480 int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
2481 {
2482         struct intel_ringbuffer *ringbuf = req->ringbuf;
2483         int remain_actual = ringbuf->size - ringbuf->tail;
2484         int remain_usable = ringbuf->effective_size - ringbuf->tail;
2485         int bytes = num_dwords * sizeof(u32);
2486         int total_bytes, wait_bytes;
2487         bool need_wrap = false;
2488
2489         total_bytes = bytes + req->reserved_space;
2490
2491         if (unlikely(bytes > remain_usable)) {
2492                 /*
2493                  * Not enough space for the basic request. So need to flush
2494                  * out the remainder and then wait for base + reserved.
2495                  */
2496                 wait_bytes = remain_actual + total_bytes;
2497                 need_wrap = true;
2498         } else if (unlikely(total_bytes > remain_usable)) {
2499                 /*
2500                  * The base request will fit but the reserved space
2501                  * falls off the end. So we don't need an immediate wrap
2502                  * and only need to effectively wait for the reserved
2503                  * size space from the start of ringbuffer.
2504                  */
2505                 wait_bytes = remain_actual + req->reserved_space;
2506         } else {
2507                 /* No wrapping required, just waiting. */
2508                 wait_bytes = total_bytes;
2509         }
2510
2511         if (wait_bytes > ringbuf->space) {
2512                 int ret = wait_for_space(req, wait_bytes);
2513                 if (unlikely(ret))
2514                         return ret;
2515
2516                 intel_ring_update_space(ringbuf);
2517                 if (unlikely(ringbuf->space < wait_bytes))
2518                         return -EAGAIN;
2519         }
2520
2521         if (unlikely(need_wrap)) {
2522                 GEM_BUG_ON(remain_actual > ringbuf->space);
2523                 GEM_BUG_ON(ringbuf->tail + remain_actual > ringbuf->size);
2524
2525                 /* Fill the tail with MI_NOOP */
2526                 memset(ringbuf->virtual_start + ringbuf->tail,
2527                        0, remain_actual);
2528                 ringbuf->tail = 0;
2529                 ringbuf->space -= remain_actual;
2530         }
2531
2532         ringbuf->space -= bytes;
2533         GEM_BUG_ON(ringbuf->space < 0);
2534         return 0;
2535 }
2536
2537 /* Align the ring tail to a cacheline boundary */
2538 int intel_ring_cacheline_align(struct drm_i915_gem_request *req)
2539 {
2540         struct intel_engine_cs *engine = req->engine;
2541         int num_dwords = (engine->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
2542         int ret;
2543
2544         if (num_dwords == 0)
2545                 return 0;
2546
2547         num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
2548         ret = intel_ring_begin(req, num_dwords);
2549         if (ret)
2550                 return ret;
2551
2552         while (num_dwords--)
2553                 intel_ring_emit(engine, MI_NOOP);
2554
2555         intel_ring_advance(engine);
2556
2557         return 0;
2558 }
2559
2560 void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno)
2561 {
2562         struct drm_i915_private *dev_priv = engine->i915;
2563
2564         /* Our semaphore implementation is strictly monotonic (i.e. we proceed
2565          * so long as the semaphore value in the register/page is greater
2566          * than the sync value), so whenever we reset the seqno,
2567          * so long as we reset the tracking semaphore value to 0, it will
2568          * always be before the next request's seqno. If we don't reset
2569          * the semaphore value, then when the seqno moves backwards all
2570          * future waits will complete instantly (causing rendering corruption).
2571          */
2572         if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
2573                 I915_WRITE(RING_SYNC_0(engine->mmio_base), 0);
2574                 I915_WRITE(RING_SYNC_1(engine->mmio_base), 0);
2575                 if (HAS_VEBOX(dev_priv))
2576                         I915_WRITE(RING_SYNC_2(engine->mmio_base), 0);
2577         }
2578         if (dev_priv->semaphore_obj) {
2579                 struct drm_i915_gem_object *obj = dev_priv->semaphore_obj;
2580                 struct page *page = i915_gem_object_get_dirty_page(obj, 0);
2581                 void *semaphores = kmap(page);
2582                 memset(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
2583                        0, I915_NUM_ENGINES * gen8_semaphore_seqno_size);
2584                 kunmap(page);
2585         }
2586         memset(engine->semaphore.sync_seqno, 0,
2587                sizeof(engine->semaphore.sync_seqno));
2588
2589         engine->set_seqno(engine, seqno);
2590         engine->last_submitted_seqno = seqno;
2591
2592         engine->hangcheck.seqno = seqno;
2593 }
2594
2595 static void gen6_bsd_ring_write_tail(struct intel_engine_cs *engine,
2596                                      u32 value)
2597 {
2598         struct drm_i915_private *dev_priv = engine->i915;
2599
2600        /* Every tail move must follow the sequence below */
2601
2602         /* Disable notification that the ring is IDLE. The GT
2603          * will then assume that it is busy and bring it out of rc6.
2604          */
2605         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
2606                    _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2607
2608         /* Clear the context id. Here be magic! */
2609         I915_WRITE64(GEN6_BSD_RNCID, 0x0);
2610
2611         /* Wait for the ring not to be idle, i.e. for it to wake up. */
2612         if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
2613                       GEN6_BSD_SLEEP_INDICATOR) == 0,
2614                      50))
2615                 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2616
2617         /* Now that the ring is fully powered up, update the tail */
2618         I915_WRITE_TAIL(engine, value);
2619         POSTING_READ(RING_TAIL(engine->mmio_base));
2620
2621         /* Let the ring send IDLE messages to the GT again,
2622          * and so let it sleep to conserve power when idle.
2623          */
2624         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
2625                    _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2626 }
2627
2628 static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
2629                                u32 invalidate, u32 flush)
2630 {
2631         struct intel_engine_cs *engine = req->engine;
2632         uint32_t cmd;
2633         int ret;
2634
2635         ret = intel_ring_begin(req, 4);
2636         if (ret)
2637                 return ret;
2638
2639         cmd = MI_FLUSH_DW;
2640         if (INTEL_GEN(req->i915) >= 8)
2641                 cmd += 1;
2642
2643         /* We always require a command barrier so that subsequent
2644          * commands, such as breadcrumb interrupts, are strictly ordered
2645          * wrt the contents of the write cache being flushed to memory
2646          * (and thus being coherent from the CPU).
2647          */
2648         cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
2649
2650         /*
2651          * Bspec vol 1c.5 - video engine command streamer:
2652          * "If ENABLED, all TLBs will be invalidated once the flush
2653          * operation is complete. This bit is only valid when the
2654          * Post-Sync Operation field is a value of 1h or 3h."
2655          */
2656         if (invalidate & I915_GEM_GPU_DOMAINS)
2657                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
2658
2659         intel_ring_emit(engine, cmd);
2660         intel_ring_emit(engine,
2661                         I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
2662         if (INTEL_GEN(req->i915) >= 8) {
2663                 intel_ring_emit(engine, 0); /* upper addr */
2664                 intel_ring_emit(engine, 0); /* value */
2665         } else  {
2666                 intel_ring_emit(engine, 0);
2667                 intel_ring_emit(engine, MI_NOOP);
2668         }
2669         intel_ring_advance(engine);
2670         return 0;
2671 }
2672
2673 static int
2674 gen8_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
2675                               u64 offset, u32 len,
2676                               unsigned dispatch_flags)
2677 {
2678         struct intel_engine_cs *engine = req->engine;
2679         bool ppgtt = USES_PPGTT(engine->dev) &&
2680                         !(dispatch_flags & I915_DISPATCH_SECURE);
2681         int ret;
2682
2683         ret = intel_ring_begin(req, 4);
2684         if (ret)
2685                 return ret;
2686
2687         /* FIXME(BDW): Address space and security selectors. */
2688         intel_ring_emit(engine, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8) |
2689                         (dispatch_flags & I915_DISPATCH_RS ?
2690                          MI_BATCH_RESOURCE_STREAMER : 0));
2691         intel_ring_emit(engine, lower_32_bits(offset));
2692         intel_ring_emit(engine, upper_32_bits(offset));
2693         intel_ring_emit(engine, MI_NOOP);
2694         intel_ring_advance(engine);
2695
2696         return 0;
2697 }
2698
2699 static int
2700 hsw_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
2701                              u64 offset, u32 len,
2702                              unsigned dispatch_flags)
2703 {
2704         struct intel_engine_cs *engine = req->engine;
2705         int ret;
2706
2707         ret = intel_ring_begin(req, 2);
2708         if (ret)
2709                 return ret;
2710
2711         intel_ring_emit(engine,
2712                         MI_BATCH_BUFFER_START |
2713                         (dispatch_flags & I915_DISPATCH_SECURE ?
2714                          0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW) |
2715                         (dispatch_flags & I915_DISPATCH_RS ?
2716                          MI_BATCH_RESOURCE_STREAMER : 0));
2717         /* bit0-7 is the length on GEN6+ */
2718         intel_ring_emit(engine, offset);
2719         intel_ring_advance(engine);
2720
2721         return 0;
2722 }
2723
2724 static int
2725 gen6_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
2726                               u64 offset, u32 len,
2727                               unsigned dispatch_flags)
2728 {
2729         struct intel_engine_cs *engine = req->engine;
2730         int ret;
2731
2732         ret = intel_ring_begin(req, 2);
2733         if (ret)
2734                 return ret;
2735
2736         intel_ring_emit(engine,
2737                         MI_BATCH_BUFFER_START |
2738                         (dispatch_flags & I915_DISPATCH_SECURE ?
2739                          0 : MI_BATCH_NON_SECURE_I965));
2740         /* bit0-7 is the length on GEN6+ */
2741         intel_ring_emit(engine, offset);
2742         intel_ring_advance(engine);
2743
2744         return 0;
2745 }
2746
2747 /* Blitter support (SandyBridge+) */
2748
2749 static int gen6_ring_flush(struct drm_i915_gem_request *req,
2750                            u32 invalidate, u32 flush)
2751 {
2752         struct intel_engine_cs *engine = req->engine;
2753         uint32_t cmd;
2754         int ret;
2755
2756         ret = intel_ring_begin(req, 4);
2757         if (ret)
2758                 return ret;
2759
2760         cmd = MI_FLUSH_DW;
2761         if (INTEL_GEN(req->i915) >= 8)
2762                 cmd += 1;
2763
2764         /* We always require a command barrier so that subsequent
2765          * commands, such as breadcrumb interrupts, are strictly ordered
2766          * wrt the contents of the write cache being flushed to memory
2767          * (and thus being coherent from the CPU).
2768          */
2769         cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
2770
2771         /*
2772          * Bspec vol 1c.3 - blitter engine command streamer:
2773          * "If ENABLED, all TLBs will be invalidated once the flush
2774          * operation is complete. This bit is only valid when the
2775          * Post-Sync Operation field is a value of 1h or 3h."
2776          */
2777         if (invalidate & I915_GEM_DOMAIN_RENDER)
2778                 cmd |= MI_INVALIDATE_TLB;
2779         intel_ring_emit(engine, cmd);
2780         intel_ring_emit(engine,
2781                         I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
2782         if (INTEL_GEN(req->i915) >= 8) {
2783                 intel_ring_emit(engine, 0); /* upper addr */
2784                 intel_ring_emit(engine, 0); /* value */
2785         } else  {
2786                 intel_ring_emit(engine, 0);
2787                 intel_ring_emit(engine, MI_NOOP);
2788         }
2789         intel_ring_advance(engine);
2790
2791         return 0;
2792 }
2793
2794 int intel_init_render_ring_buffer(struct drm_device *dev)
2795 {
2796         struct drm_i915_private *dev_priv = dev->dev_private;
2797         struct intel_engine_cs *engine = &dev_priv->engine[RCS];
2798         struct drm_i915_gem_object *obj;
2799         int ret;
2800
2801         engine->name = "render ring";
2802         engine->id = RCS;
2803         engine->exec_id = I915_EXEC_RENDER;
2804         engine->hw_id = 0;
2805         engine->mmio_base = RENDER_RING_BASE;
2806
2807         if (INTEL_GEN(dev_priv) >= 8) {
2808                 if (i915_semaphore_is_enabled(dev_priv)) {
2809                         obj = i915_gem_object_create(dev, 4096);
2810                         if (IS_ERR(obj)) {
2811                                 DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
2812                                 i915.semaphores = 0;
2813                         } else {
2814                                 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2815                                 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
2816                                 if (ret != 0) {
2817                                         drm_gem_object_unreference(&obj->base);
2818                                         DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
2819                                         i915.semaphores = 0;
2820                                 } else
2821                                         dev_priv->semaphore_obj = obj;
2822                         }
2823                 }
2824
2825                 engine->init_context = intel_rcs_ctx_init;
2826                 engine->add_request = gen8_render_add_request;
2827                 engine->flush = gen8_render_ring_flush;
2828                 engine->irq_get = gen8_ring_get_irq;
2829                 engine->irq_put = gen8_ring_put_irq;
2830                 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2831                 engine->get_seqno = ring_get_seqno;
2832                 engine->set_seqno = ring_set_seqno;
2833                 if (i915_semaphore_is_enabled(dev_priv)) {
2834                         WARN_ON(!dev_priv->semaphore_obj);
2835                         engine->semaphore.sync_to = gen8_ring_sync;
2836                         engine->semaphore.signal = gen8_rcs_signal;
2837                         GEN8_RING_SEMAPHORE_INIT(engine);
2838                 }
2839         } else if (INTEL_GEN(dev_priv) >= 6) {
2840                 engine->init_context = intel_rcs_ctx_init;
2841                 engine->add_request = gen6_add_request;
2842                 engine->flush = gen7_render_ring_flush;
2843                 if (IS_GEN6(dev_priv))
2844                         engine->flush = gen6_render_ring_flush;
2845                 engine->irq_get = gen6_ring_get_irq;
2846                 engine->irq_put = gen6_ring_put_irq;
2847                 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2848                 engine->irq_seqno_barrier = gen6_seqno_barrier;
2849                 engine->get_seqno = ring_get_seqno;
2850                 engine->set_seqno = ring_set_seqno;
2851                 if (i915_semaphore_is_enabled(dev_priv)) {
2852                         engine->semaphore.sync_to = gen6_ring_sync;
2853                         engine->semaphore.signal = gen6_signal;
2854                         /*
2855                          * The current semaphore is only applied on pre-gen8
2856                          * platform.  And there is no VCS2 ring on the pre-gen8
2857                          * platform. So the semaphore between RCS and VCS2 is
2858                          * initialized as INVALID.  Gen8 will initialize the
2859                          * sema between VCS2 and RCS later.
2860                          */
2861                         engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
2862                         engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
2863                         engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
2864                         engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
2865                         engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2866                         engine->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
2867                         engine->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
2868                         engine->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
2869                         engine->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
2870                         engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2871                 }
2872         } else if (IS_GEN5(dev_priv)) {
2873                 engine->add_request = pc_render_add_request;
2874                 engine->flush = gen4_render_ring_flush;
2875                 engine->get_seqno = pc_render_get_seqno;
2876                 engine->set_seqno = pc_render_set_seqno;
2877                 engine->irq_get = gen5_ring_get_irq;
2878                 engine->irq_put = gen5_ring_put_irq;
2879                 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
2880                                         GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
2881         } else {
2882                 engine->add_request = i9xx_add_request;
2883                 if (INTEL_GEN(dev_priv) < 4)
2884                         engine->flush = gen2_render_ring_flush;
2885                 else
2886                         engine->flush = gen4_render_ring_flush;
2887                 engine->get_seqno = ring_get_seqno;
2888                 engine->set_seqno = ring_set_seqno;
2889                 if (IS_GEN2(dev_priv)) {
2890                         engine->irq_get = i8xx_ring_get_irq;
2891                         engine->irq_put = i8xx_ring_put_irq;
2892                 } else {
2893                         engine->irq_get = i9xx_ring_get_irq;
2894                         engine->irq_put = i9xx_ring_put_irq;
2895                 }
2896                 engine->irq_enable_mask = I915_USER_INTERRUPT;
2897         }
2898         engine->write_tail = ring_write_tail;
2899
2900         if (IS_HASWELL(dev_priv))
2901                 engine->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
2902         else if (IS_GEN8(dev_priv))
2903                 engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2904         else if (INTEL_GEN(dev_priv) >= 6)
2905                 engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2906         else if (INTEL_GEN(dev_priv) >= 4)
2907                 engine->dispatch_execbuffer = i965_dispatch_execbuffer;
2908         else if (IS_I830(dev_priv) || IS_845G(dev_priv))
2909                 engine->dispatch_execbuffer = i830_dispatch_execbuffer;
2910         else
2911                 engine->dispatch_execbuffer = i915_dispatch_execbuffer;
2912         engine->init_hw = init_render_ring;
2913         engine->cleanup = render_ring_cleanup;
2914
2915         /* Workaround batchbuffer to combat CS tlb bug. */
2916         if (HAS_BROKEN_CS_TLB(dev_priv)) {
2917                 obj = i915_gem_object_create(dev, I830_WA_SIZE);
2918                 if (IS_ERR(obj)) {
2919                         DRM_ERROR("Failed to allocate batch bo\n");
2920                         return PTR_ERR(obj);
2921                 }
2922
2923                 ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
2924                 if (ret != 0) {
2925                         drm_gem_object_unreference(&obj->base);
2926                         DRM_ERROR("Failed to ping batch bo\n");
2927                         return ret;
2928                 }
2929
2930                 engine->scratch.obj = obj;
2931                 engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
2932         }
2933
2934         ret = intel_init_ring_buffer(dev, engine);
2935         if (ret)
2936                 return ret;
2937
2938         if (INTEL_GEN(dev_priv) >= 5) {
2939                 ret = intel_init_pipe_control(engine);
2940                 if (ret)
2941                         return ret;
2942         }
2943
2944         return 0;
2945 }
2946
2947 int intel_init_bsd_ring_buffer(struct drm_device *dev)
2948 {
2949         struct drm_i915_private *dev_priv = dev->dev_private;
2950         struct intel_engine_cs *engine = &dev_priv->engine[VCS];
2951
2952         engine->name = "bsd ring";
2953         engine->id = VCS;
2954         engine->exec_id = I915_EXEC_BSD;
2955         engine->hw_id = 1;
2956
2957         engine->write_tail = ring_write_tail;
2958         if (INTEL_GEN(dev_priv) >= 6) {
2959                 engine->mmio_base = GEN6_BSD_RING_BASE;
2960                 /* gen6 bsd needs a special wa for tail updates */
2961                 if (IS_GEN6(dev_priv))
2962                         engine->write_tail = gen6_bsd_ring_write_tail;
2963                 engine->flush = gen6_bsd_ring_flush;
2964                 engine->add_request = gen6_add_request;
2965                 engine->irq_seqno_barrier = gen6_seqno_barrier;
2966                 engine->get_seqno = ring_get_seqno;
2967                 engine->set_seqno = ring_set_seqno;
2968                 if (INTEL_GEN(dev_priv) >= 8) {
2969                         engine->irq_enable_mask =
2970                                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
2971                         engine->irq_get = gen8_ring_get_irq;
2972                         engine->irq_put = gen8_ring_put_irq;
2973                         engine->dispatch_execbuffer =
2974                                 gen8_ring_dispatch_execbuffer;
2975                         if (i915_semaphore_is_enabled(dev_priv)) {
2976                                 engine->semaphore.sync_to = gen8_ring_sync;
2977                                 engine->semaphore.signal = gen8_xcs_signal;
2978                                 GEN8_RING_SEMAPHORE_INIT(engine);
2979                         }
2980                 } else {
2981                         engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
2982                         engine->irq_get = gen6_ring_get_irq;
2983                         engine->irq_put = gen6_ring_put_irq;
2984                         engine->dispatch_execbuffer =
2985                                 gen6_ring_dispatch_execbuffer;
2986                         if (i915_semaphore_is_enabled(dev_priv)) {
2987                                 engine->semaphore.sync_to = gen6_ring_sync;
2988                                 engine->semaphore.signal = gen6_signal;
2989                                 engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
2990                                 engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
2991                                 engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
2992                                 engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
2993                                 engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2994                                 engine->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
2995                                 engine->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
2996                                 engine->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
2997                                 engine->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
2998                                 engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2999                         }
3000                 }
3001         } else {
3002                 engine->mmio_base = BSD_RING_BASE;
3003                 engine->flush = bsd_ring_flush;
3004                 engine->add_request = i9xx_add_request;
3005                 engine->get_seqno = ring_get_seqno;
3006                 engine->set_seqno = ring_set_seqno;
3007                 if (IS_GEN5(dev_priv)) {
3008                         engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
3009                         engine->irq_get = gen5_ring_get_irq;
3010                         engine->irq_put = gen5_ring_put_irq;
3011                 } else {
3012                         engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
3013                         engine->irq_get = i9xx_ring_get_irq;
3014                         engine->irq_put = i9xx_ring_put_irq;
3015                 }
3016                 engine->dispatch_execbuffer = i965_dispatch_execbuffer;
3017         }
3018         engine->init_hw = init_ring_common;
3019
3020         return intel_init_ring_buffer(dev, engine);
3021 }
3022
3023 /**
3024  * Initialize the second BSD ring (eg. Broadwell GT3, Skylake GT3)
3025  */
3026 int intel_init_bsd2_ring_buffer(struct drm_device *dev)
3027 {
3028         struct drm_i915_private *dev_priv = dev->dev_private;
3029         struct intel_engine_cs *engine = &dev_priv->engine[VCS2];
3030
3031         engine->name = "bsd2 ring";
3032         engine->id = VCS2;
3033         engine->exec_id = I915_EXEC_BSD;
3034         engine->hw_id = 4;
3035
3036         engine->write_tail = ring_write_tail;
3037         engine->mmio_base = GEN8_BSD2_RING_BASE;
3038         engine->flush = gen6_bsd_ring_flush;
3039         engine->add_request = gen6_add_request;
3040         engine->irq_seqno_barrier = gen6_seqno_barrier;
3041         engine->get_seqno = ring_get_seqno;
3042         engine->set_seqno = ring_set_seqno;
3043         engine->irq_enable_mask =
3044                         GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
3045         engine->irq_get = gen8_ring_get_irq;
3046         engine->irq_put = gen8_ring_put_irq;
3047         engine->dispatch_execbuffer =
3048                         gen8_ring_dispatch_execbuffer;
3049         if (i915_semaphore_is_enabled(dev_priv)) {
3050                 engine->semaphore.sync_to = gen8_ring_sync;
3051                 engine->semaphore.signal = gen8_xcs_signal;
3052                 GEN8_RING_SEMAPHORE_INIT(engine);
3053         }
3054         engine->init_hw = init_ring_common;
3055
3056         return intel_init_ring_buffer(dev, engine);
3057 }
3058
3059 int intel_init_blt_ring_buffer(struct drm_device *dev)
3060 {
3061         struct drm_i915_private *dev_priv = dev->dev_private;
3062         struct intel_engine_cs *engine = &dev_priv->engine[BCS];
3063
3064         engine->name = "blitter ring";
3065         engine->id = BCS;
3066         engine->exec_id = I915_EXEC_BLT;
3067         engine->hw_id = 2;
3068
3069         engine->mmio_base = BLT_RING_BASE;
3070         engine->write_tail = ring_write_tail;
3071         engine->flush = gen6_ring_flush;
3072         engine->add_request = gen6_add_request;
3073         engine->irq_seqno_barrier = gen6_seqno_barrier;
3074         engine->get_seqno = ring_get_seqno;
3075         engine->set_seqno = ring_set_seqno;
3076         if (INTEL_GEN(dev_priv) >= 8) {
3077                 engine->irq_enable_mask =
3078                         GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
3079                 engine->irq_get = gen8_ring_get_irq;
3080                 engine->irq_put = gen8_ring_put_irq;
3081                 engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
3082                 if (i915_semaphore_is_enabled(dev_priv)) {
3083                         engine->semaphore.sync_to = gen8_ring_sync;
3084                         engine->semaphore.signal = gen8_xcs_signal;
3085                         GEN8_RING_SEMAPHORE_INIT(engine);
3086                 }
3087         } else {
3088                 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
3089                 engine->irq_get = gen6_ring_get_irq;
3090                 engine->irq_put = gen6_ring_put_irq;
3091                 engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
3092                 if (i915_semaphore_is_enabled(dev_priv)) {
3093                         engine->semaphore.signal = gen6_signal;
3094                         engine->semaphore.sync_to = gen6_ring_sync;
3095                         /*
3096                          * The current semaphore is only applied on pre-gen8
3097                          * platform.  And there is no VCS2 ring on the pre-gen8
3098                          * platform. So the semaphore between BCS and VCS2 is
3099                          * initialized as INVALID.  Gen8 will initialize the
3100                          * sema between BCS and VCS2 later.
3101                          */
3102                         engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
3103                         engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
3104                         engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
3105                         engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
3106                         engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
3107                         engine->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
3108                         engine->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
3109                         engine->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
3110                         engine->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
3111                         engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
3112                 }
3113         }
3114         engine->init_hw = init_ring_common;
3115
3116         return intel_init_ring_buffer(dev, engine);
3117 }
3118
3119 int intel_init_vebox_ring_buffer(struct drm_device *dev)
3120 {
3121         struct drm_i915_private *dev_priv = dev->dev_private;
3122         struct intel_engine_cs *engine = &dev_priv->engine[VECS];
3123
3124         engine->name = "video enhancement ring";
3125         engine->id = VECS;
3126         engine->exec_id = I915_EXEC_VEBOX;
3127         engine->hw_id = 3;
3128
3129         engine->mmio_base = VEBOX_RING_BASE;
3130         engine->write_tail = ring_write_tail;
3131         engine->flush = gen6_ring_flush;
3132         engine->add_request = gen6_add_request;
3133         engine->irq_seqno_barrier = gen6_seqno_barrier;
3134         engine->get_seqno = ring_get_seqno;
3135         engine->set_seqno = ring_set_seqno;
3136
3137         if (INTEL_GEN(dev_priv) >= 8) {
3138                 engine->irq_enable_mask =
3139                         GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
3140                 engine->irq_get = gen8_ring_get_irq;
3141                 engine->irq_put = gen8_ring_put_irq;
3142                 engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
3143                 if (i915_semaphore_is_enabled(dev_priv)) {
3144                         engine->semaphore.sync_to = gen8_ring_sync;
3145                         engine->semaphore.signal = gen8_xcs_signal;
3146                         GEN8_RING_SEMAPHORE_INIT(engine);
3147                 }
3148         } else {
3149                 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
3150                 engine->irq_get = hsw_vebox_get_irq;
3151                 engine->irq_put = hsw_vebox_put_irq;
3152                 engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
3153                 if (i915_semaphore_is_enabled(dev_priv)) {
3154                         engine->semaphore.sync_to = gen6_ring_sync;
3155                         engine->semaphore.signal = gen6_signal;
3156                         engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
3157                         engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
3158                         engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
3159                         engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
3160                         engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
3161                         engine->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
3162                         engine->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
3163                         engine->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
3164                         engine->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
3165                         engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
3166                 }
3167         }
3168         engine->init_hw = init_ring_common;
3169
3170         return intel_init_ring_buffer(dev, engine);
3171 }
3172
3173 int
3174 intel_ring_flush_all_caches(struct drm_i915_gem_request *req)
3175 {
3176         struct intel_engine_cs *engine = req->engine;
3177         int ret;
3178
3179         if (!engine->gpu_caches_dirty)
3180                 return 0;
3181
3182         ret = engine->flush(req, 0, I915_GEM_GPU_DOMAINS);
3183         if (ret)
3184                 return ret;
3185
3186         trace_i915_gem_ring_flush(req, 0, I915_GEM_GPU_DOMAINS);
3187
3188         engine->gpu_caches_dirty = false;
3189         return 0;
3190 }
3191
3192 int
3193 intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req)
3194 {
3195         struct intel_engine_cs *engine = req->engine;
3196         uint32_t flush_domains;
3197         int ret;
3198
3199         flush_domains = 0;
3200         if (engine->gpu_caches_dirty)
3201                 flush_domains = I915_GEM_GPU_DOMAINS;
3202
3203         ret = engine->flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
3204         if (ret)
3205                 return ret;
3206
3207         trace_i915_gem_ring_flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
3208
3209         engine->gpu_caches_dirty = false;
3210         return 0;
3211 }
3212
3213 void
3214 intel_stop_engine(struct intel_engine_cs *engine)
3215 {
3216         int ret;
3217
3218         if (!intel_engine_initialized(engine))
3219                 return;
3220
3221         ret = intel_engine_idle(engine);
3222         if (ret)
3223                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
3224                           engine->name, ret);
3225
3226         stop_ring(engine);
3227 }