drivers/gpu/drm/amd/amdgpu/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include "drmP.h"
  28 #include "amdgpu.h"
  29 #include "amdgpu_atombios.h"
  30 #include "amdgpu_ih.h"
  31 #include "amdgpu_uvd.h"
  32 #include "amdgpu_vce.h"
  33 #include "cikd.h"
  34 #include "atom.h"
  35 #include "amd_pcie.h"
  36
  37 #include "cik.h"
  38 #include "gmc_v7_0.h"
  39 #include "cik_ih.h"
  40 #include "dce_v8_0.h"
  41 #include "gfx_v7_0.h"
  42 #include "cik_sdma.h"
  43 #include "uvd_v4_2.h"
  44 #include "vce_v2_0.h"
  45 #include "cik_dpm.h"
  46
  47 #include "uvd/uvd_4_2_d.h"
  48
  49 #include "smu/smu_7_0_1_d.h"
  50 #include "smu/smu_7_0_1_sh_mask.h"
  51
  52 #include "dce/dce_8_0_d.h"
  53 #include "dce/dce_8_0_sh_mask.h"
  54
  55 #include "bif/bif_4_1_d.h"
  56 #include "bif/bif_4_1_sh_mask.h"
  57
  58 #include "gca/gfx_7_2_d.h"
  59 #include "gca/gfx_7_2_enum.h"
  60 #include "gca/gfx_7_2_sh_mask.h"
  61
  62 #include "gmc/gmc_7_1_d.h"
  63 #include "gmc/gmc_7_1_sh_mask.h"
  64
  65 #include "oss/oss_2_0_d.h"
  66 #include "oss/oss_2_0_sh_mask.h"
  67
  68 #include "amdgpu_amdkfd.h"
  69 #include "amdgpu_powerplay.h"
  70
  71 /*
  72  * Indirect registers accessor
  73  */
  74 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
  75 {
  76         unsigned long flags;
  77         u32 r;
  78
  79         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  80         WREG32(mmPCIE_INDEX, reg);
  81         (void)RREG32(mmPCIE_INDEX);
  82         r = RREG32(mmPCIE_DATA);
  83         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  84         return r;
  85 }
  86
  87 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
  88 {
  89         unsigned long flags;
  90
  91         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  92         WREG32(mmPCIE_INDEX, reg);
  93         (void)RREG32(mmPCIE_INDEX);
  94         WREG32(mmPCIE_DATA, v);
  95         (void)RREG32(mmPCIE_DATA);
  96         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  97 }
  98
  99 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
 100 {
 101         unsigned long flags;
 102         u32 r;
 103
 104         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 105         WREG32(mmSMC_IND_INDEX_0, (reg));
 106         r = RREG32(mmSMC_IND_DATA_0);
 107         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 108         return r;
 109 }
 110
 111 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 112 {
 113         unsigned long flags;
 114
 115         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 116         WREG32(mmSMC_IND_INDEX_0, (reg));
 117         WREG32(mmSMC_IND_DATA_0, (v));
 118         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 119 }
 120
 121 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 122 {
 123         unsigned long flags;
 124         u32 r;
 125
 126         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 127         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 128         r = RREG32(mmUVD_CTX_DATA);
 129         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 130         return r;
 131 }
 132
 133 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 134 {
 135         unsigned long flags;
 136
 137         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 138         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 139         WREG32(mmUVD_CTX_DATA, (v));
 140         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 141 }
 142
 143 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 144 {
 145         unsigned long flags;
 146         u32 r;
 147
 148         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 149         WREG32(mmDIDT_IND_INDEX, (reg));
 150         r = RREG32(mmDIDT_IND_DATA);
 151         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 152         return r;
 153 }
 154
 155 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 156 {
 157         unsigned long flags;
 158
 159         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 160         WREG32(mmDIDT_IND_INDEX, (reg));
 161         WREG32(mmDIDT_IND_DATA, (v));
 162         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 163 }
 164
 165 static const u32 bonaire_golden_spm_registers[] =
 166 {
 167         0xc200, 0xe0ffffff, 0xe0000000
 168 };
 169
 170 static const u32 bonaire_golden_common_registers[] =
 171 {
 172         0x31dc, 0xffffffff, 0x00000800,
 173         0x31dd, 0xffffffff, 0x00000800,
 174         0x31e6, 0xffffffff, 0x00007fbf,
 175         0x31e7, 0xffffffff, 0x00007faf
 176 };
 177
 178 static const u32 bonaire_golden_registers[] =
 179 {
 180         0xcd5, 0x00000333, 0x00000333,
 181         0xcd4, 0x000c0fc0, 0x00040200,
 182         0x2684, 0x00010000, 0x00058208,
 183         0xf000, 0xffff1fff, 0x00140000,
 184         0xf080, 0xfdfc0fff, 0x00000100,
 185         0xf08d, 0x40000000, 0x40000200,
 186         0x260c, 0xffffffff, 0x00000000,
 187         0x260d, 0xf00fffff, 0x00000400,
 188         0x260e, 0x0002021c, 0x00020200,
 189         0x31e, 0x00000080, 0x00000000,
 190         0x16ec, 0x000000f0, 0x00000070,
 191         0x16f0, 0xf0311fff, 0x80300000,
 192         0x263e, 0x73773777, 0x12010001,
 193         0xd43, 0x00810000, 0x408af000,
 194         0x1c0c, 0x31000111, 0x00000011,
 195         0xbd2, 0x73773777, 0x12010001,
 196         0x883, 0x00007fb6, 0x0021a1b1,
 197         0x884, 0x00007fb6, 0x002021b1,
 198         0x860, 0x00007fb6, 0x00002191,
 199         0x886, 0x00007fb6, 0x002121b1,
 200         0x887, 0x00007fb6, 0x002021b1,
 201         0x877, 0x00007fb6, 0x00002191,
 202         0x878, 0x00007fb6, 0x00002191,
 203         0xd8a, 0x0000003f, 0x0000000a,
 204         0xd8b, 0x0000003f, 0x0000000a,
 205         0xab9, 0x00073ffe, 0x000022a2,
 206         0x903, 0x000007ff, 0x00000000,
 207         0x2285, 0xf000003f, 0x00000007,
 208         0x22fc, 0x00002001, 0x00000001,
 209         0x22c9, 0xffffffff, 0x00ffffff,
 210         0xc281, 0x0000ff0f, 0x00000000,
 211         0xa293, 0x07ffffff, 0x06000000,
 212         0x136, 0x00000fff, 0x00000100,
 213         0xf9e, 0x00000001, 0x00000002,
 214         0x2440, 0x03000000, 0x0362c688,
 215         0x2300, 0x000000ff, 0x00000001,
 216         0x390, 0x00001fff, 0x00001fff,
 217         0x2418, 0x0000007f, 0x00000020,
 218         0x2542, 0x00010000, 0x00010000,
 219         0x2b05, 0x000003ff, 0x000000f3,
 220         0x2b03, 0xffffffff, 0x00001032
 221 };
 222
 223 static const u32 bonaire_mgcg_cgcg_init[] =
 224 {
 225         0x3108, 0xffffffff, 0xfffffffc,
 226         0xc200, 0xffffffff, 0xe0000000,
 227         0xf0a8, 0xffffffff, 0x00000100,
 228         0xf082, 0xffffffff, 0x00000100,
 229         0xf0b0, 0xffffffff, 0xc0000100,
 230         0xf0b2, 0xffffffff, 0xc0000100,
 231         0xf0b1, 0xffffffff, 0xc0000100,
 232         0x1579, 0xffffffff, 0x00600100,
 233         0xf0a0, 0xffffffff, 0x00000100,
 234         0xf085, 0xffffffff, 0x06000100,
 235         0xf088, 0xffffffff, 0x00000100,
 236         0xf086, 0xffffffff, 0x06000100,
 237         0xf081, 0xffffffff, 0x00000100,
 238         0xf0b8, 0xffffffff, 0x00000100,
 239         0xf089, 0xffffffff, 0x00000100,
 240         0xf080, 0xffffffff, 0x00000100,
 241         0xf08c, 0xffffffff, 0x00000100,
 242         0xf08d, 0xffffffff, 0x00000100,
 243         0xf094, 0xffffffff, 0x00000100,
 244         0xf095, 0xffffffff, 0x00000100,
 245         0xf096, 0xffffffff, 0x00000100,
 246         0xf097, 0xffffffff, 0x00000100,
 247         0xf098, 0xffffffff, 0x00000100,
 248         0xf09f, 0xffffffff, 0x00000100,
 249         0xf09e, 0xffffffff, 0x00000100,
 250         0xf084, 0xffffffff, 0x06000100,
 251         0xf0a4, 0xffffffff, 0x00000100,
 252         0xf09d, 0xffffffff, 0x00000100,
 253         0xf0ad, 0xffffffff, 0x00000100,
 254         0xf0ac, 0xffffffff, 0x00000100,
 255         0xf09c, 0xffffffff, 0x00000100,
 256         0xc200, 0xffffffff, 0xe0000000,
 257         0xf008, 0xffffffff, 0x00010000,
 258         0xf009, 0xffffffff, 0x00030002,
 259         0xf00a, 0xffffffff, 0x00040007,
 260         0xf00b, 0xffffffff, 0x00060005,
 261         0xf00c, 0xffffffff, 0x00090008,
 262         0xf00d, 0xffffffff, 0x00010000,
 263         0xf00e, 0xffffffff, 0x00030002,
 264         0xf00f, 0xffffffff, 0x00040007,
 265         0xf010, 0xffffffff, 0x00060005,
 266         0xf011, 0xffffffff, 0x00090008,
 267         0xf012, 0xffffffff, 0x00010000,
 268         0xf013, 0xffffffff, 0x00030002,
 269         0xf014, 0xffffffff, 0x00040007,
 270         0xf015, 0xffffffff, 0x00060005,
 271         0xf016, 0xffffffff, 0x00090008,
 272         0xf017, 0xffffffff, 0x00010000,
 273         0xf018, 0xffffffff, 0x00030002,
 274         0xf019, 0xffffffff, 0x00040007,
 275         0xf01a, 0xffffffff, 0x00060005,
 276         0xf01b, 0xffffffff, 0x00090008,
 277         0xf01c, 0xffffffff, 0x00010000,
 278         0xf01d, 0xffffffff, 0x00030002,
 279         0xf01e, 0xffffffff, 0x00040007,
 280         0xf01f, 0xffffffff, 0x00060005,
 281         0xf020, 0xffffffff, 0x00090008,
 282         0xf021, 0xffffffff, 0x00010000,
 283         0xf022, 0xffffffff, 0x00030002,
 284         0xf023, 0xffffffff, 0x00040007,
 285         0xf024, 0xffffffff, 0x00060005,
 286         0xf025, 0xffffffff, 0x00090008,
 287         0xf026, 0xffffffff, 0x00010000,
 288         0xf027, 0xffffffff, 0x00030002,
 289         0xf028, 0xffffffff, 0x00040007,
 290         0xf029, 0xffffffff, 0x00060005,
 291         0xf02a, 0xffffffff, 0x00090008,
 292         0xf000, 0xffffffff, 0x96e00200,
 293         0x21c2, 0xffffffff, 0x00900100,
 294         0x3109, 0xffffffff, 0x0020003f,
 295         0xe, 0xffffffff, 0x0140001c,
 296         0xf, 0x000f0000, 0x000f0000,
 297         0x88, 0xffffffff, 0xc060000c,
 298         0x89, 0xc0000fff, 0x00000100,
 299         0x3e4, 0xffffffff, 0x00000100,
 300         0x3e6, 0x00000101, 0x00000000,
 301         0x82a, 0xffffffff, 0x00000104,
 302         0x1579, 0xff000fff, 0x00000100,
 303         0xc33, 0xc0000fff, 0x00000104,
 304         0x3079, 0x00000001, 0x00000001,
 305         0x3403, 0xff000ff0, 0x00000100,
 306         0x3603, 0xff000ff0, 0x00000100
 307 };
 308
 309 static const u32 spectre_golden_spm_registers[] =
 310 {
 311         0xc200, 0xe0ffffff, 0xe0000000
 312 };
 313
 314 static const u32 spectre_golden_common_registers[] =
 315 {
 316         0x31dc, 0xffffffff, 0x00000800,
 317         0x31dd, 0xffffffff, 0x00000800,
 318         0x31e6, 0xffffffff, 0x00007fbf,
 319         0x31e7, 0xffffffff, 0x00007faf
 320 };
 321
 322 static const u32 spectre_golden_registers[] =
 323 {
 324         0xf000, 0xffff1fff, 0x96940200,
 325         0xf003, 0xffff0001, 0xff000000,
 326         0xf080, 0xfffc0fff, 0x00000100,
 327         0x1bb6, 0x00010101, 0x00010000,
 328         0x260d, 0xf00fffff, 0x00000400,
 329         0x260e, 0xfffffffc, 0x00020200,
 330         0x16ec, 0x000000f0, 0x00000070,
 331         0x16f0, 0xf0311fff, 0x80300000,
 332         0x263e, 0x73773777, 0x12010001,
 333         0x26df, 0x00ff0000, 0x00fc0000,
 334         0xbd2, 0x73773777, 0x12010001,
 335         0x2285, 0xf000003f, 0x00000007,
 336         0x22c9, 0xffffffff, 0x00ffffff,
 337         0xa0d4, 0x3f3f3fff, 0x00000082,
 338         0xa0d5, 0x0000003f, 0x00000000,
 339         0xf9e, 0x00000001, 0x00000002,
 340         0x244f, 0xffff03df, 0x00000004,
 341         0x31da, 0x00000008, 0x00000008,
 342         0x2300, 0x000008ff, 0x00000800,
 343         0x2542, 0x00010000, 0x00010000,
 344         0x2b03, 0xffffffff, 0x54763210,
 345         0x853e, 0x01ff01ff, 0x00000002,
 346         0x8526, 0x007ff800, 0x00200000,
 347         0x8057, 0xffffffff, 0x00000f40,
 348         0xc24d, 0xffffffff, 0x00000001
 349 };
 350
 351 static const u32 spectre_mgcg_cgcg_init[] =
 352 {
 353         0x3108, 0xffffffff, 0xfffffffc,
 354         0xc200, 0xffffffff, 0xe0000000,
 355         0xf0a8, 0xffffffff, 0x00000100,
 356         0xf082, 0xffffffff, 0x00000100,
 357         0xf0b0, 0xffffffff, 0x00000100,
 358         0xf0b2, 0xffffffff, 0x00000100,
 359         0xf0b1, 0xffffffff, 0x00000100,
 360         0x1579, 0xffffffff, 0x00600100,
 361         0xf0a0, 0xffffffff, 0x00000100,
 362         0xf085, 0xffffffff, 0x06000100,
 363         0xf088, 0xffffffff, 0x00000100,
 364         0xf086, 0xffffffff, 0x06000100,
 365         0xf081, 0xffffffff, 0x00000100,
 366         0xf0b8, 0xffffffff, 0x00000100,
 367         0xf089, 0xffffffff, 0x00000100,
 368         0xf080, 0xffffffff, 0x00000100,
 369         0xf08c, 0xffffffff, 0x00000100,
 370         0xf08d, 0xffffffff, 0x00000100,
 371         0xf094, 0xffffffff, 0x00000100,
 372         0xf095, 0xffffffff, 0x00000100,
 373         0xf096, 0xffffffff, 0x00000100,
 374         0xf097, 0xffffffff, 0x00000100,
 375         0xf098, 0xffffffff, 0x00000100,
 376         0xf09f, 0xffffffff, 0x00000100,
 377         0xf09e, 0xffffffff, 0x00000100,
 378         0xf084, 0xffffffff, 0x06000100,
 379         0xf0a4, 0xffffffff, 0x00000100,
 380         0xf09d, 0xffffffff, 0x00000100,
 381         0xf0ad, 0xffffffff, 0x00000100,
 382         0xf0ac, 0xffffffff, 0x00000100,
 383         0xf09c, 0xffffffff, 0x00000100,
 384         0xc200, 0xffffffff, 0xe0000000,
 385         0xf008, 0xffffffff, 0x00010000,
 386         0xf009, 0xffffffff, 0x00030002,
 387         0xf00a, 0xffffffff, 0x00040007,
 388         0xf00b, 0xffffffff, 0x00060005,
 389         0xf00c, 0xffffffff, 0x00090008,
 390         0xf00d, 0xffffffff, 0x00010000,
 391         0xf00e, 0xffffffff, 0x00030002,
 392         0xf00f, 0xffffffff, 0x00040007,
 393         0xf010, 0xffffffff, 0x00060005,
 394         0xf011, 0xffffffff, 0x00090008,
 395         0xf012, 0xffffffff, 0x00010000,
 396         0xf013, 0xffffffff, 0x00030002,
 397         0xf014, 0xffffffff, 0x00040007,
 398         0xf015, 0xffffffff, 0x00060005,
 399         0xf016, 0xffffffff, 0x00090008,
 400         0xf017, 0xffffffff, 0x00010000,
 401         0xf018, 0xffffffff, 0x00030002,
 402         0xf019, 0xffffffff, 0x00040007,
 403         0xf01a, 0xffffffff, 0x00060005,
 404         0xf01b, 0xffffffff, 0x00090008,
 405         0xf01c, 0xffffffff, 0x00010000,
 406         0xf01d, 0xffffffff, 0x00030002,
 407         0xf01e, 0xffffffff, 0x00040007,
 408         0xf01f, 0xffffffff, 0x00060005,
 409         0xf020, 0xffffffff, 0x00090008,
 410         0xf021, 0xffffffff, 0x00010000,
 411         0xf022, 0xffffffff, 0x00030002,
 412         0xf023, 0xffffffff, 0x00040007,
 413         0xf024, 0xffffffff, 0x00060005,
 414         0xf025, 0xffffffff, 0x00090008,
 415         0xf026, 0xffffffff, 0x00010000,
 416         0xf027, 0xffffffff, 0x00030002,
 417         0xf028, 0xffffffff, 0x00040007,
 418         0xf029, 0xffffffff, 0x00060005,
 419         0xf02a, 0xffffffff, 0x00090008,
 420         0xf02b, 0xffffffff, 0x00010000,
 421         0xf02c, 0xffffffff, 0x00030002,
 422         0xf02d, 0xffffffff, 0x00040007,
 423         0xf02e, 0xffffffff, 0x00060005,
 424         0xf02f, 0xffffffff, 0x00090008,
 425         0xf000, 0xffffffff, 0x96e00200,
 426         0x21c2, 0xffffffff, 0x00900100,
 427         0x3109, 0xffffffff, 0x0020003f,
 428         0xe, 0xffffffff, 0x0140001c,
 429         0xf, 0x000f0000, 0x000f0000,
 430         0x88, 0xffffffff, 0xc060000c,
 431         0x89, 0xc0000fff, 0x00000100,
 432         0x3e4, 0xffffffff, 0x00000100,
 433         0x3e6, 0x00000101, 0x00000000,
 434         0x82a, 0xffffffff, 0x00000104,
 435         0x1579, 0xff000fff, 0x00000100,
 436         0xc33, 0xc0000fff, 0x00000104,
 437         0x3079, 0x00000001, 0x00000001,
 438         0x3403, 0xff000ff0, 0x00000100,
 439         0x3603, 0xff000ff0, 0x00000100
 440 };
 441
 442 static const u32 kalindi_golden_spm_registers[] =
 443 {
 444         0xc200, 0xe0ffffff, 0xe0000000
 445 };
 446
 447 static const u32 kalindi_golden_common_registers[] =
 448 {
 449         0x31dc, 0xffffffff, 0x00000800,
 450         0x31dd, 0xffffffff, 0x00000800,
 451         0x31e6, 0xffffffff, 0x00007fbf,
 452         0x31e7, 0xffffffff, 0x00007faf
 453 };
 454
 455 static const u32 kalindi_golden_registers[] =
 456 {
 457         0xf000, 0xffffdfff, 0x6e944040,
 458         0x1579, 0xff607fff, 0xfc000100,
 459         0xf088, 0xff000fff, 0x00000100,
 460         0xf089, 0xff000fff, 0x00000100,
 461         0xf080, 0xfffc0fff, 0x00000100,
 462         0x1bb6, 0x00010101, 0x00010000,
 463         0x260c, 0xffffffff, 0x00000000,
 464         0x260d, 0xf00fffff, 0x00000400,
 465         0x16ec, 0x000000f0, 0x00000070,
 466         0x16f0, 0xf0311fff, 0x80300000,
 467         0x263e, 0x73773777, 0x12010001,
 468         0x263f, 0xffffffff, 0x00000010,
 469         0x26df, 0x00ff0000, 0x00fc0000,
 470         0x200c, 0x00001f0f, 0x0000100a,
 471         0xbd2, 0x73773777, 0x12010001,
 472         0x902, 0x000fffff, 0x000c007f,
 473         0x2285, 0xf000003f, 0x00000007,
 474         0x22c9, 0x3fff3fff, 0x00ffcfff,
 475         0xc281, 0x0000ff0f, 0x00000000,
 476         0xa293, 0x07ffffff, 0x06000000,
 477         0x136, 0x00000fff, 0x00000100,
 478         0xf9e, 0x00000001, 0x00000002,
 479         0x31da, 0x00000008, 0x00000008,
 480         0x2300, 0x000000ff, 0x00000003,
 481         0x853e, 0x01ff01ff, 0x00000002,
 482         0x8526, 0x007ff800, 0x00200000,
 483         0x8057, 0xffffffff, 0x00000f40,
 484         0x2231, 0x001f3ae3, 0x00000082,
 485         0x2235, 0x0000001f, 0x00000010,
 486         0xc24d, 0xffffffff, 0x00000000
 487 };
 488
 489 static const u32 kalindi_mgcg_cgcg_init[] =
 490 {
 491         0x3108, 0xffffffff, 0xfffffffc,
 492         0xc200, 0xffffffff, 0xe0000000,
 493         0xf0a8, 0xffffffff, 0x00000100,
 494         0xf082, 0xffffffff, 0x00000100,
 495         0xf0b0, 0xffffffff, 0x00000100,
 496         0xf0b2, 0xffffffff, 0x00000100,
 497         0xf0b1, 0xffffffff, 0x00000100,
 498         0x1579, 0xffffffff, 0x00600100,
 499         0xf0a0, 0xffffffff, 0x00000100,
 500         0xf085, 0xffffffff, 0x06000100,
 501         0xf088, 0xffffffff, 0x00000100,
 502         0xf086, 0xffffffff, 0x06000100,
 503         0xf081, 0xffffffff, 0x00000100,
 504         0xf0b8, 0xffffffff, 0x00000100,
 505         0xf089, 0xffffffff, 0x00000100,
 506         0xf080, 0xffffffff, 0x00000100,
 507         0xf08c, 0xffffffff, 0x00000100,
 508         0xf08d, 0xffffffff, 0x00000100,
 509         0xf094, 0xffffffff, 0x00000100,
 510         0xf095, 0xffffffff, 0x00000100,
 511         0xf096, 0xffffffff, 0x00000100,
 512         0xf097, 0xffffffff, 0x00000100,
 513         0xf098, 0xffffffff, 0x00000100,
 514         0xf09f, 0xffffffff, 0x00000100,
 515         0xf09e, 0xffffffff, 0x00000100,
 516         0xf084, 0xffffffff, 0x06000100,
 517         0xf0a4, 0xffffffff, 0x00000100,
 518         0xf09d, 0xffffffff, 0x00000100,
 519         0xf0ad, 0xffffffff, 0x00000100,
 520         0xf0ac, 0xffffffff, 0x00000100,
 521         0xf09c, 0xffffffff, 0x00000100,
 522         0xc200, 0xffffffff, 0xe0000000,
 523         0xf008, 0xffffffff, 0x00010000,
 524         0xf009, 0xffffffff, 0x00030002,
 525         0xf00a, 0xffffffff, 0x00040007,
 526         0xf00b, 0xffffffff, 0x00060005,
 527         0xf00c, 0xffffffff, 0x00090008,
 528         0xf00d, 0xffffffff, 0x00010000,
 529         0xf00e, 0xffffffff, 0x00030002,
 530         0xf00f, 0xffffffff, 0x00040007,
 531         0xf010, 0xffffffff, 0x00060005,
 532         0xf011, 0xffffffff, 0x00090008,
 533         0xf000, 0xffffffff, 0x96e00200,
 534         0x21c2, 0xffffffff, 0x00900100,
 535         0x3109, 0xffffffff, 0x0020003f,
 536         0xe, 0xffffffff, 0x0140001c,
 537         0xf, 0x000f0000, 0x000f0000,
 538         0x88, 0xffffffff, 0xc060000c,
 539         0x89, 0xc0000fff, 0x00000100,
 540         0x82a, 0xffffffff, 0x00000104,
 541         0x1579, 0xff000fff, 0x00000100,
 542         0xc33, 0xc0000fff, 0x00000104,
 543         0x3079, 0x00000001, 0x00000001,
 544         0x3403, 0xff000ff0, 0x00000100,
 545         0x3603, 0xff000ff0, 0x00000100
 546 };
 547
 548 static const u32 hawaii_golden_spm_registers[] =
 549 {
 550         0xc200, 0xe0ffffff, 0xe0000000
 551 };
 552
 553 static const u32 hawaii_golden_common_registers[] =
 554 {
 555         0xc200, 0xffffffff, 0xe0000000,
 556         0xa0d4, 0xffffffff, 0x3a00161a,
 557         0xa0d5, 0xffffffff, 0x0000002e,
 558         0x2684, 0xffffffff, 0x00018208,
 559         0x263e, 0xffffffff, 0x12011003
 560 };
 561
 562 static const u32 hawaii_golden_registers[] =
 563 {
 564         0xcd5, 0x00000333, 0x00000333,
 565         0x2684, 0x00010000, 0x00058208,
 566         0x260c, 0xffffffff, 0x00000000,
 567         0x260d, 0xf00fffff, 0x00000400,
 568         0x260e, 0x0002021c, 0x00020200,
 569         0x31e, 0x00000080, 0x00000000,
 570         0x16ec, 0x000000f0, 0x00000070,
 571         0x16f0, 0xf0311fff, 0x80300000,
 572         0xd43, 0x00810000, 0x408af000,
 573         0x1c0c, 0x31000111, 0x00000011,
 574         0xbd2, 0x73773777, 0x12010001,
 575         0x848, 0x0000007f, 0x0000001b,
 576         0x877, 0x00007fb6, 0x00002191,
 577         0xd8a, 0x0000003f, 0x0000000a,
 578         0xd8b, 0x0000003f, 0x0000000a,
 579         0xab9, 0x00073ffe, 0x000022a2,
 580         0x903, 0x000007ff, 0x00000000,
 581         0x22fc, 0x00002001, 0x00000001,
 582         0x22c9, 0xffffffff, 0x00ffffff,
 583         0xc281, 0x0000ff0f, 0x00000000,
 584         0xa293, 0x07ffffff, 0x06000000,
 585         0xf9e, 0x00000001, 0x00000002,
 586         0x31da, 0x00000008, 0x00000008,
 587         0x31dc, 0x00000f00, 0x00000800,
 588         0x31dd, 0x00000f00, 0x00000800,
 589         0x31e6, 0x00ffffff, 0x00ff7fbf,
 590         0x31e7, 0x00ffffff, 0x00ff7faf,
 591         0x2300, 0x000000ff, 0x00000800,
 592         0x390, 0x00001fff, 0x00001fff,
 593         0x2418, 0x0000007f, 0x00000020,
 594         0x2542, 0x00010000, 0x00010000,
 595         0x2b80, 0x00100000, 0x000ff07c,
 596         0x2b05, 0x000003ff, 0x0000000f,
 597         0x2b04, 0xffffffff, 0x7564fdec,
 598         0x2b03, 0xffffffff, 0x3120b9a8,
 599         0x2b02, 0x20000000, 0x0f9c0000
 600 };
 601
 602 static const u32 hawaii_mgcg_cgcg_init[] =
 603 {
 604         0x3108, 0xffffffff, 0xfffffffd,
 605         0xc200, 0xffffffff, 0xe0000000,
 606         0xf0a8, 0xffffffff, 0x00000100,
 607         0xf082, 0xffffffff, 0x00000100,
 608         0xf0b0, 0xffffffff, 0x00000100,
 609         0xf0b2, 0xffffffff, 0x00000100,
 610         0xf0b1, 0xffffffff, 0x00000100,
 611         0x1579, 0xffffffff, 0x00200100,
 612         0xf0a0, 0xffffffff, 0x00000100,
 613         0xf085, 0xffffffff, 0x06000100,
 614         0xf088, 0xffffffff, 0x00000100,
 615         0xf086, 0xffffffff, 0x06000100,
 616         0xf081, 0xffffffff, 0x00000100,
 617         0xf0b8, 0xffffffff, 0x00000100,
 618         0xf089, 0xffffffff, 0x00000100,
 619         0xf080, 0xffffffff, 0x00000100,
 620         0xf08c, 0xffffffff, 0x00000100,
 621         0xf08d, 0xffffffff, 0x00000100,
 622         0xf094, 0xffffffff, 0x00000100,
 623         0xf095, 0xffffffff, 0x00000100,
 624         0xf096, 0xffffffff, 0x00000100,
 625         0xf097, 0xffffffff, 0x00000100,
 626         0xf098, 0xffffffff, 0x00000100,
 627         0xf09f, 0xffffffff, 0x00000100,
 628         0xf09e, 0xffffffff, 0x00000100,
 629         0xf084, 0xffffffff, 0x06000100,
 630         0xf0a4, 0xffffffff, 0x00000100,
 631         0xf09d, 0xffffffff, 0x00000100,
 632         0xf0ad, 0xffffffff, 0x00000100,
 633         0xf0ac, 0xffffffff, 0x00000100,
 634         0xf09c, 0xffffffff, 0x00000100,
 635         0xc200, 0xffffffff, 0xe0000000,
 636         0xf008, 0xffffffff, 0x00010000,
 637         0xf009, 0xffffffff, 0x00030002,
 638         0xf00a, 0xffffffff, 0x00040007,
 639         0xf00b, 0xffffffff, 0x00060005,
 640         0xf00c, 0xffffffff, 0x00090008,
 641         0xf00d, 0xffffffff, 0x00010000,
 642         0xf00e, 0xffffffff, 0x00030002,
 643         0xf00f, 0xffffffff, 0x00040007,
 644         0xf010, 0xffffffff, 0x00060005,
 645         0xf011, 0xffffffff, 0x00090008,
 646         0xf012, 0xffffffff, 0x00010000,
 647         0xf013, 0xffffffff, 0x00030002,
 648         0xf014, 0xffffffff, 0x00040007,
 649         0xf015, 0xffffffff, 0x00060005,
 650         0xf016, 0xffffffff, 0x00090008,
 651         0xf017, 0xffffffff, 0x00010000,
 652         0xf018, 0xffffffff, 0x00030002,
 653         0xf019, 0xffffffff, 0x00040007,
 654         0xf01a, 0xffffffff, 0x00060005,
 655         0xf01b, 0xffffffff, 0x00090008,
 656         0xf01c, 0xffffffff, 0x00010000,
 657         0xf01d, 0xffffffff, 0x00030002,
 658         0xf01e, 0xffffffff, 0x00040007,
 659         0xf01f, 0xffffffff, 0x00060005,
 660         0xf020, 0xffffffff, 0x00090008,
 661         0xf021, 0xffffffff, 0x00010000,
 662         0xf022, 0xffffffff, 0x00030002,
 663         0xf023, 0xffffffff, 0x00040007,
 664         0xf024, 0xffffffff, 0x00060005,
 665         0xf025, 0xffffffff, 0x00090008,
 666         0xf026, 0xffffffff, 0x00010000,
 667         0xf027, 0xffffffff, 0x00030002,
 668         0xf028, 0xffffffff, 0x00040007,
 669         0xf029, 0xffffffff, 0x00060005,
 670         0xf02a, 0xffffffff, 0x00090008,
 671         0xf02b, 0xffffffff, 0x00010000,
 672         0xf02c, 0xffffffff, 0x00030002,
 673         0xf02d, 0xffffffff, 0x00040007,
 674         0xf02e, 0xffffffff, 0x00060005,
 675         0xf02f, 0xffffffff, 0x00090008,
 676         0xf030, 0xffffffff, 0x00010000,
 677         0xf031, 0xffffffff, 0x00030002,
 678         0xf032, 0xffffffff, 0x00040007,
 679         0xf033, 0xffffffff, 0x00060005,
 680         0xf034, 0xffffffff, 0x00090008,
 681         0xf035, 0xffffffff, 0x00010000,
 682         0xf036, 0xffffffff, 0x00030002,
 683         0xf037, 0xffffffff, 0x00040007,
 684         0xf038, 0xffffffff, 0x00060005,
 685         0xf039, 0xffffffff, 0x00090008,
 686         0xf03a, 0xffffffff, 0x00010000,
 687         0xf03b, 0xffffffff, 0x00030002,
 688         0xf03c, 0xffffffff, 0x00040007,
 689         0xf03d, 0xffffffff, 0x00060005,
 690         0xf03e, 0xffffffff, 0x00090008,
 691         0x30c6, 0xffffffff, 0x00020200,
 692         0xcd4, 0xffffffff, 0x00000200,
 693         0x570, 0xffffffff, 0x00000400,
 694         0x157a, 0xffffffff, 0x00000000,
 695         0xbd4, 0xffffffff, 0x00000902,
 696         0xf000, 0xffffffff, 0x96940200,
 697         0x21c2, 0xffffffff, 0x00900100,
 698         0x3109, 0xffffffff, 0x0020003f,
 699         0xe, 0xffffffff, 0x0140001c,
 700         0xf, 0x000f0000, 0x000f0000,
 701         0x88, 0xffffffff, 0xc060000c,
 702         0x89, 0xc0000fff, 0x00000100,
 703         0x3e4, 0xffffffff, 0x00000100,
 704         0x3e6, 0x00000101, 0x00000000,
 705         0x82a, 0xffffffff, 0x00000104,
 706         0x1579, 0xff000fff, 0x00000100,
 707         0xc33, 0xc0000fff, 0x00000104,
 708         0x3079, 0x00000001, 0x00000001,
 709         0x3403, 0xff000ff0, 0x00000100,
 710         0x3603, 0xff000ff0, 0x00000100
 711 };
 712
 713 static const u32 godavari_golden_registers[] =
 714 {
 715         0x1579, 0xff607fff, 0xfc000100,
 716         0x1bb6, 0x00010101, 0x00010000,
 717         0x260c, 0xffffffff, 0x00000000,
 718         0x260c0, 0xf00fffff, 0x00000400,
 719         0x184c, 0xffffffff, 0x00010000,
 720         0x16ec, 0x000000f0, 0x00000070,
 721         0x16f0, 0xf0311fff, 0x80300000,
 722         0x263e, 0x73773777, 0x12010001,
 723         0x263f, 0xffffffff, 0x00000010,
 724         0x200c, 0x00001f0f, 0x0000100a,
 725         0xbd2, 0x73773777, 0x12010001,
 726         0x902, 0x000fffff, 0x000c007f,
 727         0x2285, 0xf000003f, 0x00000007,
 728         0x22c9, 0xffffffff, 0x00ff0fff,
 729         0xc281, 0x0000ff0f, 0x00000000,
 730         0xa293, 0x07ffffff, 0x06000000,
 731         0x136, 0x00000fff, 0x00000100,
 732         0x3405, 0x00010000, 0x00810001,
 733         0x3605, 0x00010000, 0x00810001,
 734         0xf9e, 0x00000001, 0x00000002,
 735         0x31da, 0x00000008, 0x00000008,
 736         0x31dc, 0x00000f00, 0x00000800,
 737         0x31dd, 0x00000f00, 0x00000800,
 738         0x31e6, 0x00ffffff, 0x00ff7fbf,
 739         0x31e7, 0x00ffffff, 0x00ff7faf,
 740         0x2300, 0x000000ff, 0x00000001,
 741         0x853e, 0x01ff01ff, 0x00000002,
 742         0x8526, 0x007ff800, 0x00200000,
 743         0x8057, 0xffffffff, 0x00000f40,
 744         0x2231, 0x001f3ae3, 0x00000082,
 745         0x2235, 0x0000001f, 0x00000010,
 746         0xc24d, 0xffffffff, 0x00000000
 747 };
 748
 749 static void cik_init_golden_registers(struct amdgpu_device *adev)
 750 {
 751         /* Some of the registers might be dependent on GRBM_GFX_INDEX */
 752         mutex_lock(&adev->grbm_idx_mutex);
 753
 754         switch (adev->asic_type) {
 755         case CHIP_BONAIRE:
 756                 amdgpu_program_register_sequence(adev,
 757                                                  bonaire_mgcg_cgcg_init,
 758                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 759                 amdgpu_program_register_sequence(adev,
 760                                                  bonaire_golden_registers,
 761                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
 762                 amdgpu_program_register_sequence(adev,
 763                                                  bonaire_golden_common_registers,
 764                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
 765                 amdgpu_program_register_sequence(adev,
 766                                                  bonaire_golden_spm_registers,
 767                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
 768                 break;
 769         case CHIP_KABINI:
 770                 amdgpu_program_register_sequence(adev,
 771                                                  kalindi_mgcg_cgcg_init,
 772                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 773                 amdgpu_program_register_sequence(adev,
 774                                                  kalindi_golden_registers,
 775                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
 776                 amdgpu_program_register_sequence(adev,
 777                                                  kalindi_golden_common_registers,
 778                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 779                 amdgpu_program_register_sequence(adev,
 780                                                  kalindi_golden_spm_registers,
 781                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 782                 break;
 783         case CHIP_MULLINS:
 784                 amdgpu_program_register_sequence(adev,
 785                                                  kalindi_mgcg_cgcg_init,
 786                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 787                 amdgpu_program_register_sequence(adev,
 788                                                  godavari_golden_registers,
 789                                                  (const u32)ARRAY_SIZE(godavari_golden_registers));
 790                 amdgpu_program_register_sequence(adev,
 791                                                  kalindi_golden_common_registers,
 792                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 793                 amdgpu_program_register_sequence(adev,
 794                                                  kalindi_golden_spm_registers,
 795                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 796                 break;
 797         case CHIP_KAVERI:
 798                 amdgpu_program_register_sequence(adev,
 799                                                  spectre_mgcg_cgcg_init,
 800                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
 801                 amdgpu_program_register_sequence(adev,
 802                                                  spectre_golden_registers,
 803                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
 804                 amdgpu_program_register_sequence(adev,
 805                                                  spectre_golden_common_registers,
 806                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
 807                 amdgpu_program_register_sequence(adev,
 808                                                  spectre_golden_spm_registers,
 809                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
 810                 break;
 811         case CHIP_HAWAII:
 812                 amdgpu_program_register_sequence(adev,
 813                                                  hawaii_mgcg_cgcg_init,
 814                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
 815                 amdgpu_program_register_sequence(adev,
 816                                                  hawaii_golden_registers,
 817                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
 818                 amdgpu_program_register_sequence(adev,
 819                                                  hawaii_golden_common_registers,
 820                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
 821                 amdgpu_program_register_sequence(adev,
 822                                                  hawaii_golden_spm_registers,
 823                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
 824                 break;
 825         default:
 826                 break;
 827         }
 828         mutex_unlock(&adev->grbm_idx_mutex);
 829 }
 830
 831 /**
 832  * cik_get_xclk - get the xclk
 833  *
 834  * @adev: amdgpu_device pointer
 835  *
 836  * Returns the reference clock used by the gfx engine
 837  * (CIK).
 838  */
 839 static u32 cik_get_xclk(struct amdgpu_device *adev)
 840 {
 841         u32 reference_clock = adev->clock.spll.reference_freq;
 842
 843         if (adev->flags & AMD_IS_APU) {
 844                 if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
 845                         return reference_clock / 2;
 846         } else {
 847                 if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
 848                         return reference_clock / 4;
 849         }
 850         return reference_clock;
 851 }
 852
 853 /**
 854  * cik_srbm_select - select specific register instances
 855  *
 856  * @adev: amdgpu_device pointer
 857  * @me: selected ME (micro engine)
 858  * @pipe: pipe
 859  * @queue: queue
 860  * @vmid: VMID
 861  *
 862  * Switches the currently active registers instances.  Some
 863  * registers are instanced per VMID, others are instanced per
 864  * me/pipe/queue combination.
 865  */
 866 void cik_srbm_select(struct amdgpu_device *adev,
 867                      u32 me, u32 pipe, u32 queue, u32 vmid)
 868 {
 869         u32 srbm_gfx_cntl =
 870                 (((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
 871                 ((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
 872                 ((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
 873                 ((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
 874         WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 875 }
 876
 877 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
 878 {
 879         uint32_t tmp;
 880
 881         tmp = RREG32(mmCONFIG_CNTL);
 882         if (state == false)
 883                 tmp |= CONFIG_CNTL__VGA_DIS_MASK;
 884         else
 885                 tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
 886         WREG32(mmCONFIG_CNTL, tmp);
 887 }
 888
 889 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 890 {
 891         u32 bus_cntl;
 892         u32 d1vga_control = 0;
 893         u32 d2vga_control = 0;
 894         u32 vga_render_control = 0;
 895         u32 rom_cntl;
 896         bool r;
 897
 898         bus_cntl = RREG32(mmBUS_CNTL);
 899         if (adev->mode_info.num_crtc) {
 900                 d1vga_control = RREG32(mmD1VGA_CONTROL);
 901                 d2vga_control = RREG32(mmD2VGA_CONTROL);
 902                 vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 903         }
 904         rom_cntl = RREG32_SMC(ixROM_CNTL);
 905
 906         /* enable the rom */
 907         WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 908         if (adev->mode_info.num_crtc) {
 909                 /* Disable VGA mode */
 910                 WREG32(mmD1VGA_CONTROL,
 911                        (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 912                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 913                 WREG32(mmD2VGA_CONTROL,
 914                        (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 915                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 916                 WREG32(mmVGA_RENDER_CONTROL,
 917                        (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 918         }
 919         WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 920
 921         r = amdgpu_read_bios(adev);
 922
 923         /* restore regs */
 924         WREG32(mmBUS_CNTL, bus_cntl);
 925         if (adev->mode_info.num_crtc) {
 926                 WREG32(mmD1VGA_CONTROL, d1vga_control);
 927                 WREG32(mmD2VGA_CONTROL, d2vga_control);
 928                 WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
 929         }
 930         WREG32_SMC(ixROM_CNTL, rom_cntl);
 931         return r;
 932 }
 933
 934 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
 935                                    u8 *bios, u32 length_bytes)
 936 {
 937         u32 *dw_ptr;
 938         unsigned long flags;
 939         u32 i, length_dw;
 940
 941         if (bios == NULL)
 942                 return false;
 943         if (length_bytes == 0)
 944                 return false;
 945         /* APU vbios image is part of sbios image */
 946         if (adev->flags & AMD_IS_APU)
 947                 return false;
 948
 949         dw_ptr = (u32 *)bios;
 950         length_dw = ALIGN(length_bytes, 4) / 4;
 951         /* take the smc lock since we are using the smc index */
 952         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 953         /* set rom index to 0 */
 954         WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
 955         WREG32(mmSMC_IND_DATA_0, 0);
 956         /* set index to data for continous read */
 957         WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
 958         for (i = 0; i < length_dw; i++)
 959                 dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
 960         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 961
 962         return true;
 963 }
 964
 965 static struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
 966         {mmGRBM_STATUS, false},
 967         {mmGB_ADDR_CONFIG, false},
 968         {mmMC_ARB_RAMCFG, false},
 969         {mmGB_TILE_MODE0, false},
 970         {mmGB_TILE_MODE1, false},
 971         {mmGB_TILE_MODE2, false},
 972         {mmGB_TILE_MODE3, false},
 973         {mmGB_TILE_MODE4, false},
 974         {mmGB_TILE_MODE5, false},
 975         {mmGB_TILE_MODE6, false},
 976         {mmGB_TILE_MODE7, false},
 977         {mmGB_TILE_MODE8, false},
 978         {mmGB_TILE_MODE9, false},
 979         {mmGB_TILE_MODE10, false},
 980         {mmGB_TILE_MODE11, false},
 981         {mmGB_TILE_MODE12, false},
 982         {mmGB_TILE_MODE13, false},
 983         {mmGB_TILE_MODE14, false},
 984         {mmGB_TILE_MODE15, false},
 985         {mmGB_TILE_MODE16, false},
 986         {mmGB_TILE_MODE17, false},
 987         {mmGB_TILE_MODE18, false},
 988         {mmGB_TILE_MODE19, false},
 989         {mmGB_TILE_MODE20, false},
 990         {mmGB_TILE_MODE21, false},
 991         {mmGB_TILE_MODE22, false},
 992         {mmGB_TILE_MODE23, false},
 993         {mmGB_TILE_MODE24, false},
 994         {mmGB_TILE_MODE25, false},
 995         {mmGB_TILE_MODE26, false},
 996         {mmGB_TILE_MODE27, false},
 997         {mmGB_TILE_MODE28, false},
 998         {mmGB_TILE_MODE29, false},
 999         {mmGB_TILE_MODE30, false},
1000         {mmGB_TILE_MODE31, false},
1001         {mmGB_MACROTILE_MODE0, false},
1002         {mmGB_MACROTILE_MODE1, false},
1003         {mmGB_MACROTILE_MODE2, false},
1004         {mmGB_MACROTILE_MODE3, false},
1005         {mmGB_MACROTILE_MODE4, false},
1006         {mmGB_MACROTILE_MODE5, false},
1007         {mmGB_MACROTILE_MODE6, false},
1008         {mmGB_MACROTILE_MODE7, false},
1009         {mmGB_MACROTILE_MODE8, false},
1010         {mmGB_MACROTILE_MODE9, false},
1011         {mmGB_MACROTILE_MODE10, false},
1012         {mmGB_MACROTILE_MODE11, false},
1013         {mmGB_MACROTILE_MODE12, false},
1014         {mmGB_MACROTILE_MODE13, false},
1015         {mmGB_MACROTILE_MODE14, false},
1016         {mmGB_MACROTILE_MODE15, false},
1017         {mmCC_RB_BACKEND_DISABLE, false, true},
1018         {mmGC_USER_RB_BACKEND_DISABLE, false, true},
1019         {mmGB_BACKEND_MAP, false, false},
1020         {mmPA_SC_RASTER_CONFIG, false, true},
1021         {mmPA_SC_RASTER_CONFIG_1, false, true},
1022 };
1023
1024 static uint32_t cik_read_indexed_register(struct amdgpu_device *adev,
1025                                           u32 se_num, u32 sh_num,
1026                                           u32 reg_offset)
1027 {
1028         uint32_t val;
1029
1030         mutex_lock(&adev->grbm_idx_mutex);
1031         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1032                 gfx_v7_0_select_se_sh(adev, se_num, sh_num);
1033
1034         val = RREG32(reg_offset);
1035
1036         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1037                 gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
1038         mutex_unlock(&adev->grbm_idx_mutex);
1039         return val;
1040 }
1041
1042 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1043                              u32 sh_num, u32 reg_offset, u32 *value)
1044 {
1045         uint32_t i;
1046
1047         *value = 0;
1048         for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1049                 if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1050                         continue;
1051
1052                 if (!cik_allowed_read_registers[i].untouched)
1053                         *value = cik_allowed_read_registers[i].grbm_indexed ?
1054                                  cik_read_indexed_register(adev, se_num,
1055                                                            sh_num, reg_offset) :
1056                                  RREG32(reg_offset);
1057                 return 0;
1058         }
1059         return -EINVAL;
1060 }
1061
1062 struct kv_reset_save_regs {
1063         u32 gmcon_reng_execute;
1064         u32 gmcon_misc;
1065         u32 gmcon_misc3;
1066 };
1067
1068 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1069                                    struct kv_reset_save_regs *save)
1070 {
1071         save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1072         save->gmcon_misc = RREG32(mmGMCON_MISC);
1073         save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1074
1075         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1076                 ~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1077         WREG32(mmGMCON_MISC, save->gmcon_misc &
1078                 ~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1079                         GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1080 }
1081
1082 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1083                                       struct kv_reset_save_regs *save)
1084 {
1085         int i;
1086
1087         WREG32(mmGMCON_PGFSM_WRITE, 0);
1088         WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1089
1090         for (i = 0; i < 5; i++)
1091                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1092
1093         WREG32(mmGMCON_PGFSM_WRITE, 0);
1094         WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1095
1096         for (i = 0; i < 5; i++)
1097                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1098
1099         WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1100         WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1101
1102         for (i = 0; i < 5; i++)
1103                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1104
1105         WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1106         WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1107
1108         for (i = 0; i < 5; i++)
1109                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1110
1111         WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1112         WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1113
1114         for (i = 0; i < 5; i++)
1115                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1116
1117         WREG32(mmGMCON_PGFSM_WRITE, 0);
1118         WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1119
1120         for (i = 0; i < 5; i++)
1121                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1122
1123         WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1124         WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1125
1126         for (i = 0; i < 5; i++)
1127                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1128
1129         WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1130         WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1131
1132         for (i = 0; i < 5; i++)
1133                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1134
1135         WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1136         WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1137
1138         for (i = 0; i < 5; i++)
1139                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1140
1141         WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1142         WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1143
1144         for (i = 0; i < 5; i++)
1145                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1146
1147         WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1148         WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1149
1150         WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1151         WREG32(mmGMCON_MISC, save->gmcon_misc);
1152         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1153 }
1154
1155 static void cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1156 {
1157         struct kv_reset_save_regs kv_save = { 0 };
1158         u32 i;
1159
1160         dev_info(adev->dev, "GPU pci config reset\n");
1161
1162         if (adev->flags & AMD_IS_APU)
1163                 kv_save_regs_for_reset(adev, &kv_save);
1164
1165         /* disable BM */
1166         pci_clear_master(adev->pdev);
1167         /* reset */
1168         amdgpu_pci_config_reset(adev);
1169
1170         udelay(100);
1171
1172         /* wait for asic to come out of reset */
1173         for (i = 0; i < adev->usec_timeout; i++) {
1174                 if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff)
1175                         break;
1176                 udelay(1);
1177         }
1178
1179         /* does asic init need to be run first??? */
1180         if (adev->flags & AMD_IS_APU)
1181                 kv_restore_regs_for_reset(adev, &kv_save);
1182 }
1183
1184 static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung)
1185 {
1186         u32 tmp = RREG32(mmBIOS_SCRATCH_3);
1187
1188         if (hung)
1189                 tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1190         else
1191                 tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1192
1193         WREG32(mmBIOS_SCRATCH_3, tmp);
1194 }
1195
1196 /**
1197  * cik_asic_reset - soft reset GPU
1198  *
1199  * @adev: amdgpu_device pointer
1200  *
1201  * Look up which blocks are hung and attempt
1202  * to reset them.
1203  * Returns 0 for success.
1204  */
1205 static int cik_asic_reset(struct amdgpu_device *adev)
1206 {
1207         cik_set_bios_scratch_engine_hung(adev, true);
1208
1209         cik_gpu_pci_config_reset(adev);
1210
1211         cik_set_bios_scratch_engine_hung(adev, false);
1212
1213         return 0;
1214 }
1215
1216 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1217                               u32 cntl_reg, u32 status_reg)
1218 {
1219         int r, i;
1220         struct atom_clock_dividers dividers;
1221         uint32_t tmp;
1222
1223         r = amdgpu_atombios_get_clock_dividers(adev,
1224                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1225                                                clock, false, &dividers);
1226         if (r)
1227                 return r;
1228
1229         tmp = RREG32_SMC(cntl_reg);
1230         tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1231                 CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1232         tmp |= dividers.post_divider;
1233         WREG32_SMC(cntl_reg, tmp);
1234
1235         for (i = 0; i < 100; i++) {
1236                 if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1237                         break;
1238                 mdelay(10);
1239         }
1240         if (i == 100)
1241                 return -ETIMEDOUT;
1242
1243         return 0;
1244 }
1245
1246 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1247 {
1248         int r = 0;
1249
1250         r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1251         if (r)
1252                 return r;
1253
1254         r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1255         return r;
1256 }
1257
1258 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1259 {
1260         int r, i;
1261         struct atom_clock_dividers dividers;
1262         u32 tmp;
1263
1264         r = amdgpu_atombios_get_clock_dividers(adev,
1265                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1266                                                ecclk, false, &dividers);
1267         if (r)
1268                 return r;
1269
1270         for (i = 0; i < 100; i++) {
1271                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1272                         break;
1273                 mdelay(10);
1274         }
1275         if (i == 100)
1276                 return -ETIMEDOUT;
1277
1278         tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1279         tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1280                 CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1281         tmp |= dividers.post_divider;
1282         WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1283
1284         for (i = 0; i < 100; i++) {
1285                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1286                         break;
1287                 mdelay(10);
1288         }
1289         if (i == 100)
1290                 return -ETIMEDOUT;
1291
1292         return 0;
1293 }
1294
1295 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1296 {
1297         struct pci_dev *root = adev->pdev->bus->self;
1298         int bridge_pos, gpu_pos;
1299         u32 speed_cntl, current_data_rate;
1300         int i;
1301         u16 tmp16;
1302
1303         if (pci_is_root_bus(adev->pdev->bus))
1304                 return;
1305
1306         if (amdgpu_pcie_gen2 == 0)
1307                 return;
1308
1309         if (adev->flags & AMD_IS_APU)
1310                 return;
1311
1312         if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1313                                         CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1314                 return;
1315
1316         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1317         current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1318                 PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1319         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1320                 if (current_data_rate == 2) {
1321                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1322                         return;
1323                 }
1324                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1325         } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1326                 if (current_data_rate == 1) {
1327                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1328                         return;
1329                 }
1330                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1331         }
1332
1333         bridge_pos = pci_pcie_cap(root);
1334         if (!bridge_pos)
1335                 return;
1336
1337         gpu_pos = pci_pcie_cap(adev->pdev);
1338         if (!gpu_pos)
1339                 return;
1340
1341         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1342                 /* re-try equalization if gen3 is not already enabled */
1343                 if (current_data_rate != 2) {
1344                         u16 bridge_cfg, gpu_cfg;
1345                         u16 bridge_cfg2, gpu_cfg2;
1346                         u32 max_lw, current_lw, tmp;
1347
1348                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1349                         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1350
1351                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1352                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1353
1354                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1355                         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1356
1357                         tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1358                         max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1359                                 PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1360                         current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1361                                 >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1362
1363                         if (current_lw < max_lw) {
1364                                 tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1365                                 if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1366                                         tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1367                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1368                                         tmp |= (max_lw <<
1369                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1370                                         tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1371                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1372                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1373                                         WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1374                                 }
1375                         }
1376
1377                         for (i = 0; i < 10; i++) {
1378                                 /* check status */
1379                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1380                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1381                                         break;
1382
1383                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1384                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1385
1386                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1387                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1388
1389                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1390                                 tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1391                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1392
1393                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1394                                 tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1395                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1396
1397                                 mdelay(100);
1398
1399                                 /* linkctl */
1400                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1401                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1402                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1403                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1404
1405                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1406                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1407                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1408                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1409
1410                                 /* linkctl2 */
1411                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1412                                 tmp16 &= ~((1 << 4) | (7 << 9));
1413                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1414                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1415
1416                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1417                                 tmp16 &= ~((1 << 4) | (7 << 9));
1418                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1419                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1420
1421                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1422                                 tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1423                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1424                         }
1425                 }
1426         }
1427
1428         /* set the link speed */
1429         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1430                 PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1431         speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1432         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1433
1434         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1435         tmp16 &= ~0xf;
1436         if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1437                 tmp16 |= 3; /* gen3 */
1438         else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1439                 tmp16 |= 2; /* gen2 */
1440         else
1441                 tmp16 |= 1; /* gen1 */
1442         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1443
1444         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1445         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1446         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1447
1448         for (i = 0; i < adev->usec_timeout; i++) {
1449                 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1450                 if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1451                         break;
1452                 udelay(1);
1453         }
1454 }
1455
1456 static void cik_program_aspm(struct amdgpu_device *adev)
1457 {
1458         u32 data, orig;
1459         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1460         bool disable_clkreq = false;
1461
1462         if (amdgpu_aspm == 0)
1463                 return;
1464
1465         if (pci_is_root_bus(adev->pdev->bus))
1466                 return;
1467
1468         /* XXX double check APUs */
1469         if (adev->flags & AMD_IS_APU)
1470                 return;
1471
1472         orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1473         data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1474         data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1475                 PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1476         if (orig != data)
1477                 WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1478
1479         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1480         data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1481         if (orig != data)
1482                 WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1483
1484         orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1485         data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1486         if (orig != data)
1487                 WREG32_PCIE(ixPCIE_P_CNTL, data);
1488
1489         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1490         data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1491                 PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1492         data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1493         if (!disable_l0s)
1494                 data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1495
1496         if (!disable_l1) {
1497                 data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1498                 data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1499                 if (orig != data)
1500                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1501
1502                 if (!disable_plloff_in_l1) {
1503                         bool clk_req_support;
1504
1505                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1506                         data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1507                                 PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1508                         data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1509                                 (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1510                         if (orig != data)
1511                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1512
1513                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1514                         data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1515                                 PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1516                         data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1517                                 (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1518                         if (orig != data)
1519                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1520
1521                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1522                         data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1523                                 PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1524                         data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1525                                 (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1526                         if (orig != data)
1527                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1528
1529                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1530                         data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1531                                 PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1532                         data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1533                                 (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1534                         if (orig != data)
1535                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1536
1537                         orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1538                         data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1539                         data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1540                         if (orig != data)
1541                                 WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1542
1543                         if (!disable_clkreq) {
1544                                 struct pci_dev *root = adev->pdev->bus->self;
1545                                 u32 lnkcap;
1546
1547                                 clk_req_support = false;
1548                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1549                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1550                                         clk_req_support = true;
1551                         } else {
1552                                 clk_req_support = false;
1553                         }
1554
1555                         if (clk_req_support) {
1556                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1557                                 data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1558                                         PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1559                                 if (orig != data)
1560                                         WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1561
1562                                 orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1563                                 data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1564                                         THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1565                                 data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1566                                         (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1567                                 if (orig != data)
1568                                         WREG32_SMC(ixTHM_CLK_CNTL, data);
1569
1570                                 orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1571                                 data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1572                                         MISC_CLK_CTRL__ZCLK_SEL_MASK);
1573                                 data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1574                                         (1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1575                                 if (orig != data)
1576                                         WREG32_SMC(ixMISC_CLK_CTRL, data);
1577
1578                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1579                                 data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1580                                 if (orig != data)
1581                                         WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1582
1583                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1584                                 data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1585                                 if (orig != data)
1586                                         WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1587
1588                                 orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1589                                 data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1590                                 data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1591                                 if (orig != data)
1592                                         WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1593                         }
1594                 }
1595         } else {
1596                 if (orig != data)
1597                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1598         }
1599
1600         orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1601         data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1602                 PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1603                 PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1604         if (orig != data)
1605                 WREG32_PCIE(ixPCIE_CNTL2, data);
1606
1607         if (!disable_l0s) {
1608                 data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1609                 if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1610                                 PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1611                         data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1612                         if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1613                         (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1614                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1615                                 data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1616                                 if (orig != data)
1617                                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1618                         }
1619                 }
1620         }
1621 }
1622
1623 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1624 {
1625         return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1626                 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1627 }
1628
1629 static const struct amdgpu_ip_block_version bonaire_ip_blocks[] =
1630 {
1631         /* ORDER MATTERS! */
1632         {
1633                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1634                 .major = 1,
1635                 .minor = 0,
1636                 .rev = 0,
1637                 .funcs = &cik_common_ip_funcs,
1638         },
1639         {
1640                 .type = AMD_IP_BLOCK_TYPE_GMC,
1641                 .major = 7,
1642                 .minor = 0,
1643                 .rev = 0,
1644                 .funcs = &gmc_v7_0_ip_funcs,
1645         },
1646         {
1647                 .type = AMD_IP_BLOCK_TYPE_IH,
1648                 .major = 2,
1649                 .minor = 0,
1650                 .rev = 0,
1651                 .funcs = &cik_ih_ip_funcs,
1652         },
1653         {
1654                 .type = AMD_IP_BLOCK_TYPE_SMC,
1655                 .major = 7,
1656                 .minor = 0,
1657                 .rev = 0,
1658                 .funcs = &amdgpu_pp_ip_funcs,
1659         },
1660         {
1661                 .type = AMD_IP_BLOCK_TYPE_DCE,
1662                 .major = 8,
1663                 .minor = 2,
1664                 .rev = 0,
1665                 .funcs = &dce_v8_0_ip_funcs,
1666         },
1667         {
1668                 .type = AMD_IP_BLOCK_TYPE_GFX,
1669                 .major = 7,
1670                 .minor = 2,
1671                 .rev = 0,
1672                 .funcs = &gfx_v7_0_ip_funcs,
1673         },
1674         {
1675                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1676                 .major = 2,
1677                 .minor = 0,
1678                 .rev = 0,
1679                 .funcs = &cik_sdma_ip_funcs,
1680         },
1681         {
1682                 .type = AMD_IP_BLOCK_TYPE_UVD,
1683                 .major = 4,
1684                 .minor = 2,
1685                 .rev = 0,
1686                 .funcs = &uvd_v4_2_ip_funcs,
1687         },
1688         {
1689                 .type = AMD_IP_BLOCK_TYPE_VCE,
1690                 .major = 2,
1691                 .minor = 0,
1692                 .rev = 0,
1693                 .funcs = &vce_v2_0_ip_funcs,
1694         },
1695 };
1696
1697 static const struct amdgpu_ip_block_version hawaii_ip_blocks[] =
1698 {
1699         /* ORDER MATTERS! */
1700         {
1701                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1702                 .major = 1,
1703                 .minor = 0,
1704                 .rev = 0,
1705                 .funcs = &cik_common_ip_funcs,
1706         },
1707         {
1708                 .type = AMD_IP_BLOCK_TYPE_GMC,
1709                 .major = 7,
1710                 .minor = 0,
1711                 .rev = 0,
1712                 .funcs = &gmc_v7_0_ip_funcs,
1713         },
1714         {
1715                 .type = AMD_IP_BLOCK_TYPE_IH,
1716                 .major = 2,
1717                 .minor = 0,
1718                 .rev = 0,
1719                 .funcs = &cik_ih_ip_funcs,
1720         },
1721         {
1722                 .type = AMD_IP_BLOCK_TYPE_SMC,
1723                 .major = 7,
1724                 .minor = 0,
1725                 .rev = 0,
1726                 .funcs = &amdgpu_pp_ip_funcs,
1727         },
1728         {
1729                 .type = AMD_IP_BLOCK_TYPE_DCE,
1730                 .major = 8,
1731                 .minor = 5,
1732                 .rev = 0,
1733                 .funcs = &dce_v8_0_ip_funcs,
1734         },
1735         {
1736                 .type = AMD_IP_BLOCK_TYPE_GFX,
1737                 .major = 7,
1738                 .minor = 3,
1739                 .rev = 0,
1740                 .funcs = &gfx_v7_0_ip_funcs,
1741         },
1742         {
1743                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1744                 .major = 2,
1745                 .minor = 0,
1746                 .rev = 0,
1747                 .funcs = &cik_sdma_ip_funcs,
1748         },
1749         {
1750                 .type = AMD_IP_BLOCK_TYPE_UVD,
1751                 .major = 4,
1752                 .minor = 2,
1753                 .rev = 0,
1754                 .funcs = &uvd_v4_2_ip_funcs,
1755         },
1756         {
1757                 .type = AMD_IP_BLOCK_TYPE_VCE,
1758                 .major = 2,
1759                 .minor = 0,
1760                 .rev = 0,
1761                 .funcs = &vce_v2_0_ip_funcs,
1762         },
1763 };
1764
1765 static const struct amdgpu_ip_block_version kabini_ip_blocks[] =
1766 {
1767         /* ORDER MATTERS! */
1768         {
1769                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1770                 .major = 1,
1771                 .minor = 0,
1772                 .rev = 0,
1773                 .funcs = &cik_common_ip_funcs,
1774         },
1775         {
1776                 .type = AMD_IP_BLOCK_TYPE_GMC,
1777                 .major = 7,
1778                 .minor = 0,
1779                 .rev = 0,
1780                 .funcs = &gmc_v7_0_ip_funcs,
1781         },
1782         {
1783                 .type = AMD_IP_BLOCK_TYPE_IH,
1784                 .major = 2,
1785                 .minor = 0,
1786                 .rev = 0,
1787                 .funcs = &cik_ih_ip_funcs,
1788         },
1789         {
1790                 .type = AMD_IP_BLOCK_TYPE_SMC,
1791                 .major = 7,
1792                 .minor = 0,
1793                 .rev = 0,
1794                 .funcs = &amdgpu_pp_ip_funcs,
1795         },
1796         {
1797                 .type = AMD_IP_BLOCK_TYPE_DCE,
1798                 .major = 8,
1799                 .minor = 3,
1800                 .rev = 0,
1801                 .funcs = &dce_v8_0_ip_funcs,
1802         },
1803         {
1804                 .type = AMD_IP_BLOCK_TYPE_GFX,
1805                 .major = 7,
1806                 .minor = 2,
1807                 .rev = 0,
1808                 .funcs = &gfx_v7_0_ip_funcs,
1809         },
1810         {
1811                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1812                 .major = 2,
1813                 .minor = 0,
1814                 .rev = 0,
1815                 .funcs = &cik_sdma_ip_funcs,
1816         },
1817         {
1818                 .type = AMD_IP_BLOCK_TYPE_UVD,
1819                 .major = 4,
1820                 .minor = 2,
1821                 .rev = 0,
1822                 .funcs = &uvd_v4_2_ip_funcs,
1823         },
1824         {
1825                 .type = AMD_IP_BLOCK_TYPE_VCE,
1826                 .major = 2,
1827                 .minor = 0,
1828                 .rev = 0,
1829                 .funcs = &vce_v2_0_ip_funcs,
1830         },
1831 };
1832
1833 static const struct amdgpu_ip_block_version mullins_ip_blocks[] =
1834 {
1835         /* ORDER MATTERS! */
1836         {
1837                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1838                 .major = 1,
1839                 .minor = 0,
1840                 .rev = 0,
1841                 .funcs = &cik_common_ip_funcs,
1842         },
1843         {
1844                 .type = AMD_IP_BLOCK_TYPE_GMC,
1845                 .major = 7,
1846                 .minor = 0,
1847                 .rev = 0,
1848                 .funcs = &gmc_v7_0_ip_funcs,
1849         },
1850         {
1851                 .type = AMD_IP_BLOCK_TYPE_IH,
1852                 .major = 2,
1853                 .minor = 0,
1854                 .rev = 0,
1855                 .funcs = &cik_ih_ip_funcs,
1856         },
1857         {
1858                 .type = AMD_IP_BLOCK_TYPE_SMC,
1859                 .major = 7,
1860                 .minor = 0,
1861                 .rev = 0,
1862                 .funcs = &amdgpu_pp_ip_funcs,
1863         },
1864         {
1865                 .type = AMD_IP_BLOCK_TYPE_DCE,
1866                 .major = 8,
1867                 .minor = 3,
1868                 .rev = 0,
1869                 .funcs = &dce_v8_0_ip_funcs,
1870         },
1871         {
1872                 .type = AMD_IP_BLOCK_TYPE_GFX,
1873                 .major = 7,
1874                 .minor = 2,
1875                 .rev = 0,
1876                 .funcs = &gfx_v7_0_ip_funcs,
1877         },
1878         {
1879                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1880                 .major = 2,
1881                 .minor = 0,
1882                 .rev = 0,
1883                 .funcs = &cik_sdma_ip_funcs,
1884         },
1885         {
1886                 .type = AMD_IP_BLOCK_TYPE_UVD,
1887                 .major = 4,
1888                 .minor = 2,
1889                 .rev = 0,
1890                 .funcs = &uvd_v4_2_ip_funcs,
1891         },
1892         {
1893                 .type = AMD_IP_BLOCK_TYPE_VCE,
1894                 .major = 2,
1895                 .minor = 0,
1896                 .rev = 0,
1897                 .funcs = &vce_v2_0_ip_funcs,
1898         },
1899 };
1900
1901 static const struct amdgpu_ip_block_version kaveri_ip_blocks[] =
1902 {
1903         /* ORDER MATTERS! */
1904         {
1905                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1906                 .major = 1,
1907                 .minor = 0,
1908                 .rev = 0,
1909                 .funcs = &cik_common_ip_funcs,
1910         },
1911         {
1912                 .type = AMD_IP_BLOCK_TYPE_GMC,
1913                 .major = 7,
1914                 .minor = 0,
1915                 .rev = 0,
1916                 .funcs = &gmc_v7_0_ip_funcs,
1917         },
1918         {
1919                 .type = AMD_IP_BLOCK_TYPE_IH,
1920                 .major = 2,
1921                 .minor = 0,
1922                 .rev = 0,
1923                 .funcs = &cik_ih_ip_funcs,
1924         },
1925         {
1926                 .type = AMD_IP_BLOCK_TYPE_SMC,
1927                 .major = 7,
1928                 .minor = 0,
1929                 .rev = 0,
1930                 .funcs = &amdgpu_pp_ip_funcs,
1931         },
1932         {
1933                 .type = AMD_IP_BLOCK_TYPE_DCE,
1934                 .major = 8,
1935                 .minor = 1,
1936                 .rev = 0,
1937                 .funcs = &dce_v8_0_ip_funcs,
1938         },
1939         {
1940                 .type = AMD_IP_BLOCK_TYPE_GFX,
1941                 .major = 7,
1942                 .minor = 1,
1943                 .rev = 0,
1944                 .funcs = &gfx_v7_0_ip_funcs,
1945         },
1946         {
1947                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1948                 .major = 2,
1949                 .minor = 0,
1950                 .rev = 0,
1951                 .funcs = &cik_sdma_ip_funcs,
1952         },
1953         {
1954                 .type = AMD_IP_BLOCK_TYPE_UVD,
1955                 .major = 4,
1956                 .minor = 2,
1957                 .rev = 0,
1958                 .funcs = &uvd_v4_2_ip_funcs,
1959         },
1960         {
1961                 .type = AMD_IP_BLOCK_TYPE_VCE,
1962                 .major = 2,
1963                 .minor = 0,
1964                 .rev = 0,
1965                 .funcs = &vce_v2_0_ip_funcs,
1966         },
1967 };
1968
1969 int cik_set_ip_blocks(struct amdgpu_device *adev)
1970 {
1971         switch (adev->asic_type) {
1972         case CHIP_BONAIRE:
1973                 adev->ip_blocks = bonaire_ip_blocks;
1974                 adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks);
1975                 break;
1976         case CHIP_HAWAII:
1977                 adev->ip_blocks = hawaii_ip_blocks;
1978                 adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks);
1979                 break;
1980         case CHIP_KAVERI:
1981                 adev->ip_blocks = kaveri_ip_blocks;
1982                 adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks);
1983                 break;
1984         case CHIP_KABINI:
1985                 adev->ip_blocks = kabini_ip_blocks;
1986                 adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks);
1987                 break;
1988         case CHIP_MULLINS:
1989                 adev->ip_blocks = mullins_ip_blocks;
1990                 adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks);
1991                 break;
1992         default:
1993                 /* FIXME: not supported yet */
1994                 return -EINVAL;
1995         }
1996
1997         return 0;
1998 }
1999
2000 static const struct amdgpu_asic_funcs cik_asic_funcs =
2001 {
2002         .read_disabled_bios = &cik_read_disabled_bios,
2003         .read_bios_from_rom = &cik_read_bios_from_rom,
2004         .read_register = &cik_read_register,
2005         .reset = &cik_asic_reset,
2006         .set_vga_state = &cik_vga_set_state,
2007         .get_xclk = &cik_get_xclk,
2008         .set_uvd_clocks = &cik_set_uvd_clocks,
2009         .set_vce_clocks = &cik_set_vce_clocks,
2010         .get_cu_info = &gfx_v7_0_get_cu_info,
2011         /* these should be moved to their own ip modules */
2012         .get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
2013         .wait_for_mc_idle = &gmc_v7_0_mc_wait_for_idle,
2014 };
2015
2016 static int cik_common_early_init(void *handle)
2017 {
2018         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2019
2020         adev->smc_rreg = &cik_smc_rreg;
2021         adev->smc_wreg = &cik_smc_wreg;
2022         adev->pcie_rreg = &cik_pcie_rreg;
2023         adev->pcie_wreg = &cik_pcie_wreg;
2024         adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2025         adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2026         adev->didt_rreg = &cik_didt_rreg;
2027         adev->didt_wreg = &cik_didt_wreg;
2028
2029         adev->asic_funcs = &cik_asic_funcs;
2030
2031         adev->rev_id = cik_get_rev_id(adev);
2032         adev->external_rev_id = 0xFF;
2033         switch (adev->asic_type) {
2034         case CHIP_BONAIRE:
2035                 adev->cg_flags =
2036                         AMD_CG_SUPPORT_GFX_MGCG |
2037                         AMD_CG_SUPPORT_GFX_MGLS |
2038                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2039                         AMD_CG_SUPPORT_GFX_CGLS |
2040                         AMD_CG_SUPPORT_GFX_CGTS |
2041                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2042                         AMD_CG_SUPPORT_GFX_CP_LS |
2043                         AMD_CG_SUPPORT_MC_LS |
2044                         AMD_CG_SUPPORT_MC_MGCG |
2045                         AMD_CG_SUPPORT_SDMA_MGCG |
2046                         AMD_CG_SUPPORT_SDMA_LS |
2047                         AMD_CG_SUPPORT_BIF_LS |
2048                         AMD_CG_SUPPORT_VCE_MGCG |
2049                         AMD_CG_SUPPORT_UVD_MGCG |
2050                         AMD_CG_SUPPORT_HDP_LS |
2051                         AMD_CG_SUPPORT_HDP_MGCG;
2052                 adev->pg_flags = 0;
2053                 adev->external_rev_id = adev->rev_id + 0x14;
2054                 break;
2055         case CHIP_HAWAII:
2056                 adev->cg_flags =
2057                         AMD_CG_SUPPORT_GFX_MGCG |
2058                         AMD_CG_SUPPORT_GFX_MGLS |
2059                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2060                         AMD_CG_SUPPORT_GFX_CGLS |
2061                         AMD_CG_SUPPORT_GFX_CGTS |
2062                         AMD_CG_SUPPORT_GFX_CP_LS |
2063                         AMD_CG_SUPPORT_MC_LS |
2064                         AMD_CG_SUPPORT_MC_MGCG |
2065                         AMD_CG_SUPPORT_SDMA_MGCG |
2066                         AMD_CG_SUPPORT_SDMA_LS |
2067                         AMD_CG_SUPPORT_BIF_LS |
2068                         AMD_CG_SUPPORT_VCE_MGCG |
2069                         AMD_CG_SUPPORT_UVD_MGCG |
2070                         AMD_CG_SUPPORT_HDP_LS |
2071                         AMD_CG_SUPPORT_HDP_MGCG;
2072                 adev->pg_flags = 0;
2073                 adev->external_rev_id = 0x28;
2074                 break;
2075         case CHIP_KAVERI:
2076                 adev->cg_flags =
2077                         AMD_CG_SUPPORT_GFX_MGCG |
2078                         AMD_CG_SUPPORT_GFX_MGLS |
2079                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2080                         AMD_CG_SUPPORT_GFX_CGLS |
2081                         AMD_CG_SUPPORT_GFX_CGTS |
2082                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2083                         AMD_CG_SUPPORT_GFX_CP_LS |
2084                         AMD_CG_SUPPORT_SDMA_MGCG |
2085                         AMD_CG_SUPPORT_SDMA_LS |
2086                         AMD_CG_SUPPORT_BIF_LS |
2087                         AMD_CG_SUPPORT_VCE_MGCG |
2088                         AMD_CG_SUPPORT_UVD_MGCG |
2089                         AMD_CG_SUPPORT_HDP_LS |
2090                         AMD_CG_SUPPORT_HDP_MGCG;
2091                 adev->pg_flags =
2092                         /*AMD_PG_SUPPORT_GFX_PG |
2093                           AMD_PG_SUPPORT_GFX_SMG |
2094                           AMD_PG_SUPPORT_GFX_DMG |*/
2095                         AMD_PG_SUPPORT_UVD |
2096                         /*AMD_PG_SUPPORT_VCE |
2097                           AMD_PG_SUPPORT_CP |
2098                           AMD_PG_SUPPORT_GDS |
2099                           AMD_PG_SUPPORT_RLC_SMU_HS |
2100                           AMD_PG_SUPPORT_ACP |
2101                           AMD_PG_SUPPORT_SAMU |*/
2102                         0;
2103                 if (adev->pdev->device == 0x1312 ||
2104                         adev->pdev->device == 0x1316 ||
2105                         adev->pdev->device == 0x1317)
2106                         adev->external_rev_id = 0x41;
2107                 else
2108                         adev->external_rev_id = 0x1;
2109                 break;
2110         case CHIP_KABINI:
2111         case CHIP_MULLINS:
2112                 adev->cg_flags =
2113                         AMD_CG_SUPPORT_GFX_MGCG |
2114                         AMD_CG_SUPPORT_GFX_MGLS |
2115                         /*AMD_CG_SUPPORT_GFX_CGCG |*/
2116                         AMD_CG_SUPPORT_GFX_CGLS |
2117                         AMD_CG_SUPPORT_GFX_CGTS |
2118                         AMD_CG_SUPPORT_GFX_CGTS_LS |
2119                         AMD_CG_SUPPORT_GFX_CP_LS |
2120                         AMD_CG_SUPPORT_SDMA_MGCG |
2121                         AMD_CG_SUPPORT_SDMA_LS |
2122                         AMD_CG_SUPPORT_BIF_LS |
2123                         AMD_CG_SUPPORT_VCE_MGCG |
2124                         AMD_CG_SUPPORT_UVD_MGCG |
2125                         AMD_CG_SUPPORT_HDP_LS |
2126                         AMD_CG_SUPPORT_HDP_MGCG;
2127                 adev->pg_flags =
2128                         /*AMD_PG_SUPPORT_GFX_PG |
2129                           AMD_PG_SUPPORT_GFX_SMG | */
2130                         AMD_PG_SUPPORT_UVD |
2131                         /*AMD_PG_SUPPORT_VCE |
2132                           AMD_PG_SUPPORT_CP |
2133                           AMD_PG_SUPPORT_GDS |
2134                           AMD_PG_SUPPORT_RLC_SMU_HS |
2135                           AMD_PG_SUPPORT_SAMU |*/
2136                         0;
2137                 if (adev->asic_type == CHIP_KABINI) {
2138                         if (adev->rev_id == 0)
2139                                 adev->external_rev_id = 0x81;
2140                         else if (adev->rev_id == 1)
2141                                 adev->external_rev_id = 0x82;
2142                         else if (adev->rev_id == 2)
2143                                 adev->external_rev_id = 0x85;
2144                 } else
2145                         adev->external_rev_id = adev->rev_id + 0xa1;
2146                 break;
2147         default:
2148                 /* FIXME: not supported yet */
2149                 return -EINVAL;
2150         }
2151
2152         amdgpu_get_pcie_info(adev);
2153
2154         return 0;
2155 }
2156
2157 static int cik_common_sw_init(void *handle)
2158 {
2159         return 0;
2160 }
2161
2162 static int cik_common_sw_fini(void *handle)
2163 {
2164         return 0;
2165 }
2166
2167 static int cik_common_hw_init(void *handle)
2168 {
2169         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2170
2171         /* move the golden regs per IP block */
2172         cik_init_golden_registers(adev);
2173         /* enable pcie gen2/3 link */
2174         cik_pcie_gen3_enable(adev);
2175         /* enable aspm */
2176         cik_program_aspm(adev);
2177
2178         return 0;
2179 }
2180
2181 static int cik_common_hw_fini(void *handle)
2182 {
2183         return 0;
2184 }
2185
2186 static int cik_common_suspend(void *handle)
2187 {
2188         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2189
2190         amdgpu_amdkfd_suspend(adev);
2191
2192         return cik_common_hw_fini(adev);
2193 }
2194
2195 static int cik_common_resume(void *handle)
2196 {
2197         int r;
2198         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2199
2200         r = cik_common_hw_init(adev);
2201         if (r)
2202                 return r;
2203
2204         return amdgpu_amdkfd_resume(adev);
2205 }
2206
2207 static bool cik_common_is_idle(void *handle)
2208 {
2209         return true;
2210 }
2211
2212 static int cik_common_wait_for_idle(void *handle)
2213 {
2214         return 0;
2215 }
2216
2217 static void cik_common_print_status(void *handle)
2218 {
2219
2220 }
2221
2222 static int cik_common_soft_reset(void *handle)
2223 {
2224         /* XXX hard reset?? */
2225         return 0;
2226 }
2227
2228 static int cik_common_set_clockgating_state(void *handle,
2229                                             enum amd_clockgating_state state)
2230 {
2231         return 0;
2232 }
2233
2234 static int cik_common_set_powergating_state(void *handle,
2235                                             enum amd_powergating_state state)
2236 {
2237         return 0;
2238 }
2239
2240 const struct amd_ip_funcs cik_common_ip_funcs = {
2241         .early_init = cik_common_early_init,
2242         .late_init = NULL,
2243         .sw_init = cik_common_sw_init,
2244         .sw_fini = cik_common_sw_fini,
2245         .hw_init = cik_common_hw_init,
2246         .hw_fini = cik_common_hw_fini,
2247         .suspend = cik_common_suspend,
2248         .resume = cik_common_resume,
2249         .is_idle = cik_common_is_idle,
2250         .wait_for_idle = cik_common_wait_for_idle,
2251         .soft_reset = cik_common_soft_reset,
2252         .print_status = cik_common_print_status,
2253         .set_clockgating_state = cik_common_set_clockgating_state,
2254         .set_powergating_state = cik_common_set_powergating_state,
2255 };