drivers/gpu/drm/radeon/rv770.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/firmware.h>
  29 #include <linux/slab.h>
  30 #include <drm/drmP.h>
  31 #include "radeon.h"
  32 #include "radeon_asic.h"
  33 #include "radeon_audio.h"
  34 #include <drm/radeon_drm.h>
  35 #include "rv770d.h"
  36 #include "atom.h"
  37 #include "avivod.h"
  38
  39 #define R700_PFP_UCODE_SIZE 848
  40 #define R700_PM4_UCODE_SIZE 1360
  41
  42 static void rv770_gpu_init(struct radeon_device *rdev);
  43 void rv770_fini(struct radeon_device *rdev);
  44 static void rv770_pcie_gen2_enable(struct radeon_device *rdev);
  45 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk);
  46
  47 int rv770_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
  48 {
  49         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
  50         int r;
  51
  52         /* RV740 uses evergreen uvd clk programming */
  53         if (rdev->family == CHIP_RV740)
  54                 return evergreen_set_uvd_clocks(rdev, vclk, dclk);
  55
  56         /* bypass vclk and dclk with bclk */
  57         WREG32_P(CG_UPLL_FUNC_CNTL_2,
  58                  VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
  59                  ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
  60
  61         if (!vclk || !dclk) {
  62                 /* keep the Bypass mode, put PLL to sleep */
  63                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
  64                 return 0;
  65         }
  66
  67         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000,
  68                                           43663, 0x03FFFFFE, 1, 30, ~0,
  69                                           &fb_div, &vclk_div, &dclk_div);
  70         if (r)
  71                 return r;
  72
  73         fb_div |= 1;
  74         vclk_div -= 1;
  75         dclk_div -= 1;
  76
  77         /* set UPLL_FB_DIV to 0x50000 */
  78         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(0x50000), ~UPLL_FB_DIV_MASK);
  79
  80         /* deassert UPLL_RESET and UPLL_SLEEP */
  81         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~(UPLL_RESET_MASK | UPLL_SLEEP_MASK));
  82
  83         /* assert BYPASS EN and FB_DIV[0] <- ??? why? */
  84         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
  85         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(1), ~UPLL_FB_DIV(1));
  86
  87         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
  88         if (r)
  89                 return r;
  90
  91         /* assert PLL_RESET */
  92         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
  93
  94         /* set the required FB_DIV, REF_DIV, Post divder values */
  95         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REF_DIV(1), ~UPLL_REF_DIV_MASK);
  96         WREG32_P(CG_UPLL_FUNC_CNTL_2,
  97                  UPLL_SW_HILEN(vclk_div >> 1) |
  98                  UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) |
  99                  UPLL_SW_HILEN2(dclk_div >> 1) |
 100                  UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)),
 101                  ~UPLL_SW_MASK);
 102
 103         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div),
 104                  ~UPLL_FB_DIV_MASK);
 105
 106         /* give the PLL some time to settle */
 107         mdelay(15);
 108
 109         /* deassert PLL_RESET */
 110         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
 111
 112         mdelay(15);
 113
 114         /* deassert BYPASS EN and FB_DIV[0] <- ??? why? */
 115         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
 116         WREG32_P(CG_UPLL_FUNC_CNTL_3, 0, ~UPLL_FB_DIV(1));
 117
 118         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
 119         if (r)
 120                 return r;
 121
 122         /* switch VCLK and DCLK selection */
 123         WREG32_P(CG_UPLL_FUNC_CNTL_2,
 124                  VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
 125                  ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
 126
 127         mdelay(100);
 128
 129         return 0;
 130 }
 131
 132 static const u32 r7xx_golden_registers[] =
 133 {
 134         0x8d00, 0xffffffff, 0x0e0e0074,
 135         0x8d04, 0xffffffff, 0x013a2b34,
 136         0x9508, 0xffffffff, 0x00000002,
 137         0x8b20, 0xffffffff, 0,
 138         0x88c4, 0xffffffff, 0x000000c2,
 139         0x28350, 0xffffffff, 0,
 140         0x9058, 0xffffffff, 0x0fffc40f,
 141         0x240c, 0xffffffff, 0x00000380,
 142         0x733c, 0xffffffff, 0x00000002,
 143         0x2650, 0x00040000, 0,
 144         0x20bc, 0x00040000, 0,
 145         0x7300, 0xffffffff, 0x001000f0
 146 };
 147
 148 static const u32 r7xx_golden_dyn_gpr_registers[] =
 149 {
 150         0x8db0, 0xffffffff, 0x98989898,
 151         0x8db4, 0xffffffff, 0x98989898,
 152         0x8db8, 0xffffffff, 0x98989898,
 153         0x8dbc, 0xffffffff, 0x98989898,
 154         0x8dc0, 0xffffffff, 0x98989898,
 155         0x8dc4, 0xffffffff, 0x98989898,
 156         0x8dc8, 0xffffffff, 0x98989898,
 157         0x8dcc, 0xffffffff, 0x98989898,
 158         0x88c4, 0xffffffff, 0x00000082
 159 };
 160
 161 static const u32 rv770_golden_registers[] =
 162 {
 163         0x562c, 0xffffffff, 0,
 164         0x3f90, 0xffffffff, 0,
 165         0x9148, 0xffffffff, 0,
 166         0x3f94, 0xffffffff, 0,
 167         0x914c, 0xffffffff, 0,
 168         0x9698, 0x18000000, 0x18000000
 169 };
 170
 171 static const u32 rv770ce_golden_registers[] =
 172 {
 173         0x562c, 0xffffffff, 0,
 174         0x3f90, 0xffffffff, 0x00cc0000,
 175         0x9148, 0xffffffff, 0x00cc0000,
 176         0x3f94, 0xffffffff, 0x00cc0000,
 177         0x914c, 0xffffffff, 0x00cc0000,
 178         0x9b7c, 0xffffffff, 0x00fa0000,
 179         0x3f8c, 0xffffffff, 0x00fa0000,
 180         0x9698, 0x18000000, 0x18000000
 181 };
 182
 183 static const u32 rv770_mgcg_init[] =
 184 {
 185         0x8bcc, 0xffffffff, 0x130300f9,
 186         0x5448, 0xffffffff, 0x100,
 187         0x55e4, 0xffffffff, 0x100,
 188         0x160c, 0xffffffff, 0x100,
 189         0x5644, 0xffffffff, 0x100,
 190         0xc164, 0xffffffff, 0x100,
 191         0x8a18, 0xffffffff, 0x100,
 192         0x897c, 0xffffffff, 0x8000100,
 193         0x8b28, 0xffffffff, 0x3c000100,
 194         0x9144, 0xffffffff, 0x100,
 195         0x9a1c, 0xffffffff, 0x10000,
 196         0x9a50, 0xffffffff, 0x100,
 197         0x9a1c, 0xffffffff, 0x10001,
 198         0x9a50, 0xffffffff, 0x100,
 199         0x9a1c, 0xffffffff, 0x10002,
 200         0x9a50, 0xffffffff, 0x100,
 201         0x9a1c, 0xffffffff, 0x10003,
 202         0x9a50, 0xffffffff, 0x100,
 203         0x9a1c, 0xffffffff, 0x0,
 204         0x9870, 0xffffffff, 0x100,
 205         0x8d58, 0xffffffff, 0x100,
 206         0x9500, 0xffffffff, 0x0,
 207         0x9510, 0xffffffff, 0x100,
 208         0x9500, 0xffffffff, 0x1,
 209         0x9510, 0xffffffff, 0x100,
 210         0x9500, 0xffffffff, 0x2,
 211         0x9510, 0xffffffff, 0x100,
 212         0x9500, 0xffffffff, 0x3,
 213         0x9510, 0xffffffff, 0x100,
 214         0x9500, 0xffffffff, 0x4,
 215         0x9510, 0xffffffff, 0x100,
 216         0x9500, 0xffffffff, 0x5,
 217         0x9510, 0xffffffff, 0x100,
 218         0x9500, 0xffffffff, 0x6,
 219         0x9510, 0xffffffff, 0x100,
 220         0x9500, 0xffffffff, 0x7,
 221         0x9510, 0xffffffff, 0x100,
 222         0x9500, 0xffffffff, 0x8,
 223         0x9510, 0xffffffff, 0x100,
 224         0x9500, 0xffffffff, 0x9,
 225         0x9510, 0xffffffff, 0x100,
 226         0x9500, 0xffffffff, 0x8000,
 227         0x9490, 0xffffffff, 0x0,
 228         0x949c, 0xffffffff, 0x100,
 229         0x9490, 0xffffffff, 0x1,
 230         0x949c, 0xffffffff, 0x100,
 231         0x9490, 0xffffffff, 0x2,
 232         0x949c, 0xffffffff, 0x100,
 233         0x9490, 0xffffffff, 0x3,
 234         0x949c, 0xffffffff, 0x100,
 235         0x9490, 0xffffffff, 0x4,
 236         0x949c, 0xffffffff, 0x100,
 237         0x9490, 0xffffffff, 0x5,
 238         0x949c, 0xffffffff, 0x100,
 239         0x9490, 0xffffffff, 0x6,
 240         0x949c, 0xffffffff, 0x100,
 241         0x9490, 0xffffffff, 0x7,
 242         0x949c, 0xffffffff, 0x100,
 243         0x9490, 0xffffffff, 0x8,
 244         0x949c, 0xffffffff, 0x100,
 245         0x9490, 0xffffffff, 0x9,
 246         0x949c, 0xffffffff, 0x100,
 247         0x9490, 0xffffffff, 0x8000,
 248         0x9604, 0xffffffff, 0x0,
 249         0x9654, 0xffffffff, 0x100,
 250         0x9604, 0xffffffff, 0x1,
 251         0x9654, 0xffffffff, 0x100,
 252         0x9604, 0xffffffff, 0x2,
 253         0x9654, 0xffffffff, 0x100,
 254         0x9604, 0xffffffff, 0x3,
 255         0x9654, 0xffffffff, 0x100,
 256         0x9604, 0xffffffff, 0x4,
 257         0x9654, 0xffffffff, 0x100,
 258         0x9604, 0xffffffff, 0x5,
 259         0x9654, 0xffffffff, 0x100,
 260         0x9604, 0xffffffff, 0x6,
 261         0x9654, 0xffffffff, 0x100,
 262         0x9604, 0xffffffff, 0x7,
 263         0x9654, 0xffffffff, 0x100,
 264         0x9604, 0xffffffff, 0x8,
 265         0x9654, 0xffffffff, 0x100,
 266         0x9604, 0xffffffff, 0x9,
 267         0x9654, 0xffffffff, 0x100,
 268         0x9604, 0xffffffff, 0x80000000,
 269         0x9030, 0xffffffff, 0x100,
 270         0x9034, 0xffffffff, 0x100,
 271         0x9038, 0xffffffff, 0x100,
 272         0x903c, 0xffffffff, 0x100,
 273         0x9040, 0xffffffff, 0x100,
 274         0xa200, 0xffffffff, 0x100,
 275         0xa204, 0xffffffff, 0x100,
 276         0xa208, 0xffffffff, 0x100,
 277         0xa20c, 0xffffffff, 0x100,
 278         0x971c, 0xffffffff, 0x100,
 279         0x915c, 0xffffffff, 0x00020001,
 280         0x9160, 0xffffffff, 0x00040003,
 281         0x916c, 0xffffffff, 0x00060005,
 282         0x9170, 0xffffffff, 0x00080007,
 283         0x9174, 0xffffffff, 0x000a0009,
 284         0x9178, 0xffffffff, 0x000c000b,
 285         0x917c, 0xffffffff, 0x000e000d,
 286         0x9180, 0xffffffff, 0x0010000f,
 287         0x918c, 0xffffffff, 0x00120011,
 288         0x9190, 0xffffffff, 0x00140013,
 289         0x9194, 0xffffffff, 0x00020001,
 290         0x9198, 0xffffffff, 0x00040003,
 291         0x919c, 0xffffffff, 0x00060005,
 292         0x91a8, 0xffffffff, 0x00080007,
 293         0x91ac, 0xffffffff, 0x000a0009,
 294         0x91b0, 0xffffffff, 0x000c000b,
 295         0x91b4, 0xffffffff, 0x000e000d,
 296         0x91b8, 0xffffffff, 0x0010000f,
 297         0x91c4, 0xffffffff, 0x00120011,
 298         0x91c8, 0xffffffff, 0x00140013,
 299         0x91cc, 0xffffffff, 0x00020001,
 300         0x91d0, 0xffffffff, 0x00040003,
 301         0x91d4, 0xffffffff, 0x00060005,
 302         0x91e0, 0xffffffff, 0x00080007,
 303         0x91e4, 0xffffffff, 0x000a0009,
 304         0x91e8, 0xffffffff, 0x000c000b,
 305         0x91ec, 0xffffffff, 0x00020001,
 306         0x91f0, 0xffffffff, 0x00040003,
 307         0x91f4, 0xffffffff, 0x00060005,
 308         0x9200, 0xffffffff, 0x00080007,
 309         0x9204, 0xffffffff, 0x000a0009,
 310         0x9208, 0xffffffff, 0x000c000b,
 311         0x920c, 0xffffffff, 0x000e000d,
 312         0x9210, 0xffffffff, 0x0010000f,
 313         0x921c, 0xffffffff, 0x00120011,
 314         0x9220, 0xffffffff, 0x00140013,
 315         0x9224, 0xffffffff, 0x00020001,
 316         0x9228, 0xffffffff, 0x00040003,
 317         0x922c, 0xffffffff, 0x00060005,
 318         0x9238, 0xffffffff, 0x00080007,
 319         0x923c, 0xffffffff, 0x000a0009,
 320         0x9240, 0xffffffff, 0x000c000b,
 321         0x9244, 0xffffffff, 0x000e000d,
 322         0x9248, 0xffffffff, 0x0010000f,
 323         0x9254, 0xffffffff, 0x00120011,
 324         0x9258, 0xffffffff, 0x00140013,
 325         0x925c, 0xffffffff, 0x00020001,
 326         0x9260, 0xffffffff, 0x00040003,
 327         0x9264, 0xffffffff, 0x00060005,
 328         0x9270, 0xffffffff, 0x00080007,
 329         0x9274, 0xffffffff, 0x000a0009,
 330         0x9278, 0xffffffff, 0x000c000b,
 331         0x927c, 0xffffffff, 0x000e000d,
 332         0x9280, 0xffffffff, 0x0010000f,
 333         0x928c, 0xffffffff, 0x00120011,
 334         0x9290, 0xffffffff, 0x00140013,
 335         0x9294, 0xffffffff, 0x00020001,
 336         0x929c, 0xffffffff, 0x00040003,
 337         0x92a0, 0xffffffff, 0x00060005,
 338         0x92a4, 0xffffffff, 0x00080007
 339 };
 340
 341 static const u32 rv710_golden_registers[] =
 342 {
 343         0x3f90, 0x00ff0000, 0x00fc0000,
 344         0x9148, 0x00ff0000, 0x00fc0000,
 345         0x3f94, 0x00ff0000, 0x00fc0000,
 346         0x914c, 0x00ff0000, 0x00fc0000,
 347         0xb4c, 0x00000020, 0x00000020,
 348         0xa180, 0xffffffff, 0x00003f3f
 349 };
 350
 351 static const u32 rv710_mgcg_init[] =
 352 {
 353         0x8bcc, 0xffffffff, 0x13030040,
 354         0x5448, 0xffffffff, 0x100,
 355         0x55e4, 0xffffffff, 0x100,
 356         0x160c, 0xffffffff, 0x100,
 357         0x5644, 0xffffffff, 0x100,
 358         0xc164, 0xffffffff, 0x100,
 359         0x8a18, 0xffffffff, 0x100,
 360         0x897c, 0xffffffff, 0x8000100,
 361         0x8b28, 0xffffffff, 0x3c000100,
 362         0x9144, 0xffffffff, 0x100,
 363         0x9a1c, 0xffffffff, 0x10000,
 364         0x9a50, 0xffffffff, 0x100,
 365         0x9a1c, 0xffffffff, 0x0,
 366         0x9870, 0xffffffff, 0x100,
 367         0x8d58, 0xffffffff, 0x100,
 368         0x9500, 0xffffffff, 0x0,
 369         0x9510, 0xffffffff, 0x100,
 370         0x9500, 0xffffffff, 0x1,
 371         0x9510, 0xffffffff, 0x100,
 372         0x9500, 0xffffffff, 0x8000,
 373         0x9490, 0xffffffff, 0x0,
 374         0x949c, 0xffffffff, 0x100,
 375         0x9490, 0xffffffff, 0x1,
 376         0x949c, 0xffffffff, 0x100,
 377         0x9490, 0xffffffff, 0x8000,
 378         0x9604, 0xffffffff, 0x0,
 379         0x9654, 0xffffffff, 0x100,
 380         0x9604, 0xffffffff, 0x1,
 381         0x9654, 0xffffffff, 0x100,
 382         0x9604, 0xffffffff, 0x80000000,
 383         0x9030, 0xffffffff, 0x100,
 384         0x9034, 0xffffffff, 0x100,
 385         0x9038, 0xffffffff, 0x100,
 386         0x903c, 0xffffffff, 0x100,
 387         0x9040, 0xffffffff, 0x100,
 388         0xa200, 0xffffffff, 0x100,
 389         0xa204, 0xffffffff, 0x100,
 390         0xa208, 0xffffffff, 0x100,
 391         0xa20c, 0xffffffff, 0x100,
 392         0x971c, 0xffffffff, 0x100,
 393         0x915c, 0xffffffff, 0x00020001,
 394         0x9174, 0xffffffff, 0x00000003,
 395         0x9178, 0xffffffff, 0x00050001,
 396         0x917c, 0xffffffff, 0x00030002,
 397         0x918c, 0xffffffff, 0x00000004,
 398         0x9190, 0xffffffff, 0x00070006,
 399         0x9194, 0xffffffff, 0x00050001,
 400         0x9198, 0xffffffff, 0x00030002,
 401         0x91a8, 0xffffffff, 0x00000004,
 402         0x91ac, 0xffffffff, 0x00070006,
 403         0x91e8, 0xffffffff, 0x00000001,
 404         0x9294, 0xffffffff, 0x00000001,
 405         0x929c, 0xffffffff, 0x00000002,
 406         0x92a0, 0xffffffff, 0x00040003,
 407         0x9150, 0xffffffff, 0x4d940000
 408 };
 409
 410 static const u32 rv730_golden_registers[] =
 411 {
 412         0x3f90, 0x00ff0000, 0x00f00000,
 413         0x9148, 0x00ff0000, 0x00f00000,
 414         0x3f94, 0x00ff0000, 0x00f00000,
 415         0x914c, 0x00ff0000, 0x00f00000,
 416         0x900c, 0xffffffff, 0x003b033f,
 417         0xb4c, 0x00000020, 0x00000020,
 418         0xa180, 0xffffffff, 0x00003f3f
 419 };
 420
 421 static const u32 rv730_mgcg_init[] =
 422 {
 423         0x8bcc, 0xffffffff, 0x130300f9,
 424         0x5448, 0xffffffff, 0x100,
 425         0x55e4, 0xffffffff, 0x100,
 426         0x160c, 0xffffffff, 0x100,
 427         0x5644, 0xffffffff, 0x100,
 428         0xc164, 0xffffffff, 0x100,
 429         0x8a18, 0xffffffff, 0x100,
 430         0x897c, 0xffffffff, 0x8000100,
 431         0x8b28, 0xffffffff, 0x3c000100,
 432         0x9144, 0xffffffff, 0x100,
 433         0x9a1c, 0xffffffff, 0x10000,
 434         0x9a50, 0xffffffff, 0x100,
 435         0x9a1c, 0xffffffff, 0x10001,
 436         0x9a50, 0xffffffff, 0x100,
 437         0x9a1c, 0xffffffff, 0x0,
 438         0x9870, 0xffffffff, 0x100,
 439         0x8d58, 0xffffffff, 0x100,
 440         0x9500, 0xffffffff, 0x0,
 441         0x9510, 0xffffffff, 0x100,
 442         0x9500, 0xffffffff, 0x1,
 443         0x9510, 0xffffffff, 0x100,
 444         0x9500, 0xffffffff, 0x2,
 445         0x9510, 0xffffffff, 0x100,
 446         0x9500, 0xffffffff, 0x3,
 447         0x9510, 0xffffffff, 0x100,
 448         0x9500, 0xffffffff, 0x4,
 449         0x9510, 0xffffffff, 0x100,
 450         0x9500, 0xffffffff, 0x5,
 451         0x9510, 0xffffffff, 0x100,
 452         0x9500, 0xffffffff, 0x6,
 453         0x9510, 0xffffffff, 0x100,
 454         0x9500, 0xffffffff, 0x7,
 455         0x9510, 0xffffffff, 0x100,
 456         0x9500, 0xffffffff, 0x8000,
 457         0x9490, 0xffffffff, 0x0,
 458         0x949c, 0xffffffff, 0x100,
 459         0x9490, 0xffffffff, 0x1,
 460         0x949c, 0xffffffff, 0x100,
 461         0x9490, 0xffffffff, 0x2,
 462         0x949c, 0xffffffff, 0x100,
 463         0x9490, 0xffffffff, 0x3,
 464         0x949c, 0xffffffff, 0x100,
 465         0x9490, 0xffffffff, 0x4,
 466         0x949c, 0xffffffff, 0x100,
 467         0x9490, 0xffffffff, 0x5,
 468         0x949c, 0xffffffff, 0x100,
 469         0x9490, 0xffffffff, 0x6,
 470         0x949c, 0xffffffff, 0x100,
 471         0x9490, 0xffffffff, 0x7,
 472         0x949c, 0xffffffff, 0x100,
 473         0x9490, 0xffffffff, 0x8000,
 474         0x9604, 0xffffffff, 0x0,
 475         0x9654, 0xffffffff, 0x100,
 476         0x9604, 0xffffffff, 0x1,
 477         0x9654, 0xffffffff, 0x100,
 478         0x9604, 0xffffffff, 0x2,
 479         0x9654, 0xffffffff, 0x100,
 480         0x9604, 0xffffffff, 0x3,
 481         0x9654, 0xffffffff, 0x100,
 482         0x9604, 0xffffffff, 0x4,
 483         0x9654, 0xffffffff, 0x100,
 484         0x9604, 0xffffffff, 0x5,
 485         0x9654, 0xffffffff, 0x100,
 486         0x9604, 0xffffffff, 0x6,
 487         0x9654, 0xffffffff, 0x100,
 488         0x9604, 0xffffffff, 0x7,
 489         0x9654, 0xffffffff, 0x100,
 490         0x9604, 0xffffffff, 0x80000000,
 491         0x9030, 0xffffffff, 0x100,
 492         0x9034, 0xffffffff, 0x100,
 493         0x9038, 0xffffffff, 0x100,
 494         0x903c, 0xffffffff, 0x100,
 495         0x9040, 0xffffffff, 0x100,
 496         0xa200, 0xffffffff, 0x100,
 497         0xa204, 0xffffffff, 0x100,
 498         0xa208, 0xffffffff, 0x100,
 499         0xa20c, 0xffffffff, 0x100,
 500         0x971c, 0xffffffff, 0x100,
 501         0x915c, 0xffffffff, 0x00020001,
 502         0x916c, 0xffffffff, 0x00040003,
 503         0x9170, 0xffffffff, 0x00000005,
 504         0x9178, 0xffffffff, 0x00050001,
 505         0x917c, 0xffffffff, 0x00030002,
 506         0x918c, 0xffffffff, 0x00000004,
 507         0x9190, 0xffffffff, 0x00070006,
 508         0x9194, 0xffffffff, 0x00050001,
 509         0x9198, 0xffffffff, 0x00030002,
 510         0x91a8, 0xffffffff, 0x00000004,
 511         0x91ac, 0xffffffff, 0x00070006,
 512         0x91b0, 0xffffffff, 0x00050001,
 513         0x91b4, 0xffffffff, 0x00030002,
 514         0x91c4, 0xffffffff, 0x00000004,
 515         0x91c8, 0xffffffff, 0x00070006,
 516         0x91cc, 0xffffffff, 0x00050001,
 517         0x91d0, 0xffffffff, 0x00030002,
 518         0x91e0, 0xffffffff, 0x00000004,
 519         0x91e4, 0xffffffff, 0x00070006,
 520         0x91e8, 0xffffffff, 0x00000001,
 521         0x91ec, 0xffffffff, 0x00050001,
 522         0x91f0, 0xffffffff, 0x00030002,
 523         0x9200, 0xffffffff, 0x00000004,
 524         0x9204, 0xffffffff, 0x00070006,
 525         0x9208, 0xffffffff, 0x00050001,
 526         0x920c, 0xffffffff, 0x00030002,
 527         0x921c, 0xffffffff, 0x00000004,
 528         0x9220, 0xffffffff, 0x00070006,
 529         0x9224, 0xffffffff, 0x00050001,
 530         0x9228, 0xffffffff, 0x00030002,
 531         0x9238, 0xffffffff, 0x00000004,
 532         0x923c, 0xffffffff, 0x00070006,
 533         0x9240, 0xffffffff, 0x00050001,
 534         0x9244, 0xffffffff, 0x00030002,
 535         0x9254, 0xffffffff, 0x00000004,
 536         0x9258, 0xffffffff, 0x00070006,
 537         0x9294, 0xffffffff, 0x00000001,
 538         0x929c, 0xffffffff, 0x00000002,
 539         0x92a0, 0xffffffff, 0x00040003,
 540         0x92a4, 0xffffffff, 0x00000005
 541 };
 542
 543 static const u32 rv740_golden_registers[] =
 544 {
 545         0x88c4, 0xffffffff, 0x00000082,
 546         0x28a50, 0xfffffffc, 0x00000004,
 547         0x2650, 0x00040000, 0,
 548         0x20bc, 0x00040000, 0,
 549         0x733c, 0xffffffff, 0x00000002,
 550         0x7300, 0xffffffff, 0x001000f0,
 551         0x3f90, 0x00ff0000, 0,
 552         0x9148, 0x00ff0000, 0,
 553         0x3f94, 0x00ff0000, 0,
 554         0x914c, 0x00ff0000, 0,
 555         0x240c, 0xffffffff, 0x00000380,
 556         0x8a14, 0x00000007, 0x00000007,
 557         0x8b24, 0xffffffff, 0x00ff0fff,
 558         0x28a4c, 0xffffffff, 0x00004000,
 559         0xa180, 0xffffffff, 0x00003f3f,
 560         0x8d00, 0xffffffff, 0x0e0e003a,
 561         0x8d04, 0xffffffff, 0x013a0e2a,
 562         0x8c00, 0xffffffff, 0xe400000f,
 563         0x8db0, 0xffffffff, 0x98989898,
 564         0x8db4, 0xffffffff, 0x98989898,
 565         0x8db8, 0xffffffff, 0x98989898,
 566         0x8dbc, 0xffffffff, 0x98989898,
 567         0x8dc0, 0xffffffff, 0x98989898,
 568         0x8dc4, 0xffffffff, 0x98989898,
 569         0x8dc8, 0xffffffff, 0x98989898,
 570         0x8dcc, 0xffffffff, 0x98989898,
 571         0x9058, 0xffffffff, 0x0fffc40f,
 572         0x900c, 0xffffffff, 0x003b033f,
 573         0x28350, 0xffffffff, 0,
 574         0x8cf0, 0x1fffffff, 0x08e00420,
 575         0x9508, 0xffffffff, 0x00000002,
 576         0x88c4, 0xffffffff, 0x000000c2,
 577         0x9698, 0x18000000, 0x18000000
 578 };
 579
 580 static const u32 rv740_mgcg_init[] =
 581 {
 582         0x8bcc, 0xffffffff, 0x13030100,
 583         0x5448, 0xffffffff, 0x100,
 584         0x55e4, 0xffffffff, 0x100,
 585         0x160c, 0xffffffff, 0x100,
 586         0x5644, 0xffffffff, 0x100,
 587         0xc164, 0xffffffff, 0x100,
 588         0x8a18, 0xffffffff, 0x100,
 589         0x897c, 0xffffffff, 0x100,
 590         0x8b28, 0xffffffff, 0x100,
 591         0x9144, 0xffffffff, 0x100,
 592         0x9a1c, 0xffffffff, 0x10000,
 593         0x9a50, 0xffffffff, 0x100,
 594         0x9a1c, 0xffffffff, 0x10001,
 595         0x9a50, 0xffffffff, 0x100,
 596         0x9a1c, 0xffffffff, 0x10002,
 597         0x9a50, 0xffffffff, 0x100,
 598         0x9a1c, 0xffffffff, 0x10003,
 599         0x9a50, 0xffffffff, 0x100,
 600         0x9a1c, 0xffffffff, 0x0,
 601         0x9870, 0xffffffff, 0x100,
 602         0x8d58, 0xffffffff, 0x100,
 603         0x9500, 0xffffffff, 0x0,
 604         0x9510, 0xffffffff, 0x100,
 605         0x9500, 0xffffffff, 0x1,
 606         0x9510, 0xffffffff, 0x100,
 607         0x9500, 0xffffffff, 0x2,
 608         0x9510, 0xffffffff, 0x100,
 609         0x9500, 0xffffffff, 0x3,
 610         0x9510, 0xffffffff, 0x100,
 611         0x9500, 0xffffffff, 0x4,
 612         0x9510, 0xffffffff, 0x100,
 613         0x9500, 0xffffffff, 0x5,
 614         0x9510, 0xffffffff, 0x100,
 615         0x9500, 0xffffffff, 0x6,
 616         0x9510, 0xffffffff, 0x100,
 617         0x9500, 0xffffffff, 0x7,
 618         0x9510, 0xffffffff, 0x100,
 619         0x9500, 0xffffffff, 0x8000,
 620         0x9490, 0xffffffff, 0x0,
 621         0x949c, 0xffffffff, 0x100,
 622         0x9490, 0xffffffff, 0x1,
 623         0x949c, 0xffffffff, 0x100,
 624         0x9490, 0xffffffff, 0x2,
 625         0x949c, 0xffffffff, 0x100,
 626         0x9490, 0xffffffff, 0x3,
 627         0x949c, 0xffffffff, 0x100,
 628         0x9490, 0xffffffff, 0x4,
 629         0x949c, 0xffffffff, 0x100,
 630         0x9490, 0xffffffff, 0x5,
 631         0x949c, 0xffffffff, 0x100,
 632         0x9490, 0xffffffff, 0x6,
 633         0x949c, 0xffffffff, 0x100,
 634         0x9490, 0xffffffff, 0x7,
 635         0x949c, 0xffffffff, 0x100,
 636         0x9490, 0xffffffff, 0x8000,
 637         0x9604, 0xffffffff, 0x0,
 638         0x9654, 0xffffffff, 0x100,
 639         0x9604, 0xffffffff, 0x1,
 640         0x9654, 0xffffffff, 0x100,
 641         0x9604, 0xffffffff, 0x2,
 642         0x9654, 0xffffffff, 0x100,
 643         0x9604, 0xffffffff, 0x3,
 644         0x9654, 0xffffffff, 0x100,
 645         0x9604, 0xffffffff, 0x4,
 646         0x9654, 0xffffffff, 0x100,
 647         0x9604, 0xffffffff, 0x5,
 648         0x9654, 0xffffffff, 0x100,
 649         0x9604, 0xffffffff, 0x6,
 650         0x9654, 0xffffffff, 0x100,
 651         0x9604, 0xffffffff, 0x7,
 652         0x9654, 0xffffffff, 0x100,
 653         0x9604, 0xffffffff, 0x80000000,
 654         0x9030, 0xffffffff, 0x100,
 655         0x9034, 0xffffffff, 0x100,
 656         0x9038, 0xffffffff, 0x100,
 657         0x903c, 0xffffffff, 0x100,
 658         0x9040, 0xffffffff, 0x100,
 659         0xa200, 0xffffffff, 0x100,
 660         0xa204, 0xffffffff, 0x100,
 661         0xa208, 0xffffffff, 0x100,
 662         0xa20c, 0xffffffff, 0x100,
 663         0x971c, 0xffffffff, 0x100,
 664         0x915c, 0xffffffff, 0x00020001,
 665         0x9160, 0xffffffff, 0x00040003,
 666         0x916c, 0xffffffff, 0x00060005,
 667         0x9170, 0xffffffff, 0x00080007,
 668         0x9174, 0xffffffff, 0x000a0009,
 669         0x9178, 0xffffffff, 0x000c000b,
 670         0x917c, 0xffffffff, 0x000e000d,
 671         0x9180, 0xffffffff, 0x0010000f,
 672         0x918c, 0xffffffff, 0x00120011,
 673         0x9190, 0xffffffff, 0x00140013,
 674         0x9194, 0xffffffff, 0x00020001,
 675         0x9198, 0xffffffff, 0x00040003,
 676         0x919c, 0xffffffff, 0x00060005,
 677         0x91a8, 0xffffffff, 0x00080007,
 678         0x91ac, 0xffffffff, 0x000a0009,
 679         0x91b0, 0xffffffff, 0x000c000b,
 680         0x91b4, 0xffffffff, 0x000e000d,
 681         0x91b8, 0xffffffff, 0x0010000f,
 682         0x91c4, 0xffffffff, 0x00120011,
 683         0x91c8, 0xffffffff, 0x00140013,
 684         0x91cc, 0xffffffff, 0x00020001,
 685         0x91d0, 0xffffffff, 0x00040003,
 686         0x91d4, 0xffffffff, 0x00060005,
 687         0x91e0, 0xffffffff, 0x00080007,
 688         0x91e4, 0xffffffff, 0x000a0009,
 689         0x91e8, 0xffffffff, 0x000c000b,
 690         0x91ec, 0xffffffff, 0x00020001,
 691         0x91f0, 0xffffffff, 0x00040003,
 692         0x91f4, 0xffffffff, 0x00060005,
 693         0x9200, 0xffffffff, 0x00080007,
 694         0x9204, 0xffffffff, 0x000a0009,
 695         0x9208, 0xffffffff, 0x000c000b,
 696         0x920c, 0xffffffff, 0x000e000d,
 697         0x9210, 0xffffffff, 0x0010000f,
 698         0x921c, 0xffffffff, 0x00120011,
 699         0x9220, 0xffffffff, 0x00140013,
 700         0x9224, 0xffffffff, 0x00020001,
 701         0x9228, 0xffffffff, 0x00040003,
 702         0x922c, 0xffffffff, 0x00060005,
 703         0x9238, 0xffffffff, 0x00080007,
 704         0x923c, 0xffffffff, 0x000a0009,
 705         0x9240, 0xffffffff, 0x000c000b,
 706         0x9244, 0xffffffff, 0x000e000d,
 707         0x9248, 0xffffffff, 0x0010000f,
 708         0x9254, 0xffffffff, 0x00120011,
 709         0x9258, 0xffffffff, 0x00140013,
 710         0x9294, 0xffffffff, 0x00020001,
 711         0x929c, 0xffffffff, 0x00040003,
 712         0x92a0, 0xffffffff, 0x00060005,
 713         0x92a4, 0xffffffff, 0x00080007
 714 };
 715
 716 static void rv770_init_golden_registers(struct radeon_device *rdev)
 717 {
 718         switch (rdev->family) {
 719         case CHIP_RV770:
 720                 radeon_program_register_sequence(rdev,
 721                                                  r7xx_golden_registers,
 722                                                  (const u32)ARRAY_SIZE(r7xx_golden_registers));
 723                 radeon_program_register_sequence(rdev,
 724                                                  r7xx_golden_dyn_gpr_registers,
 725                                                  (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 726                 if (rdev->pdev->device == 0x994e)
 727                         radeon_program_register_sequence(rdev,
 728                                                          rv770ce_golden_registers,
 729                                                          (const u32)ARRAY_SIZE(rv770ce_golden_registers));
 730                 else
 731                         radeon_program_register_sequence(rdev,
 732                                                          rv770_golden_registers,
 733                                                          (const u32)ARRAY_SIZE(rv770_golden_registers));
 734                 radeon_program_register_sequence(rdev,
 735                                                  rv770_mgcg_init,
 736                                                  (const u32)ARRAY_SIZE(rv770_mgcg_init));
 737                 break;
 738         case CHIP_RV730:
 739                 radeon_program_register_sequence(rdev,
 740                                                  r7xx_golden_registers,
 741                                                  (const u32)ARRAY_SIZE(r7xx_golden_registers));
 742                 radeon_program_register_sequence(rdev,
 743                                                  r7xx_golden_dyn_gpr_registers,
 744                                                  (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 745                 radeon_program_register_sequence(rdev,
 746                                                  rv730_golden_registers,
 747                                                  (const u32)ARRAY_SIZE(rv730_golden_registers));
 748                 radeon_program_register_sequence(rdev,
 749                                                  rv730_mgcg_init,
 750                                                  (const u32)ARRAY_SIZE(rv730_mgcg_init));
 751                 break;
 752         case CHIP_RV710:
 753                 radeon_program_register_sequence(rdev,
 754                                                  r7xx_golden_registers,
 755                                                  (const u32)ARRAY_SIZE(r7xx_golden_registers));
 756                 radeon_program_register_sequence(rdev,
 757                                                  r7xx_golden_dyn_gpr_registers,
 758                                                  (const u32)ARRAY_SIZE(r7xx_golden_dyn_gpr_registers));
 759                 radeon_program_register_sequence(rdev,
 760                                                  rv710_golden_registers,
 761                                                  (const u32)ARRAY_SIZE(rv710_golden_registers));
 762                 radeon_program_register_sequence(rdev,
 763                                                  rv710_mgcg_init,
 764                                                  (const u32)ARRAY_SIZE(rv710_mgcg_init));
 765                 break;
 766         case CHIP_RV740:
 767                 radeon_program_register_sequence(rdev,
 768                                                  rv740_golden_registers,
 769                                                  (const u32)ARRAY_SIZE(rv740_golden_registers));
 770                 radeon_program_register_sequence(rdev,
 771                                                  rv740_mgcg_init,
 772                                                  (const u32)ARRAY_SIZE(rv740_mgcg_init));
 773                 break;
 774         default:
 775                 break;
 776         }
 777 }
 778
 779 #define PCIE_BUS_CLK                10000
 780 #define TCLK                        (PCIE_BUS_CLK / 10)
 781
 782 /**
 783  * rv770_get_xclk - get the xclk
 784  *
 785  * @rdev: radeon_device pointer
 786  *
 787  * Returns the reference clock used by the gfx engine
 788  * (r7xx-cayman).
 789  */
 790 u32 rv770_get_xclk(struct radeon_device *rdev)
 791 {
 792         u32 reference_clock = rdev->clock.spll.reference_freq;
 793         u32 tmp = RREG32(CG_CLKPIN_CNTL);
 794
 795         if (tmp & MUX_TCLK_TO_XCLK)
 796                 return TCLK;
 797
 798         if (tmp & XTALIN_DIVIDE)
 799                 return reference_clock / 4;
 800
 801         return reference_clock;
 802 }
 803
 804 void rv770_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base, bool async)
 805 {
 806         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 807         u32 tmp = RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset);
 808         int i;
 809
 810         /* Lock the graphics update lock */
 811         tmp |= AVIVO_D1GRPH_UPDATE_LOCK;
 812         WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 813
 814         /* update the scanout addresses */
 815         WREG32(AVIVO_D1GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
 816                async ? AVIVO_D1GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
 817         if (radeon_crtc->crtc_id) {
 818                 WREG32(D2GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 819                 WREG32(D2GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 820         } else {
 821                 WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 822                 WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS_HIGH, upper_32_bits(crtc_base));
 823         }
 824         WREG32(D1GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 825                (u32)crtc_base);
 826         WREG32(D1GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
 827                (u32)crtc_base);
 828
 829         /* Wait for update_pending to go high. */
 830         for (i = 0; i < rdev->usec_timeout; i++) {
 831                 if (RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING)
 832                         break;
 833                 udelay(1);
 834         }
 835         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
 836
 837         /* Unlock the lock, so double-buffering can take place inside vblank */
 838         tmp &= ~AVIVO_D1GRPH_UPDATE_LOCK;
 839         WREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
 840 }
 841
 842 bool rv770_page_flip_pending(struct radeon_device *rdev, int crtc_id)
 843 {
 844         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
 845
 846         /* Return current update_pending status: */
 847         return !!(RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) &
 848                 AVIVO_D1GRPH_SURFACE_UPDATE_PENDING);
 849 }
 850
 851 /* get temperature in millidegrees */
 852 int rv770_get_temp(struct radeon_device *rdev)
 853 {
 854         u32 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
 855                 ASIC_T_SHIFT;
 856         int actual_temp;
 857
 858         if (temp & 0x400)
 859                 actual_temp = -256;
 860         else if (temp & 0x200)
 861                 actual_temp = 255;
 862         else if (temp & 0x100) {
 863                 actual_temp = temp & 0x1ff;
 864                 actual_temp |= ~0x1ff;
 865         } else
 866                 actual_temp = temp & 0xff;
 867
 868         return (actual_temp * 1000) / 2;
 869 }
 870
 871 void rv770_pm_misc(struct radeon_device *rdev)
 872 {
 873         int req_ps_idx = rdev->pm.requested_power_state_index;
 874         int req_cm_idx = rdev->pm.requested_clock_mode_index;
 875         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 876         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 877
 878         if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
 879                 /* 0xff01 is a flag rather then an actual voltage */
 880                 if (voltage->voltage == 0xff01)
 881                         return;
 882                 if (voltage->voltage != rdev->pm.current_vddc) {
 883                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 884                         rdev->pm.current_vddc = voltage->voltage;
 885                         DRM_DEBUG("Setting: v: %d\n", voltage->voltage);
 886                 }
 887         }
 888 }
 889
 890 /*
 891  * GART
 892  */
 893 static int rv770_pcie_gart_enable(struct radeon_device *rdev)
 894 {
 895         u32 tmp;
 896         int r, i;
 897
 898         if (rdev->gart.robj == NULL) {
 899                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 900                 return -EINVAL;
 901         }
 902         r = radeon_gart_table_vram_pin(rdev);
 903         if (r)
 904                 return r;
 905         /* Setup L2 cache */
 906         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 907                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 908                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 909         WREG32(VM_L2_CNTL2, 0);
 910         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 911         /* Setup TLB control */
 912         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 913                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 914                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 915                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 916         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 917         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 918         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 919         if (rdev->family == CHIP_RV740)
 920                 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
 921         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 922         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 923         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 924         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 925         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 926         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 927         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 928         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 929                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 930         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 931                         (u32)(rdev->dummy_page.addr >> 12));
 932         for (i = 1; i < 7; i++)
 933                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 934
 935         r600_pcie_gart_tlb_flush(rdev);
 936         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
 937                  (unsigned)(rdev->mc.gtt_size >> 20),
 938                  (unsigned long long)rdev->gart.table_addr);
 939         rdev->gart.ready = true;
 940         return 0;
 941 }
 942
 943 static void rv770_pcie_gart_disable(struct radeon_device *rdev)
 944 {
 945         u32 tmp;
 946         int i;
 947
 948         /* Disable all tables */
 949         for (i = 0; i < 7; i++)
 950                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 951
 952         /* Setup L2 cache */
 953         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 954                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 955         WREG32(VM_L2_CNTL2, 0);
 956         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 957         /* Setup TLB control */
 958         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 959         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 960         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 961         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 962         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 963         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 964         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 965         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
 966         radeon_gart_table_vram_unpin(rdev);
 967 }
 968
 969 static void rv770_pcie_gart_fini(struct radeon_device *rdev)
 970 {
 971         radeon_gart_fini(rdev);
 972         rv770_pcie_gart_disable(rdev);
 973         radeon_gart_table_vram_free(rdev);
 974 }
 975
 976
 977 static void rv770_agp_enable(struct radeon_device *rdev)
 978 {
 979         u32 tmp;
 980         int i;
 981
 982         /* Setup L2 cache */
 983         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 984                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 985                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 986         WREG32(VM_L2_CNTL2, 0);
 987         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
 988         /* Setup TLB control */
 989         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 990                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 991                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 992                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
 993         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
 994         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
 995         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
 996         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
 997         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
 998         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
 999         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1000         for (i = 0; i < 7; i++)
1001                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1002 }
1003
1004 static void rv770_mc_program(struct radeon_device *rdev)
1005 {
1006         struct rv515_mc_save save;
1007         u32 tmp;
1008         int i, j;
1009
1010         /* Initialize HDP */
1011         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1012                 WREG32((0x2c14 + j), 0x00000000);
1013                 WREG32((0x2c18 + j), 0x00000000);
1014                 WREG32((0x2c1c + j), 0x00000000);
1015                 WREG32((0x2c20 + j), 0x00000000);
1016                 WREG32((0x2c24 + j), 0x00000000);
1017         }
1018         /* r7xx hw bug.  Read from HDP_DEBUG1 rather
1019          * than writing to HDP_REG_COHERENCY_FLUSH_CNTL
1020          */
1021         tmp = RREG32(HDP_DEBUG1);
1022
1023         rv515_mc_stop(rdev, &save);
1024         if (r600_mc_wait_for_idle(rdev)) {
1025                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1026         }
1027         /* Lockout access through VGA aperture*/
1028         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1029         /* Update configuration */
1030         if (rdev->flags & RADEON_IS_AGP) {
1031                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1032                         /* VRAM before AGP */
1033                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1034                                 rdev->mc.vram_start >> 12);
1035                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1036                                 rdev->mc.gtt_end >> 12);
1037                 } else {
1038                         /* VRAM after AGP */
1039                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1040                                 rdev->mc.gtt_start >> 12);
1041                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1042                                 rdev->mc.vram_end >> 12);
1043                 }
1044         } else {
1045                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1046                         rdev->mc.vram_start >> 12);
1047                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1048                         rdev->mc.vram_end >> 12);
1049         }
1050         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1051         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1052         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1053         WREG32(MC_VM_FB_LOCATION, tmp);
1054         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1055         WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1056         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1057         if (rdev->flags & RADEON_IS_AGP) {
1058                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1059                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1060                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1061         } else {
1062                 WREG32(MC_VM_AGP_BASE, 0);
1063                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1064                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1065         }
1066         if (r600_mc_wait_for_idle(rdev)) {
1067                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1068         }
1069         rv515_mc_resume(rdev, &save);
1070         /* we need to own VRAM, so turn off the VGA renderer here
1071          * to stop it overwriting our objects */
1072         rv515_vga_render_disable(rdev);
1073 }
1074
1075
1076 /*
1077  * CP.
1078  */
1079 void r700_cp_stop(struct radeon_device *rdev)
1080 {
1081         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1082                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1083         WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1084         WREG32(SCRATCH_UMSK, 0);
1085         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1086 }
1087
1088 static int rv770_cp_load_microcode(struct radeon_device *rdev)
1089 {
1090         const __be32 *fw_data;
1091         int i;
1092
1093         if (!rdev->me_fw || !rdev->pfp_fw)
1094                 return -EINVAL;
1095
1096         r700_cp_stop(rdev);
1097         WREG32(CP_RB_CNTL,
1098 #ifdef __BIG_ENDIAN
1099                BUF_SWAP_32BIT |
1100 #endif
1101                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1102
1103         /* Reset cp */
1104         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
1105         RREG32(GRBM_SOFT_RESET);
1106         mdelay(15);
1107         WREG32(GRBM_SOFT_RESET, 0);
1108
1109         fw_data = (const __be32 *)rdev->pfp_fw->data;
1110         WREG32(CP_PFP_UCODE_ADDR, 0);
1111         for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
1112                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1113         WREG32(CP_PFP_UCODE_ADDR, 0);
1114
1115         fw_data = (const __be32 *)rdev->me_fw->data;
1116         WREG32(CP_ME_RAM_WADDR, 0);
1117         for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
1118                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1119
1120         WREG32(CP_PFP_UCODE_ADDR, 0);
1121         WREG32(CP_ME_RAM_WADDR, 0);
1122         WREG32(CP_ME_RAM_RADDR, 0);
1123         return 0;
1124 }
1125
1126 void r700_cp_fini(struct radeon_device *rdev)
1127 {
1128         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1129         r700_cp_stop(rdev);
1130         radeon_ring_fini(rdev, ring);
1131         radeon_scratch_free(rdev, ring->rptr_save_reg);
1132 }
1133
1134 void rv770_set_clk_bypass_mode(struct radeon_device *rdev)
1135 {
1136         u32 tmp, i;
1137
1138         if (rdev->flags & RADEON_IS_IGP)
1139                 return;
1140
1141         tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
1142         tmp &= SCLK_MUX_SEL_MASK;
1143         tmp |= SCLK_MUX_SEL(1) | SCLK_MUX_UPDATE;
1144         WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1145
1146         for (i = 0; i < rdev->usec_timeout; i++) {
1147                 if (RREG32(CG_SPLL_STATUS) & SPLL_CHG_STATUS)
1148                         break;
1149                 udelay(1);
1150         }
1151
1152         tmp &= ~SCLK_MUX_UPDATE;
1153         WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
1154
1155         tmp = RREG32(MPLL_CNTL_MODE);
1156         if ((rdev->family == CHIP_RV710) || (rdev->family == CHIP_RV730))
1157                 tmp &= ~RV730_MPLL_MCLK_SEL;
1158         else
1159                 tmp &= ~MPLL_MCLK_SEL;
1160         WREG32(MPLL_CNTL_MODE, tmp);
1161 }
1162
1163 /*
1164  * Core functions
1165  */
1166 static void rv770_gpu_init(struct radeon_device *rdev)
1167 {
1168         int i, j, num_qd_pipes;
1169         u32 ta_aux_cntl;
1170         u32 sx_debug_1;
1171         u32 smx_dc_ctl0;
1172         u32 db_debug3;
1173         u32 num_gs_verts_per_thread;
1174         u32 vgt_gs_per_es;
1175         u32 gs_prim_buffer_depth = 0;
1176         u32 sq_ms_fifo_sizes;
1177         u32 sq_config;
1178         u32 sq_thread_resource_mgmt;
1179         u32 hdp_host_path_cntl;
1180         u32 sq_dyn_gpr_size_simd_ab_0;
1181         u32 gb_tiling_config = 0;
1182         u32 cc_gc_shader_pipe_config = 0;
1183         u32 mc_arb_ramcfg;
1184         u32 db_debug4, tmp;
1185         u32 inactive_pipes, shader_pipe_config;
1186         u32 disabled_rb_mask;
1187         unsigned active_number;
1188
1189         /* setup chip specs */
1190         rdev->config.rv770.tiling_group_size = 256;
1191         switch (rdev->family) {
1192         case CHIP_RV770:
1193                 rdev->config.rv770.max_pipes = 4;
1194                 rdev->config.rv770.max_tile_pipes = 8;
1195                 rdev->config.rv770.max_simds = 10;
1196                 rdev->config.rv770.max_backends = 4;
1197                 rdev->config.rv770.max_gprs = 256;
1198                 rdev->config.rv770.max_threads = 248;
1199                 rdev->config.rv770.max_stack_entries = 512;
1200                 rdev->config.rv770.max_hw_contexts = 8;
1201                 rdev->config.rv770.max_gs_threads = 16 * 2;
1202                 rdev->config.rv770.sx_max_export_size = 128;
1203                 rdev->config.rv770.sx_max_export_pos_size = 16;
1204                 rdev->config.rv770.sx_max_export_smx_size = 112;
1205                 rdev->config.rv770.sq_num_cf_insts = 2;
1206
1207                 rdev->config.rv770.sx_num_of_sets = 7;
1208                 rdev->config.rv770.sc_prim_fifo_size = 0xF9;
1209                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1210                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1211                 break;
1212         case CHIP_RV730:
1213                 rdev->config.rv770.max_pipes = 2;
1214                 rdev->config.rv770.max_tile_pipes = 4;
1215                 rdev->config.rv770.max_simds = 8;
1216                 rdev->config.rv770.max_backends = 2;
1217                 rdev->config.rv770.max_gprs = 128;
1218                 rdev->config.rv770.max_threads = 248;
1219                 rdev->config.rv770.max_stack_entries = 256;
1220                 rdev->config.rv770.max_hw_contexts = 8;
1221                 rdev->config.rv770.max_gs_threads = 16 * 2;
1222                 rdev->config.rv770.sx_max_export_size = 256;
1223                 rdev->config.rv770.sx_max_export_pos_size = 32;
1224                 rdev->config.rv770.sx_max_export_smx_size = 224;
1225                 rdev->config.rv770.sq_num_cf_insts = 2;
1226
1227                 rdev->config.rv770.sx_num_of_sets = 7;
1228                 rdev->config.rv770.sc_prim_fifo_size = 0xf9;
1229                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1230                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1231                 if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1232                         rdev->config.rv770.sx_max_export_pos_size -= 16;
1233                         rdev->config.rv770.sx_max_export_smx_size += 16;
1234                 }
1235                 break;
1236         case CHIP_RV710:
1237                 rdev->config.rv770.max_pipes = 2;
1238                 rdev->config.rv770.max_tile_pipes = 2;
1239                 rdev->config.rv770.max_simds = 2;
1240                 rdev->config.rv770.max_backends = 1;
1241                 rdev->config.rv770.max_gprs = 256;
1242                 rdev->config.rv770.max_threads = 192;
1243                 rdev->config.rv770.max_stack_entries = 256;
1244                 rdev->config.rv770.max_hw_contexts = 4;
1245                 rdev->config.rv770.max_gs_threads = 8 * 2;
1246                 rdev->config.rv770.sx_max_export_size = 128;
1247                 rdev->config.rv770.sx_max_export_pos_size = 16;
1248                 rdev->config.rv770.sx_max_export_smx_size = 112;
1249                 rdev->config.rv770.sq_num_cf_insts = 1;
1250
1251                 rdev->config.rv770.sx_num_of_sets = 7;
1252                 rdev->config.rv770.sc_prim_fifo_size = 0x40;
1253                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1254                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1255                 break;
1256         case CHIP_RV740:
1257                 rdev->config.rv770.max_pipes = 4;
1258                 rdev->config.rv770.max_tile_pipes = 4;
1259                 rdev->config.rv770.max_simds = 8;
1260                 rdev->config.rv770.max_backends = 4;
1261                 rdev->config.rv770.max_gprs = 256;
1262                 rdev->config.rv770.max_threads = 248;
1263                 rdev->config.rv770.max_stack_entries = 512;
1264                 rdev->config.rv770.max_hw_contexts = 8;
1265                 rdev->config.rv770.max_gs_threads = 16 * 2;
1266                 rdev->config.rv770.sx_max_export_size = 256;
1267                 rdev->config.rv770.sx_max_export_pos_size = 32;
1268                 rdev->config.rv770.sx_max_export_smx_size = 224;
1269                 rdev->config.rv770.sq_num_cf_insts = 2;
1270
1271                 rdev->config.rv770.sx_num_of_sets = 7;
1272                 rdev->config.rv770.sc_prim_fifo_size = 0x100;
1273                 rdev->config.rv770.sc_hiz_tile_fifo_size = 0x30;
1274                 rdev->config.rv770.sc_earlyz_tile_fifo_fize = 0x130;
1275
1276                 if (rdev->config.rv770.sx_max_export_pos_size > 16) {
1277                         rdev->config.rv770.sx_max_export_pos_size -= 16;
1278                         rdev->config.rv770.sx_max_export_smx_size += 16;
1279                 }
1280                 break;
1281         default:
1282                 break;
1283         }
1284
1285         /* Initialize HDP */
1286         j = 0;
1287         for (i = 0; i < 32; i++) {
1288                 WREG32((0x2c14 + j), 0x00000000);
1289                 WREG32((0x2c18 + j), 0x00000000);
1290                 WREG32((0x2c1c + j), 0x00000000);
1291                 WREG32((0x2c20 + j), 0x00000000);
1292                 WREG32((0x2c24 + j), 0x00000000);
1293                 j += 0x18;
1294         }
1295
1296         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1297
1298         /* setup tiling, simd, pipe config */
1299         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1300
1301         shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG);
1302         inactive_pipes = (shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> INACTIVE_QD_PIPES_SHIFT;
1303         for (i = 0, tmp = 1, active_number = 0; i < R7XX_MAX_PIPES; i++) {
1304                 if (!(inactive_pipes & tmp)) {
1305                         active_number++;
1306                 }
1307                 tmp <<= 1;
1308         }
1309         if (active_number == 1) {
1310                 WREG32(SPI_CONFIG_CNTL, DISABLE_INTERP_1);
1311         } else {
1312                 WREG32(SPI_CONFIG_CNTL, 0);
1313         }
1314
1315         cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1316         tmp = rdev->config.rv770.max_simds -
1317                 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R7XX_MAX_SIMDS_MASK);
1318         rdev->config.rv770.active_simds = tmp;
1319
1320         switch (rdev->config.rv770.max_tile_pipes) {
1321         case 1:
1322         default:
1323                 gb_tiling_config = PIPE_TILING(0);
1324                 break;
1325         case 2:
1326                 gb_tiling_config = PIPE_TILING(1);
1327                 break;
1328         case 4:
1329                 gb_tiling_config = PIPE_TILING(2);
1330                 break;
1331         case 8:
1332                 gb_tiling_config = PIPE_TILING(3);
1333                 break;
1334         }
1335         rdev->config.rv770.tiling_npipes = rdev->config.rv770.max_tile_pipes;
1336
1337         disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R7XX_MAX_BACKENDS_MASK;
1338         tmp = 0;
1339         for (i = 0; i < rdev->config.rv770.max_backends; i++)
1340                 tmp |= (1 << i);
1341         /* if all the backends are disabled, fix it up here */
1342         if ((disabled_rb_mask & tmp) == tmp) {
1343                 for (i = 0; i < rdev->config.rv770.max_backends; i++)
1344                         disabled_rb_mask &= ~(1 << i);
1345         }
1346         tmp = (gb_tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1347         tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.rv770.max_backends,
1348                                         R7XX_MAX_BACKENDS, disabled_rb_mask);
1349         gb_tiling_config |= tmp << 16;
1350         rdev->config.rv770.backend_map = tmp;
1351
1352         if (rdev->family == CHIP_RV770)
1353                 gb_tiling_config |= BANK_TILING(1);
1354         else {
1355                 if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
1356                         gb_tiling_config |= BANK_TILING(1);
1357                 else
1358                         gb_tiling_config |= BANK_TILING(0);
1359         }
1360         rdev->config.rv770.tiling_nbanks = 4 << ((gb_tiling_config >> 4) & 0x3);
1361         gb_tiling_config |= GROUP_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1362         if (((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT) > 3) {
1363                 gb_tiling_config |= ROW_TILING(3);
1364                 gb_tiling_config |= SAMPLE_SPLIT(3);
1365         } else {
1366                 gb_tiling_config |=
1367                         ROW_TILING(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1368                 gb_tiling_config |=
1369                         SAMPLE_SPLIT(((mc_arb_ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT));
1370         }
1371
1372         gb_tiling_config |= BANK_SWAPS(1);
1373         rdev->config.rv770.tile_config = gb_tiling_config;
1374
1375         WREG32(GB_TILING_CONFIG, gb_tiling_config);
1376         WREG32(DCP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1377         WREG32(HDP_TILING_CONFIG, (gb_tiling_config & 0xffff));
1378         WREG32(DMA_TILING_CONFIG, (gb_tiling_config & 0xffff));
1379         WREG32(DMA_TILING_CONFIG2, (gb_tiling_config & 0xffff));
1380         if (rdev->family == CHIP_RV730) {
1381                 WREG32(UVD_UDEC_DB_TILING_CONFIG, (gb_tiling_config & 0xffff));
1382                 WREG32(UVD_UDEC_DBW_TILING_CONFIG, (gb_tiling_config & 0xffff));
1383                 WREG32(UVD_UDEC_TILING_CONFIG, (gb_tiling_config & 0xffff));
1384         }
1385
1386         WREG32(CGTS_SYS_TCC_DISABLE, 0);
1387         WREG32(CGTS_TCC_DISABLE, 0);
1388         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
1389         WREG32(CGTS_USER_TCC_DISABLE, 0);
1390
1391
1392         num_qd_pipes = R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1393         WREG32(VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & DEALLOC_DIST_MASK);
1394         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1395
1396         /* set HW defaults for 3D engine */
1397         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
1398                                      ROQ_IB2_START(0x2b)));
1399
1400         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
1401
1402         ta_aux_cntl = RREG32(TA_CNTL_AUX);
1403         WREG32(TA_CNTL_AUX, ta_aux_cntl | DISABLE_CUBE_ANISO);
1404
1405         sx_debug_1 = RREG32(SX_DEBUG_1);
1406         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1407         WREG32(SX_DEBUG_1, sx_debug_1);
1408
1409         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1410         smx_dc_ctl0 &= ~CACHE_DEPTH(0x1ff);
1411         smx_dc_ctl0 |= CACHE_DEPTH((rdev->config.rv770.sx_num_of_sets * 64) - 1);
1412         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1413
1414         if (rdev->family != CHIP_RV740)
1415                 WREG32(SMX_EVENT_CTL, (ES_FLUSH_CTL(4) |
1416                                        GS_FLUSH_CTL(4) |
1417                                        ACK_FLUSH_CTL(3) |
1418                                        SYNC_FLUSH_CTL));
1419
1420         if (rdev->family != CHIP_RV770)
1421                 WREG32(SMX_SAR_CTL0, 0x00003f3f);
1422
1423         db_debug3 = RREG32(DB_DEBUG3);
1424         db_debug3 &= ~DB_CLK_OFF_DELAY(0x1f);
1425         switch (rdev->family) {
1426         case CHIP_RV770:
1427         case CHIP_RV740:
1428                 db_debug3 |= DB_CLK_OFF_DELAY(0x1f);
1429                 break;
1430         case CHIP_RV710:
1431         case CHIP_RV730:
1432         default:
1433                 db_debug3 |= DB_CLK_OFF_DELAY(2);
1434                 break;
1435         }
1436         WREG32(DB_DEBUG3, db_debug3);
1437
1438         if (rdev->family != CHIP_RV770) {
1439                 db_debug4 = RREG32(DB_DEBUG4);
1440                 db_debug4 |= DISABLE_TILE_COVERED_FOR_PS_ITER;
1441                 WREG32(DB_DEBUG4, db_debug4);
1442         }
1443
1444         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.rv770.sx_max_export_size / 4) - 1) |
1445                                         POSITION_BUFFER_SIZE((rdev->config.rv770.sx_max_export_pos_size / 4) - 1) |
1446                                         SMX_BUFFER_SIZE((rdev->config.rv770.sx_max_export_smx_size / 4) - 1)));
1447
1448         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.rv770.sc_prim_fifo_size) |
1449                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_hiz_tile_fifo_size) |
1450                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.rv770.sc_earlyz_tile_fifo_fize)));
1451
1452         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1453
1454         WREG32(VGT_NUM_INSTANCES, 1);
1455
1456         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
1457
1458         WREG32(CP_PERFMON_CNTL, 0);
1459
1460         sq_ms_fifo_sizes = (CACHE_FIFO_SIZE(16 * rdev->config.rv770.sq_num_cf_insts) |
1461                             DONE_FIFO_HIWATER(0xe0) |
1462                             ALU_UPDATE_FIFO_HIWATER(0x8));
1463         switch (rdev->family) {
1464         case CHIP_RV770:
1465         case CHIP_RV730:
1466         case CHIP_RV710:
1467                 sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x1);
1468                 break;
1469         case CHIP_RV740:
1470         default:
1471                 sq_ms_fifo_sizes |= FETCH_FIFO_HIWATER(0x4);
1472                 break;
1473         }
1474         WREG32(SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
1475
1476         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1477          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1478          */
1479         sq_config = RREG32(SQ_CONFIG);
1480         sq_config &= ~(PS_PRIO(3) |
1481                        VS_PRIO(3) |
1482                        GS_PRIO(3) |
1483                        ES_PRIO(3));
1484         sq_config |= (DX9_CONSTS |
1485                       VC_ENABLE |
1486                       EXPORT_SRC_C |
1487                       PS_PRIO(0) |
1488                       VS_PRIO(1) |
1489                       GS_PRIO(2) |
1490                       ES_PRIO(3));
1491         if (rdev->family == CHIP_RV710)
1492                 /* no vertex cache */
1493                 sq_config &= ~VC_ENABLE;
1494
1495         WREG32(SQ_CONFIG, sq_config);
1496
1497         WREG32(SQ_GPR_RESOURCE_MGMT_1,  (NUM_PS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1498                                          NUM_VS_GPRS((rdev->config.rv770.max_gprs * 24)/64) |
1499                                          NUM_CLAUSE_TEMP_GPRS(((rdev->config.rv770.max_gprs * 24)/64)/2)));
1500
1501         WREG32(SQ_GPR_RESOURCE_MGMT_2,  (NUM_GS_GPRS((rdev->config.rv770.max_gprs * 7)/64) |
1502                                          NUM_ES_GPRS((rdev->config.rv770.max_gprs * 7)/64)));
1503
1504         sq_thread_resource_mgmt = (NUM_PS_THREADS((rdev->config.rv770.max_threads * 4)/8) |
1505                                    NUM_VS_THREADS((rdev->config.rv770.max_threads * 2)/8) |
1506                                    NUM_ES_THREADS((rdev->config.rv770.max_threads * 1)/8));
1507         if (((rdev->config.rv770.max_threads * 1) / 8) > rdev->config.rv770.max_gs_threads)
1508                 sq_thread_resource_mgmt |= NUM_GS_THREADS(rdev->config.rv770.max_gs_threads);
1509         else
1510                 sq_thread_resource_mgmt |= NUM_GS_THREADS((rdev->config.rv770.max_gs_threads * 1)/8);
1511         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1512
1513         WREG32(SQ_STACK_RESOURCE_MGMT_1, (NUM_PS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1514                                                      NUM_VS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1515
1516         WREG32(SQ_STACK_RESOURCE_MGMT_2, (NUM_GS_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4) |
1517                                                      NUM_ES_STACK_ENTRIES((rdev->config.rv770.max_stack_entries * 1)/4)));
1518
1519         sq_dyn_gpr_size_simd_ab_0 = (SIMDA_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1520                                      SIMDA_RING1((rdev->config.rv770.max_gprs * 38)/64) |
1521                                      SIMDB_RING0((rdev->config.rv770.max_gprs * 38)/64) |
1522                                      SIMDB_RING1((rdev->config.rv770.max_gprs * 38)/64));
1523
1524         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
1525         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
1526         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
1527         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
1528         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
1529         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
1530         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
1531         WREG32(SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
1532
1533         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1534                                           FORCE_EOV_MAX_REZ_CNT(255)));
1535
1536         if (rdev->family == CHIP_RV710)
1537                 WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(TC_ONLY) |
1538                                                 AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1539         else
1540                 WREG32(VGT_CACHE_INVALIDATION, (CACHE_INVALIDATION(VC_AND_TC) |
1541                                                 AUTO_INVLD_EN(ES_AND_GS_AUTO)));
1542
1543         switch (rdev->family) {
1544         case CHIP_RV770:
1545         case CHIP_RV730:
1546         case CHIP_RV740:
1547                 gs_prim_buffer_depth = 384;
1548                 break;
1549         case CHIP_RV710:
1550                 gs_prim_buffer_depth = 128;
1551                 break;
1552         default:
1553                 break;
1554         }
1555
1556         num_gs_verts_per_thread = rdev->config.rv770.max_pipes * 16;
1557         vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1558         /* Max value for this is 256 */
1559         if (vgt_gs_per_es > 256)
1560                 vgt_gs_per_es = 256;
1561
1562         WREG32(VGT_ES_PER_GS, 128);
1563         WREG32(VGT_GS_PER_ES, vgt_gs_per_es);
1564         WREG32(VGT_GS_PER_VS, 2);
1565
1566         /* more default values. 2D/3D driver should adjust as needed */
1567         WREG32(VGT_GS_VERTEX_REUSE, 16);
1568         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1569         WREG32(VGT_STRMOUT_EN, 0);
1570         WREG32(SX_MISC, 0);
1571         WREG32(PA_SC_MODE_CNTL, 0);
1572         WREG32(PA_SC_EDGERULE, 0xaaaaaaaa);
1573         WREG32(PA_SC_AA_CONFIG, 0);
1574         WREG32(PA_SC_CLIPRECT_RULE, 0xffff);
1575         WREG32(PA_SC_LINE_STIPPLE, 0);
1576         WREG32(SPI_INPUT_Z, 0);
1577         WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
1578         WREG32(CB_COLOR7_FRAG, 0);
1579
1580         /* clear render buffer base addresses */
1581         WREG32(CB_COLOR0_BASE, 0);
1582         WREG32(CB_COLOR1_BASE, 0);
1583         WREG32(CB_COLOR2_BASE, 0);
1584         WREG32(CB_COLOR3_BASE, 0);
1585         WREG32(CB_COLOR4_BASE, 0);
1586         WREG32(CB_COLOR5_BASE, 0);
1587         WREG32(CB_COLOR6_BASE, 0);
1588         WREG32(CB_COLOR7_BASE, 0);
1589
1590         WREG32(TCP_CNTL, 0);
1591
1592         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1593         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1594
1595         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1596
1597         WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
1598                                           NUM_CLIP_SEQ(3)));
1599         WREG32(VC_ENHANCE, 0);
1600 }
1601
1602 void r700_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1603 {
1604         u64 size_bf, size_af;
1605
1606         if (mc->mc_vram_size > 0xE0000000) {
1607                 /* leave room for at least 512M GTT */
1608                 dev_warn(rdev->dev, "limiting VRAM\n");
1609                 mc->real_vram_size = 0xE0000000;
1610                 mc->mc_vram_size = 0xE0000000;
1611         }
1612         if (rdev->flags & RADEON_IS_AGP) {
1613                 size_bf = mc->gtt_start;
1614                 size_af = mc->mc_mask - mc->gtt_end;
1615                 if (size_bf > size_af) {
1616                         if (mc->mc_vram_size > size_bf) {
1617                                 dev_warn(rdev->dev, "limiting VRAM\n");
1618                                 mc->real_vram_size = size_bf;
1619                                 mc->mc_vram_size = size_bf;
1620                         }
1621                         mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1622                 } else {
1623                         if (mc->mc_vram_size > size_af) {
1624                                 dev_warn(rdev->dev, "limiting VRAM\n");
1625                                 mc->real_vram_size = size_af;
1626                                 mc->mc_vram_size = size_af;
1627                         }
1628                         mc->vram_start = mc->gtt_end + 1;
1629                 }
1630                 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1631                 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1632                                 mc->mc_vram_size >> 20, mc->vram_start,
1633                                 mc->vram_end, mc->real_vram_size >> 20);
1634         } else {
1635                 radeon_vram_location(rdev, &rdev->mc, 0);
1636                 rdev->mc.gtt_base_align = 0;
1637                 radeon_gtt_location(rdev, mc);
1638         }
1639 }
1640
1641 static int rv770_mc_init(struct radeon_device *rdev)
1642 {
1643         u32 tmp;
1644         int chansize, numchan;
1645
1646         /* Get VRAM informations */
1647         rdev->mc.vram_is_ddr = true;
1648         tmp = RREG32(MC_ARB_RAMCFG);
1649         if (tmp & CHANSIZE_OVERRIDE) {
1650                 chansize = 16;
1651         } else if (tmp & CHANSIZE_MASK) {
1652                 chansize = 64;
1653         } else {
1654                 chansize = 32;
1655         }
1656         tmp = RREG32(MC_SHARED_CHMAP);
1657         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1658         case 0:
1659         default:
1660                 numchan = 1;
1661                 break;
1662         case 1:
1663                 numchan = 2;
1664                 break;
1665         case 2:
1666                 numchan = 4;
1667                 break;
1668         case 3:
1669                 numchan = 8;
1670                 break;
1671         }
1672         rdev->mc.vram_width = numchan * chansize;
1673         /* Could aper size report 0 ? */
1674         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1675         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1676         /* Setup GPU memory space */
1677         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1678         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1679         rdev->mc.visible_vram_size = rdev->mc.aper_size;
1680         r700_vram_gtt_location(rdev, &rdev->mc);
1681         radeon_update_bandwidth_info(rdev);
1682
1683         return 0;
1684 }
1685
1686 static void rv770_uvd_init(struct radeon_device *rdev)
1687 {
1688         int r;
1689
1690         if (!rdev->has_uvd)
1691                 return;
1692
1693         r = radeon_uvd_init(rdev);
1694         if (r) {
1695                 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
1696                 /*
1697                  * At this point rdev->uvd.vcpu_bo is NULL which trickles down
1698                  * to early fails uvd_v2_2_resume() and thus nothing happens
1699                  * there. So it is pointless to try to go through that code
1700                  * hence why we disable uvd here.
1701                  */
1702                 rdev->has_uvd = 0;
1703                 return;
1704         }
1705         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
1706         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
1707 }
1708
1709 static void rv770_uvd_start(struct radeon_device *rdev)
1710 {
1711         int r;
1712
1713         if (!rdev->has_uvd)
1714                 return;
1715
1716         r = uvd_v2_2_resume(rdev);
1717         if (r) {
1718                 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
1719                 goto error;
1720         }
1721         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
1722         if (r) {
1723                 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
1724                 goto error;
1725         }
1726         return;
1727
1728 error:
1729         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
1730 }
1731
1732 static void rv770_uvd_resume(struct radeon_device *rdev)
1733 {
1734         struct radeon_ring *ring;
1735         int r;
1736
1737         if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
1738                 return;
1739
1740         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
1741         r = radeon_ring_init(rdev, ring, ring->ring_size, 0, RADEON_CP_PACKET2);
1742         if (r) {
1743                 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
1744                 return;
1745         }
1746         r = uvd_v1_0_init(rdev);
1747         if (r) {
1748                 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
1749                 return;
1750         }
1751 }
1752
1753 static int rv770_startup(struct radeon_device *rdev)
1754 {
1755         struct radeon_ring *ring;
1756         int r;
1757
1758         /* enable pcie gen2 link */
1759         rv770_pcie_gen2_enable(rdev);
1760
1761         /* scratch needs to be initialized before MC */
1762         r = r600_vram_scratch_init(rdev);
1763         if (r)
1764                 return r;
1765
1766         rv770_mc_program(rdev);
1767
1768         if (rdev->flags & RADEON_IS_AGP) {
1769                 rv770_agp_enable(rdev);
1770         } else {
1771                 r = rv770_pcie_gart_enable(rdev);
1772                 if (r)
1773                         return r;
1774         }
1775
1776         rv770_gpu_init(rdev);
1777
1778         /* allocate wb buffer */
1779         r = radeon_wb_init(rdev);
1780         if (r)
1781                 return r;
1782
1783         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1784         if (r) {
1785                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1786                 return r;
1787         }
1788
1789         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
1790         if (r) {
1791                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
1792                 return r;
1793         }
1794
1795         rv770_uvd_start(rdev);
1796
1797         /* Enable IRQ */
1798         if (!rdev->irq.installed) {
1799                 r = radeon_irq_kms_init(rdev);
1800                 if (r)
1801                         return r;
1802         }
1803
1804         r = r600_irq_init(rdev);
1805         if (r) {
1806                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
1807                 radeon_irq_kms_fini(rdev);
1808                 return r;
1809         }
1810         r600_irq_set(rdev);
1811
1812         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1813         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
1814                              RADEON_CP_PACKET2);
1815         if (r)
1816                 return r;
1817
1818         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
1819         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
1820                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1821         if (r)
1822                 return r;
1823
1824         r = rv770_cp_load_microcode(rdev);
1825         if (r)
1826                 return r;
1827         r = r600_cp_resume(rdev);
1828         if (r)
1829                 return r;
1830
1831         r = r600_dma_resume(rdev);
1832         if (r)
1833                 return r;
1834
1835         rv770_uvd_resume(rdev);
1836
1837         r = radeon_ib_pool_init(rdev);
1838         if (r) {
1839                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
1840                 return r;
1841         }
1842
1843         r = radeon_audio_init(rdev);
1844         if (r) {
1845                 DRM_ERROR("radeon: audio init failed\n");
1846                 return r;
1847         }
1848
1849         return 0;
1850 }
1851
1852 int rv770_resume(struct radeon_device *rdev)
1853 {
1854         int r;
1855
1856         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
1857          * posting will perform necessary task to bring back GPU into good
1858          * shape.
1859          */
1860         /* post card */
1861         atom_asic_init(rdev->mode_info.atom_context);
1862
1863         /* init golden registers */
1864         rv770_init_golden_registers(rdev);
1865
1866         if (rdev->pm.pm_method == PM_METHOD_DPM)
1867                 radeon_pm_resume(rdev);
1868
1869         rdev->accel_working = true;
1870         r = rv770_startup(rdev);
1871         if (r) {
1872                 DRM_ERROR("r600 startup failed on resume\n");
1873                 rdev->accel_working = false;
1874                 return r;
1875         }
1876
1877         return r;
1878
1879 }
1880
1881 int rv770_suspend(struct radeon_device *rdev)
1882 {
1883         radeon_pm_suspend(rdev);
1884         radeon_audio_fini(rdev);
1885         if (rdev->has_uvd) {
1886                 uvd_v1_0_fini(rdev);
1887                 radeon_uvd_suspend(rdev);
1888         }
1889         r700_cp_stop(rdev);
1890         r600_dma_stop(rdev);
1891         r600_irq_suspend(rdev);
1892         radeon_wb_disable(rdev);
1893         rv770_pcie_gart_disable(rdev);
1894
1895         return 0;
1896 }
1897
1898 /* Plan is to move initialization in that function and use
1899  * helper function so that radeon_device_init pretty much
1900  * do nothing more than calling asic specific function. This
1901  * should also allow to remove a bunch of callback function
1902  * like vram_info.
1903  */
1904 int rv770_init(struct radeon_device *rdev)
1905 {
1906         int r;
1907
1908         /* Read BIOS */
1909         if (!radeon_get_bios(rdev)) {
1910                 if (ASIC_IS_AVIVO(rdev))
1911                         return -EINVAL;
1912         }
1913         /* Must be an ATOMBIOS */
1914         if (!rdev->is_atom_bios) {
1915                 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
1916                 return -EINVAL;
1917         }
1918         r = radeon_atombios_init(rdev);
1919         if (r)
1920                 return r;
1921         /* Post card if necessary */
1922         if (!radeon_card_posted(rdev)) {
1923                 if (!rdev->bios) {
1924                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
1925                         return -EINVAL;
1926                 }
1927                 DRM_INFO("GPU not posted. posting now...\n");
1928                 atom_asic_init(rdev->mode_info.atom_context);
1929         }
1930         /* init golden registers */
1931         rv770_init_golden_registers(rdev);
1932         /* Initialize scratch registers */
1933         r600_scratch_init(rdev);
1934         /* Initialize surface registers */
1935         radeon_surface_init(rdev);
1936         /* Initialize clocks */
1937         radeon_get_clock_info(rdev->ddev);
1938         /* Fence driver */
1939         r = radeon_fence_driver_init(rdev);
1940         if (r)
1941                 return r;
1942         /* initialize AGP */
1943         if (rdev->flags & RADEON_IS_AGP) {
1944                 r = radeon_agp_init(rdev);
1945                 if (r)
1946                         radeon_agp_disable(rdev);
1947         }
1948         r = rv770_mc_init(rdev);
1949         if (r)
1950                 return r;
1951         /* Memory manager */
1952         r = radeon_bo_init(rdev);
1953         if (r)
1954                 return r;
1955
1956         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
1957                 r = r600_init_microcode(rdev);
1958                 if (r) {
1959                         DRM_ERROR("Failed to load firmware!\n");
1960                         return r;
1961                 }
1962         }
1963
1964         /* Initialize power management */
1965         radeon_pm_init(rdev);
1966
1967         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
1968         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1969
1970         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
1971         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
1972
1973         rv770_uvd_init(rdev);
1974
1975         rdev->ih.ring_obj = NULL;
1976         r600_ih_ring_init(rdev, 64 * 1024);
1977
1978         r = r600_pcie_gart_init(rdev);
1979         if (r)
1980                 return r;
1981
1982         rdev->accel_working = true;
1983         r = rv770_startup(rdev);
1984         if (r) {
1985                 dev_err(rdev->dev, "disabling GPU acceleration\n");
1986                 r700_cp_fini(rdev);
1987                 r600_dma_fini(rdev);
1988                 r600_irq_fini(rdev);
1989                 radeon_wb_fini(rdev);
1990                 radeon_ib_pool_fini(rdev);
1991                 radeon_irq_kms_fini(rdev);
1992                 rv770_pcie_gart_fini(rdev);
1993                 rdev->accel_working = false;
1994         }
1995
1996         return 0;
1997 }
1998
1999 void rv770_fini(struct radeon_device *rdev)
2000 {
2001         radeon_pm_fini(rdev);
2002         r700_cp_fini(rdev);
2003         r600_dma_fini(rdev);
2004         r600_irq_fini(rdev);
2005         radeon_wb_fini(rdev);
2006         radeon_ib_pool_fini(rdev);
2007         radeon_irq_kms_fini(rdev);
2008         uvd_v1_0_fini(rdev);
2009         radeon_uvd_fini(rdev);
2010         rv770_pcie_gart_fini(rdev);
2011         r600_vram_scratch_fini(rdev);
2012         radeon_gem_fini(rdev);
2013         radeon_fence_driver_fini(rdev);
2014         radeon_agp_fini(rdev);
2015         radeon_bo_fini(rdev);
2016         radeon_atombios_fini(rdev);
2017         kfree(rdev->bios);
2018         rdev->bios = NULL;
2019 }
2020
2021 static void rv770_pcie_gen2_enable(struct radeon_device *rdev)
2022 {
2023         u32 link_width_cntl, lanes, speed_cntl, tmp;
2024         u16 link_cntl2;
2025
2026         if (radeon_pcie_gen2 == 0)
2027                 return;
2028
2029         if (rdev->flags & RADEON_IS_IGP)
2030                 return;
2031
2032         if (!(rdev->flags & RADEON_IS_PCIE))
2033                 return;
2034
2035         /* x2 cards have a special sequence */
2036         if (ASIC_IS_X2(rdev))
2037                 return;
2038
2039         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
2040                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
2041                 return;
2042
2043         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
2044
2045         /* advertise upconfig capability */
2046         link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2047         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2048         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2049         link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2050         if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
2051                 lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
2052                 link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
2053                                      LC_RECONFIG_ARC_MISSING_ESCAPE);
2054                 link_width_cntl |= lanes | LC_RECONFIG_NOW |
2055                         LC_RENEGOTIATE_EN | LC_UPCONFIGURE_SUPPORT;
2056                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2057         } else {
2058                 link_width_cntl |= LC_UPCONFIGURE_DIS;
2059                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2060         }
2061
2062         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2063         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
2064             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
2065
2066                 tmp = RREG32(0x541c);
2067                 WREG32(0x541c, tmp | 0x8);
2068                 WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
2069                 link_cntl2 = RREG16(0x4088);
2070                 link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
2071                 link_cntl2 |= 0x2;
2072                 WREG16(0x4088, link_cntl2);
2073                 WREG32(MM_CFGREGS_CNTL, 0);
2074
2075                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2076                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
2077                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2078
2079                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2080                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
2081                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2082
2083                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2084                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
2085                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2086
2087                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2088                 speed_cntl |= LC_GEN2_EN_STRAP;
2089                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
2090
2091         } else {
2092                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
2093                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
2094                 if (1)
2095                         link_width_cntl |= LC_UPCONFIGURE_DIS;
2096                 else
2097                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
2098                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
2099         }
2100 }