drivers/gpu/arm/t6xx/kbase/src/linux/config/tpip/mali_kbase_config_exynos5.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2012 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   6  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   7  *
   8  * A copy of the licence is included with the program, and can also be obtained from Free Software
   9  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  10  *
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/init.h>
  15 #include <linux/poll.h>
  16 #include <linux/kernel.h>
  17 #include <linux/errno.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/pci.h>
  20 #include <linux/miscdevice.h>
  21 #include <linux/list.h>
  22 #include <linux/semaphore.h>
  23 #include <linux/fs.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/io.h>
  27 #include <linux/ioport.h>
  28 #include <linux/spinlock.h>
  29
  30 #include <mach/map.h>
  31 #include <linux/fb.h>
  32 #include <linux/clk.h>
  33 #include <mach/regs-clock.h>
  34 #include <mach/pmu.h>
  35 #include <mach/regs-pmu.h>
  36 #include <asm/delay.h>
  37 #include <mach/map.h>
  38 #include <generated/autoconf.h>
  39
  40 #include <linux/timer.h>
  41 #include <linux/pm_runtime.h>
  42 #include <linux/workqueue.h>
  43 #include <linux/regulator/consumer.h>
  44 #include <linux/regulator/driver.h>
  45
  46 #include <osk/mali_osk.h>
  47 #include <kbase/src/common/mali_kbase.h>
  48 #include <kbase/src/common/mali_kbase_pm.h>
  49 #include <kbase/src/common/mali_kbase_uku.h>
  50 #include <kbase/src/common/mali_kbase_mem.h>
  51 #include <kbase/src/common/mali_midg_regmap.h>
  52 #include <kbase/src/linux/mali_kbase_mem_linux.h>
  53 #include <kbase/src/common/mali_kbase_defs.h>
  54 #include <kbase/src/linux/mali_kbase_config_linux.h>
  55
  56 #include <kbase/src/linux/mali_linux_dvfs_trace.h>
  57
  58 #if CONFIG_MALI_UNCACHED == 0
  59 #error CONFIG_MALI_UNCACHED should equal 1 for Exynos5 support, your scons commandline should contain 'no_syncsets=1'
  60 #endif
  61
  62 #define MALI_DVFS_DEBUG 0
  63 #define MALI_DVFS_STEP 7
  64 #define MALI_DVFS_KEEP_STAY_CNT 10
  65
  66 #ifdef CONFIG_T6XX_DVFS
  67 #ifdef CONFIG_CPU_FREQ
  68 #define MALI_DVFS_ASV_ENABLE
  69 #endif
  70 #endif
  71
  72 #ifdef MALI_DVFS_ASV_ENABLE
  73 #include <mach/busfreq_exynos5.h>
  74 #define MALI_DVFS_ASV_GROUP_SPECIAL_NUM 10
  75 #define MALI_DVFS_ASV_GROUP_NUM 12
  76 #endif
  77
  78 #define HZ_IN_MHZ            (1000000)
  79 #define MALI_RTPM_DEBUG      0
  80 #define VITHAR_DEFAULT_CLOCK 533000000
  81 #define RUNTIME_PM_DELAY_TIME 10
  82 #define CONFIG_T6XX_HWVER_R0P0 1
  83
  84 struct regulator *kbase_platform_get_regulator(void);
  85 int kbase_platform_regulator_init(void);
  86 int kbase_platform_regulator_disable(void);
  87 int kbase_platform_regulator_enable(void);
  88 int kbase_platform_get_default_voltage(struct device *dev, int *vol);
  89 int kbase_platform_get_voltage(struct device *dev, int *vol);
  90 int kbase_platform_set_voltage(struct device *dev, int vol);
  91 void kbase_platform_dvfs_set_clock(kbase_device *kbdev, int freq);
  92 void kbase_platform_dvfs_set_level(kbase_device *kbdev, int level);
  93 int kbase_platform_dvfs_get_level(int freq);
  94
  95 #ifdef CONFIG_T6XX_DVFS
  96 int kbase_platform_dvfs_init(kbase_device *kbdev);
  97 void kbase_platform_dvfs_term(void);
  98 int kbase_platform_dvfs_get_control_status(void);
  99 int kbase_platform_dvfs_get_utilisation(void);
 100 #ifdef MALI_DVFS_ASV_ENABLE
 101 static int kbase_platform_asv_set(int enable);
 102 static int kbase_platform_dvfs_sprint_avs_table(char *buf);
 103 #endif /* MALI_DVFS_ASV_ENABLE */
 104 #endif /* CONFIG_T6XX_DVFS */
 105
 106 int kbase_platform_cmu_pmu_control(struct kbase_device *kbdev, int control);
 107 void kbase_platform_remove_sysfs_file(struct device *dev);
 108 mali_error kbase_platform_init(struct kbase_device *kbdev);
 109 static int kbase_platform_is_power_on(void);
 110 void kbase_platform_term(struct kbase_device *kbdev);
 111
 112 #ifdef CONFIG_T6XX_DEBUG_SYS
 113 static int kbase_platform_create_sysfs_file(struct device *dev);
 114 #endif /* CONFIG_T6XX_DEBUG_SYS */
 115
 116 static int mali_setup_system_tracing(struct device *dev);
 117 static void mali_cleanup_system_tracing(struct device *dev);
 118
 119 struct exynos_context
 120 {
 121         /** Indicator if system clock to mail-t604 is active */
 122         int cmu_pmu_status;
 123         /** cmd & pmu lock */
 124         spinlock_t cmu_pmu_lock;
 125         struct clk *sclk_g3d;
 126 };
 127
 128 static kbase_io_resources io_resources =
 129 {
 130         .job_irq_number   = EXYNOS5_JOB_IRQ_NUMBER,
 131         .mmu_irq_number   = EXYNOS5_MMU_IRQ_NUMBER,
 132         .gpu_irq_number   = EXYNOS5_GPU_IRQ_NUMBER,
 133         .io_memory_region =
 134         {
 135                 .start = EXYNOS5_PA_G3D,
 136                 .end   = EXYNOS5_PA_G3D + (4096 * 5) - 1
 137         }
 138 };
 139
 140 /**
 141  * Read the CPU clock speed
 142  */
 143 int get_cpu_clock_speed(u32* cpu_clock)
 144 {
 145         struct clk * cpu_clk;
 146         u32 freq=0;
 147         cpu_clk = clk_get(NULL, "armclk");
 148         if (IS_ERR(cpu_clk))
 149                 return 1;
 150         freq = clk_get_rate(cpu_clk);
 151         *cpu_clock = (freq/HZ_IN_MHZ);
 152         return 0;
 153 }
 154
 155 /**
 156  * Power Management callback - power ON
 157  */
 158 static int pm_callback_power_on(kbase_device *kbdev)
 159 {
 160 #ifdef CONFIG_PM_RUNTIME
 161         pm_runtime_resume(kbdev->osdev.dev);
 162 #endif /* CONFIG_PM_RUNTIME */
 163         return 0;
 164 }
 165
 166 /**
 167  * Power Management callback - power OFF
 168  */
 169 static void pm_callback_power_off(kbase_device *kbdev)
 170 {
 171 #ifdef CONFIG_PM_RUNTIME
 172         pm_schedule_suspend(kbdev->osdev.dev, RUNTIME_PM_DELAY_TIME);
 173 #endif /* CONFIG_PM_RUNTIME */
 174 }
 175
 176 /**
 177  * Power Management callback - runtime power ON
 178  */
 179 #ifdef CONFIG_PM_RUNTIME
 180 static int pm_callback_runtime_power_on(kbase_device *kbdev)
 181 {
 182 #if MALI_RTPM_DEBUG
 183         printk("kbase_device_runtime_resume\n");
 184 #endif /* MALI_RTPM_DEBUG */
 185         return kbase_platform_cmu_pmu_control(kbdev, 1);
 186 }
 187 #endif /* CONFIG_PM_RUNTIME */
 188
 189 /**
 190  * Power Management callback - runtime power OFF
 191  */
 192 #ifdef CONFIG_PM_RUNTIME
 193 static void pm_callback_runtime_power_off(kbase_device *kbdev)
 194 {
 195 #if MALI_RTPM_DEBUG
 196         printk("kbase_device_runtime_suspend\n");
 197 #endif /* MALI_RTPM_DEBUG */
 198         kbase_platform_cmu_pmu_control(kbdev, 0);
 199 }
 200 #endif /* CONFIG_PM_RUNTIME */
 201
 202 static kbase_pm_callback_conf pm_callbacks =
 203 {
 204         .power_on_callback = pm_callback_power_on,
 205         .power_off_callback = pm_callback_power_off,
 206 };
 207
 208 /**
 209  * Exynos5 hardware specific initialization
 210  */
 211 mali_bool kbase_platform_exynos5_init(kbase_device *kbdev)
 212 {
 213         if(MALI_ERROR_NONE == kbase_platform_init(kbdev))
 214         {
 215 #ifdef CONFIG_T6XX_DEBUG_SYS
 216                 if(kbase_platform_create_sysfs_file(kbdev->osdev.dev))
 217                 {
 218                         return MALI_TRUE;
 219                 }
 220 #endif /* CONFIG_T6XX_DEBUG_SYS */
 221                 return MALI_TRUE;
 222         }
 223
 224         return MALI_FALSE;
 225 }
 226
 227 /**
 228  * Exynos5 hardware specific termination
 229  */
 230 void kbase_platform_exynos5_term(kbase_device *kbdev)
 231 {
 232 #ifdef CONFIG_T6XX_DEBUG_SYS
 233         kbase_platform_remove_sysfs_file(kbdev->osdev.dev);
 234 #endif /* CONFIG_T6XX_DEBUG_SYS */
 235         kbase_platform_term(kbdev);
 236 }
 237
 238 kbase_platform_funcs_conf platform_funcs =
 239 {
 240         .platform_init_func = &kbase_platform_exynos5_init,
 241         .platform_term_func = &kbase_platform_exynos5_term,
 242 };
 243
 244 static kbase_attribute config_attributes[] = {
 245         {
 246                 KBASE_CONFIG_ATTR_MEMORY_OS_SHARED_MAX,
 247                 2048 * 1024 * 1024UL /* 2048MB */
 248         },
 249
 250         {
 251                 KBASE_CONFIG_ATTR_MEMORY_OS_SHARED_PERF_GPU,
 252                 KBASE_MEM_PERF_FAST
 253         },
 254         {
 255                 KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS,
 256                 (uintptr_t)&pm_callbacks
 257         },
 258         {
 259                 KBASE_CONFIG_ATTR_PLATFORM_FUNCS,
 260                 (uintptr_t)&platform_funcs
 261         },
 262
 263         {
 264                 KBASE_CONFIG_ATTR_GPU_FREQ_KHZ_MAX,
 265                 533000
 266         },
 267         {
 268                 KBASE_CONFIG_ATTR_GPU_FREQ_KHZ_MIN,
 269                 100000
 270         },
 271         {
 272                 KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS,
 273                 500 /* 500ms before cancelling stuck jobs */
 274         },
 275         {
 276                 KBASE_CONFIG_ATTR_CPU_SPEED_FUNC,
 277                 (uintptr_t)&get_cpu_clock_speed
 278         },
 279         {
 280                 KBASE_CONFIG_ATTR_END,
 281                 0
 282         }
 283 };
 284
 285 kbase_platform_config platform_config =
 286 {
 287         .attributes   = config_attributes,
 288         .io_resources = &io_resources,
 289         .midgard_type = KBASE_MALI_T604
 290 };
 291
 292 static struct clk *clk_g3d = NULL;
 293
 294 /**
 295  * Initialize GPU clocks
 296  */
 297 static int kbase_platform_power_clock_init(kbase_device *kbdev)
 298 {
 299         struct device *dev =  kbdev->osdev.dev;
 300         int timeout;
 301         struct exynos_context *platform;
 302
 303         platform = (struct exynos_context *) kbdev->platform_context;
 304         if(NULL == platform)
 305         {
 306                 panic("oops");
 307         }
 308
 309         /* Turn on G3D power */
 310         __raw_writel(0x7, EXYNOS5_G3D_CONFIGURATION);
 311
 312         /* Wait for G3D power stability for 1ms */
 313         timeout = 10;
 314         while((__raw_readl(EXYNOS5_G3D_STATUS) & 0x7) != 0x7) {
 315                 if(timeout == 0) {
 316                         /* need to call panic  */
 317                         panic("failed to turn on g3d power\n");
 318                         goto out;
 319                 }
 320                 timeout--;
 321                 udelay(100);
 322         }
 323
 324         /* Turn on G3D clock */
 325         clk_g3d = clk_get(dev, "g3d");
 326         if(IS_ERR(clk_g3d)) {
 327                 clk_g3d = NULL;
 328                 OSK_PRINT_ERROR(OSK_BASE_PM, "failed to clk_get [clk_g3d]\n");
 329                 /* chrome linux does not have this clock */
 330         }
 331         else
 332         {
 333                 /* android_v4 support */
 334                 clk_enable(clk_g3d);
 335                 printk("v4 support\n");
 336         }
 337
 338 #ifdef CONFIG_T6XX_HWVER_R0P0
 339         platform->sclk_g3d = clk_get(dev, "aclk_400_g3d");
 340         if(IS_ERR(platform->sclk_g3d)) {
 341                 OSK_PRINT_ERROR(OSK_BASE_PM, "failed to clk_get [sclk_g3d]\n");
 342                 goto out;
 343         }
 344 #else /* CONFIG_T6XX_HWVER_R0P0 */
 345         {
 346                 struct clk *mpll = NULL;
 347                 mpll = clk_get(dev, "mout_mpll_user");
 348                 if(IS_ERR(mpll)) {
 349                         OSK_PRINT_ERROR(OSK_BASE_PM, "failed to clk_get [mout_mpll_user]\n");
 350                         goto out;
 351                 }
 352
 353                 platform->sclk_g3d = clk_get(dev, "sclk_g3d");
 354                 if(IS_ERR(platform->sclk_g3d)) {
 355                         OSK_PRINT_ERROR(OSK_BASE_PM, "failed to clk_get [sclk_g3d]\n");
 356                         goto out;
 357                 }
 358
 359                 clk_set_parent(platform->sclk_g3d, mpll);
 360                 if(IS_ERR(platform->sclk_g3d)) {
 361                         OSK_PRINT_ERROR(OSK_BASE_PM, "failed to clk_set_parent\n");
 362                         goto out;
 363                 }
 364
 365                 clk_set_rate(platform->sclk_g3d, VITHAR_DEFAULT_CLOCK);
 366                 if(IS_ERR(platform->sclk_g3d)) {
 367                         OSK_PRINT_ERROR(OSK_BASE_PM, "failed to clk_set_rate [sclk_g3d] = %d\n", VITHAR_DEFAULT_CLOCK);
 368                         goto out;
 369                 }
 370         }
 371 #endif /*  CONFIG_T6XX_HWVER_R0P0 */
 372         (void) clk_enable(platform->sclk_g3d);
 373         return 0;
 374 out:
 375         return -EPERM;
 376 }
 377
 378 /**
 379  * Enable GPU clocks
 380  */
 381 static int kbase_platform_clock_on(struct kbase_device *kbdev)
 382 {
 383         struct exynos_context *platform;
 384         if (!kbdev)
 385                 return -ENODEV;
 386
 387         platform = (struct exynos_context *) kbdev->platform_context;
 388         if (!platform)
 389                 return -ENODEV;
 390
 391         if(clk_g3d)
 392         {
 393                 /* android_v4 support */
 394                 (void) clk_enable(clk_g3d);
 395         }
 396         else
 397         {
 398                 /* chrome support */
 399                 (void) clk_enable(platform->sclk_g3d);
 400         }
 401
 402         return 0;
 403 }
 404
 405 /**
 406  * Disable GPU clocks
 407  */
 408 static int kbase_platform_clock_off(struct kbase_device *kbdev)
 409 {
 410         struct exynos_context *platform;
 411         if (!kbdev)
 412                 return -ENODEV;
 413
 414         platform = (struct exynos_context *) kbdev->platform_context;
 415         if (!platform)
 416                 return -ENODEV;
 417
 418         if(clk_g3d)
 419         {
 420                 /* android_v4 support */
 421                 (void)clk_disable(clk_g3d);
 422         }
 423         else
 424         {
 425                 /* chrome support */
 426                 (void)clk_disable(platform->sclk_g3d);
 427         }
 428         return 0;
 429 }
 430
 431 /**
 432  * Report GPU power status
 433  */
 434 static inline int kbase_platform_is_power_on(void)
 435 {
 436         return ((__raw_readl(EXYNOS5_G3D_STATUS) & 0x7) == 0x7) ? 1 : 0;
 437 }
 438
 439 /**
 440  * Enable GPU power
 441  */
 442 static int kbase_platform_power_on(void)
 443 {
 444         int timeout;
 445
 446         /* Turn on G3D  */
 447         __raw_writel(0x7, EXYNOS5_G3D_CONFIGURATION);
 448
 449         /* Wait for G3D power stability */
 450         timeout = 1000;
 451
 452         while((__raw_readl(EXYNOS5_G3D_STATUS) & 0x7) != 0x7) {
 453                 if(timeout == 0) {
 454                         /* need to call panic  */
 455                         panic("failed to turn on g3d via g3d_configuration\n");
 456                         return -ETIMEDOUT;
 457                 }
 458                 timeout--;
 459                 udelay(10);
 460         }
 461
 462         return 0;
 463 }
 464
 465 /**
 466  * Disable GPU power
 467  */
 468 static int kbase_platform_power_off(void)
 469 {
 470         int timeout;
 471
 472         /* Turn off G3D  */
 473         __raw_writel(0x0, EXYNOS5_G3D_CONFIGURATION);
 474
 475         /* Wait for G3D power stability */
 476         timeout = 1000;
 477
 478         while(__raw_readl(EXYNOS5_G3D_STATUS) & 0x7) {
 479                 if(timeout == 0) {
 480                         /* need to call panic */
 481                         panic( "failed to turn off g3d via g3d_configuration\n");
 482                         return -ETIMEDOUT;
 483                 }
 484                 timeout--;
 485                 udelay(10);
 486         }
 487
 488         return 0;
 489 }
 490
 491 /**
 492  * Power Management unit control. Enable/disable power and clocks to GPU
 493  */
 494 int kbase_platform_cmu_pmu_control(struct kbase_device *kbdev, int control)
 495 {
 496         unsigned long flags;
 497         struct exynos_context *platform;
 498         if (!kbdev)
 499         {
 500                 return -ENODEV;
 501         }
 502
 503         platform = (struct exynos_context *) kbdev->platform_context;
 504         if (!platform)
 505         {
 506                 return -ENODEV;
 507         }
 508
 509         spin_lock_irqsave(&platform->cmu_pmu_lock, flags);
 510
 511         /* off */
 512         if(control == 0)
 513         {
 514                 if(platform->cmu_pmu_status == 0)
 515                 {
 516                         spin_unlock_irqrestore(&platform->cmu_pmu_lock, flags);
 517                         return 0;
 518                 }
 519
 520                 if(kbase_platform_power_off())
 521                         panic("failed to turn off g3d power\n");
 522                 if(kbase_platform_clock_off(kbdev))
 523
 524                         panic("failed to turn off sclk_g3d\n");
 525
 526                 platform->cmu_pmu_status = 0;
 527 #if MALI_RTPM_DEBUG
 528                 printk( KERN_ERR "3D cmu_pmu_control - off\n" );
 529 #endif /* MALI_RTPM_DEBUG */
 530         }
 531         else
 532         {
 533                 /* on */
 534                 if(platform->cmu_pmu_status == 1)
 535                 {
 536                         spin_unlock_irqrestore(&platform->cmu_pmu_lock, flags);
 537                         return 0;
 538                 }
 539
 540                 if(kbase_platform_clock_on(kbdev))
 541                         panic("failed to turn on sclk_g3d\n");
 542                 if(kbase_platform_power_on())
 543                         panic("failed to turn on g3d power\n");
 544
 545                 platform->cmu_pmu_status = 1;
 546 #if MALI_RTPM_DEBUG
 547                 printk( KERN_ERR "3D cmu_pmu_control - on\n");
 548 #endif /* MALI_RTPM_DEBUG */
 549         }
 550
 551         spin_unlock_irqrestore(&platform->cmu_pmu_lock, flags);
 552
 553         return 0;
 554 }
 555
 556 #ifdef CONFIG_T6XX_DEBUG_SYS
 557 /** The sysfs file @c clock, fbdev.
 558  *
 559  * This is used for obtaining information about the vithar
 560  * operating clock & framebuffer address,
 561  */
 562
 563 static ssize_t mali_sysfs_show_clock(struct device *dev,
 564         struct device_attribute *attr, char *buf)
 565 {
 566         struct kbase_device *kbdev;
 567         struct exynos_context *platform;
 568         ssize_t ret = 0;
 569         unsigned int clkrate;
 570
 571         kbdev = dev_get_drvdata(dev);
 572
 573         if (!kbdev)
 574                 return -ENODEV;
 575
 576         platform = (struct exynos_context *) kbdev->platform_context;
 577         if(!platform)
 578                 return -ENODEV;
 579
 580         if(!platform->sclk_g3d)
 581                 return -ENODEV;
 582
 583         clkrate = clk_get_rate(platform->sclk_g3d);
 584         ret += snprintf(buf+ret, PAGE_SIZE-ret, "Current sclk_g3d[G3D_BLK] = %dMhz", clkrate/1000000);
 585
 586         /* To be revised  */
 587         ret += snprintf(buf+ret, PAGE_SIZE-ret, "\nPossible settings : 533, 450, 400, 350, 266, 160, 100Mhz");
 588         if (ret < PAGE_SIZE - 1)
 589                 ret += snprintf(buf+ret, PAGE_SIZE-ret, "\n");
 590         else
 591         {
 592                 buf[PAGE_SIZE-2] = '\n';
 593                 buf[PAGE_SIZE-1] = '\0';
 594                 ret = PAGE_SIZE-1;
 595         }
 596
 597         return ret;
 598 }
 599
 600 static ssize_t mali_sysfs_set_clock(struct device *dev,
 601         struct device_attribute *attr, const char *buf, size_t count)
 602 {
 603         struct kbase_device *kbdev = dev_get_drvdata(dev);
 604         struct exynos_context *platform;
 605         unsigned int tmp = 0, freq = 0 ;
 606
 607         if (!kbdev) {
 608                 pr_err("%s: no kbdev\n", __func__);
 609                 return -ENODEV;
 610         }
 611
 612         platform = (struct exynos_context *) kbdev->platform_context;
 613         if (platform == NULL) {
 614                 pr_err("%s: no platform\n", __func__);
 615                 return -ENODEV;
 616         }
 617         if (!platform->sclk_g3d) {
 618                 pr_info("%s: clkout not 3d\n", __func__);
 619                 return -ENODEV;
 620         }
 621
 622         /* TODO(dianders) need to be more careful fiddling voltage+clock */
 623         if (sysfs_streq("533", buf)) {
 624                 freq = 533;
 625         } else if (sysfs_streq("450", buf)) {
 626                 freq = 450;
 627         } else if (sysfs_streq("400", buf)) {
 628                 freq = 400;
 629         } else if (sysfs_streq("350", buf)) {
 630                 freq = 350;
 631         } else if (sysfs_streq("266", buf)) {
 632                 freq = 266;
 633         } else if (sysfs_streq("160", buf)) {
 634                 freq = 160;
 635         } else if (sysfs_streq("100", buf)) {
 636                 freq = 100;
 637         } else {
 638                 pr_err("%s: invalid value\n", __func__);
 639                 return -ENOENT;
 640         }
 641
 642         pr_info("aclk400 %u\n", (unsigned int)clk_get_rate(platform->sclk_g3d));
 643         kbase_platform_dvfs_set_level(kbdev,
 644                 kbase_platform_dvfs_get_level(freq));
 645         /* Waiting for clock is stable */
 646         do {
 647                 tmp = __raw_readl(EXYNOS5_CLKDIV_STAT_TOP0);
 648         } while (tmp & 0x1000000);
 649
 650         return count;
 651 }
 652 DEVICE_ATTR(clock, S_IRUGO|S_IWUSR, mali_sysfs_show_clock,
 653         mali_sysfs_set_clock);
 654
 655 static ssize_t mali_sysfs_show_fbdev(struct device *dev,
 656         struct device_attribute *attr, char *buf)
 657 {
 658         struct kbase_device *kbdev;
 659         ssize_t ret = 0;
 660         int i;
 661
 662         kbdev = dev_get_drvdata(dev);
 663
 664         if (!kbdev)
 665                 return -ENODEV;
 666
 667         for(i = 0 ; i < num_registered_fb ; i++) {
 668                 ret += snprintf(buf+ret, PAGE_SIZE-ret, "fb[%d] xres=%d, yres=%d, addr=0x%lx\n", i, registered_fb[i]->var.xres, registered_fb[i]->var.yres, registered_fb[i]->fix.smem_start);
 669         }
 670
 671         if (ret < PAGE_SIZE - 1)
 672                 ret += snprintf(buf+ret, PAGE_SIZE-ret, "\n");
 673         else
 674         {
 675                 buf[PAGE_SIZE-2] = '\n';
 676                 buf[PAGE_SIZE-1] = '\0';
 677                 ret = PAGE_SIZE-1;
 678         }
 679
 680         return ret;
 681 }
 682 DEVICE_ATTR(fbdev, S_IRUGO, mali_sysfs_show_fbdev, NULL);
 683
 684 typedef enum {
 685         L1_I_tag_RAM = 0x00,
 686         L1_I_data_RAM = 0x01,
 687         L1_I_BTB_RAM = 0x02,
 688         L1_I_GHB_RAM = 0x03,
 689         L1_I_TLB_RAM = 0x04,
 690         L1_I_indirect_predictor_RAM = 0x05,
 691         L1_D_tag_RAM = 0x08,
 692         L1_D_data_RAM = 0x09,
 693         L1_D_load_TLB_array = 0x0A,
 694         L1_D_store_TLB_array = 0x0B,
 695         L2_tag_RAM = 0x10,
 696         L2_data_RAM = 0x11,
 697         L2_snoop_tag_RAM = 0x12,
 698         L2_data_ECC_RAM = 0x13,
 699         L2_dirty_RAM = 0x14,
 700         L2_TLB_RAM = 0x18
 701 } RAMID_type;
 702
 703 static inline void asm_ramindex_mrc(u32 *DL1Data0, u32 *DL1Data1,
 704         u32 *DL1Data2, u32 *DL1Data3)
 705 {
 706         u32 val;
 707
 708         if(DL1Data0)
 709         {
 710                 asm volatile("mrc p15, 0, %0, c15, c1, 0" : "=r" (val));
 711                 *DL1Data0 = val;
 712         }
 713         if(DL1Data1)
 714         {
 715                 asm volatile("mrc p15, 0, %0, c15, c1, 1" : "=r" (val));
 716                 *DL1Data1 = val;
 717         }
 718         if(DL1Data2)
 719         {
 720                 asm volatile("mrc p15, 0, %0, c15, c1, 2" : "=r" (val));
 721                 *DL1Data2 = val;
 722         }
 723         if(DL1Data3)
 724         {
 725                 asm volatile("mrc p15, 0, %0, c15, c1, 3" : "=r" (val));
 726                 *DL1Data3 = val;
 727         }
 728 }
 729
 730 static inline void asm_ramindex_mcr(u32 val)
 731 {
 732         asm volatile("mcr p15, 0, %0, c15, c4, 0" : : "r" (val));
 733         asm volatile("dsb");
 734         asm volatile("isb");
 735 }
 736
 737 static void get_tlb_array(u32 val, u32 *DL1Data0, u32 *DL1Data1,
 738         u32 *DL1Data2, u32 *DL1Data3)
 739 {
 740         asm_ramindex_mcr(val);
 741         asm_ramindex_mrc(DL1Data0, DL1Data1, DL1Data2, DL1Data3);
 742 }
 743
 744 static RAMID_type ramindex = L1_D_load_TLB_array;
 745 static ssize_t mali_sysfs_show_dtlb(struct device *dev,
 746         struct device_attribute *attr, char *buf)
 747 {
 748         struct kbase_device *kbdev;
 749         ssize_t ret = 0;
 750         int entries, ways;
 751         u32 DL1Data0 = 0, DL1Data1 = 0, DL1Data2 = 0, DL1Data3 = 0;
 752
 753         kbdev = dev_get_drvdata(dev);
 754
 755         if (!kbdev)
 756                 return -ENODEV;
 757
 758         /* L1-I tag RAM */
 759         if(ramindex == L1_I_tag_RAM)
 760         {
 761                 printk("Not implemented yet\n");
 762         }
 763         /* L1-I data RAM */
 764         else if(ramindex == L1_I_data_RAM)
 765         {
 766                 printk("Not implemented yet\n");
 767         }
 768         /* L1-I BTB RAM */
 769         else if(ramindex == L1_I_BTB_RAM)
 770         {
 771                 printk("Not implemented yet\n");
 772         }
 773         /* L1-I GHB RAM */
 774         else if(ramindex == L1_I_GHB_RAM)
 775         {
 776                 printk("Not implemented yet\n");
 777         }
 778         /* L1-I TLB RAM */
 779         else if(ramindex == L1_I_TLB_RAM)
 780         {
 781                 printk("L1-I TLB RAM\n");
 782                 for(entries = 0 ; entries < 32 ; entries++)
 783                 {
 784                         get_tlb_array((((u8)ramindex) << 24) + entries, &DL1Data0, &DL1Data1, &DL1Data2, NULL);
 785                         printk("entries[%d], DL1Data0=%08x, DL1Data1=%08x DL1Data2=%08x\n", entries, DL1Data0, DL1Data1 & 0xffff, 0x0);
 786                 }
 787         }
 788         /* L1-I indirect predictor RAM */
 789         else if(ramindex == L1_I_indirect_predictor_RAM)
 790         {
 791                 printk("Not implemented yet\n");
 792         }
 793         /* L1-D tag RAM */
 794         else if(ramindex == L1_D_tag_RAM)
 795         {
 796                 printk("Not implemented yet\n");
 797         }
 798         /* L1-D data RAM */
 799         else if(ramindex == L1_D_data_RAM)
 800         {
 801                 printk("Not implemented yet\n");
 802         }
 803         /* L1-D load TLB array */
 804         else if(ramindex == L1_D_load_TLB_array)
 805         {
 806                 printk("L1-D load TLB array\n");
 807                 for(entries = 0 ; entries < 32 ; entries++)
 808                 {
 809                 get_tlb_array((((u8)ramindex) << 24) + entries, &DL1Data0, &DL1Data1, &DL1Data2, &DL1Data3);
 810                 printk("entries[%d], DL1Data0=%08x, DL1Data1=%08x, DL1Data2=%08x, DL1Data3=%08x\n", entries, DL1Data0, DL1Data1, DL1Data2, DL1Data3 & 0x3f);
 811                 }
 812         }
 813         /* L1-D store TLB array */
 814         else if(ramindex == L1_D_store_TLB_array)
 815         {
 816                 printk("\nL1-D store TLB array\n");
 817                 for(entries = 0 ; entries < 32 ; entries++)
 818                 {
 819                         get_tlb_array((((u8)ramindex) << 24) + entries, &DL1Data0, &DL1Data1, &DL1Data2, &DL1Data3);
 820                         printk("entries[%d], DL1Data0=%08x, DL1Data1=%08x, DL1Data2=%08x, DL1Data3=%08x\n", entries, DL1Data0, DL1Data1, DL1Data2, DL1Data3 & 0x3f);
 821                 }
 822         }
 823         /* L2 tag RAM */
 824         else if(ramindex == L2_tag_RAM)
 825         {
 826                 printk("Not implemented yet\n");
 827         }
 828         /* L2 data RAM */
 829         else if(ramindex == L2_data_RAM)
 830         {
 831                 printk("Not implemented yet\n");
 832         }
 833         /* L2 snoop tag RAM */
 834         else if(ramindex == L2_snoop_tag_RAM)
 835         {
 836                 printk("Not implemented yet\n");
 837         }
 838         /* L2 data ECC RAM */
 839         else if(ramindex == L2_data_ECC_RAM)
 840         {
 841                 printk("Not implemented yet\n");
 842         }
 843         /* L2 dirty RAM */
 844         else if(ramindex == L2_dirty_RAM)
 845         {
 846                 printk("Not implemented yet\n");
 847         }
 848         /* L2 TLB array */
 849         else if(ramindex == L2_TLB_RAM)
 850         {
 851                 printk("\nL2 TLB array\n");
 852                 for(ways = 0 ; ways < 4 ; ways++)
 853                 {
 854                         for(entries = 0 ; entries < 512 ; entries++)
 855                         {
 856                                 get_tlb_array((ramindex << 24) + (ways << 18) + entries, &DL1Data0, &DL1Data1, &DL1Data2, &DL1Data3);
 857                                 printk("ways[%d]:entries[%d], DL1Data0=%08x, DL1Data1=%08x, DL1Data2=%08x, DL1Data3=%08x\n", ways, entries, DL1Data0, DL1Data1, DL1Data2, DL1Data3);
 858                         }
 859                 }
 860         }
 861
 862         ret += snprintf(buf+ret, PAGE_SIZE-ret, "Succeeded...\n");
 863
 864         if (ret < PAGE_SIZE - 1)
 865                 ret += snprintf(buf+ret, PAGE_SIZE-ret, "\n");
 866         else
 867         {
 868                 buf[PAGE_SIZE-2] = '\n';
 869                 buf[PAGE_SIZE-1] = '\0';
 870                 ret = PAGE_SIZE-1;
 871         }
 872         return ret;
 873 }
 874
 875 static ssize_t mali_sysfs_set_dtlb(struct device *dev,
 876         struct device_attribute *attr, const char *buf, size_t count)
 877 {
 878         struct kbase_device *kbdev;
 879         kbdev = dev_get_drvdata(dev);
 880
 881         if (!kbdev)
 882                 return -ENODEV;
 883
 884         if (sysfs_streq("L1_I_tag_RAM", buf)) {
 885                 ramindex = L1_I_tag_RAM;
 886         } else if (sysfs_streq("L1_I_data_RAM", buf)) {
 887                 ramindex = L1_I_data_RAM;
 888         } else if (sysfs_streq("L1_I_BTB_RAM", buf)) {
 889                 ramindex = L1_I_BTB_RAM;
 890         } else if (sysfs_streq("L1_I_GHB_RAM", buf)) {
 891                 ramindex = L1_I_GHB_RAM;
 892         } else if (sysfs_streq("L1_I_TLB_RAM", buf)) {
 893                 ramindex = L1_I_TLB_RAM;
 894         } else if (sysfs_streq("L1_I_indirect_predictor_RAM", buf)) {
 895                 ramindex = L1_I_indirect_predictor_RAM;
 896         } else if (sysfs_streq("L1_D_tag_RAM", buf)) {
 897                 ramindex = L1_D_tag_RAM;
 898         } else if (sysfs_streq("L1_D_data_RAM", buf)) {
 899                 ramindex = L1_D_data_RAM;
 900         } else if (sysfs_streq("L1_D_load_TLB_array", buf)) {
 901                 ramindex = L1_D_load_TLB_array;
 902         } else if (sysfs_streq("L1_D_store_TLB_array", buf)) {
 903                 ramindex = L1_D_store_TLB_array;
 904         } else if (sysfs_streq("L2_tag_RAM", buf)) {
 905                 ramindex = L2_tag_RAM;
 906         } else if (sysfs_streq("L2_data_RAM", buf)) {
 907                 ramindex = L2_data_RAM;
 908         } else if (sysfs_streq("L2_snoop_tag_RAM", buf)) {
 909                 ramindex = L2_snoop_tag_RAM;
 910         } else if (sysfs_streq("L2_data_ECC_RAM", buf)) {
 911                 ramindex = L2_data_ECC_RAM;
 912         } else if (sysfs_streq("L2_dirty_RAM", buf)) {
 913                 ramindex = L2_dirty_RAM;
 914         } else if (sysfs_streq("L2_TLB_RAM", buf)) {
 915                 ramindex = L2_TLB_RAM;
 916         } else {
 917                 printk("Invalid value....\n\n");
 918                 printk("Available options are one of below\n");
 919                 printk("L1_I_tag_RAM, L1_I_data_RAM, L1_I_BTB_RAM\n");
 920                 printk("L1_I_GHB_RAM, L1_I_TLB_RAM, L1_I_indirect_predictor_RAM\n");
 921                 printk("L1_D_tag_RAM, L1_D_data_RAM, L1_D_load_TLB_array, L1_D_store_TLB_array\n");
 922                 printk("L2_tag_RAM, L2_data_RAM, L2_snoop_tag_RAM, L2_data_ECC_RAM\n");
 923                 printk("L2_dirty_RAM, L2_TLB_RAM\n");
 924         }
 925
 926         return count;
 927 }
 928 DEVICE_ATTR(dtlb, S_IRUGO|S_IWUSR, mali_sysfs_show_dtlb, mali_sysfs_set_dtlb);
 929
 930 static ssize_t mali_sysfs_show_vol(struct device *dev,
 931         struct device_attribute *attr, char *buf)
 932 {
 933         struct kbase_device *kbdev;
 934         ssize_t ret = 0;
 935         int vol;
 936
 937         kbdev = dev_get_drvdata(dev);
 938
 939         if (!kbdev)
 940                 return -ENODEV;
 941
 942         kbase_platform_get_voltage(dev, &vol);
 943         ret += snprintf(buf+ret, PAGE_SIZE-ret, "Current operating voltage for mali t6xx = %d", vol);
 944
 945         if (ret < PAGE_SIZE - 1)
 946                 ret += snprintf(buf+ret, PAGE_SIZE-ret, "\n");
 947         else
 948         {
 949                 buf[PAGE_SIZE-2] = '\n';
 950                 buf[PAGE_SIZE-1] = '\0';
 951                 ret = PAGE_SIZE-1;
 952         }
 953
 954         return ret;
 955 }
 956 DEVICE_ATTR(vol, S_IRUGO|S_IWUSR, mali_sysfs_show_vol, NULL);
 957
 958 static int get_clkout_cmu_top(int *val)
 959 {
 960         *val = __raw_readl(/*EXYNOS5_CLKOUT_CMU_TOP*/EXYNOS_CLKREG(0x10A00));
 961         if((*val & 0x1f) == 0xB) /* CLKOUT is ACLK_400 in CLKOUT_CMU_TOP */
 962                 return 1;
 963         else
 964                 return 0;
 965 }
 966
 967 static void set_clkout_for_3d(void)
 968 {
 969         int tmp;
 970
 971         tmp = 0x0;
 972         tmp |= 0x1000B; /* ACLK_400 selected */
 973         tmp |= 9 << 8;  /* divided by (9 + 1) */
 974         __raw_writel(tmp, /*EXYNOS5_CLKOUT_CMU_TOP*/EXYNOS_CLKREG(0x10A00));
 975
 976 #ifdef PMU_XCLKOUT_SET
 977         exynos5_pmu_xclkout_set(1, XCLKOUT_CMU_TOP);
 978 #else /* PMU_XCLKOUT_SET */
 979         tmp = 0x0;
 980         tmp |= 7 << 8; /* CLKOUT_CMU_TOP selected */
 981         __raw_writel(tmp, /*S5P_PMU_DEBUG*/S5P_PMUREG(0x0A00));
 982 #endif /* PMU_XCLKOUT_SET */
 983 }
 984
 985 static ssize_t mali_sysfs_show_clkout(struct device *dev,
 986         struct device_attribute *attr, char *buf)
 987 {
 988         struct kbase_device *kbdev;
 989         ssize_t ret = 0;
 990         int val;
 991
 992         kbdev = dev_get_drvdata(dev);
 993
 994         if (!kbdev)
 995                 return -ENODEV;
 996
 997         if(get_clkout_cmu_top(&val))
 998                 ret += snprintf(buf+ret, PAGE_SIZE-ret, "Current CLKOUT is g3d divided by 10, CLKOUT_CMU_TOP=0x%x", val);
 999         else
1000                 ret += snprintf(buf+ret, PAGE_SIZE-ret, "Current CLKOUT is not g3d, CLKOUT_CMU_TOP=0x%x", val);
1001
1002         if (ret < PAGE_SIZE - 1)
1003                 ret += snprintf(buf+ret, PAGE_SIZE-ret, "\n");
1004         else
1005         {
1006                 buf[PAGE_SIZE-2] = '\n';
1007                 buf[PAGE_SIZE-1] = '\0';
1008                 ret = PAGE_SIZE-1;
1009         }
1010
1011         return ret;
1012 }
1013
1014 static ssize_t mali_sysfs_set_clkout(struct device *dev,
1015         struct device_attribute *attr, const char *buf, size_t count)
1016 {
1017         struct kbase_device *kbdev = dev_get_drvdata(dev);
1018
1019         if (!kbdev)
1020                 return -ENODEV;
1021
1022         set_clkout_for_3d();
1023         pr_info("clkout set to 3d\n");
1024         return count;
1025 }
1026 DEVICE_ATTR(clkout, S_IRUGO|S_IWUSR, mali_sysfs_show_clkout,
1027         mali_sysfs_set_clkout);
1028
1029 #ifdef CONFIG_T6XX_DVFS
1030 static ssize_t mali_sysfs_show_dvfs(struct device *dev,
1031         struct device_attribute *attr, char *buf)
1032 {
1033         struct kbase_device *kbdev;
1034         ssize_t ret = 0;
1035
1036         kbdev = dev_get_drvdata(dev);
1037
1038         if (!kbdev)
1039                 return -ENODEV;
1040
1041         if (!kbasep_pm_metrics_isactive(kbdev))
1042                 ret += snprintf(buf+ret, PAGE_SIZE-ret, "G3D DVFS is off\n");
1043         else
1044                 ret += snprintf(buf+ret, PAGE_SIZE-ret, "G3D DVFS is on\n");
1045
1046         return ret;
1047 }
1048
1049 static ssize_t mali_sysfs_set_dvfs(struct device *dev,
1050         struct device_attribute *attr, const char *buf, size_t count)
1051 {
1052         mali_error ret;
1053         struct kbase_device *kbdev;
1054         kbdev = dev_get_drvdata(dev);
1055
1056         if (!kbdev)
1057                 return -ENODEV;
1058
1059         if (sysfs_streq("off", buf)) {
1060                 if (kbasep_pm_metrics_isactive(kbdev)) {
1061                         kbasep_pm_metrics_term(kbdev);
1062                         kbase_platform_dvfs_set_level(kbdev,
1063                                 kbase_platform_dvfs_get_level(VITHAR_DEFAULT_CLOCK / 1000000));
1064                         pr_info("G3D DVFS is disabled\n");
1065                 }
1066         } else if (sysfs_streq("on", buf)) {
1067                 if (!kbasep_pm_metrics_isactive(kbdev)) {
1068                         ret = kbasep_pm_metrics_init(kbdev);
1069                         if (ret != MALI_ERROR_NONE)
1070                                 pr_warning("kbase_pm_metrics_init failed,"
1071                                     " error %u\n", ret);
1072                         else
1073                                 pr_info("G3D DVFS is enabled\n");
1074                 }
1075         } else {
1076                 pr_info("%s: invalid, only [on, off] is accepted\n", buf);
1077         }
1078
1079         return count;
1080 }
1081 DEVICE_ATTR(dvfs, S_IRUGO|S_IWUSR, mali_sysfs_show_dvfs, mali_sysfs_set_dvfs);
1082 #endif /* CONFIG_T6XX_DVFS */
1083
1084 #ifdef MALI_DVFS_ASV_ENABLE
1085 static ssize_t mali_sysfs_show_asv(struct device *dev,
1086         struct device_attribute *attr, char *buf)
1087 {
1088
1089         struct kbase_device *kbdev;
1090         ssize_t ret = 0;
1091         kbdev = dev_get_drvdata(dev);
1092
1093         if (!kbdev)
1094                 return -ENODEV;
1095
1096         ret = kbase_platform_dvfs_sprint_avs_table(buf);
1097
1098         return ret;
1099 }
1100
1101 static ssize_t mali_sysfs_set_asv(struct device *dev,
1102         struct device_attribute *attr, const char *buf, size_t count)
1103 {
1104         if (sysfs_streq("off", buf))
1105                 kbase_platform_asv_set(0);
1106         else if (sysfs_streq("on", buf))
1107                 kbase_platform_asv_set(1);
1108         else
1109                 printk(KERN_ERR "invalid val -only [on, off] is accepted\n");
1110
1111         return count;
1112 }
1113 DEVICE_ATTR(asv, S_IRUGO|S_IWUSR, mali_sysfs_show_asv, mali_sysfs_set_asv);
1114 #endif
1115
1116 static int kbase_platform_create_sysfs_file(struct device *dev)
1117 {
1118         if (device_create_file(dev, &dev_attr_clock))
1119         {
1120                 dev_err(dev, "Couldn't create sysfs file [clock]\n");
1121                 goto out;
1122         }
1123
1124         if (device_create_file(dev, &dev_attr_fbdev))
1125         {
1126                 dev_err(dev, "Couldn't create sysfs file [fbdev]\n");
1127                 goto out;
1128         }
1129
1130         if (device_create_file(dev, &dev_attr_dtlb))
1131         {
1132                 dev_err(dev, "Couldn't create sysfs file [dtlb]\n");
1133                 goto out;
1134         }
1135
1136         if (device_create_file(dev, &dev_attr_vol))
1137         {
1138                 dev_err(dev, "Couldn't create sysfs file [vol]\n");
1139                 goto out;
1140         }
1141
1142         if (device_create_file(dev, &dev_attr_clkout))
1143         {
1144                 dev_err(dev, "Couldn't create sysfs file [clkout]\n");
1145                 goto out;
1146         }
1147 #ifdef CONFIG_T6XX_DVFS
1148         if (device_create_file(dev, &dev_attr_dvfs))
1149         {
1150                 dev_err(dev, "Couldn't create sysfs file [dvfs]\n");
1151                 goto out;
1152         }
1153 #ifdef MALI_DVFS_ASV_ENABLE
1154         if (device_create_file(dev, &dev_attr_asv)) {
1155                 dev_err(dev, "Couldn't create sysfs file [asv]\n");
1156                 goto out;
1157         }
1158 #endif
1159 #endif /* CONFIG_T6XX_DVFS */
1160         if (!mali_setup_system_tracing(dev))
1161                 goto out;
1162
1163
1164         return 0;
1165 out:
1166         return -ENOENT;
1167 }
1168
1169 void kbase_platform_remove_sysfs_file(struct device *dev)
1170 {
1171         device_remove_file(dev, &dev_attr_clock);
1172         device_remove_file(dev, &dev_attr_fbdev);
1173         device_remove_file(dev, &dev_attr_dtlb);
1174         device_remove_file(dev, &dev_attr_vol);
1175         device_remove_file(dev, &dev_attr_clkout);
1176 #ifdef CONFIG_T6XX_DVFS
1177         device_remove_file(dev, &dev_attr_dvfs);
1178 #ifdef MALI_DVFS_ASV_ENABLE
1179         device_remove_file(dev, &dev_attr_asv);
1180 #endif
1181 #endif /* CONFIG_T6XX_DVFS */
1182         mali_cleanup_system_tracing(dev);
1183 }
1184 #endif /* CONFIG_T6XX_DEBUG_SYS */
1185
1186 #include "osk/include/mali_osk_lock_order.h"
1187
1188 #ifdef CONFIG_PM_RUNTIME
1189 static void kbase_platform_runtime_term(struct kbase_device *kbdev)
1190 {
1191         pm_runtime_disable(kbdev->osdev.dev);
1192 }
1193 #endif /* CONFIG_PM_RUNTIME */
1194
1195 #ifdef CONFIG_PM_RUNTIME
1196 extern void pm_runtime_init(struct device *dev);
1197
1198 static mali_error kbase_platform_runtime_init(struct kbase_device *kbdev)
1199 {
1200         pm_runtime_init(kbdev->osdev.dev);
1201         pm_suspend_ignore_children(kbdev->osdev.dev, true);
1202         pm_runtime_enable(kbdev->osdev.dev);
1203         return MALI_ERROR_NONE;
1204 }
1205 #endif /* CONFIG_PM_RUNTIME */
1206
1207 mali_error kbase_platform_init(kbase_device *kbdev)
1208 {
1209         struct exynos_context *platform;
1210
1211         platform = osk_malloc(sizeof(struct exynos_context));
1212
1213         if(NULL == platform)
1214         {
1215                 return MALI_ERROR_OUT_OF_MEMORY;
1216         }
1217
1218         kbdev->platform_context = (void *) platform;
1219
1220         platform->cmu_pmu_status = 0;
1221         spin_lock_init(&platform->cmu_pmu_lock);
1222
1223         if(kbase_platform_power_clock_init(kbdev))
1224         {
1225                 goto clock_init_fail;
1226         }
1227
1228 #ifdef CONFIG_REGULATOR
1229         if(kbase_platform_regulator_init())
1230         {
1231                 goto regulator_init_fail;
1232         }
1233 #endif /* CONFIG_REGULATOR */
1234
1235 #ifdef CONFIG_T6XX_DVFS
1236         kbase_platform_dvfs_init(kbdev);
1237 #endif /* CONFIG_T6XX_DVFS */
1238
1239         /* Enable power */
1240         kbase_platform_cmu_pmu_control(kbdev, 1);
1241         return MALI_ERROR_NONE;
1242
1243 #ifdef CONFIG_REGULATOR
1244         kbase_platform_regulator_disable();
1245 #endif /* CONFIG_REGULATOR */
1246 regulator_init_fail:
1247 clock_init_fail:
1248         osk_free(platform);
1249
1250         return MALI_ERROR_FUNCTION_FAILED;
1251 }
1252
1253 void kbase_platform_term(kbase_device *kbdev)
1254 {
1255         struct exynos_context *platform;
1256
1257         platform = (struct exynos_context *) kbdev->platform_context;
1258
1259 #ifdef CONFIG_T6XX_DVFS
1260         kbase_platform_dvfs_term();
1261 #endif /* CONFIG_T6XX_DVFS */
1262
1263         /* Disable power */
1264         kbase_platform_cmu_pmu_control(kbdev, 0);
1265 #ifdef CONFIG_REGULATOR
1266         kbase_platform_regulator_disable();
1267 #endif /* CONFIG_REGULATOR */
1268         osk_free(kbdev->platform_context);
1269         kbdev->platform_context = 0;
1270         return;
1271 }
1272
1273 #ifdef CONFIG_REGULATOR
1274 static struct regulator *g3d_regulator=NULL;
1275 #ifdef CONFIG_T6XX_HWVER_R0P0
1276 static int mali_gpu_vol = 1250000; /* 1.25V @ 533 MHz */
1277 #else
1278 static int mali_gpu_vol = 1050000; /* 1.05V @ 266 MHz */
1279 #endif /*  CONFIG_T6XX_HWVER_R0P0 */
1280 #endif /* CONFIG_REGULATOR */
1281
1282 #ifdef CONFIG_T6XX_DVFS
1283 /*
1284  * Weighted moving average support for signed integer data
1285  * with 7-bits of precision (not currently used; all data
1286  * are integers).
1287  */
1288 #define DVFS_AVG_DUMMY_MARKER   (~0)
1289 #define DVFS_AVG_LPF_LEN        4       /* NB: best to be pow2 */
1290 #define DVFS_AVG_EP_MULTIPLIER  (1<<7)  /* 7 fractional bits */
1291
1292 #define DVFS_AVG_RESET(x)       ((x) = DVFS_AVG_DUMMY_MARKER)
1293 #define _DVFS_AVG_IN(x)         ((x) * DVFS_AVG_EP_MULTIPLIER)
1294 #define _DVFS_LPF_UTIL(x, y, len) \
1295         ((x != DVFS_AVG_DUMMY_MARKER) ? \
1296          (((x) * ((len) - 1) + (y)) / (len)) : (y))
1297 #define DVFS_AVG_LPF(x, y) do { \
1298         x = _DVFS_LPF_UTIL((x), _DVFS_AVG_IN((y)), DVFS_AVG_LPF_LEN); \
1299 } while (0)
1300 #define DVFS_TO_AVG(x)          DIV_ROUND_CLOSEST(x, DVFS_AVG_EP_MULTIPLIER)
1301
1302 #ifdef MALI_DVFS_ASV_ENABLE
1303 enum asv_update_val {
1304         DVFS_NOT_UPDATE_ASV_TBL = 0,
1305         DVFS_UPDATE_ASV_TBL = 1,
1306         DVFS_UPDATE_ASV_DEFAULT_TBL = 2,
1307 };
1308 #endif /* MALI_DVFS_ASV_ENABLE */
1309
1310 struct mali_dvfs_status {
1311         kbase_device *kbdev;
1312         int step;
1313         int utilisation;
1314         uint nsamples;
1315         u32 avg_utilisation;
1316 #ifdef MALI_DVFS_ASV_ENABLE
1317         enum asv_update_val asv_need_update;
1318         int asv_group;
1319 #endif
1320 };
1321
1322 static struct workqueue_struct *mali_dvfs_wq = 0;
1323 int mali_dvfs_control=0;
1324 osk_spinlock_irq mali_dvfs_spinlock;
1325
1326
1327 static struct mali_dvfs_status mali_dvfs_status_current;
1328 /*
1329  * Governor parameters.  The governor gets periodic samples of the
1330  * GPU utilisation (%busy) and maintains a weighted average over the
1331  * last DVFS_AVG_LPF_LEN values.  When the average is in the range
1332  * [min_threshold..max_threshold] we maintain the current clock+voltage.
1333  * If the utilisation drops below min for down_cnt_threshold samples
1334  * we step down.  If the utilisation exceeds max_threshold for
1335  * up_cnt_threshold samples we step up.
1336  *
1337  * The up/down thresholds are chosen to enable fast step up under
1338  * load with a longer step down; this optimizes for performance over
1339  * power consumption.  266MHz is the "sweet spot"; it has the best
1340  * performance/power ratio.  For this reason it has slightly extended
1341  * up/down thresholds to make it "sticky".
1342  */
1343 struct mali_dvfs_info {
1344         unsigned int voltage;
1345         unsigned int clock;
1346         int min_threshold;
1347         int max_threshold;
1348         int up_cnt_threshold;
1349         int down_cnt_threshold;
1350 };
1351
1352 /* TODO(sleffler) round or verify time is a multiple of frequency */
1353 /* convert a time in milliseconds to a dvfs sample count */
1354 #define DVFS_TIME_TO_CNT(t)     ((t) / KBASE_PM_DVFS_FREQUENCY)
1355
1356 /* TODO(sleffler) should be const but for voltage */
1357 static struct mali_dvfs_info mali_dvfs_infotbl[MALI_DVFS_STEP] = {
1358 #if (MALI_DVFS_STEP == 7)
1359         { 912500, 100,  0,  60, DVFS_TIME_TO_CNT(750), DVFS_TIME_TO_CNT(2000)},
1360         { 925000, 160, 40,  75, DVFS_TIME_TO_CNT(750), DVFS_TIME_TO_CNT(2000)},
1361         {1025000, 266, 65,  85, DVFS_TIME_TO_CNT(1000), DVFS_TIME_TO_CNT(3000)},
1362         {1075000, 350, 65,  85, DVFS_TIME_TO_CNT(750), DVFS_TIME_TO_CNT(1500)},
1363         {1125000, 400, 65,  85, DVFS_TIME_TO_CNT(750), DVFS_TIME_TO_CNT(1500)},
1364         {1150000, 450, 65,  90, DVFS_TIME_TO_CNT(1000), DVFS_TIME_TO_CNT(1500)},
1365         {1250000, 533, 75, 100, DVFS_TIME_TO_CNT(750), DVFS_TIME_TO_CNT(1500)}
1366 #else
1367 #error no table
1368 #endif
1369 };
1370
1371 #ifdef MALI_DVFS_ASV_ENABLE
1372 static const unsigned int mali_dvfs_asv_vol_tbl_special
1373         [MALI_DVFS_ASV_GROUP_SPECIAL_NUM][MALI_DVFS_STEP] = {
1374         /*  100Mh   160Mh     266Mh   350Mh             400Mh   450Mh   533Mh*/
1375         {/*Group 1*/
1376                 912500, 925000, 1025000, 1075000, 1100000, 1150000, 1225000,
1377         },
1378         {/*Group 2*/
1379                 900000, 900000, 1000000, 1037500, 1087500, 1125000, 1200000,
1380         },
1381         {/*Group 3*/
1382                912500, 925000, 1025000, 1037500, 1100000, 1150000, 1225000,
1383         },
1384         {/*Group 4*/
1385                 900000, 900000, 1000000, 1025000, 1087500, 1125000, 1200000,
1386         },
1387         {/*Group 5*/
1388                 912500, 925000, 1000000, 1000000, 1125000, 1150000, 1250000,
1389         },
1390         {/*Group 6*/
1391                 900000, 912500, 987500, 987500, 1112500, 1150000, 1237500,
1392         },
1393         {/*Group 7*/
1394                 900000, 900000, 975000, 987500, 1100000, 1137500, 1225000,
1395         },
1396         {/*Group 8*/
1397                 900000, 900000, 975000, 987500, 1100000, 1137500, 1225000,
1398         },
1399         {/*Group 9*/
1400                 887500, 900000, 962500, 975000, 1087500, 1125000, 1212500,
1401         },
1402         {/*Group 10*/
1403                 887500, 900000, 962500, 962500, 1087500, 1125000, 1212500,
1404         },
1405 };
1406
1407 static const unsigned int mali_dvfs_asv_vol_tbl
1408         [MALI_DVFS_ASV_GROUP_NUM][MALI_DVFS_STEP] = {
1409        /*  100Mh       160Mh      266Mh        350Mh,  400Mh   450Mh   533Mh*/
1410         {/*Group 0*/
1411                 925000, 925000, 1025000, 1075000, 1125000, 1150000, 1200000,
1412         },
1413         {/*Group 1*/
1414                 900000, 900000, 1000000, 1037500, 1087500, 1137500, 1187500,
1415         },
1416         {/*Group 2*/
1417                 900000, 900000, 950000, 1037500, 1075000, 1125000, 1187500,
1418         },
1419         {/*Group 3*/
1420                 900000, 900000, 950000, 1037500, 1075000, 1125000, 1187500,
1421         },
1422         {/*Group 4*/
1423                 900000, 900000, 937500, 1025000, 1075000, 1112500, 1175000,
1424         },
1425         {/*Group 5*/
1426                 900000, 900000, 937500, 1000000, 1050000, 1100000, 1150000,
1427         },
1428         {/*Group 6*/
1429                 900000, 900000, 925000, 987500, 1037500, 1087500, 1137500,
1430         },
1431         {/*Group 7*/
1432                 900000, 900000, 912500, 987500, 1025000, 1075000, 1125000,
1433         },
1434         {/*Group 8*/
1435                 900000, 900000, 912500, 987500, 1012500, 1075000, 1125000,
1436         },
1437         {/*Group 9*/
1438                 900000, 900000, 900000, 975000, 1012500, 1050000, 1125000,
1439         },
1440         {/*Group 10*/
1441                 875000, 900000, 900000, 962500, 1000000, 1050000, 1112500,
1442         },
1443         {/*Group 11*/
1444                 875000, 900000, 900000, 962500, 1000000, 1050000, 1112500,
1445         },
1446 };
1447
1448 static const unsigned int mali_dvfs_vol_default[MALI_DVFS_STEP] = {
1449          925000, 925000, 1025000, 1075000, 1125000, 1150000, 120000
1450 };
1451
1452 static int mali_dvfs_update_asv(int group)
1453 {
1454         int i;
1455
1456         /* TODO(Shariq): Replace exynos_lot_id with api call once ASV code is rebased */
1457         if (exynos_lot_id && group == 0) {
1458                 /* wrong group: Use default table to keep the system running smoothly */
1459                 for (i = 0; i < MALI_DVFS_STEP; i++)
1460                         mali_dvfs_infotbl[i].voltage = mali_dvfs_vol_default[i];
1461                 pr_err("exynos_lot_id has group 0. Using default gpu asv table\n");
1462                 return 1;
1463         }
1464
1465         if (group == -1) {
1466                 for (i = 0; i < MALI_DVFS_STEP; i++)
1467                         mali_dvfs_infotbl[i].voltage = mali_dvfs_vol_default[i];
1468                 pr_err("mali_dvfs_update_asv use default table\n");
1469                 return 1;
1470         }
1471         if (group > MALI_DVFS_ASV_GROUP_NUM) {
1472                 /* unknown group: Use default table to keep the system running smoothly */
1473                 for (i = 0; i < MALI_DVFS_STEP; i++)
1474                         mali_dvfs_infotbl[i].voltage = mali_dvfs_vol_default[i];
1475                 OSK_PRINT_ERROR(OSK_BASE_PM, "invalid asv group (%d)\n", group);
1476                 return 1;
1477         }
1478         for (i = 0; i < MALI_DVFS_STEP; i++) {
1479                 if (exynos_lot_id)
1480                         mali_dvfs_infotbl[i].voltage =
1481                                 mali_dvfs_asv_vol_tbl_special[group-1][i];
1482                  else
1483                          mali_dvfs_infotbl[i].voltage =
1484                                 mali_dvfs_asv_vol_tbl[group][i];
1485         }
1486
1487         return 0;
1488 }
1489 #endif /* MALI_DVFS_ASV_ENABLE */
1490
1491 static void mali_dvfs_event_proc(struct work_struct *w)
1492 {
1493         struct mali_dvfs_status dvfs_status;
1494         const struct mali_dvfs_info *info;
1495         int avg_utilisation;
1496
1497         osk_spinlock_irq_lock(&mali_dvfs_spinlock);
1498         dvfs_status = mali_dvfs_status_current;
1499 #ifdef MALI_DVFS_ASV_ENABLE
1500         if (dvfs_status.asv_need_update == DVFS_UPDATE_ASV_DEFAULT_TBL) {
1501                 mali_dvfs_update_asv(-1);
1502                 dvfs_status.asv_need_update = DVFS_NOT_UPDATE_ASV_TBL;
1503         } else if (dvfs_status.asv_need_update == DVFS_UPDATE_ASV_TBL) {
1504                 if (mali_dvfs_update_asv(exynos_result_of_asv&0xf) == 0)
1505                         dvfs_status.asv_group = (exynos_result_of_asv&0xf);
1506                 dvfs_status.asv_need_update = DVFS_NOT_UPDATE_ASV_TBL;
1507         }
1508 #endif
1509         osk_spinlock_irq_unlock(&mali_dvfs_spinlock);
1510
1511         info = &mali_dvfs_infotbl[dvfs_status.step];
1512
1513         OSK_ASSERT(dvfs_status.utilisation <= 100);
1514         dvfs_status.nsamples++;
1515         DVFS_AVG_LPF(dvfs_status.avg_utilisation, dvfs_status.utilisation);
1516         OSK_ASSERT(dvfs_status.nsamples > 0);
1517         avg_utilisation = DVFS_TO_AVG(dvfs_status.avg_utilisation);
1518 #if MALI_DVFS_DEBUG
1519         pr_debug("%s: step %d utilisation %d avg %d max %d min %d "
1520             "nsamples %u up_cnt %d down_cnt %d\n", __func__,
1521             dvfs_status.step, dvfs_status.utilisation,
1522             avg_utilisation, info->max_threshold, info->min_threshold,
1523             dvfs_status.nsamples, allow_up_cnt, allow_down_cnt);
1524 #endif
1525         trace_mali_dvfs_event(dvfs_status.utilisation, avg_utilisation);
1526
1527         if (avg_utilisation > info->max_threshold &&
1528             dvfs_status.nsamples >= info->up_cnt_threshold) {
1529                 dvfs_status.step++;
1530                 /*
1531                  * NB: max clock should have max_threshold of 100
1532                  * so this should never trip.
1533                  */
1534                 OSK_ASSERT(dvfs_status.step < MALI_DVFS_STEP);
1535                 DVFS_AVG_RESET(dvfs_status.avg_utilisation);
1536                 dvfs_status.nsamples = 0;
1537         } else if (dvfs_status.step > 0 &&
1538             avg_utilisation < info->min_threshold &&
1539             dvfs_status.nsamples >= info->down_cnt_threshold) {
1540                 OSK_ASSERT(dvfs_status.step > 0);
1541                 dvfs_status.step--;
1542                 DVFS_AVG_RESET(dvfs_status.avg_utilisation);
1543                 dvfs_status.nsamples = 0;
1544         }
1545
1546         if (kbasep_pm_metrics_isactive(dvfs_status.kbdev))
1547                 kbase_platform_dvfs_set_level(dvfs_status.kbdev,
1548                 dvfs_status.step);
1549
1550         osk_spinlock_irq_lock(&mali_dvfs_spinlock);
1551         mali_dvfs_status_current = dvfs_status;
1552         osk_spinlock_irq_unlock(&mali_dvfs_spinlock);
1553
1554 }
1555
1556 static DECLARE_WORK(mali_dvfs_work, mali_dvfs_event_proc);
1557
1558 /**
1559  * Exynos5 alternative dvfs_callback imlpementation.
1560  * instead of:
1561  *    action = kbase_pm_get_dvfs_action(kbdev);
1562  * use this:
1563  *    kbase_platform_dvfs_event(kbdev);
1564  */
1565
1566 int kbase_platform_dvfs_get_control_status(void)
1567 {
1568         return mali_dvfs_control;
1569 }
1570
1571 int kbase_platform_dvfs_init(kbase_device *kbdev)
1572 {
1573         /*default status
1574         add here with the right function to get initilization value.
1575         */
1576         if (!mali_dvfs_wq)
1577                 mali_dvfs_wq = create_singlethread_workqueue("mali_dvfs");
1578
1579         osk_spinlock_irq_init(&mali_dvfs_spinlock, OSK_LOCK_ORDER_PM_METRICS);
1580
1581         /*add a error handling here*/
1582         osk_spinlock_irq_lock(&mali_dvfs_spinlock);
1583         mali_dvfs_status_current.kbdev = kbdev;
1584         mali_dvfs_status_current.step = MALI_DVFS_STEP-1;
1585         mali_dvfs_status_current.utilisation = 100;
1586         DVFS_AVG_RESET(mali_dvfs_status_current.avg_utilisation);
1587 #ifdef MALI_DVFS_ASV_ENABLE
1588         mali_dvfs_status_current.asv_need_update = DVFS_UPDATE_ASV_TBL;
1589         mali_dvfs_status_current.asv_group = -1;
1590 #endif
1591         mali_dvfs_control = 1;
1592         osk_spinlock_irq_unlock(&mali_dvfs_spinlock);
1593
1594         return MALI_TRUE;
1595 }
1596
1597 void kbase_platform_dvfs_term(void)
1598 {
1599         if (mali_dvfs_wq)
1600                 destroy_workqueue(mali_dvfs_wq);
1601
1602         mali_dvfs_wq = NULL;
1603 }
1604 #endif /* CONFIG_T6XX_DVFS */
1605
1606
1607 int kbase_platform_dvfs_event(struct kbase_device *kbdev)
1608 {
1609 #ifdef CONFIG_T6XX_DVFS
1610         int utilisation = kbase_pm_get_dvfs_utilisation(kbdev);
1611
1612         osk_spinlock_irq_lock(&mali_dvfs_spinlock);
1613         mali_dvfs_status_current.utilisation = utilisation;
1614         osk_spinlock_irq_unlock(&mali_dvfs_spinlock);
1615         queue_work_on(0, mali_dvfs_wq, &mali_dvfs_work);
1616
1617         /*add error handle here*/
1618         return MALI_TRUE;
1619 #else
1620         return MALI_FALSE;
1621 #endif
1622 }
1623
1624 int kbase_platform_regulator_init(void)
1625 {
1626 #ifdef CONFIG_REGULATOR
1627         g3d_regulator = regulator_get(NULL, "vdd_g3d");
1628         if(IS_ERR(g3d_regulator))
1629         {
1630                 printk("[kbase_platform_regulator_init] failed to get mali t6xx regulator\n");
1631                 return -1;
1632         }
1633
1634         if(regulator_enable(g3d_regulator) != 0)
1635         {
1636                 printk("[kbase_platform_regulator_init] failed to enable mali t6xx regulator\n");
1637                 return -1;
1638         }
1639
1640         if(regulator_set_voltage(g3d_regulator, mali_gpu_vol, mali_gpu_vol) != 0)
1641         {
1642                 printk("[kbase_platform_regulator_init] failed to set mali t6xx operating voltage [%d]\n", mali_gpu_vol);
1643                 return -1;
1644         }
1645 #endif /* CONFIG_REGULATOR */
1646
1647         return 0;
1648 }
1649
1650 int kbase_platform_regulator_disable(void)
1651 {
1652 #ifdef CONFIG_REGULATOR
1653         if(!g3d_regulator)
1654         {
1655                 printk("[kbase_platform_regulator_disable] g3d_regulator is not initialized\n");
1656                 return -1;
1657         }
1658
1659         if(regulator_disable(g3d_regulator) != 0)
1660         {
1661                 printk("[kbase_platform_regulator_disable] failed to disable g3d regulator\n");
1662                 return -1;
1663         }
1664 #endif /* CONFIG_REGULATOR */
1665         return 0;
1666 }
1667
1668 int kbase_platform_regulator_enable(void)
1669 {
1670 #ifdef CONFIG_REGULATOR
1671         if(!g3d_regulator)
1672         {
1673                 printk("[kbase_platform_regulator_enable] g3d_regulator is not initialized\n");
1674                 return -1;
1675         }
1676
1677         if(regulator_enable(g3d_regulator) != 0)
1678         {
1679                 printk("[kbase_platform_regulator_enable] failed to enable g3d regulator\n");
1680                 return -1;
1681         }
1682 #endif /* CONFIG_REGULATOR */
1683         return 0;
1684 }
1685
1686 int kbase_platform_get_default_voltage(struct device *dev, int *vol)
1687 {
1688 #ifdef CONFIG_REGULATOR
1689         *vol = mali_gpu_vol;
1690 #else /* CONFIG_REGULATOR */
1691         *vol = 0;
1692 #endif /* CONFIG_REGULATOR */
1693         return 0;
1694 }
1695
1696 #ifdef CONFIG_T6XX_DEBUG_SYS
1697 int kbase_platform_get_voltage(struct device *dev, int *vol)
1698 {
1699 #ifdef CONFIG_REGULATOR
1700         if(!g3d_regulator)
1701         {
1702                 printk("[kbase_platform_get_voltage] g3d_regulator is not initialized\n");
1703                 return -1;
1704         }
1705
1706         *vol = regulator_get_voltage(g3d_regulator);
1707 #else /* CONFIG_REGULATOR */
1708         *vol = 0;
1709 #endif /* CONFIG_REGULATOR */
1710         return 0;
1711 }
1712 #endif /* CONFIG_T6XX_DEBUG_SYS */
1713
1714 #if defined CONFIG_T6XX_DEBUG_SYS || defined CONFIG_T6XX_DVFS
1715 int kbase_platform_set_voltage(struct device *dev, int vol)
1716 {
1717 #ifdef CONFIG_REGULATOR
1718         if(!g3d_regulator)
1719         {
1720                 printk("[kbase_platform_set_voltage] g3d_regulator is not initialized\n");
1721                 return -1;
1722         }
1723
1724         if(regulator_set_voltage(g3d_regulator, vol, vol) != 0)
1725         {
1726                 printk("[kbase_platform_set_voltage] failed to set voltage\n");
1727                 return -1;
1728         }
1729 #endif /* CONFIG_REGULATOR */
1730         return 0;
1731 }
1732 #endif /* CONFIG_T6XX_DEBUG_SYS */
1733
1734 void kbase_platform_dvfs_set_clock(kbase_device *kbdev, int freq)
1735 {
1736         static struct clk * mout_gpll = NULL;
1737         static struct clk * fin_gpll = NULL;
1738         static struct clk * fout_gpll = NULL;
1739         static int _freq = -1;
1740         static unsigned long gpll_rate_prev = 0;
1741         unsigned long gpll_rate = 0, aclk_400_rate = 0;
1742         unsigned long tmp = 0;
1743         struct exynos_context *platform;
1744
1745
1746         if (!kbdev)
1747                 panic("no kbdev");
1748
1749         platform = (struct exynos_context *) kbdev->platform_context;
1750         if (NULL == platform)
1751                 panic("no platform");
1752
1753         if (platform->sclk_g3d == 0)
1754                 return;
1755
1756         if (mout_gpll == NULL) {
1757                 mout_gpll = clk_get(kbdev->osdev.dev, "mout_gpll");
1758                 fin_gpll = clk_get(kbdev->osdev.dev, "ext_xtal");
1759                 fout_gpll = clk_get(kbdev->osdev.dev, "fout_gpll");
1760                 if (IS_ERR(mout_gpll) || IS_ERR(fin_gpll) || IS_ERR(fout_gpll))
1761                         panic("clk_get ERROR");
1762         }
1763
1764         if (freq == _freq)
1765                 return;
1766
1767         trace_mali_dvfs_set_clock(freq);
1768
1769         switch(freq)
1770         {
1771                 case 533:
1772                         gpll_rate = 533000000;
1773                         aclk_400_rate = 533000000;
1774                         break;
1775                 case 450:
1776                         gpll_rate = 450000000;
1777                         aclk_400_rate = 450000000;
1778                         break;
1779                 case 400:
1780                         gpll_rate = 800000000;
1781                         aclk_400_rate = 400000000;
1782                         break;
1783                 case 350:
1784                         gpll_rate = 1400000000;
1785                         aclk_400_rate = 350000000;
1786                         break;
1787                 case 266:
1788                         gpll_rate = 800000000;
1789                         aclk_400_rate = 267000000;
1790                         break;
1791                 case 160:
1792                         gpll_rate = 800000000;
1793                         aclk_400_rate = 160000000;
1794                         break;
1795                 case 100:
1796                         gpll_rate = 800000000;
1797                         aclk_400_rate = 100000000;
1798                         break;
1799                 default:
1800                         return;
1801         }
1802
1803         /* if changed the GPLL rate, set rate for GPLL and wait for lock time */
1804         if( gpll_rate != gpll_rate_prev) {
1805                 /*for stable clock input.*/
1806                 clk_set_rate(platform->sclk_g3d, 100000000);
1807                 clk_set_parent(mout_gpll, fin_gpll);
1808
1809                 /*change gpll*/
1810                 clk_set_rate( fout_gpll, gpll_rate );
1811
1812                 /*restore parent*/
1813                 clk_set_parent(mout_gpll, fout_gpll);
1814                 gpll_rate_prev = gpll_rate;
1815         }
1816
1817         _freq = freq;
1818         clk_set_rate(platform->sclk_g3d, aclk_400_rate);
1819
1820         /* Waiting for clock is stable */
1821         do {
1822                 tmp = __raw_readl(/*EXYNOS5_CLKDIV_STAT_TOP0*/EXYNOS_CLKREG(0x10610));
1823         } while (tmp & 0x1000000);
1824 #ifdef CONFIG_T6XX_DVFS
1825 #if MALI_DVFS_DEBUG
1826         printk("aclk400 %u[%d]\n", (unsigned int)clk_get_rate(platform->sclk_g3d),mali_dvfs_status_current.utilisation);
1827         printk("dvfs_set_clock GPLL : %lu, ACLK_400 : %luMhz\n", gpll_rate, aclk_400_rate );
1828 #endif /* MALI_DVFS_DEBUG */
1829 #endif /* CONFIG_T6XX_DVFS */
1830         return;
1831 }
1832
1833 static void kbase_platform_dvfs_set_vol(unsigned int vol)
1834 {
1835         static int _vol = -1;
1836
1837         if (_vol == vol)
1838                 return;
1839         trace_mali_dvfs_set_voltage(vol);
1840
1841         kbase_platform_set_voltage(NULL, vol);
1842
1843         _vol = vol;
1844 #if MALI_DVFS_DEBUG
1845         printk(KERN_DEBUG "dvfs_set_vol %dmV\n", vol);
1846 #endif
1847         return;
1848 }
1849
1850 int kbase_platform_dvfs_get_level(int freq)
1851 {
1852         int i;
1853         for (i = 0; i < MALI_DVFS_STEP; i++) {
1854                 if (mali_dvfs_infotbl[i].clock == freq)
1855                         return i;
1856         }
1857         return -1;
1858 }
1859
1860 #if defined CONFIG_T6XX_DVFS || defined CONFIG_T6XX_DEBUG_SYS
1861 void kbase_platform_dvfs_set_level(kbase_device *kbdev, int level)
1862 {
1863         static int level_prev = -1;
1864
1865         if (level == level_prev)
1866                 return;
1867         if (WARN_ON((level >= MALI_DVFS_STEP) || (level < 0)))
1868                 panic("invalid level");
1869
1870         if (level > level_prev) {
1871                 kbase_platform_dvfs_set_vol(mali_dvfs_infotbl[level].voltage);
1872                 kbase_platform_dvfs_set_clock(kbdev,
1873                         mali_dvfs_infotbl[level].clock);
1874         } else{
1875                 kbase_platform_dvfs_set_clock(kbdev,
1876                         mali_dvfs_infotbl[level].clock);
1877                 kbase_platform_dvfs_set_vol(mali_dvfs_infotbl[level].voltage);
1878         }
1879         level_prev = level;
1880 }
1881 #endif
1882
1883 #ifdef MALI_DVFS_ASV_ENABLE
1884 static int kbase_platform_dvfs_sprint_avs_table(char *buf)
1885 {
1886         int i, cnt = 0;
1887         if (buf == NULL)
1888                 return 0;
1889
1890         cnt += sprintf(buf, "asv group:%d exynos_lot_id:%d\n",
1891                 exynos_result_of_asv&0xf, exynos_lot_id);
1892         for (i = MALI_DVFS_STEP-1; i >= 0; i--) {
1893                 cnt += sprintf(buf+cnt, "%dMhz:%d\n",
1894                         mali_dvfs_infotbl[i].clock, mali_dvfs_infotbl[i].voltage);
1895         }
1896         return cnt;
1897 }
1898
1899 static int kbase_platform_asv_set(int enable)
1900 {
1901         osk_spinlock_irq_lock(&mali_dvfs_spinlock);
1902         if (enable) {
1903                 mali_dvfs_status_current.asv_need_update = DVFS_UPDATE_ASV_TBL;
1904                 mali_dvfs_status_current.asv_group = -1;
1905         } else{
1906                 mali_dvfs_status_current.asv_need_update = DVFS_UPDATE_ASV_DEFAULT_TBL;
1907         }
1908         osk_spinlock_irq_unlock(&mali_dvfs_spinlock);
1909         return 0;
1910 }
1911 #endif /* MALI_DVFS_ASV_ENABLE */
1912
1913 /**
1914  * Exynos5 alternative dvfs_callback imlpementation.
1915  * instead of:
1916  *    action = kbase_pm_get_dvfs_action(kbdev);
1917  * use this:
1918  *    kbase_platform_dvfs_event(kbdev);
1919  */
1920
1921 #ifdef CONFIG_T6XX_DVFS
1922 int kbase_pm_get_dvfs_utilisation(kbase_device *kbdev)
1923 {
1924         unsigned long flags;
1925         int utilisation=0;
1926         ktime_t now = ktime_get();
1927         ktime_t diff;
1928
1929         OSK_ASSERT(kbdev != NULL);
1930
1931         spin_lock_irqsave(&kbdev->pm.metrics.lock, flags);
1932
1933         if (kbdev->pm.metrics.gpu_active)
1934         {
1935                 diff = ktime_sub(now, kbdev->pm.metrics.time_period_start);
1936                 kbdev->pm.metrics.time_busy += (u32)(ktime_to_ns( diff ) >> KBASE_PM_TIME_SHIFT);
1937                 kbdev->pm.metrics.time_period_start = now;
1938         }
1939         else
1940         {
1941                 diff = ktime_sub(now, kbdev->pm.metrics.time_period_start);
1942                 kbdev->pm.metrics.time_idle += (u32)(ktime_to_ns( diff ) >> KBASE_PM_TIME_SHIFT);
1943                 kbdev->pm.metrics.time_period_start = now;
1944         }
1945
1946         if (kbdev->pm.metrics.time_idle + kbdev->pm.metrics.time_busy == 0)
1947         {
1948                 /* No data - so we return NOP */
1949                 goto out;
1950         }
1951
1952         utilisation = (100*kbdev->pm.metrics.time_busy) / (kbdev->pm.metrics.time_idle + kbdev->pm.metrics.time_busy);
1953
1954 out:
1955
1956         kbdev->pm.metrics.time_idle = 0;
1957         kbdev->pm.metrics.time_busy = 0;
1958
1959         spin_unlock_irqrestore(&kbdev->pm.metrics.lock, flags);
1960
1961         return utilisation;
1962 }
1963 #endif
1964
1965 #ifdef CONFIG_MALI_HWC_TRACE
1966 /*
1967  * Mali hardware performance counter trace support.  Each counter
1968  * has a corresponding trace event.  To use, enable events and write
1969  * 1 to the hwc_enable sysfs file to start polling for counter data
1970  * at vblank.  An event is dispatched each time a counter's value
1971  * changes.  Note system event tracing must be enabled to get the
1972  * events; otherwise the polling will not generate events.  When done
1973  * turn off polling.  Note also that the GPU_ACTIVE counter causes
1974  * the GPU to never be idle; this causes the governor to push the
1975  * clock to max (since it sees 100% utilization).
1976  *
1977  * This code is derived from ARM's gator driver.
1978  */
1979 #include <drm/drm_notifier.h>
1980 #include <kbase/src/linux/mali_linux_hwc_trace.h>
1981
1982 /*
1983  * Mali hw counters by block.  Note that each hw counter enable bit
1984  * covers 4 counters.  There are 64 counters / block for 256 total
1985  * counters.
1986  */
1987 #define MALI_HWC_TOTAL  (4*64)                  /* total # hw counters */
1988
1989 #define hwc_off(x)      ((x) & 0x3f)            /* offset in block */
1990 #define hwc_bit(x)      (hwc_off(x) >> 2)       /* block bm enable bit */
1991
1992 #define BLOCK_OFF(b)    ((b) << 6)
1993
1994 /* Job Manager (Block 0) */
1995 #define BLOCK_JM_OFF    BLOCK_OFF(0)
1996 #define MALI_HW_MESSAGES_SENT           (BLOCK_JM_OFF + 4)
1997 #define MALI_HW_MESSAGES_RECEIVED       (BLOCK_JM_OFF + 5)
1998 /* NB: if GPU_ACTIVE is enabled the GPU will stay active when idle */
1999 #define MALI_HW_GPU_ACTIVE              (BLOCK_JM_OFF + 6)
2000 #define MALI_HW_IRQ_ACTIVE              (BLOCK_JM_OFF + 7)
2001
2002 #define MALI_HW_JS0_JOBS                (BLOCK_JM_OFF + 8*0 + 8)
2003 #define MALI_HW_JS0_TASKS               (BLOCK_JM_OFF + 8*0 + 9)
2004 #define MALI_HW_JS0_ACTIVE              (BLOCK_JM_OFF + 8*0 + 10)
2005 #define MALI_HW_JS0_WAIT_READ           (BLOCK_JM_OFF + 8*0 + 11)
2006 #define MALI_HW_JS0_WAIT_ISSUE          (BLOCK_JM_OFF + 8*0 + 12)
2007 #define MALI_HW_JS0_WAIT_DEPEND         (BLOCK_JM_OFF + 8*0 + 13)
2008 #define MALI_HW_JS0_WAIT_FINISH         (BLOCK_JM_OFF + 8*0 + 14)
2009
2010 #define MALI_HW_JS1_JOBS                (BLOCK_JM_OFF + 8*1 + 8)
2011 #define MALI_HW_JS1_TASKS               (BLOCK_JM_OFF + 8*1 + 9)
2012 #define MALI_HW_JS1_ACTIVE              (BLOCK_JM_OFF + 8*1 + 10)
2013 #define MALI_HW_JS1_WAIT_READ           (BLOCK_JM_OFF + 8*1 + 11)
2014 #define MALI_HW_JS1_WAIT_ISSUE          (BLOCK_JM_OFF + 8*1 + 12)
2015 #define MALI_HW_JS1_WAIT_DEPEND         (BLOCK_JM_OFF + 8*1 + 13)
2016 #define MALI_HW_JS1_WAIT_FINISH         (BLOCK_JM_OFF + 8*1 + 14)
2017
2018 #define MALI_HW_JS2_JOBS                (BLOCK_JM_OFF + 8*2 + 8)
2019 #define MALI_HW_JS2_TASKS               (BLOCK_JM_OFF + 8*2 + 9)
2020 #define MALI_HW_JS2_ACTIVE              (BLOCK_JM_OFF + 8*2 + 10)
2021 #define MALI_HW_JS2_WAIT_READ           (BLOCK_JM_OFF + 8*2 + 11)
2022 #define MALI_HW_JS2_WAIT_ISSUE          (BLOCK_JM_OFF + 8*2 + 12)
2023 #define MALI_HW_JS2_WAIT_DEPEND         (BLOCK_JM_OFF + 8*2 + 13)
2024 #define MALI_HW_JS2_WAIT_FINISH         (BLOCK_JM_OFF + 8*2 + 14)
2025
2026 #define MALI_HW_JS3_JOBS                (BLOCK_JM_OFF + 8*3 + 8)
2027 #define MALI_HW_JS3_TASKS               (BLOCK_JM_OFF + 8*3 + 9)
2028 #define MALI_HW_JS3_ACTIVE              (BLOCK_JM_OFF + 8*3 + 10)
2029 #define MALI_HW_JS3_WAIT_READ           (BLOCK_JM_OFF + 8*3 + 11)
2030 #define MALI_HW_JS3_WAIT_ISSUE          (BLOCK_JM_OFF + 8*3 + 12)
2031 #define MALI_HW_JS3_WAIT_DEPEND         (BLOCK_JM_OFF + 8*3 + 13)
2032 #define MALI_HW_JS3_WAIT_FINISH         (BLOCK_JM_OFF + 8*3 + 14)
2033
2034 #define MALI_HW_JS4_JOBS                (BLOCK_JM_OFF + 8*4 + 8)
2035 #define MALI_HW_JS4_TASKS               (BLOCK_JM_OFF + 8*4 + 9)
2036 #define MALI_HW_JS4_ACTIVE              (BLOCK_JM_OFF + 8*4 + 10)
2037 #define MALI_HW_JS4_WAIT_READ           (BLOCK_JM_OFF + 8*4 + 11)
2038 #define MALI_HW_JS4_WAIT_ISSUE          (BLOCK_JM_OFF + 8*4 + 12)
2039 #define MALI_HW_JS4_WAIT_DEPEND         (BLOCK_JM_OFF + 8*4 + 13)
2040 #define MALI_HW_JS4_WAIT_FINISH         (BLOCK_JM_OFF + 8*4 + 14)
2041
2042 #define MALI_HW_JS5_JOBS                (BLOCK_JM_OFF + 8*5 + 8)
2043 #define MALI_HW_JS5_TASKS               (BLOCK_JM_OFF + 8*5 + 9)
2044 #define MALI_HW_JS5_ACTIVE              (BLOCK_JM_OFF + 8*5 + 10)
2045 #define MALI_HW_JS5_WAIT_READ           (BLOCK_JM_OFF + 8*5 + 11)
2046 #define MALI_HW_JS5_WAIT_ISSUE          (BLOCK_JM_OFF + 8*5 + 12)
2047 #define MALI_HW_JS5_WAIT_DEPEND         (BLOCK_JM_OFF + 8*5 + 13)
2048 #define MALI_HW_JS5_WAIT_FINISH         (BLOCK_JM_OFF + 8*5 + 14)
2049
2050 #define MALI_HW_JS6_JOBS                (BLOCK_JM_OFF + 8*6 + 8)
2051 #define MALI_HW_JS6_TASKS               (BLOCK_JM_OFF + 8*6 + 9)
2052 #define MALI_HW_JS6_ACTIVE              (BLOCK_JM_OFF + 8*6 + 10)
2053 #define MALI_HW_JS6_WAIT_READ           (BLOCK_JM_OFF + 8*6 + 11)
2054 #define MALI_HW_JS6_WAIT_ISSUE          (BLOCK_JM_OFF + 8*6 + 12)
2055 #define MALI_HW_JS6_WAIT_DEPEND         (BLOCK_JM_OFF + 8*6 + 13)
2056 #define MALI_HW_JS6_WAIT_FINISH         (BLOCK_JM_OFF + 8*6 + 14)
2057
2058 #define MALI_HW_JM_FIRST        MALI_HW_GPU_ACTIVE
2059 #define MALI_HW_JM_LAST         MALI_HW_JS6_WAIT_FINISH
2060
2061 /* Tiler (Block 1) */
2062 #define BLOCK_TILER_OFF BLOCK_OFF(1)
2063 #define MALI_HW_JOBS_PROCESSED          (BLOCK_TILER_OFF + 3)
2064 #define MALI_HW_TRIANGLES               (BLOCK_TILER_OFF + 4)
2065 #define MALI_HW_QUADS                   (BLOCK_TILER_OFF + 5)
2066 #define MALI_HW_POLYGONS                (BLOCK_TILER_OFF + 6)
2067 #define MALI_HW_POINTS                  (BLOCK_TILER_OFF + 7)
2068 #define MALI_HW_LINES                   (BLOCK_TILER_OFF + 8)
2069 #define MALI_HW_VCACHE_HIT              (BLOCK_TILER_OFF + 9)
2070 #define MALI_HW_VCACHE_MISS             (BLOCK_TILER_OFF + 10)
2071 #define MALI_HW_FRONT_FACING            (BLOCK_TILER_OFF + 11)
2072 #define MALI_HW_BACK_FACING             (BLOCK_TILER_OFF + 12)
2073 #define MALI_HW_PRIM_VISIBLE            (BLOCK_TILER_OFF + 13)
2074 #define MALI_HW_PRIM_CULLED             (BLOCK_TILER_OFF + 14)
2075 #define MALI_HW_PRIM_CLIPPED            (BLOCK_TILER_OFF + 15)
2076
2077 #define MALI_HW_COMPRESS_IN             (BLOCK_TILER_OFF + 32)
2078 #define MALI_HW_COMPRESS_OUT            (BLOCK_TILER_OFF + 33)
2079 #define MALI_HW_COMPRESS_FLUSH          (BLOCK_TILER_OFF + 34)
2080 #define MALI_HW_TIMESTAMPS              (BLOCK_TILER_OFF + 35)
2081 #define MALI_HW_PCACHE_HIT              (BLOCK_TILER_OFF + 36)
2082 #define MALI_HW_PCACHE_MISS             (BLOCK_TILER_OFF + 37)
2083 #define MALI_HW_PCACHE_LINE             (BLOCK_TILER_OFF + 38)
2084 #define MALI_HW_PCACHE_STALL            (BLOCK_TILER_OFF + 39)
2085 #define MALI_HW_WRBUF_HIT               (BLOCK_TILER_OFF + 40)
2086 #define MALI_HW_WRBUF_MISS              (BLOCK_TILER_OFF + 41)
2087 #define MALI_HW_WRBUF_LINE              (BLOCK_TILER_OFF + 42)
2088 #define MALI_HW_WRBUF_PARTIAL           (BLOCK_TILER_OFF + 43)
2089 #define MALI_HW_WRBUF_STALL             (BLOCK_TILER_OFF + 44)
2090 #define MALI_HW_ACTIVE                  (BLOCK_TILER_OFF + 45)
2091 #define MALI_HW_LOADING_DESC            (BLOCK_TILER_OFF + 46)
2092 #define MALI_HW_INDEX_WAIT              (BLOCK_TILER_OFF + 47)
2093 #define MALI_HW_INDEX_RANGE_WAIT        (BLOCK_TILER_OFF + 48)
2094 #define MALI_HW_VERTEX_WAIT             (BLOCK_TILER_OFF + 49)
2095 #define MALI_HW_PCACHE_WAIT             (BLOCK_TILER_OFF + 50)
2096 #define MALI_HW_WRBUF_WAIT              (BLOCK_TILER_OFF + 51)
2097 #define MALI_HW_BUS_READ                (BLOCK_TILER_OFF + 52)
2098 #define MALI_HW_BUS_WRITE               (BLOCK_TILER_OFF + 53)
2099
2100 #define MALI_HW_TILER_UTLB_STALL        (BLOCK_TILER_OFF + 59)
2101 #define MALI_HW_TILER_UTLB_REPLAY_MISS  (BLOCK_TILER_OFF + 60)
2102 #define MALI_HW_TILER_UTLB_REPLAY_FULL  (BLOCK_TILER_OFF + 61)
2103 #define MALI_HW_TILER_UTLB_NEW_MISS     (BLOCK_TILER_OFF + 62)
2104 #define MALI_HW_TILER_UTLB_HIT          (BLOCK_TILER_OFF + 63)
2105
2106 #define MALI_HW_TILER_FIRST     MALI_HW_JOBS_PROCESSED
2107 #define MALI_HW_TILER_LAST      MALI_HW_TILER_UTLB_HIT
2108
2109 /* Shader Core (Block 2) */
2110 #define BLOCK_SHADER_OFF        BLOCK_OFF(2)
2111 #define MALI_HW_SHADER_CORE_ACTIVE      (BLOCK_SHADER_OFF + 3)
2112 #define MALI_HW_FRAG_ACTIVE             (BLOCK_SHADER_OFF + 4)
2113 #define MALI_HW_FRAG_PRIMATIVES         (BLOCK_SHADER_OFF + 5)
2114 #define MALI_HW_FRAG_PRIMATIVES_DROPPED (BLOCK_SHADER_OFF + 6)
2115 #define MALI_HW_FRAG_CYCLE_DESC         (BLOCK_SHADER_OFF + 7)
2116 #define MALI_HW_FRAG_CYCLES_PLR         (BLOCK_SHADER_OFF + 8)
2117 #define MALI_HW_FRAG_CYCLES_VERT        (BLOCK_SHADER_OFF + 9)
2118 #define MALI_HW_FRAG_CYCLES_TRISETUP    (BLOCK_SHADER_OFF + 10)
2119 #define MALI_HW_FRAG_CYCLES_RAST        (BLOCK_SHADER_OFF + 11)
2120 #define MALI_HW_FRAG_THREADS            (BLOCK_SHADER_OFF + 12)
2121 #define MALI_HW_FRAG_DUMMY_THREADS      (BLOCK_SHADER_OFF + 13)
2122 #define MALI_HW_FRAG_QUADS_RAST         (BLOCK_SHADER_OFF + 14)
2123 #define MALI_HW_FRAG_QUADS_EZS_TEST     (BLOCK_SHADER_OFF + 15)
2124 #define MALI_HW_FRAG_QUADS_EZS_KILLED   (BLOCK_SHADER_OFF + 16)
2125 #define MALI_HW_FRAG_QUADS_LZS_TEST     (BLOCK_SHADER_OFF + 17)
2126 #define MALI_HW_FRAG_QUADS_LZS_KILLED   (BLOCK_SHADER_OFF + 18)
2127 #define MALI_HW_FRAG_CYCLE_NO_TILE      (BLOCK_SHADER_OFF + 19)
2128 #define MALI_HW_FRAG_NUM_TILES          (BLOCK_SHADER_OFF + 20)
2129 #define MALI_HW_FRAG_TRANS_ELIM         (BLOCK_SHADER_OFF + 21)
2130 #define MALI_HW_COMPUTE_ACTIVE          (BLOCK_SHADER_OFF + 22)
2131 #define MALI_HW_COMPUTE_TASKS           (BLOCK_SHADER_OFF + 23)
2132 #define MALI_HW_COMPUTE_THREADS         (BLOCK_SHADER_OFF + 24)
2133 #define MALI_HW_COMPUTE_CYCLES_DESC     (BLOCK_SHADER_OFF + 25)
2134 #define MALI_HW_TRIPIPE_ACTIVE          (BLOCK_SHADER_OFF + 26)
2135 #define MALI_HW_ARITH_WORDS             (BLOCK_SHADER_OFF + 27)
2136 #define MALI_HW_ARITH_CYCLES_REG        (BLOCK_SHADER_OFF + 28)
2137 #define MALI_HW_ARITH_CYCLES_L0         (BLOCK_SHADER_OFF + 29)
2138 #define MALI_HW_ARITH_FRAG_DEPEND       (BLOCK_SHADER_OFF + 30)
2139 #define MALI_HW_LS_WORDS                (BLOCK_SHADER_OFF + 31)
2140 #define MALI_HW_LS_ISSUES               (BLOCK_SHADER_OFF + 32)
2141 #define MALI_HW_LS_RESTARTS             (BLOCK_SHADER_OFF + 33)
2142 #define MALI_HW_LS_REISSUES_MISS        (BLOCK_SHADER_OFF + 34)
2143 #define MALI_HW_LS_REISSUES_VD          (BLOCK_SHADER_OFF + 35)
2144 #define MALI_HW_LS_REISSUE_ATTRIB_MISS  (BLOCK_SHADER_OFF + 36)
2145 #define MALI_HW_LS_NO_WB                (BLOCK_SHADER_OFF + 37)
2146 #define MALI_HW_TEX_WORDS               (BLOCK_SHADER_OFF + 38)
2147 #define MALI_HW_TEX_BUBBLES             (BLOCK_SHADER_OFF + 39)
2148 #define MALI_HW_TEX_WORDS_L0            (BLOCK_SHADER_OFF + 40)
2149 #define MALI_HW_TEX_WORDS_DESC          (BLOCK_SHADER_OFF + 41)
2150 #define MALI_HW_TEX_THREADS             (BLOCK_SHADER_OFF + 42)
2151 #define MALI_HW_TEX_RECIRC_FMISS        (BLOCK_SHADER_OFF + 43)
2152 #define MALI_HW_TEX_RECIRC_DESC         (BLOCK_SHADER_OFF + 44)
2153 #define MALI_HW_TEX_RECIRC_MULTI        (BLOCK_SHADER_OFF + 45)
2154 #define MALI_HW_TEX_RECIRC_PMISS        (BLOCK_SHADER_OFF + 46)
2155 #define MALI_HW_TEX_RECIRC_CONF         (BLOCK_SHADER_OFF + 47)
2156 #define MALI_HW_LSC_READ_HITS           (BLOCK_SHADER_OFF + 48)
2157 #define MALI_HW_LSC_READ_MISSES         (BLOCK_SHADER_OFF + 49)
2158 #define MALI_HW_LSC_WRITE_HITS          (BLOCK_SHADER_OFF + 50)
2159 #define MALI_HW_LSC_WRITE_MISSES        (BLOCK_SHADER_OFF + 51)
2160 #define MALI_HW_LSC_ATOMIC_HITS         (BLOCK_SHADER_OFF + 52)
2161 #define MALI_HW_LSC_ATOMIC_MISSES       (BLOCK_SHADER_OFF + 53)
2162 #define MALI_HW_LSC_LINE_FETCHES        (BLOCK_SHADER_OFF + 54)
2163 #define MALI_HW_LSC_DIRTY_LINE          (BLOCK_SHADER_OFF + 55)
2164 #define MALI_HW_LSC_SNOOPS              (BLOCK_SHADER_OFF + 56)
2165 #define MALI_HW_AXI_TLB_STALL           (BLOCK_SHADER_OFF + 57)
2166 #define MALI_HW_AXI_TLB_MISS            (BLOCK_SHADER_OFF + 58)
2167 #define MALI_HW_AXI_TLB_TRANSACTION     (BLOCK_SHADER_OFF + 59)
2168 #define MALI_HW_LS_TLB_MISS             (BLOCK_SHADER_OFF + 60)
2169 #define MALI_HW_LS_TLB_HIT              (BLOCK_SHADER_OFF + 61)
2170 #define MALI_HW_AXI_BEATS_READ          (BLOCK_SHADER_OFF + 62)
2171 #define MALI_HW_AXI_BEATS_WRITE         (BLOCK_SHADER_OFF + 63)
2172
2173 #define MALI_HW_SHADER_FIRST    MALI_HW_SHADER_CORE_ACTIVE
2174 #define MALI_HW_SHADER_LAST     MALI_HW_AXI_BEATS_WRITE
2175
2176 /* L2 and MMU (Block 3) */
2177 #define BLOCK_MMU_OFF   BLOCK_OFF(3)
2178 #define MALI_HW_MMU_TABLE_WALK          (BLOCK_MMU_OFF + 4)
2179 #define MALI_HW_MMU_REPLAY_MISS         (BLOCK_MMU_OFF + 5)
2180 #define MALI_HW_MMU_REPLAY_FULL         (BLOCK_MMU_OFF + 6)
2181 #define MALI_HW_MMU_NEW_MISS            (BLOCK_MMU_OFF + 7)
2182 #define MALI_HW_MMU_HIT                 (BLOCK_MMU_OFF + 8)
2183
2184 #define MALI_HW_UTLB_STALL              (BLOCK_MMU_OFF + 16)
2185 #define MALI_HW_UTLB_REPLAY_MISS        (BLOCK_MMU_OFF + 17)
2186 #define MALI_HW_UTLB_REPLAY_FULL        (BLOCK_MMU_OFF + 18)
2187 #define MALI_HW_UTLB_NEW_MISS           (BLOCK_MMU_OFF + 19)
2188 #define MALI_HW_UTLB_HIT                (BLOCK_MMU_OFF + 20)
2189
2190 #define MALI_HW_L2_WRITE_BEATS          (BLOCK_MMU_OFF + 30)
2191 #define MALI_HW_L2_READ_BEATS           (BLOCK_MMU_OFF + 31)
2192 #define MALI_HW_L2_ANY_LOOKUP           (BLOCK_MMU_OFF + 32)
2193 #define MALI_HW_L2_READ_LOOKUP          (BLOCK_MMU_OFF + 33)
2194 #define MALI_HW_L2_SREAD_LOOKUP         (BLOCK_MMU_OFF + 34)
2195 #define MALI_HW_L2_READ_REPLAY          (BLOCK_MMU_OFF + 35)
2196 #define MALI_HW_L2_READ_SNOOP           (BLOCK_MMU_OFF + 36)
2197 #define MALI_HW_L2_READ_HIT             (BLOCK_MMU_OFF + 37)
2198 #define MALI_HW_L2_CLEAN_MISS           (BLOCK_MMU_OFF + 38)
2199 #define MALI_HW_L2_WRITE_LOOKUP         (BLOCK_MMU_OFF + 39)
2200 #define MALI_HW_L2_SWRITE_LOOKUP        (BLOCK_MMU_OFF + 40)
2201 #define MALI_HW_L2_WRITE_REPLAY         (BLOCK_MMU_OFF + 41)
2202 #define MALI_HW_L2_WRITE_SNOOP          (BLOCK_MMU_OFF + 42)
2203 #define MALI_HW_L2_WRITE_HIT            (BLOCK_MMU_OFF + 43)
2204 #define MALI_HW_L2_EXT_READ_FULL        (BLOCK_MMU_OFF + 44)
2205 #define MALI_HW_L2_EXT_READ_HALF        (BLOCK_MMU_OFF + 45)
2206 #define MALI_HW_L2_EXT_WRITE_FULL       (BLOCK_MMU_OFF + 46)
2207 #define MALI_HW_L2_EXT_WRITE_HALF       (BLOCK_MMU_OFF + 47)
2208 #define MALI_HW_L2_EXT_READ             (BLOCK_MMU_OFF + 48)
2209 #define MALI_HW_L2_EXT_READ_LINE        (BLOCK_MMU_OFF + 49)
2210 #define MALI_HW_L2_EXT_WRITE            (BLOCK_MMU_OFF + 50)
2211 #define MALI_HW_L2_EXT_WRITE_LINE       (BLOCK_MMU_OFF + 51)
2212 #define MALI_HW_L2_EXT_WRITE_SMALL      (BLOCK_MMU_OFF + 52)
2213 #define MALI_HW_L2_EXT_BARRIER          (BLOCK_MMU_OFF + 53)
2214 #define MALI_HW_L2_EXT_AR_STALL         (BLOCK_MMU_OFF + 54)
2215 #define MALI_HW_L2_EXT_R_BUF_FULL       (BLOCK_MMU_OFF + 55)
2216 #define MALI_HW_L2_EXT_RD_BUF_FULL      (BLOCK_MMU_OFF + 56)
2217 #define MALI_HW_L2_EXT_R_RAW            (BLOCK_MMU_OFF + 57)
2218 #define MALI_HW_L2_EXT_W_STALL          (BLOCK_MMU_OFF + 58)
2219 #define MALI_HW_L2_EXT_W_BUF_FULL       (BLOCK_MMU_OFF + 59)
2220 #define MALI_HW_L2_EXT_R_W_HAZARD       (BLOCK_MMU_OFF + 60)
2221 #define MALI_HW_L2_TAG_HAZARD           (BLOCK_MMU_OFF + 61)
2222 #define MALI_HW_L2_SNOOP_FULL           (BLOCK_MMU_OFF + 62)
2223 #define MALI_HW_L2_REPLAY_FULL          (BLOCK_MMU_OFF + 63)
2224
2225 #define MALI_HW_MMU_FIRST       MALI_HW_MMU_TABLE_WALK
2226 #define MALI_HW_MMU_LAST        MALI_HW_L2_REPLAY_FULL
2227
2228 /*
2229  * The amount of memory required to store a hwc dump is:
2230  *    # "core groups" (1)
2231  *  x # blocks (always 8 for midgard arch)
2232  *  x # counters / block (always 64)
2233  *  x # bytes / counter (4 for 32-bit counters)
2234  */
2235 #define MALI_HWC_DUMP_SIZE      (1*8*64*4)
2236
2237 struct mali_hwcounter_state {
2238         struct workqueue_struct *wq;            /* collection context */
2239         struct kbase_context *ctx;              /* kbase device context */
2240         void *buf;                              /* counter data buffer */
2241         kbase_uk_hwcnt_setup setup;             /* hwcounter setup block */
2242         bool active;                            /* collecting data */
2243         u32 last_read[MALI_HWC_TOTAL];          /* last counter value read */
2244 };
2245 static struct mali_hwcounter_state mali_hwcs;
2246 static osk_spinlock mali_hwcounter_spinlock;
2247
2248 /*
2249  * Support for mapping between hw counters and trace events.
2250  */
2251 struct mali_hwcounter_trace_map {
2252         void (*do_trace)(unsigned int val);     /* NB: all the same prototype */
2253         struct static_key *key;
2254 };
2255 #define HWC_EVENT_MAP(event)                                            \
2256         [MALI_HW_##event] = {                                           \
2257                 .do_trace = &trace_mali_hwc_##event,                    \
2258                 .key = &__tracepoint_mali_hwc_##event.key               \
2259         }
2260
2261 static struct mali_hwcounter_trace_map mali_hwcounter_map[256] = {
2262         /* Job Manager */
2263         HWC_EVENT_MAP(MESSAGES_SENT),
2264         HWC_EVENT_MAP(MESSAGES_RECEIVED),
2265
2266         HWC_EVENT_MAP(GPU_ACTIVE),
2267         HWC_EVENT_MAP(IRQ_ACTIVE),
2268
2269         HWC_EVENT_MAP(JS0_JOBS),
2270         HWC_EVENT_MAP(JS0_TASKS),
2271         HWC_EVENT_MAP(JS0_ACTIVE),
2272         HWC_EVENT_MAP(JS0_WAIT_READ),
2273         HWC_EVENT_MAP(JS0_WAIT_ISSUE),
2274         HWC_EVENT_MAP(JS0_WAIT_DEPEND),
2275         HWC_EVENT_MAP(JS0_WAIT_FINISH),
2276         HWC_EVENT_MAP(JS1_JOBS),
2277         HWC_EVENT_MAP(JS1_TASKS),
2278         HWC_EVENT_MAP(JS1_ACTIVE),
2279         HWC_EVENT_MAP(JS1_WAIT_READ),
2280         HWC_EVENT_MAP(JS1_WAIT_ISSUE),
2281         HWC_EVENT_MAP(JS1_WAIT_DEPEND),
2282         HWC_EVENT_MAP(JS1_WAIT_FINISH),
2283         HWC_EVENT_MAP(JS2_JOBS),
2284         HWC_EVENT_MAP(JS2_TASKS),
2285         HWC_EVENT_MAP(JS2_ACTIVE),
2286         HWC_EVENT_MAP(JS2_WAIT_READ),
2287         HWC_EVENT_MAP(JS2_WAIT_ISSUE),
2288         HWC_EVENT_MAP(JS2_WAIT_DEPEND),
2289         HWC_EVENT_MAP(JS2_WAIT_FINISH),
2290         HWC_EVENT_MAP(JS3_JOBS),
2291         HWC_EVENT_MAP(JS3_TASKS),
2292         HWC_EVENT_MAP(JS3_ACTIVE),
2293         HWC_EVENT_MAP(JS3_WAIT_READ),
2294         HWC_EVENT_MAP(JS3_WAIT_ISSUE),
2295         HWC_EVENT_MAP(JS3_WAIT_DEPEND),
2296         HWC_EVENT_MAP(JS3_WAIT_FINISH),
2297         HWC_EVENT_MAP(JS4_JOBS),
2298         HWC_EVENT_MAP(JS4_TASKS),
2299         HWC_EVENT_MAP(JS4_ACTIVE),
2300         HWC_EVENT_MAP(JS4_WAIT_READ),
2301         HWC_EVENT_MAP(JS4_WAIT_ISSUE),
2302         HWC_EVENT_MAP(JS4_WAIT_DEPEND),
2303         HWC_EVENT_MAP(JS4_WAIT_FINISH),
2304         HWC_EVENT_MAP(JS5_JOBS),
2305         HWC_EVENT_MAP(JS5_TASKS),
2306         HWC_EVENT_MAP(JS5_ACTIVE),
2307         HWC_EVENT_MAP(JS5_WAIT_READ),
2308         HWC_EVENT_MAP(JS5_WAIT_ISSUE),
2309         HWC_EVENT_MAP(JS5_WAIT_DEPEND),
2310         HWC_EVENT_MAP(JS5_WAIT_FINISH),
2311         HWC_EVENT_MAP(JS6_JOBS),
2312         HWC_EVENT_MAP(JS6_TASKS),
2313         HWC_EVENT_MAP(JS6_ACTIVE),
2314         HWC_EVENT_MAP(JS6_WAIT_READ),
2315         HWC_EVENT_MAP(JS6_WAIT_ISSUE),
2316         HWC_EVENT_MAP(JS6_WAIT_DEPEND),
2317         HWC_EVENT_MAP(JS6_WAIT_FINISH),
2318
2319         /* Tiler */
2320         HWC_EVENT_MAP(JOBS_PROCESSED),
2321         HWC_EVENT_MAP(TRIANGLES),
2322         HWC_EVENT_MAP(QUADS),
2323         HWC_EVENT_MAP(POLYGONS),
2324         HWC_EVENT_MAP(POINTS),
2325         HWC_EVENT_MAP(LINES),
2326         HWC_EVENT_MAP(VCACHE_HIT),
2327         HWC_EVENT_MAP(VCACHE_MISS),
2328         HWC_EVENT_MAP(FRONT_FACING),
2329         HWC_EVENT_MAP(BACK_FACING),
2330         HWC_EVENT_MAP(PRIM_VISIBLE),
2331         HWC_EVENT_MAP(PRIM_CULLED),
2332         HWC_EVENT_MAP(PRIM_CLIPPED),
2333
2334         HWC_EVENT_MAP(COMPRESS_IN),
2335         HWC_EVENT_MAP(COMPRESS_OUT),
2336         HWC_EVENT_MAP(COMPRESS_FLUSH),
2337         HWC_EVENT_MAP(TIMESTAMPS),
2338         HWC_EVENT_MAP(PCACHE_HIT),
2339         HWC_EVENT_MAP(PCACHE_MISS),
2340         HWC_EVENT_MAP(PCACHE_LINE),
2341         HWC_EVENT_MAP(PCACHE_STALL),
2342         HWC_EVENT_MAP(WRBUF_HIT),
2343         HWC_EVENT_MAP(WRBUF_MISS),
2344         HWC_EVENT_MAP(WRBUF_LINE),
2345         HWC_EVENT_MAP(WRBUF_PARTIAL),
2346         HWC_EVENT_MAP(WRBUF_STALL),
2347         HWC_EVENT_MAP(ACTIVE),
2348         HWC_EVENT_MAP(LOADING_DESC),
2349         HWC_EVENT_MAP(INDEX_WAIT),
2350         HWC_EVENT_MAP(INDEX_RANGE_WAIT),
2351         HWC_EVENT_MAP(VERTEX_WAIT),
2352         HWC_EVENT_MAP(PCACHE_WAIT),
2353         HWC_EVENT_MAP(WRBUF_WAIT),
2354         HWC_EVENT_MAP(BUS_READ),
2355         HWC_EVENT_MAP(BUS_WRITE),
2356
2357         HWC_EVENT_MAP(TILER_UTLB_STALL),
2358         HWC_EVENT_MAP(TILER_UTLB_REPLAY_MISS),
2359         HWC_EVENT_MAP(TILER_UTLB_REPLAY_FULL),
2360         HWC_EVENT_MAP(TILER_UTLB_NEW_MISS),
2361         HWC_EVENT_MAP(TILER_UTLB_HIT),
2362
2363         /* Shader */
2364         HWC_EVENT_MAP(SHADER_CORE_ACTIVE),
2365         HWC_EVENT_MAP(FRAG_ACTIVE),
2366         HWC_EVENT_MAP(FRAG_PRIMATIVES),
2367         HWC_EVENT_MAP(FRAG_PRIMATIVES_DROPPED),
2368         HWC_EVENT_MAP(FRAG_CYCLE_DESC),
2369         HWC_EVENT_MAP(FRAG_CYCLES_PLR),
2370         HWC_EVENT_MAP(FRAG_CYCLES_VERT),
2371         HWC_EVENT_MAP(FRAG_CYCLES_TRISETUP),
2372         HWC_EVENT_MAP(FRAG_CYCLES_RAST),
2373         HWC_EVENT_MAP(FRAG_THREADS),
2374         HWC_EVENT_MAP(FRAG_DUMMY_THREADS),
2375         HWC_EVENT_MAP(FRAG_QUADS_RAST),
2376         HWC_EVENT_MAP(FRAG_QUADS_EZS_TEST),
2377         HWC_EVENT_MAP(FRAG_QUADS_EZS_KILLED),
2378         HWC_EVENT_MAP(FRAG_QUADS_LZS_TEST),
2379         HWC_EVENT_MAP(FRAG_QUADS_LZS_KILLED),
2380         HWC_EVENT_MAP(FRAG_CYCLE_NO_TILE),
2381         HWC_EVENT_MAP(FRAG_NUM_TILES),
2382         HWC_EVENT_MAP(FRAG_TRANS_ELIM),
2383         HWC_EVENT_MAP(COMPUTE_ACTIVE),
2384         HWC_EVENT_MAP(COMPUTE_TASKS),
2385         HWC_EVENT_MAP(COMPUTE_THREADS),
2386         HWC_EVENT_MAP(COMPUTE_CYCLES_DESC),
2387         HWC_EVENT_MAP(TRIPIPE_ACTIVE),
2388         HWC_EVENT_MAP(ARITH_WORDS),
2389         HWC_EVENT_MAP(ARITH_CYCLES_REG),
2390         HWC_EVENT_MAP(ARITH_CYCLES_L0),
2391         HWC_EVENT_MAP(ARITH_FRAG_DEPEND),
2392         HWC_EVENT_MAP(LS_WORDS),
2393         HWC_EVENT_MAP(LS_ISSUES),
2394         HWC_EVENT_MAP(LS_RESTARTS),
2395         HWC_EVENT_MAP(LS_REISSUES_MISS),
2396         HWC_EVENT_MAP(LS_REISSUES_VD),
2397         HWC_EVENT_MAP(LS_REISSUE_ATTRIB_MISS),
2398         HWC_EVENT_MAP(LS_NO_WB),
2399         HWC_EVENT_MAP(TEX_WORDS),
2400         HWC_EVENT_MAP(TEX_BUBBLES),
2401         HWC_EVENT_MAP(TEX_WORDS_L0),
2402         HWC_EVENT_MAP(TEX_WORDS_DESC),
2403         HWC_EVENT_MAP(TEX_THREADS),
2404         HWC_EVENT_MAP(TEX_RECIRC_FMISS),
2405         HWC_EVENT_MAP(TEX_RECIRC_DESC),
2406         HWC_EVENT_MAP(TEX_RECIRC_MULTI),
2407         HWC_EVENT_MAP(TEX_RECIRC_PMISS),
2408         HWC_EVENT_MAP(TEX_RECIRC_CONF),
2409         HWC_EVENT_MAP(LSC_READ_HITS),
2410         HWC_EVENT_MAP(LSC_READ_MISSES),
2411         HWC_EVENT_MAP(LSC_WRITE_HITS),
2412         HWC_EVENT_MAP(LSC_WRITE_MISSES),
2413         HWC_EVENT_MAP(LSC_ATOMIC_HITS),
2414         HWC_EVENT_MAP(LSC_ATOMIC_MISSES),
2415         HWC_EVENT_MAP(LSC_LINE_FETCHES),
2416         HWC_EVENT_MAP(LSC_DIRTY_LINE),
2417         HWC_EVENT_MAP(LSC_SNOOPS),
2418         HWC_EVENT_MAP(AXI_TLB_STALL),
2419         HWC_EVENT_MAP(AXI_TLB_MISS),
2420         HWC_EVENT_MAP(AXI_TLB_TRANSACTION),
2421         HWC_EVENT_MAP(LS_TLB_MISS),
2422         HWC_EVENT_MAP(LS_TLB_HIT),
2423         HWC_EVENT_MAP(AXI_BEATS_READ),
2424         HWC_EVENT_MAP(AXI_BEATS_WRITE),
2425
2426         /* MMU */
2427         HWC_EVENT_MAP(MMU_TABLE_WALK),
2428         HWC_EVENT_MAP(MMU_REPLAY_MISS),
2429         HWC_EVENT_MAP(MMU_REPLAY_FULL),
2430         HWC_EVENT_MAP(MMU_NEW_MISS),
2431         HWC_EVENT_MAP(MMU_HIT),
2432
2433         HWC_EVENT_MAP(UTLB_STALL),
2434         HWC_EVENT_MAP(UTLB_REPLAY_MISS),
2435         HWC_EVENT_MAP(UTLB_REPLAY_FULL),
2436         HWC_EVENT_MAP(UTLB_NEW_MISS),
2437         HWC_EVENT_MAP(UTLB_HIT),
2438
2439         HWC_EVENT_MAP(L2_WRITE_BEATS),
2440         HWC_EVENT_MAP(L2_READ_BEATS),
2441         HWC_EVENT_MAP(L2_ANY_LOOKUP),
2442         HWC_EVENT_MAP(L2_READ_LOOKUP),
2443         HWC_EVENT_MAP(L2_SREAD_LOOKUP),
2444         HWC_EVENT_MAP(L2_READ_REPLAY),
2445         HWC_EVENT_MAP(L2_READ_SNOOP),
2446         HWC_EVENT_MAP(L2_READ_HIT),
2447         HWC_EVENT_MAP(L2_CLEAN_MISS),
2448         HWC_EVENT_MAP(L2_WRITE_LOOKUP),
2449         HWC_EVENT_MAP(L2_SWRITE_LOOKUP),
2450         HWC_EVENT_MAP(L2_WRITE_REPLAY),
2451         HWC_EVENT_MAP(L2_WRITE_SNOOP),
2452         HWC_EVENT_MAP(L2_WRITE_HIT),
2453         HWC_EVENT_MAP(L2_EXT_READ_FULL),
2454         HWC_EVENT_MAP(L2_EXT_READ_HALF),
2455         HWC_EVENT_MAP(L2_EXT_WRITE_FULL),
2456         HWC_EVENT_MAP(L2_EXT_WRITE_HALF),
2457         HWC_EVENT_MAP(L2_EXT_READ),
2458         HWC_EVENT_MAP(L2_EXT_READ_LINE),
2459         HWC_EVENT_MAP(L2_EXT_WRITE),
2460         HWC_EVENT_MAP(L2_EXT_WRITE_LINE),
2461         HWC_EVENT_MAP(L2_EXT_WRITE_SMALL),
2462         HWC_EVENT_MAP(L2_EXT_BARRIER),
2463         HWC_EVENT_MAP(L2_EXT_AR_STALL),
2464         HWC_EVENT_MAP(L2_EXT_R_BUF_FULL),
2465         HWC_EVENT_MAP(L2_EXT_RD_BUF_FULL),
2466         HWC_EVENT_MAP(L2_EXT_R_RAW),
2467         HWC_EVENT_MAP(L2_EXT_W_STALL),
2468         HWC_EVENT_MAP(L2_EXT_W_BUF_FULL),
2469         HWC_EVENT_MAP(L2_EXT_R_W_HAZARD),
2470         HWC_EVENT_MAP(L2_TAG_HAZARD),
2471         HWC_EVENT_MAP(L2_SNOOP_FULL),
2472         HWC_EVENT_MAP(L2_REPLAY_FULL),
2473 };
2474
2475 /*
2476  * Support for building hw counter enable bitmaps from enabled trace events.
2477  */
2478 #define __BLOCK_CHECK_ENABLED(block, bm_name) do {                      \
2479         int ev;                                                         \
2480         for (ev = block##_FIRST; ev <= block##_LAST; ev++) {            \
2481                 const struct mali_hwcounter_trace_map *map =            \
2482                     &mali_hwcounter_map[ev];                            \
2483                 if (map->key != NULL && static_key_false(map->key)) {   \
2484                         mali_hwcs.setup.bm_name |= 1<<hwc_bit(ev);      \
2485                         ncounters++;                                    \
2486                 }                                                       \
2487         }                                                               \
2488 } while (0)
2489 #define JM_CHECK_ENABLED() __BLOCK_CHECK_ENABLED(MALI_HW_JM, jm_bm)
2490 #define TILER_CHECK_ENABLED() __BLOCK_CHECK_ENABLED(MALI_HW_TILER, tiler_bm)
2491 #define SHADER_CHECK_ENABLED() __BLOCK_CHECK_ENABLED(MALI_HW_SHADER, shader_bm)
2492 #define MMU_CHECK_ENABLED() __BLOCK_CHECK_ENABLED(MALI_HW_MMU, mmu_l2_bm)
2493
2494 /*
2495  * Support for dispatching trace events based on hw counter data.
2496  * Note we dispatch an event only when a counter changes value.
2497  */
2498 static int block_update(const int boff, const int event_num)
2499 {
2500         const u32 *block = ((const u32 *)((uintptr_t)mali_hwcs.buf + boff));
2501         u32 value = block[hwc_off(event_num)];
2502         if (value != mali_hwcs.last_read[event_num]) {
2503                 mali_hwcs.last_read[event_num] = value;
2504                 return true;
2505         } else
2506                 return false;
2507 }
2508
2509 #define __BLOCK_DISPATCH_EVENTS(block, bm_name, boff) do {              \
2510         int ev;                                                         \
2511         for (ev = block##_FIRST; ev <= block##_LAST; ev++) {            \
2512                 const struct mali_hwcounter_trace_map *map =            \
2513                     &mali_hwcounter_map[ev];                            \
2514                 /* NB: each enable bit covers 4 counters */             \
2515                 if ((mali_hwcs.setup.bm_name & (1<<hwc_bit(ev))) &&     \
2516                     map->key != NULL && static_key_false(map->key) &&   \
2517                     block_update(boff, ev))                             \
2518                         map->do_trace(mali_hwcs.last_read[ev]);         \
2519         }                                                               \
2520 } while (0)
2521 #define JM_DISPATCH_EVENTS() \
2522         __BLOCK_DISPATCH_EVENTS(MALI_HW_JM, jm_bm, 0x700)
2523 #define TILER_DISPATCH_EVENTS() \
2524         __BLOCK_DISPATCH_EVENTS(MALI_HW_TILER, tiler_bm, 0x400)
2525 #define MMU_DISPATCH_EVENTS() \
2526         __BLOCK_DISPATCH_EVENTS(MALI_HW_MMU, mmu_l2_bm, 0x500)
2527
2528 static int shader_block_update(const int event_num)
2529 {
2530         const u32 *block = ((const u32 *)((uintptr_t)mali_hwcs.buf + 0x000));
2531         u32 value = block[hwc_off(event_num)]
2532                   + block[hwc_off(event_num) + 0x100]
2533                   + block[hwc_off(event_num) + 0x200]
2534                   + block[hwc_off(event_num) + 0x300]
2535                   ;
2536         if (value != mali_hwcs.last_read[event_num]) {
2537                 mali_hwcs.last_read[event_num] = value;
2538                 return true;
2539         } else
2540                 return false;
2541 }
2542
2543 #define SHADER_DISPATCH_EVENTS() do {                                   \
2544         int ev;                                                         \
2545         for (ev = MALI_HW_SHADER_FIRST; ev <= MALI_HW_SHADER_LAST; ev++) {\
2546                 const struct mali_hwcounter_trace_map *map =            \
2547                     &mali_hwcounter_map[ev];                            \
2548                 /* NB: each enable bit covers 4 counters */             \
2549                 if ((mali_hwcs.setup.shader_bm & (1<<hwc_bit(ev))) &&   \
2550                     map->key != NULL && static_key_false(map->key) &&   \
2551                     shader_block_update(ev))                            \
2552                         map->do_trace(mali_hwcs.last_read[ev]);         \
2553         }                                                               \
2554 } while (0)
2555
2556 /*
2557  * Collect hw counter data and dispatch trace events.
2558  * This runs from the workqueue so it can block when
2559  * fetching hw counter data.
2560  */
2561 static void mali_hwcounter_collect(struct work_struct *w)
2562 {
2563         mali_error error;
2564
2565         osk_spinlock_lock(&mali_hwcounter_spinlock);
2566         if (!mali_hwcs.active) {
2567                 osk_spinlock_unlock(&mali_hwcounter_spinlock);
2568                 return;
2569         }
2570         error = kbase_instr_hwcnt_dump(mali_hwcs.ctx);
2571         osk_spinlock_unlock(&mali_hwcounter_spinlock);
2572         if (error == MALI_ERROR_NONE) {
2573                 /* extract data and generate trace events */
2574                 JM_DISPATCH_EVENTS();
2575                 TILER_DISPATCH_EVENTS();
2576                 SHADER_DISPATCH_EVENTS();
2577                 MMU_DISPATCH_EVENTS();
2578         } else
2579                 pr_err("%s: failed to dump hw counters\n", __func__);
2580 }
2581 static DECLARE_WORK(mali_hwcounter_work, mali_hwcounter_collect);
2582
2583 /*
2584  * drm vblank notifier; used to trigger hw counter polling.
2585  */
2586 static int mali_hwcounter_vblank(struct notifier_block *nb,
2587         unsigned long unused, void *data)
2588 {
2589         /* punt to work queue where we can block */
2590         queue_work_on(0, mali_hwcs.wq, &mali_hwcounter_work);
2591         return 0;
2592 }
2593 static struct notifier_block mali_vblank_notifier = {
2594         .notifier_call  = mali_hwcounter_vblank,
2595 };
2596
2597 /*
2598  * Start hw counter polling.  Construct the counter enable bitmaps
2599  * based on the enabled trace events, call kbase to turn on counters,
2600  * and arrange polling at vblank.
2601  */
2602 static int mali_hwcounter_polling_start(struct kbase_device *kbdev)
2603 {
2604         int ncounters = 0;
2605
2606         if (mali_hwcs.active)
2607                 return -EALREADY;
2608
2609         /* Construct hw counter bitmaps from enabled trace events */
2610         memset(&mali_hwcs.setup, 0, sizeof(mali_hwcs.setup));
2611
2612         JM_CHECK_ENABLED();
2613         TILER_CHECK_ENABLED();
2614         SHADER_CHECK_ENABLED();
2615         MMU_CHECK_ENABLED();
2616
2617         if (ncounters > 0) {
2618                 mali_error error;
2619
2620                 mali_hwcs.ctx = kbase_create_context(kbdev);
2621                 if (mali_hwcs.ctx == NULL) {
2622                         pr_err("%s: cannot create context\n", __func__);
2623                         return -EIO;
2624                 }
2625                 mali_hwcs.buf = kbase_va_alloc(mali_hwcs.ctx,
2626                                                MALI_HWC_DUMP_SIZE);
2627                 mali_hwcs.setup.dump_buffer = (uintptr_t) mali_hwcs.buf;
2628
2629                 error = kbase_instr_hwcnt_enable(mali_hwcs.ctx,
2630                                                  &mali_hwcs.setup);
2631                 if (error != MALI_ERROR_NONE) {
2632                         pr_err("%s: cannot enable hw counters\n", __func__);
2633
2634                         kbase_va_free(mali_hwcs.ctx, mali_hwcs.buf);
2635                         mali_hwcs.buf = NULL;
2636
2637                         kbase_destroy_context(mali_hwcs.ctx);
2638                         mali_hwcs.ctx = NULL;
2639
2640                         return -EIO;/*TODO make unique*/
2641                 }
2642                 kbase_instr_hwcnt_clear(mali_hwcs.ctx);
2643                 mali_hwcs.active = true;
2644                 /* NB: use 0 to minimize meaningless events */
2645                 memset(mali_hwcs.last_read, 0, sizeof(mali_hwcs.last_read));
2646
2647                 drm_vblank_register_notifier(&mali_vblank_notifier);
2648         }
2649         return 0;
2650 }
2651
2652 /*
2653  * Stop hw counter polling. Disable polling, turn off hw counters,
2654  * and release our resources.
2655  */
2656 static void mali_hwcounter_polling_stop(struct kbase_device *kbdev)
2657 {
2658         osk_spinlock_lock(&mali_hwcounter_spinlock);
2659         if (mali_hwcs.active) {
2660                 drm_vblank_unregister_notifier(&mali_vblank_notifier);
2661
2662                 kbase_instr_hwcnt_disable(mali_hwcs.ctx);
2663
2664                 kbase_va_free(mali_hwcs.ctx, mali_hwcs.buf);
2665                 mali_hwcs.buf = NULL;
2666
2667                 kbase_destroy_context(mali_hwcs.ctx);
2668                 mali_hwcs.ctx = NULL;
2669
2670                 mali_hwcs.active = false;
2671         }
2672         osk_spinlock_unlock(&mali_hwcounter_spinlock);
2673 }
2674
2675 static ssize_t mali_sysfs_show_hwc_enable(struct device *dev,
2676         struct device_attribute *attr, char *buf)
2677 {
2678         return snprintf(buf, PAGE_SIZE, "%d\n", (mali_hwcs.active == true));
2679 }
2680
2681 static ssize_t mali_sysfs_set_hwc_enable(struct device *dev,
2682         struct device_attribute *attr, const char *buf, size_t count)
2683 {
2684         struct kbase_device *kbdev = dev_get_drvdata(dev);
2685         int error;
2686         bool enable;
2687
2688         if (strtobool(buf, &enable) < 0)
2689                 return -EINVAL;
2690         if (enable) {
2691                 error = mali_hwcounter_polling_start(kbdev);
2692                 if (error < 0)
2693                         return error;
2694         } else
2695                 mali_hwcounter_polling_stop(kbdev);
2696         return count;
2697 }
2698 DEVICE_ATTR(hwc_enable, S_IRUGO|S_IWUSR, mali_sysfs_show_hwc_enable,
2699         mali_sysfs_set_hwc_enable);
2700
2701 static int mali_setup_system_tracing(struct device *dev)
2702 {
2703         osk_spinlock_init(&mali_hwcounter_spinlock, OSK_LOCK_ORDER_PM_METRICS);
2704         mali_hwcs.wq = create_singlethread_workqueue("mali_hwc");
2705         mali_hwcs.active = false;
2706
2707         if (device_create_file(dev, &dev_attr_hwc_enable)) {
2708                 dev_err(dev, "Couldn't create sysfs file [hwc_enable]\n");
2709                 return -ENOENT;
2710         }
2711         return 0;
2712 }
2713
2714 static void mali_cleanup_system_tracing(struct device *dev)
2715 {
2716         struct kbase_device *kbdev = dev_get_drvdata(dev);
2717
2718         mali_hwcounter_polling_stop(kbdev);
2719         destroy_workqueue(mali_hwcs.wq);
2720         device_remove_file(dev, &dev_attr_hwc_enable);
2721 }
2722 #else /* CONFIG_MALI_HWC_TRACE */
2723 static int mali_setup_system_tracing(struct device *dev)
2724 {
2725         return 0;
2726 }
2727
2728 static void mali_cleanup_system_tracing(struct device *dev)
2729 {
2730 }
2731 #endif /* CONFIG_MALI_HWC_TRACE */