arch/mips/mm/c-r4k.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
   7  * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org)
   8  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
   9  */
  10 #include <linux/cpu_pm.h>
  11 #include <linux/hardirq.h>
  12 #include <linux/init.h>
  13 #include <linux/highmem.h>
  14 #include <linux/kernel.h>
  15 #include <linux/linkage.h>
  16 #include <linux/preempt.h>
  17 #include <linux/sched.h>
  18 #include <linux/smp.h>
  19 #include <linux/mm.h>
  20 #include <linux/module.h>
  21 #include <linux/bitops.h>
  22
  23 #include <asm/bcache.h>
  24 #include <asm/bootinfo.h>
  25 #include <asm/cache.h>
  26 #include <asm/cacheops.h>
  27 #include <asm/cpu.h>
  28 #include <asm/cpu-features.h>
  29 #include <asm/cpu-type.h>
  30 #include <asm/io.h>
  31 #include <asm/page.h>
  32 #include <asm/pgtable.h>
  33 #include <asm/r4kcache.h>
  34 #include <asm/sections.h>
  35 #include <asm/mmu_context.h>
  36 #include <asm/war.h>
  37 #include <asm/cacheflush.h> /* for run_uncached() */
  38 #include <asm/traps.h>
  39 #include <asm/dma-coherence.h>
  40 #include <asm/mips-cm.h>
  41
  42 /*
  43  * Special Variant of smp_call_function for use by cache functions:
  44  *
  45  *  o No return value
  46  *  o collapses to normal function call on UP kernels
  47  *  o collapses to normal function call on systems with a single shared
  48  *    primary cache.
  49  *  o doesn't disable interrupts on the local CPU
  50  */
  51 static inline void r4k_on_each_cpu(void (*func) (void *info), void *info)
  52 {
  53         preempt_disable();
  54
  55         /*
  56          * The Coherent Manager propagates address-based cache ops to other
  57          * cores but not index-based ops. However, r4k_on_each_cpu is used
  58          * in both cases so there is no easy way to tell what kind of op is
  59          * executed to the other cores. The best we can probably do is
  60          * to restrict that call when a CM is not present because both
  61          * CM-based SMP protocols (CMP & CPS) restrict index-based cache ops.
  62          */
  63         if (!mips_cm_present())
  64                 smp_call_function_many(&cpu_foreign_map, func, info, 1);
  65         func(info);
  66         preempt_enable();
  67 }
  68
  69 /*
  70  * Must die.
  71  */
  72 static unsigned long icache_size __read_mostly;
  73 static unsigned long dcache_size __read_mostly;
  74 static unsigned long vcache_size __read_mostly;
  75 static unsigned long scache_size __read_mostly;
  76
  77 /*
  78  * Dummy cache handling routines for machines without boardcaches
  79  */
  80 static void cache_noop(void) {}
  81
  82 static struct bcache_ops no_sc_ops = {
  83         .bc_enable = (void *)cache_noop,
  84         .bc_disable = (void *)cache_noop,
  85         .bc_wback_inv = (void *)cache_noop,
  86         .bc_inv = (void *)cache_noop
  87 };
  88
  89 struct bcache_ops *bcops = &no_sc_ops;
  90
  91 #define cpu_is_r4600_v1_x()     ((read_c0_prid() & 0xfffffff0) == 0x00002010)
  92 #define cpu_is_r4600_v2_x()     ((read_c0_prid() & 0xfffffff0) == 0x00002020)
  93
  94 #define R4600_HIT_CACHEOP_WAR_IMPL                                      \
  95 do {                                                                    \
  96         if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())            \
  97                 *(volatile unsigned long *)CKSEG1;                      \
  98         if (R4600_V1_HIT_CACHEOP_WAR)                                   \
  99                 __asm__ __volatile__("nop;nop;nop;nop");                \
 100 } while (0)
 101
 102 static void (*r4k_blast_dcache_page)(unsigned long addr);
 103
 104 static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
 105 {
 106         R4600_HIT_CACHEOP_WAR_IMPL;
 107         blast_dcache32_page(addr);
 108 }
 109
 110 static inline void r4k_blast_dcache_page_dc64(unsigned long addr)
 111 {
 112         blast_dcache64_page(addr);
 113 }
 114
 115 static inline void r4k_blast_dcache_page_dc128(unsigned long addr)
 116 {
 117         blast_dcache128_page(addr);
 118 }
 119
 120 static void r4k_blast_dcache_page_setup(void)
 121 {
 122         unsigned long  dc_lsize = cpu_dcache_line_size();
 123
 124         switch (dc_lsize) {
 125         case 0:
 126                 r4k_blast_dcache_page = (void *)cache_noop;
 127                 break;
 128         case 16:
 129                 r4k_blast_dcache_page = blast_dcache16_page;
 130                 break;
 131         case 32:
 132                 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
 133                 break;
 134         case 64:
 135                 r4k_blast_dcache_page = r4k_blast_dcache_page_dc64;
 136                 break;
 137         case 128:
 138                 r4k_blast_dcache_page = r4k_blast_dcache_page_dc128;
 139                 break;
 140         default:
 141                 break;
 142         }
 143 }
 144
 145 #ifndef CONFIG_EVA
 146 #define r4k_blast_dcache_user_page  r4k_blast_dcache_page
 147 #else
 148
 149 static void (*r4k_blast_dcache_user_page)(unsigned long addr);
 150
 151 static void r4k_blast_dcache_user_page_setup(void)
 152 {
 153         unsigned long  dc_lsize = cpu_dcache_line_size();
 154
 155         if (dc_lsize == 0)
 156                 r4k_blast_dcache_user_page = (void *)cache_noop;
 157         else if (dc_lsize == 16)
 158                 r4k_blast_dcache_user_page = blast_dcache16_user_page;
 159         else if (dc_lsize == 32)
 160                 r4k_blast_dcache_user_page = blast_dcache32_user_page;
 161         else if (dc_lsize == 64)
 162                 r4k_blast_dcache_user_page = blast_dcache64_user_page;
 163 }
 164
 165 #endif
 166
 167 static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
 168
 169 static void r4k_blast_dcache_page_indexed_setup(void)
 170 {
 171         unsigned long dc_lsize = cpu_dcache_line_size();
 172
 173         if (dc_lsize == 0)
 174                 r4k_blast_dcache_page_indexed = (void *)cache_noop;
 175         else if (dc_lsize == 16)
 176                 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
 177         else if (dc_lsize == 32)
 178                 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
 179         else if (dc_lsize == 64)
 180                 r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed;
 181         else if (dc_lsize == 128)
 182                 r4k_blast_dcache_page_indexed = blast_dcache128_page_indexed;
 183 }
 184
 185 void (* r4k_blast_dcache)(void);
 186 EXPORT_SYMBOL(r4k_blast_dcache);
 187
 188 static void r4k_blast_dcache_setup(void)
 189 {
 190         unsigned long dc_lsize = cpu_dcache_line_size();
 191
 192         if (dc_lsize == 0)
 193                 r4k_blast_dcache = (void *)cache_noop;
 194         else if (dc_lsize == 16)
 195                 r4k_blast_dcache = blast_dcache16;
 196         else if (dc_lsize == 32)
 197                 r4k_blast_dcache = blast_dcache32;
 198         else if (dc_lsize == 64)
 199                 r4k_blast_dcache = blast_dcache64;
 200         else if (dc_lsize == 128)
 201                 r4k_blast_dcache = blast_dcache128;
 202 }
 203
 204 /* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */
 205 #define JUMP_TO_ALIGN(order) \
 206         __asm__ __volatile__( \
 207                 "b\t1f\n\t" \
 208                 ".align\t" #order "\n\t" \
 209                 "1:\n\t" \
 210                 )
 211 #define CACHE32_UNROLL32_ALIGN  JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */
 212 #define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11)
 213
 214 static inline void blast_r4600_v1_icache32(void)
 215 {
 216         unsigned long flags;
 217
 218         local_irq_save(flags);
 219         blast_icache32();
 220         local_irq_restore(flags);
 221 }
 222
 223 static inline void tx49_blast_icache32(void)
 224 {
 225         unsigned long start = INDEX_BASE;
 226         unsigned long end = start + current_cpu_data.icache.waysize;
 227         unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
 228         unsigned long ws_end = current_cpu_data.icache.ways <<
 229                                current_cpu_data.icache.waybit;
 230         unsigned long ws, addr;
 231
 232         CACHE32_UNROLL32_ALIGN2;
 233         /* I'm in even chunk.  blast odd chunks */
 234         for (ws = 0; ws < ws_end; ws += ws_inc)
 235                 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
 236                         cache32_unroll32(addr|ws, Index_Invalidate_I);
 237         CACHE32_UNROLL32_ALIGN;
 238         /* I'm in odd chunk.  blast even chunks */
 239         for (ws = 0; ws < ws_end; ws += ws_inc)
 240                 for (addr = start; addr < end; addr += 0x400 * 2)
 241                         cache32_unroll32(addr|ws, Index_Invalidate_I);
 242 }
 243
 244 static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page)
 245 {
 246         unsigned long flags;
 247
 248         local_irq_save(flags);
 249         blast_icache32_page_indexed(page);
 250         local_irq_restore(flags);
 251 }
 252
 253 static inline void tx49_blast_icache32_page_indexed(unsigned long page)
 254 {
 255         unsigned long indexmask = current_cpu_data.icache.waysize - 1;
 256         unsigned long start = INDEX_BASE + (page & indexmask);
 257         unsigned long end = start + PAGE_SIZE;
 258         unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit;
 259         unsigned long ws_end = current_cpu_data.icache.ways <<
 260                                current_cpu_data.icache.waybit;
 261         unsigned long ws, addr;
 262
 263         CACHE32_UNROLL32_ALIGN2;
 264         /* I'm in even chunk.  blast odd chunks */
 265         for (ws = 0; ws < ws_end; ws += ws_inc)
 266                 for (addr = start + 0x400; addr < end; addr += 0x400 * 2)
 267                         cache32_unroll32(addr|ws, Index_Invalidate_I);
 268         CACHE32_UNROLL32_ALIGN;
 269         /* I'm in odd chunk.  blast even chunks */
 270         for (ws = 0; ws < ws_end; ws += ws_inc)
 271                 for (addr = start; addr < end; addr += 0x400 * 2)
 272                         cache32_unroll32(addr|ws, Index_Invalidate_I);
 273 }
 274
 275 static void (* r4k_blast_icache_page)(unsigned long addr);
 276
 277 static void r4k_blast_icache_page_setup(void)
 278 {
 279         unsigned long ic_lsize = cpu_icache_line_size();
 280
 281         if (ic_lsize == 0)
 282                 r4k_blast_icache_page = (void *)cache_noop;
 283         else if (ic_lsize == 16)
 284                 r4k_blast_icache_page = blast_icache16_page;
 285         else if (ic_lsize == 32 && current_cpu_type() == CPU_LOONGSON2)
 286                 r4k_blast_icache_page = loongson2_blast_icache32_page;
 287         else if (ic_lsize == 32)
 288                 r4k_blast_icache_page = blast_icache32_page;
 289         else if (ic_lsize == 64)
 290                 r4k_blast_icache_page = blast_icache64_page;
 291         else if (ic_lsize == 128)
 292                 r4k_blast_icache_page = blast_icache128_page;
 293 }
 294
 295 #ifndef CONFIG_EVA
 296 #define r4k_blast_icache_user_page  r4k_blast_icache_page
 297 #else
 298
 299 static void (*r4k_blast_icache_user_page)(unsigned long addr);
 300
 301 static void r4k_blast_icache_user_page_setup(void)
 302 {
 303         unsigned long ic_lsize = cpu_icache_line_size();
 304
 305         if (ic_lsize == 0)
 306                 r4k_blast_icache_user_page = (void *)cache_noop;
 307         else if (ic_lsize == 16)
 308                 r4k_blast_icache_user_page = blast_icache16_user_page;
 309         else if (ic_lsize == 32)
 310                 r4k_blast_icache_user_page = blast_icache32_user_page;
 311         else if (ic_lsize == 64)
 312                 r4k_blast_icache_user_page = blast_icache64_user_page;
 313 }
 314
 315 #endif
 316
 317 static void (* r4k_blast_icache_page_indexed)(unsigned long addr);
 318
 319 static void r4k_blast_icache_page_indexed_setup(void)
 320 {
 321         unsigned long ic_lsize = cpu_icache_line_size();
 322
 323         if (ic_lsize == 0)
 324                 r4k_blast_icache_page_indexed = (void *)cache_noop;
 325         else if (ic_lsize == 16)
 326                 r4k_blast_icache_page_indexed = blast_icache16_page_indexed;
 327         else if (ic_lsize == 32) {
 328                 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
 329                         r4k_blast_icache_page_indexed =
 330                                 blast_icache32_r4600_v1_page_indexed;
 331                 else if (TX49XX_ICACHE_INDEX_INV_WAR)
 332                         r4k_blast_icache_page_indexed =
 333                                 tx49_blast_icache32_page_indexed;
 334                 else if (current_cpu_type() == CPU_LOONGSON2)
 335                         r4k_blast_icache_page_indexed =
 336                                 loongson2_blast_icache32_page_indexed;
 337                 else
 338                         r4k_blast_icache_page_indexed =
 339                                 blast_icache32_page_indexed;
 340         } else if (ic_lsize == 64)
 341                 r4k_blast_icache_page_indexed = blast_icache64_page_indexed;
 342 }
 343
 344 void (* r4k_blast_icache)(void);
 345 EXPORT_SYMBOL(r4k_blast_icache);
 346
 347 static void r4k_blast_icache_setup(void)
 348 {
 349         unsigned long ic_lsize = cpu_icache_line_size();
 350
 351         if (ic_lsize == 0)
 352                 r4k_blast_icache = (void *)cache_noop;
 353         else if (ic_lsize == 16)
 354                 r4k_blast_icache = blast_icache16;
 355         else if (ic_lsize == 32) {
 356                 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x())
 357                         r4k_blast_icache = blast_r4600_v1_icache32;
 358                 else if (TX49XX_ICACHE_INDEX_INV_WAR)
 359                         r4k_blast_icache = tx49_blast_icache32;
 360                 else if (current_cpu_type() == CPU_LOONGSON2)
 361                         r4k_blast_icache = loongson2_blast_icache32;
 362                 else
 363                         r4k_blast_icache = blast_icache32;
 364         } else if (ic_lsize == 64)
 365                 r4k_blast_icache = blast_icache64;
 366         else if (ic_lsize == 128)
 367                 r4k_blast_icache = blast_icache128;
 368 }
 369
 370 static void (* r4k_blast_scache_page)(unsigned long addr);
 371
 372 static void r4k_blast_scache_page_setup(void)
 373 {
 374         unsigned long sc_lsize = cpu_scache_line_size();
 375
 376         if (scache_size == 0)
 377                 r4k_blast_scache_page = (void *)cache_noop;
 378         else if (sc_lsize == 16)
 379                 r4k_blast_scache_page = blast_scache16_page;
 380         else if (sc_lsize == 32)
 381                 r4k_blast_scache_page = blast_scache32_page;
 382         else if (sc_lsize == 64)
 383                 r4k_blast_scache_page = blast_scache64_page;
 384         else if (sc_lsize == 128)
 385                 r4k_blast_scache_page = blast_scache128_page;
 386 }
 387
 388 static void (* r4k_blast_scache_page_indexed)(unsigned long addr);
 389
 390 static void r4k_blast_scache_page_indexed_setup(void)
 391 {
 392         unsigned long sc_lsize = cpu_scache_line_size();
 393
 394         if (scache_size == 0)
 395                 r4k_blast_scache_page_indexed = (void *)cache_noop;
 396         else if (sc_lsize == 16)
 397                 r4k_blast_scache_page_indexed = blast_scache16_page_indexed;
 398         else if (sc_lsize == 32)
 399                 r4k_blast_scache_page_indexed = blast_scache32_page_indexed;
 400         else if (sc_lsize == 64)
 401                 r4k_blast_scache_page_indexed = blast_scache64_page_indexed;
 402         else if (sc_lsize == 128)
 403                 r4k_blast_scache_page_indexed = blast_scache128_page_indexed;
 404 }
 405
 406 static void (* r4k_blast_scache)(void);
 407
 408 static void r4k_blast_scache_setup(void)
 409 {
 410         unsigned long sc_lsize = cpu_scache_line_size();
 411
 412         if (scache_size == 0)
 413                 r4k_blast_scache = (void *)cache_noop;
 414         else if (sc_lsize == 16)
 415                 r4k_blast_scache = blast_scache16;
 416         else if (sc_lsize == 32)
 417                 r4k_blast_scache = blast_scache32;
 418         else if (sc_lsize == 64)
 419                 r4k_blast_scache = blast_scache64;
 420         else if (sc_lsize == 128)
 421                 r4k_blast_scache = blast_scache128;
 422 }
 423
 424 static inline void local_r4k___flush_cache_all(void * args)
 425 {
 426         switch (current_cpu_type()) {
 427         case CPU_LOONGSON2:
 428         case CPU_LOONGSON3:
 429         case CPU_R4000SC:
 430         case CPU_R4000MC:
 431         case CPU_R4400SC:
 432         case CPU_R4400MC:
 433         case CPU_R10000:
 434         case CPU_R12000:
 435         case CPU_R14000:
 436         case CPU_R16000:
 437                 /*
 438                  * These caches are inclusive caches, that is, if something
 439                  * is not cached in the S-cache, we know it also won't be
 440                  * in one of the primary caches.
 441                  */
 442                 r4k_blast_scache();
 443                 break;
 444
 445         case CPU_BMIPS5000:
 446                 r4k_blast_scache();
 447                 __sync();
 448                 break;
 449
 450         default:
 451                 r4k_blast_dcache();
 452                 r4k_blast_icache();
 453                 break;
 454         }
 455 }
 456
 457 static void r4k___flush_cache_all(void)
 458 {
 459         r4k_on_each_cpu(local_r4k___flush_cache_all, NULL);
 460 }
 461
 462 static inline int has_valid_asid(const struct mm_struct *mm)
 463 {
 464 #ifdef CONFIG_MIPS_MT_SMP
 465         int i;
 466
 467         for_each_online_cpu(i)
 468                 if (cpu_context(i, mm))
 469                         return 1;
 470
 471         return 0;
 472 #else
 473         return cpu_context(smp_processor_id(), mm);
 474 #endif
 475 }
 476
 477 static void r4k__flush_cache_vmap(void)
 478 {
 479         r4k_blast_dcache();
 480 }
 481
 482 static void r4k__flush_cache_vunmap(void)
 483 {
 484         r4k_blast_dcache();
 485 }
 486
 487 /*
 488  * Note: flush_tlb_range() assumes flush_cache_range() sufficiently flushes
 489  * whole caches when vma is executable.
 490  */
 491 static inline void local_r4k_flush_cache_range(void * args)
 492 {
 493         struct vm_area_struct *vma = args;
 494         int exec = vma->vm_flags & VM_EXEC;
 495
 496         if (!(has_valid_asid(vma->vm_mm)))
 497                 return;
 498
 499         /*
 500          * If dcache can alias, we must blast it since mapping is changing.
 501          * If executable, we must ensure any dirty lines are written back far
 502          * enough to be visible to icache.
 503          */
 504         if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc))
 505                 r4k_blast_dcache();
 506         /* If executable, blast stale lines from icache */
 507         if (exec)
 508                 r4k_blast_icache();
 509 }
 510
 511 static void r4k_flush_cache_range(struct vm_area_struct *vma,
 512         unsigned long start, unsigned long end)
 513 {
 514         int exec = vma->vm_flags & VM_EXEC;
 515
 516         if (cpu_has_dc_aliases || exec)
 517                 r4k_on_each_cpu(local_r4k_flush_cache_range, vma);
 518 }
 519
 520 static inline void local_r4k_flush_cache_mm(void * args)
 521 {
 522         struct mm_struct *mm = args;
 523
 524         if (!has_valid_asid(mm))
 525                 return;
 526
 527         /*
 528          * Kludge alert.  For obscure reasons R4000SC and R4400SC go nuts if we
 529          * only flush the primary caches but R1x000 behave sane ...
 530          * R4000SC and R4400SC indexed S-cache ops also invalidate primary
 531          * caches, so we can bail out early.
 532          */
 533         if (current_cpu_type() == CPU_R4000SC ||
 534             current_cpu_type() == CPU_R4000MC ||
 535             current_cpu_type() == CPU_R4400SC ||
 536             current_cpu_type() == CPU_R4400MC) {
 537                 r4k_blast_scache();
 538                 return;
 539         }
 540
 541         r4k_blast_dcache();
 542 }
 543
 544 static void r4k_flush_cache_mm(struct mm_struct *mm)
 545 {
 546         if (!cpu_has_dc_aliases)
 547                 return;
 548
 549         r4k_on_each_cpu(local_r4k_flush_cache_mm, mm);
 550 }
 551
 552 struct flush_cache_page_args {
 553         struct vm_area_struct *vma;
 554         unsigned long addr;
 555         unsigned long pfn;
 556 };
 557
 558 static inline void local_r4k_flush_cache_page(void *args)
 559 {
 560         struct flush_cache_page_args *fcp_args = args;
 561         struct vm_area_struct *vma = fcp_args->vma;
 562         unsigned long addr = fcp_args->addr;
 563         struct page *page = pfn_to_page(fcp_args->pfn);
 564         int exec = vma->vm_flags & VM_EXEC;
 565         struct mm_struct *mm = vma->vm_mm;
 566         int map_coherent = 0;
 567         pgd_t *pgdp;
 568         pud_t *pudp;
 569         pmd_t *pmdp;
 570         pte_t *ptep;
 571         void *vaddr;
 572
 573         /*
 574          * If ownes no valid ASID yet, cannot possibly have gotten
 575          * this page into the cache.
 576          */
 577         if (!has_valid_asid(mm))
 578                 return;
 579
 580         addr &= PAGE_MASK;
 581         pgdp = pgd_offset(mm, addr);
 582         pudp = pud_offset(pgdp, addr);
 583         pmdp = pmd_offset(pudp, addr);
 584         ptep = pte_offset(pmdp, addr);
 585
 586         /*
 587          * If the page isn't marked valid, the page cannot possibly be
 588          * in the cache.
 589          */
 590         if (!(pte_present(*ptep)))
 591                 return;
 592
 593         if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID))
 594                 vaddr = NULL;
 595         else {
 596                 /*
 597                  * Use kmap_coherent or kmap_atomic to do flushes for
 598                  * another ASID than the current one.
 599                  */
 600                 map_coherent = (cpu_has_dc_aliases &&
 601                                 page_mapcount(page) &&
 602                                 !Page_dcache_dirty(page));
 603                 if (map_coherent)
 604                         vaddr = kmap_coherent(page, addr);
 605                 else
 606                         vaddr = kmap_atomic(page);
 607                 addr = (unsigned long)vaddr;
 608         }
 609
 610         if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
 611                 vaddr ? r4k_blast_dcache_page(addr) :
 612                         r4k_blast_dcache_user_page(addr);
 613                 if (exec && !cpu_icache_snoops_remote_store)
 614                         r4k_blast_scache_page(addr);
 615         }
 616         if (exec) {
 617                 if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) {
 618                         int cpu = smp_processor_id();
 619
 620                         if (cpu_context(cpu, mm) != 0)
 621                                 drop_mmu_context(mm, cpu);
 622                 } else
 623                         vaddr ? r4k_blast_icache_page(addr) :
 624                                 r4k_blast_icache_user_page(addr);
 625         }
 626
 627         if (vaddr) {
 628                 if (map_coherent)
 629                         kunmap_coherent();
 630                 else
 631                         kunmap_atomic(vaddr);
 632         }
 633 }
 634
 635 static void r4k_flush_cache_page(struct vm_area_struct *vma,
 636         unsigned long addr, unsigned long pfn)
 637 {
 638         struct flush_cache_page_args args;
 639
 640         args.vma = vma;
 641         args.addr = addr;
 642         args.pfn = pfn;
 643
 644         r4k_on_each_cpu(local_r4k_flush_cache_page, &args);
 645 }
 646
 647 static inline void local_r4k_flush_data_cache_page(void * addr)
 648 {
 649         r4k_blast_dcache_page((unsigned long) addr);
 650 }
 651
 652 static void r4k_flush_data_cache_page(unsigned long addr)
 653 {
 654         if (in_atomic())
 655                 local_r4k_flush_data_cache_page((void *)addr);
 656         else
 657                 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr);
 658 }
 659
 660 struct flush_icache_range_args {
 661         unsigned long start;
 662         unsigned long end;
 663 };
 664
 665 static inline void local_r4k_flush_icache_range(unsigned long start, unsigned long end)
 666 {
 667         if (!cpu_has_ic_fills_f_dc) {
 668                 if (end - start >= dcache_size) {
 669                         r4k_blast_dcache();
 670                 } else {
 671                         R4600_HIT_CACHEOP_WAR_IMPL;
 672                         protected_blast_dcache_range(start, end);
 673                 }
 674         }
 675
 676         if (end - start > icache_size)
 677                 r4k_blast_icache();
 678         else {
 679                 switch (boot_cpu_type()) {
 680                 case CPU_LOONGSON2:
 681                         protected_loongson2_blast_icache_range(start, end);
 682                         break;
 683
 684                 default:
 685                         protected_blast_icache_range(start, end);
 686                         break;
 687                 }
 688         }
 689 #ifdef CONFIG_EVA
 690         /*
 691          * Due to all possible segment mappings, there might cache aliases
 692          * caused by the bootloader being in non-EVA mode, and the CPU switching
 693          * to EVA during early kernel init. It's best to flush the scache
 694          * to avoid having secondary cores fetching stale data and lead to
 695          * kernel crashes.
 696          */
 697         bc_wback_inv(start, (end - start));
 698         __sync();
 699 #endif
 700 }
 701
 702 static inline void local_r4k_flush_icache_range_ipi(void *args)
 703 {
 704         struct flush_icache_range_args *fir_args = args;
 705         unsigned long start = fir_args->start;
 706         unsigned long end = fir_args->end;
 707
 708         local_r4k_flush_icache_range(start, end);
 709 }
 710
 711 static void r4k_flush_icache_range(unsigned long start, unsigned long end)
 712 {
 713         struct flush_icache_range_args args;
 714
 715         args.start = start;
 716         args.end = end;
 717
 718         r4k_on_each_cpu(local_r4k_flush_icache_range_ipi, &args);
 719         instruction_hazard();
 720 }
 721
 722 #if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT)
 723
 724 static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
 725 {
 726         /* Catch bad driver code */
 727         BUG_ON(size == 0);
 728
 729         preempt_disable();
 730         if (cpu_has_inclusive_pcaches) {
 731                 if (size >= scache_size)
 732                         r4k_blast_scache();
 733                 else
 734                         blast_scache_range(addr, addr + size);
 735                 preempt_enable();
 736                 __sync();
 737                 return;
 738         }
 739
 740         /*
 741          * Either no secondary cache or the available caches don't have the
 742          * subset property so we have to flush the primary caches
 743          * explicitly
 744          */
 745         if (size >= dcache_size) {
 746                 r4k_blast_dcache();
 747         } else {
 748                 R4600_HIT_CACHEOP_WAR_IMPL;
 749                 blast_dcache_range(addr, addr + size);
 750         }
 751         preempt_enable();
 752
 753         bc_wback_inv(addr, size);
 754         __sync();
 755 }
 756
 757 static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
 758 {
 759         /* Catch bad driver code */
 760         BUG_ON(size == 0);
 761
 762         preempt_disable();
 763         if (cpu_has_inclusive_pcaches) {
 764                 if (size >= scache_size)
 765                         r4k_blast_scache();
 766                 else {
 767                         /*
 768                          * There is no clearly documented alignment requirement
 769                          * for the cache instruction on MIPS processors and
 770                          * some processors, among them the RM5200 and RM7000
 771                          * QED processors will throw an address error for cache
 772                          * hit ops with insufficient alignment.  Solved by
 773                          * aligning the address to cache line size.
 774                          */
 775                         blast_inv_scache_range(addr, addr + size);
 776                 }
 777                 preempt_enable();
 778                 __sync();
 779                 return;
 780         }
 781
 782         if (size >= dcache_size) {
 783                 r4k_blast_dcache();
 784         } else {
 785                 R4600_HIT_CACHEOP_WAR_IMPL;
 786                 blast_inv_dcache_range(addr, addr + size);
 787         }
 788         preempt_enable();
 789
 790         bc_inv(addr, size);
 791         __sync();
 792 }
 793 #endif /* CONFIG_DMA_NONCOHERENT || CONFIG_DMA_MAYBE_COHERENT */
 794
 795 /*
 796  * While we're protected against bad userland addresses we don't care
 797  * very much about what happens in that case.  Usually a segmentation
 798  * fault will dump the process later on anyway ...
 799  */
 800 static void local_r4k_flush_cache_sigtramp(void * arg)
 801 {
 802         unsigned long ic_lsize = cpu_icache_line_size();
 803         unsigned long dc_lsize = cpu_dcache_line_size();
 804         unsigned long sc_lsize = cpu_scache_line_size();
 805         unsigned long addr = (unsigned long) arg;
 806
 807         R4600_HIT_CACHEOP_WAR_IMPL;
 808         if (dc_lsize)
 809                 protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
 810         if (!cpu_icache_snoops_remote_store && scache_size)
 811                 protected_writeback_scache_line(addr & ~(sc_lsize - 1));
 812         if (ic_lsize)
 813                 protected_flush_icache_line(addr & ~(ic_lsize - 1));
 814         if (MIPS4K_ICACHE_REFILL_WAR) {
 815                 __asm__ __volatile__ (
 816                         ".set push\n\t"
 817                         ".set noat\n\t"
 818                         ".set "MIPS_ISA_LEVEL"\n\t"
 819 #ifdef CONFIG_32BIT
 820                         "la     $at,1f\n\t"
 821 #endif
 822 #ifdef CONFIG_64BIT
 823                         "dla    $at,1f\n\t"
 824 #endif
 825                         "cache  %0,($at)\n\t"
 826                         "nop; nop; nop\n"
 827                         "1:\n\t"
 828                         ".set pop"
 829                         :
 830                         : "i" (Hit_Invalidate_I));
 831         }
 832         if (MIPS_CACHE_SYNC_WAR)
 833                 __asm__ __volatile__ ("sync");
 834 }
 835
 836 static void r4k_flush_cache_sigtramp(unsigned long addr)
 837 {
 838         r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr);
 839 }
 840
 841 static void r4k_flush_icache_all(void)
 842 {
 843         if (cpu_has_vtag_icache)
 844                 r4k_blast_icache();
 845 }
 846
 847 struct flush_kernel_vmap_range_args {
 848         unsigned long   vaddr;
 849         int             size;
 850 };
 851
 852 static inline void local_r4k_flush_kernel_vmap_range(void *args)
 853 {
 854         struct flush_kernel_vmap_range_args *vmra = args;
 855         unsigned long vaddr = vmra->vaddr;
 856         int size = vmra->size;
 857
 858         /*
 859          * Aliases only affect the primary caches so don't bother with
 860          * S-caches or T-caches.
 861          */
 862         if (size >= dcache_size)
 863                 r4k_blast_dcache();
 864         else {
 865                 R4600_HIT_CACHEOP_WAR_IMPL;
 866                 blast_dcache_range(vaddr, vaddr + size);
 867         }
 868 }
 869
 870 static void r4k_flush_kernel_vmap_range(unsigned long vaddr, int size)
 871 {
 872         struct flush_kernel_vmap_range_args args;
 873
 874         args.vaddr = (unsigned long) vaddr;
 875         args.size = size;
 876
 877         r4k_on_each_cpu(local_r4k_flush_kernel_vmap_range, &args);
 878 }
 879
 880 static inline void rm7k_erratum31(void)
 881 {
 882         const unsigned long ic_lsize = 32;
 883         unsigned long addr;
 884
 885         /* RM7000 erratum #31. The icache is screwed at startup. */
 886         write_c0_taglo(0);
 887         write_c0_taghi(0);
 888
 889         for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) {
 890                 __asm__ __volatile__ (
 891                         ".set push\n\t"
 892                         ".set noreorder\n\t"
 893                         ".set mips3\n\t"
 894                         "cache\t%1, 0(%0)\n\t"
 895                         "cache\t%1, 0x1000(%0)\n\t"
 896                         "cache\t%1, 0x2000(%0)\n\t"
 897                         "cache\t%1, 0x3000(%0)\n\t"
 898                         "cache\t%2, 0(%0)\n\t"
 899                         "cache\t%2, 0x1000(%0)\n\t"
 900                         "cache\t%2, 0x2000(%0)\n\t"
 901                         "cache\t%2, 0x3000(%0)\n\t"
 902                         "cache\t%1, 0(%0)\n\t"
 903                         "cache\t%1, 0x1000(%0)\n\t"
 904                         "cache\t%1, 0x2000(%0)\n\t"
 905                         "cache\t%1, 0x3000(%0)\n\t"
 906                         ".set pop\n"
 907                         :
 908                         : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill));
 909         }
 910 }
 911
 912 static inline int alias_74k_erratum(struct cpuinfo_mips *c)
 913 {
 914         unsigned int imp = c->processor_id & PRID_IMP_MASK;
 915         unsigned int rev = c->processor_id & PRID_REV_MASK;
 916         int present = 0;
 917
 918         /*
 919          * Early versions of the 74K do not update the cache tags on a
 920          * vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG
 921          * aliases.  In this case it is better to treat the cache as always
 922          * having aliases.  Also disable the synonym tag update feature
 923          * where available.  In this case no opportunistic tag update will
 924          * happen where a load causes a virtual address miss but a physical
 925          * address hit during a D-cache look-up.
 926          */
 927         switch (imp) {
 928         case PRID_IMP_74K:
 929                 if (rev <= PRID_REV_ENCODE_332(2, 4, 0))
 930                         present = 1;
 931                 if (rev == PRID_REV_ENCODE_332(2, 4, 0))
 932                         write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
 933                 break;
 934         case PRID_IMP_1074K:
 935                 if (rev <= PRID_REV_ENCODE_332(1, 1, 0)) {
 936                         present = 1;
 937                         write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
 938                 }
 939                 break;
 940         default:
 941                 BUG();
 942         }
 943
 944         return present;
 945 }
 946
 947 static void b5k_instruction_hazard(void)
 948 {
 949         __sync();
 950         __sync();
 951         __asm__ __volatile__(
 952         "       nop; nop; nop; nop; nop; nop; nop; nop\n"
 953         "       nop; nop; nop; nop; nop; nop; nop; nop\n"
 954         "       nop; nop; nop; nop; nop; nop; nop; nop\n"
 955         "       nop; nop; nop; nop; nop; nop; nop; nop\n"
 956         : : : "memory");
 957 }
 958
 959 static char *way_string[] = { NULL, "direct mapped", "2-way",
 960         "3-way", "4-way", "5-way", "6-way", "7-way", "8-way",
 961         "9-way", "10-way", "11-way", "12-way",
 962         "13-way", "14-way", "15-way", "16-way",
 963 };
 964
 965 static void probe_pcache(void)
 966 {
 967         struct cpuinfo_mips *c = &current_cpu_data;
 968         unsigned int config = read_c0_config();
 969         unsigned int prid = read_c0_prid();
 970         int has_74k_erratum = 0;
 971         unsigned long config1;
 972         unsigned int lsize;
 973
 974         switch (current_cpu_type()) {
 975         case CPU_R4600:                 /* QED style two way caches? */
 976         case CPU_R4700:
 977         case CPU_R5000:
 978         case CPU_NEVADA:
 979                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
 980                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
 981                 c->icache.ways = 2;
 982                 c->icache.waybit = __ffs(icache_size/2);
 983
 984                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
 985                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
 986                 c->dcache.ways = 2;
 987                 c->dcache.waybit= __ffs(dcache_size/2);
 988
 989                 c->options |= MIPS_CPU_CACHE_CDEX_P;
 990                 break;
 991
 992         case CPU_R5432:
 993         case CPU_R5500:
 994                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
 995                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
 996                 c->icache.ways = 2;
 997                 c->icache.waybit= 0;
 998
 999                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1000                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1001                 c->dcache.ways = 2;
1002                 c->dcache.waybit = 0;
1003
1004                 c->options |= MIPS_CPU_CACHE_CDEX_P | MIPS_CPU_PREFETCH;
1005                 break;
1006
1007         case CPU_TX49XX:
1008                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1009                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1010                 c->icache.ways = 4;
1011                 c->icache.waybit= 0;
1012
1013                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1014                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1015                 c->dcache.ways = 4;
1016                 c->dcache.waybit = 0;
1017
1018                 c->options |= MIPS_CPU_CACHE_CDEX_P;
1019                 c->options |= MIPS_CPU_PREFETCH;
1020                 break;
1021
1022         case CPU_R4000PC:
1023         case CPU_R4000SC:
1024         case CPU_R4000MC:
1025         case CPU_R4400PC:
1026         case CPU_R4400SC:
1027         case CPU_R4400MC:
1028         case CPU_R4300:
1029                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1030                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1031                 c->icache.ways = 1;
1032                 c->icache.waybit = 0;   /* doesn't matter */
1033
1034                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1035                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1036                 c->dcache.ways = 1;
1037                 c->dcache.waybit = 0;   /* does not matter */
1038
1039                 c->options |= MIPS_CPU_CACHE_CDEX_P;
1040                 break;
1041
1042         case CPU_R10000:
1043         case CPU_R12000:
1044         case CPU_R14000:
1045         case CPU_R16000:
1046                 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29));
1047                 c->icache.linesz = 64;
1048                 c->icache.ways = 2;
1049                 c->icache.waybit = 0;
1050
1051                 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26));
1052                 c->dcache.linesz = 32;
1053                 c->dcache.ways = 2;
1054                 c->dcache.waybit = 0;
1055
1056                 c->options |= MIPS_CPU_PREFETCH;
1057                 break;
1058
1059         case CPU_VR4133:
1060                 write_c0_config(config & ~VR41_CONF_P4K);
1061         case CPU_VR4131:
1062                 /* Workaround for cache instruction bug of VR4131 */
1063                 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U ||
1064                     c->processor_id == 0x0c82U) {
1065                         config |= 0x00400000U;
1066                         if (c->processor_id == 0x0c80U)
1067                                 config |= VR41_CONF_BP;
1068                         write_c0_config(config);
1069                 } else
1070                         c->options |= MIPS_CPU_CACHE_CDEX_P;
1071
1072                 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
1073                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1074                 c->icache.ways = 2;
1075                 c->icache.waybit = __ffs(icache_size/2);
1076
1077                 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
1078                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1079                 c->dcache.ways = 2;
1080                 c->dcache.waybit = __ffs(dcache_size/2);
1081                 break;
1082
1083         case CPU_VR41XX:
1084         case CPU_VR4111:
1085         case CPU_VR4121:
1086         case CPU_VR4122:
1087         case CPU_VR4181:
1088         case CPU_VR4181A:
1089                 icache_size = 1 << (10 + ((config & CONF_IC) >> 9));
1090                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1091                 c->icache.ways = 1;
1092                 c->icache.waybit = 0;   /* doesn't matter */
1093
1094                 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6));
1095                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1096                 c->dcache.ways = 1;
1097                 c->dcache.waybit = 0;   /* does not matter */
1098
1099                 c->options |= MIPS_CPU_CACHE_CDEX_P;
1100                 break;
1101
1102         case CPU_RM7000:
1103                 rm7k_erratum31();
1104
1105                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1106                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1107                 c->icache.ways = 4;
1108                 c->icache.waybit = __ffs(icache_size / c->icache.ways);
1109
1110                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1111                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1112                 c->dcache.ways = 4;
1113                 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways);
1114
1115                 c->options |= MIPS_CPU_CACHE_CDEX_P;
1116                 c->options |= MIPS_CPU_PREFETCH;
1117                 break;
1118
1119         case CPU_LOONGSON2:
1120                 icache_size = 1 << (12 + ((config & CONF_IC) >> 9));
1121                 c->icache.linesz = 16 << ((config & CONF_IB) >> 5);
1122                 if (prid & 0x3)
1123                         c->icache.ways = 4;
1124                 else
1125                         c->icache.ways = 2;
1126                 c->icache.waybit = 0;
1127
1128                 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6));
1129                 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4);
1130                 if (prid & 0x3)
1131                         c->dcache.ways = 4;
1132                 else
1133                         c->dcache.ways = 2;
1134                 c->dcache.waybit = 0;
1135                 break;
1136
1137         case CPU_LOONGSON3:
1138                 config1 = read_c0_config1();
1139                 lsize = (config1 >> 19) & 7;
1140                 if (lsize)
1141                         c->icache.linesz = 2 << lsize;
1142                 else
1143                         c->icache.linesz = 0;
1144                 c->icache.sets = 64 << ((config1 >> 22) & 7);
1145                 c->icache.ways = 1 + ((config1 >> 16) & 7);
1146                 icache_size = c->icache.sets *
1147                                           c->icache.ways *
1148                                           c->icache.linesz;
1149                 c->icache.waybit = 0;
1150
1151                 lsize = (config1 >> 10) & 7;
1152                 if (lsize)
1153                         c->dcache.linesz = 2 << lsize;
1154                 else
1155                         c->dcache.linesz = 0;
1156                 c->dcache.sets = 64 << ((config1 >> 13) & 7);
1157                 c->dcache.ways = 1 + ((config1 >> 7) & 7);
1158                 dcache_size = c->dcache.sets *
1159                                           c->dcache.ways *
1160                                           c->dcache.linesz;
1161                 c->dcache.waybit = 0;
1162                 if ((prid & PRID_REV_MASK) >= PRID_REV_LOONGSON3A_R2)
1163                         c->options |= MIPS_CPU_PREFETCH;
1164                 break;
1165
1166         case CPU_CAVIUM_OCTEON3:
1167                 /* For now lie about the number of ways. */
1168                 c->icache.linesz = 128;
1169                 c->icache.sets = 16;
1170                 c->icache.ways = 8;
1171                 c->icache.flags |= MIPS_CACHE_VTAG;
1172                 icache_size = c->icache.sets * c->icache.ways * c->icache.linesz;
1173
1174                 c->dcache.linesz = 128;
1175                 c->dcache.ways = 8;
1176                 c->dcache.sets = 8;
1177                 dcache_size = c->dcache.sets * c->dcache.ways * c->dcache.linesz;
1178                 c->options |= MIPS_CPU_PREFETCH;
1179                 break;
1180
1181         default:
1182                 if (!(config & MIPS_CONF_M))
1183                         panic("Don't know how to probe P-caches on this cpu.");
1184
1185                 /*
1186                  * So we seem to be a MIPS32 or MIPS64 CPU
1187                  * So let's probe the I-cache ...
1188                  */
1189                 config1 = read_c0_config1();
1190
1191                 lsize = (config1 >> 19) & 7;
1192
1193                 /* IL == 7 is reserved */
1194                 if (lsize == 7)
1195                         panic("Invalid icache line size");
1196
1197                 c->icache.linesz = lsize ? 2 << lsize : 0;
1198
1199                 c->icache.sets = 32 << (((config1 >> 22) + 1) & 7);
1200                 c->icache.ways = 1 + ((config1 >> 16) & 7);
1201
1202                 icache_size = c->icache.sets *
1203                               c->icache.ways *
1204                               c->icache.linesz;
1205                 c->icache.waybit = __ffs(icache_size/c->icache.ways);
1206
1207                 if (config & 0x8)               /* VI bit */
1208                         c->icache.flags |= MIPS_CACHE_VTAG;
1209
1210                 /*
1211                  * Now probe the MIPS32 / MIPS64 data cache.
1212                  */
1213                 c->dcache.flags = 0;
1214
1215                 lsize = (config1 >> 10) & 7;
1216
1217                 /* DL == 7 is reserved */
1218                 if (lsize == 7)
1219                         panic("Invalid dcache line size");
1220
1221                 c->dcache.linesz = lsize ? 2 << lsize : 0;
1222
1223                 c->dcache.sets = 32 << (((config1 >> 13) + 1) & 7);
1224                 c->dcache.ways = 1 + ((config1 >> 7) & 7);
1225
1226                 dcache_size = c->dcache.sets *
1227                               c->dcache.ways *
1228                               c->dcache.linesz;
1229                 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways);
1230
1231                 c->options |= MIPS_CPU_PREFETCH;
1232                 break;
1233         }
1234
1235         /*
1236          * Processor configuration sanity check for the R4000SC erratum
1237          * #5.  With page sizes larger than 32kB there is no possibility
1238          * to get a VCE exception anymore so we don't care about this
1239          * misconfiguration.  The case is rather theoretical anyway;
1240          * presumably no vendor is shipping his hardware in the "bad"
1241          * configuration.
1242          */
1243         if ((prid & PRID_IMP_MASK) == PRID_IMP_R4000 &&
1244             (prid & PRID_REV_MASK) < PRID_REV_R4400 &&
1245             !(config & CONF_SC) && c->icache.linesz != 16 &&
1246             PAGE_SIZE <= 0x8000)
1247                 panic("Improper R4000SC processor configuration detected");
1248
1249         /* compute a couple of other cache variables */
1250         c->icache.waysize = icache_size / c->icache.ways;
1251         c->dcache.waysize = dcache_size / c->dcache.ways;
1252
1253         c->icache.sets = c->icache.linesz ?
1254                 icache_size / (c->icache.linesz * c->icache.ways) : 0;
1255         c->dcache.sets = c->dcache.linesz ?
1256                 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0;
1257
1258         /*
1259          * R1x000 P-caches are odd in a positive way.  They're 32kB 2-way
1260          * virtually indexed so normally would suffer from aliases.  So
1261          * normally they'd suffer from aliases but magic in the hardware deals
1262          * with that for us so we don't need to take care ourselves.
1263          */
1264         switch (current_cpu_type()) {
1265         case CPU_20KC:
1266         case CPU_25KF:
1267         case CPU_SB1:
1268         case CPU_SB1A:
1269         case CPU_XLR:
1270                 c->dcache.flags |= MIPS_CACHE_PINDEX;
1271                 break;
1272
1273         case CPU_R10000:
1274         case CPU_R12000:
1275         case CPU_R14000:
1276         case CPU_R16000:
1277                 break;
1278
1279         case CPU_74K:
1280         case CPU_1074K:
1281                 has_74k_erratum = alias_74k_erratum(c);
1282                 /* Fall through. */
1283         case CPU_M14KC:
1284         case CPU_M14KEC:
1285         case CPU_24K:
1286         case CPU_34K:
1287         case CPU_1004K:
1288         case CPU_INTERAPTIV:
1289         case CPU_P5600:
1290         case CPU_PROAPTIV:
1291         case CPU_M5150:
1292         case CPU_QEMU_GENERIC:
1293         case CPU_I6400:
1294         case CPU_P6600:
1295         case CPU_M6250:
1296                 if (!(read_c0_config7() & MIPS_CONF7_IAR) &&
1297                     (c->icache.waysize > PAGE_SIZE))
1298                         c->icache.flags |= MIPS_CACHE_ALIASES;
1299                 if (!has_74k_erratum && (read_c0_config7() & MIPS_CONF7_AR)) {
1300                         /*
1301                          * Effectively physically indexed dcache,
1302                          * thus no virtual aliases.
1303                         */
1304                         c->dcache.flags |= MIPS_CACHE_PINDEX;
1305                         break;
1306                 }
1307         default:
1308                 if (has_74k_erratum || c->dcache.waysize > PAGE_SIZE)
1309                         c->dcache.flags |= MIPS_CACHE_ALIASES;
1310         }
1311
1312         switch (current_cpu_type()) {
1313         case CPU_20KC:
1314                 /*
1315                  * Some older 20Kc chips doesn't have the 'VI' bit in
1316                  * the config register.
1317                  */
1318                 c->icache.flags |= MIPS_CACHE_VTAG;
1319                 break;
1320
1321         case CPU_ALCHEMY:
1322         case CPU_I6400:
1323                 c->icache.flags |= MIPS_CACHE_IC_F_DC;
1324                 break;
1325
1326         case CPU_BMIPS5000:
1327                 c->icache.flags |= MIPS_CACHE_IC_F_DC;
1328                 /* Cache aliases are handled in hardware; allow HIGHMEM */
1329                 c->dcache.flags &= ~MIPS_CACHE_ALIASES;
1330                 break;
1331
1332         case CPU_LOONGSON2:
1333                 /*
1334                  * LOONGSON2 has 4 way icache, but when using indexed cache op,
1335                  * one op will act on all 4 ways
1336                  */
1337                 c->icache.ways = 1;
1338         }
1339
1340         printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
1341                icache_size >> 10,
1342                c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT",
1343                way_string[c->icache.ways], c->icache.linesz);
1344
1345         printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n",
1346                dcache_size >> 10, way_string[c->dcache.ways],
1347                (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT",
1348                (c->dcache.flags & MIPS_CACHE_ALIASES) ?
1349                         "cache aliases" : "no aliases",
1350                c->dcache.linesz);
1351 }
1352
1353 static void probe_vcache(void)
1354 {
1355         struct cpuinfo_mips *c = &current_cpu_data;
1356         unsigned int config2, lsize;
1357
1358         if (current_cpu_type() != CPU_LOONGSON3)
1359                 return;
1360
1361         config2 = read_c0_config2();
1362         if ((lsize = ((config2 >> 20) & 15)))
1363                 c->vcache.linesz = 2 << lsize;
1364         else
1365                 c->vcache.linesz = lsize;
1366
1367         c->vcache.sets = 64 << ((config2 >> 24) & 15);
1368         c->vcache.ways = 1 + ((config2 >> 16) & 15);
1369
1370         vcache_size = c->vcache.sets * c->vcache.ways * c->vcache.linesz;
1371
1372         c->vcache.waybit = 0;
1373
1374         pr_info("Unified victim cache %ldkB %s, linesize %d bytes.\n",
1375                 vcache_size >> 10, way_string[c->vcache.ways], c->vcache.linesz);
1376 }
1377
1378 /*
1379  * If you even _breathe_ on this function, look at the gcc output and make sure
1380  * it does not pop things on and off the stack for the cache sizing loop that
1381  * executes in KSEG1 space or else you will crash and burn badly.  You have
1382  * been warned.
1383  */
1384 static int probe_scache(void)
1385 {
1386         unsigned long flags, addr, begin, end, pow2;
1387         unsigned int config = read_c0_config();
1388         struct cpuinfo_mips *c = &current_cpu_data;
1389
1390         if (config & CONF_SC)
1391                 return 0;
1392
1393         begin = (unsigned long) &_stext;
1394         begin &= ~((4 * 1024 * 1024) - 1);
1395         end = begin + (4 * 1024 * 1024);
1396
1397         /*
1398          * This is such a bitch, you'd think they would make it easy to do
1399          * this.  Away you daemons of stupidity!
1400          */
1401         local_irq_save(flags);
1402
1403         /* Fill each size-multiple cache line with a valid tag. */
1404         pow2 = (64 * 1024);
1405         for (addr = begin; addr < end; addr = (begin + pow2)) {
1406                 unsigned long *p = (unsigned long *) addr;
1407                 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */
1408                 pow2 <<= 1;
1409         }
1410
1411         /* Load first line with zero (therefore invalid) tag. */
1412         write_c0_taglo(0);
1413         write_c0_taghi(0);
1414         __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */
1415         cache_op(Index_Store_Tag_I, begin);
1416         cache_op(Index_Store_Tag_D, begin);
1417         cache_op(Index_Store_Tag_SD, begin);
1418
1419         /* Now search for the wrap around point. */
1420         pow2 = (128 * 1024);
1421         for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) {
1422                 cache_op(Index_Load_Tag_SD, addr);
1423                 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
1424                 if (!read_c0_taglo())
1425                         break;
1426                 pow2 <<= 1;
1427         }
1428         local_irq_restore(flags);
1429         addr -= begin;
1430
1431         scache_size = addr;
1432         c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22);
1433         c->scache.ways = 1;
1434         c->scache.waybit = 0;           /* does not matter */
1435
1436         return 1;
1437 }
1438
1439 static void __init loongson2_sc_init(void)
1440 {
1441         struct cpuinfo_mips *c = &current_cpu_data;
1442
1443         scache_size = 512*1024;
1444         c->scache.linesz = 32;
1445         c->scache.ways = 4;
1446         c->scache.waybit = 0;
1447         c->scache.waysize = scache_size / (c->scache.ways);
1448         c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1449         pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1450                scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1451
1452         c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1453 }
1454
1455 static void __init loongson3_sc_init(void)
1456 {
1457         struct cpuinfo_mips *c = &current_cpu_data;
1458         unsigned int config2, lsize;
1459
1460         config2 = read_c0_config2();
1461         lsize = (config2 >> 4) & 15;
1462         if (lsize)
1463                 c->scache.linesz = 2 << lsize;
1464         else
1465                 c->scache.linesz = 0;
1466         c->scache.sets = 64 << ((config2 >> 8) & 15);
1467         c->scache.ways = 1 + (config2 & 15);
1468
1469         scache_size = c->scache.sets *
1470                                   c->scache.ways *
1471                                   c->scache.linesz;
1472         /* Loongson-3 has 4 cores, 1MB scache for each. scaches are shared */
1473         scache_size *= 4;
1474         c->scache.waybit = 0;
1475         pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1476                scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1477         if (scache_size)
1478                 c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1479         return;
1480 }
1481
1482 extern int r5k_sc_init(void);
1483 extern int rm7k_sc_init(void);
1484 extern int mips_sc_init(void);
1485
1486 static void setup_scache(void)
1487 {
1488         struct cpuinfo_mips *c = &current_cpu_data;
1489         unsigned int config = read_c0_config();
1490         int sc_present = 0;
1491
1492         /*
1493          * Do the probing thing on R4000SC and R4400SC processors.  Other
1494          * processors don't have a S-cache that would be relevant to the
1495          * Linux memory management.
1496          */
1497         switch (current_cpu_type()) {
1498         case CPU_R4000SC:
1499         case CPU_R4000MC:
1500         case CPU_R4400SC:
1501         case CPU_R4400MC:
1502                 sc_present = run_uncached(probe_scache);
1503                 if (sc_present)
1504                         c->options |= MIPS_CPU_CACHE_CDEX_S;
1505                 break;
1506
1507         case CPU_R10000:
1508         case CPU_R12000:
1509         case CPU_R14000:
1510         case CPU_R16000:
1511                 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16);
1512                 c->scache.linesz = 64 << ((config >> 13) & 1);
1513                 c->scache.ways = 2;
1514                 c->scache.waybit= 0;
1515                 sc_present = 1;
1516                 break;
1517
1518         case CPU_R5000:
1519         case CPU_NEVADA:
1520 #ifdef CONFIG_R5000_CPU_SCACHE
1521                 r5k_sc_init();
1522 #endif
1523                 return;
1524
1525         case CPU_RM7000:
1526 #ifdef CONFIG_RM7000_CPU_SCACHE
1527                 rm7k_sc_init();
1528 #endif
1529                 return;
1530
1531         case CPU_LOONGSON2:
1532                 loongson2_sc_init();
1533                 return;
1534
1535         case CPU_LOONGSON3:
1536                 loongson3_sc_init();
1537                 return;
1538
1539         case CPU_CAVIUM_OCTEON3:
1540         case CPU_XLP:
1541                 /* don't need to worry about L2, fully coherent */
1542                 return;
1543
1544         default:
1545                 if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M32R2 |
1546                                     MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R1 |
1547                                     MIPS_CPU_ISA_M64R2 | MIPS_CPU_ISA_M64R6)) {
1548 #ifdef CONFIG_MIPS_CPU_SCACHE
1549                         if (mips_sc_init ()) {
1550                                 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz;
1551                                 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n",
1552                                        scache_size >> 10,
1553                                        way_string[c->scache.ways], c->scache.linesz);
1554                         }
1555 #else
1556                         if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT))
1557                                 panic("Dunno how to handle MIPS32 / MIPS64 second level cache");
1558 #endif
1559                         return;
1560                 }
1561                 sc_present = 0;
1562         }
1563
1564         if (!sc_present)
1565                 return;
1566
1567         /* compute a couple of other cache variables */
1568         c->scache.waysize = scache_size / c->scache.ways;
1569
1570         c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways);
1571
1572         printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
1573                scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
1574
1575         c->options |= MIPS_CPU_INCLUSIVE_CACHES;
1576 }
1577
1578 void au1x00_fixup_config_od(void)
1579 {
1580         /*
1581          * c0_config.od (bit 19) was write only (and read as 0)
1582          * on the early revisions of Alchemy SOCs.  It disables the bus
1583          * transaction overlapping and needs to be set to fix various errata.
1584          */
1585         switch (read_c0_prid()) {
1586         case 0x00030100: /* Au1000 DA */
1587         case 0x00030201: /* Au1000 HA */
1588         case 0x00030202: /* Au1000 HB */
1589         case 0x01030200: /* Au1500 AB */
1590         /*
1591          * Au1100 errata actually keeps silence about this bit, so we set it
1592          * just in case for those revisions that require it to be set according
1593          * to the (now gone) cpu table.
1594          */
1595         case 0x02030200: /* Au1100 AB */
1596         case 0x02030201: /* Au1100 BA */
1597         case 0x02030202: /* Au1100 BC */
1598                 set_c0_config(1 << 19);
1599                 break;
1600         }
1601 }
1602
1603 /* CP0 hazard avoidance. */
1604 #define NXP_BARRIER()                                                   \
1605          __asm__ __volatile__(                                          \
1606         ".set noreorder\n\t"                                            \
1607         "nop; nop; nop; nop; nop; nop;\n\t"                             \
1608         ".set reorder\n\t")
1609
1610 static void nxp_pr4450_fixup_config(void)
1611 {
1612         unsigned long config0;
1613
1614         config0 = read_c0_config();
1615
1616         /* clear all three cache coherency fields */
1617         config0 &= ~(0x7 | (7 << 25) | (7 << 28));
1618         config0 |= (((_page_cachable_default >> _CACHE_SHIFT) <<  0) |
1619                     ((_page_cachable_default >> _CACHE_SHIFT) << 25) |
1620                     ((_page_cachable_default >> _CACHE_SHIFT) << 28));
1621         write_c0_config(config0);
1622         NXP_BARRIER();
1623 }
1624
1625 static int cca = -1;
1626
1627 static int __init cca_setup(char *str)
1628 {
1629         get_option(&str, &cca);
1630
1631         return 0;
1632 }
1633
1634 early_param("cca", cca_setup);
1635
1636 static void coherency_setup(void)
1637 {
1638         if (cca < 0 || cca > 7)
1639                 cca = read_c0_config() & CONF_CM_CMASK;
1640         _page_cachable_default = cca << _CACHE_SHIFT;
1641
1642         pr_debug("Using cache attribute %d\n", cca);
1643         change_c0_config(CONF_CM_CMASK, cca);
1644
1645         /*
1646          * c0_status.cu=0 specifies that updates by the sc instruction use
1647          * the coherency mode specified by the TLB; 1 means cachable
1648          * coherent update on write will be used.  Not all processors have
1649          * this bit and; some wire it to zero, others like Toshiba had the
1650          * silly idea of putting something else there ...
1651          */
1652         switch (current_cpu_type()) {
1653         case CPU_R4000PC:
1654         case CPU_R4000SC:
1655         case CPU_R4000MC:
1656         case CPU_R4400PC:
1657         case CPU_R4400SC:
1658         case CPU_R4400MC:
1659                 clear_c0_config(CONF_CU);
1660                 break;
1661         /*
1662          * We need to catch the early Alchemy SOCs with
1663          * the write-only co_config.od bit and set it back to one on:
1664          * Au1000 rev DA, HA, HB;  Au1100 AB, BA, BC, Au1500 AB
1665          */
1666         case CPU_ALCHEMY:
1667                 au1x00_fixup_config_od();
1668                 break;
1669
1670         case PRID_IMP_PR4450:
1671                 nxp_pr4450_fixup_config();
1672                 break;
1673         }
1674 }
1675
1676 static void r4k_cache_error_setup(void)
1677 {
1678         extern char __weak except_vec2_generic;
1679         extern char __weak except_vec2_sb1;
1680
1681         switch (current_cpu_type()) {
1682         case CPU_SB1:
1683         case CPU_SB1A:
1684                 set_uncached_handler(0x100, &except_vec2_sb1, 0x80);
1685                 break;
1686
1687         default:
1688                 set_uncached_handler(0x100, &except_vec2_generic, 0x80);
1689                 break;
1690         }
1691 }
1692
1693 void r4k_cache_init(void)
1694 {
1695         extern void build_clear_page(void);
1696         extern void build_copy_page(void);
1697         struct cpuinfo_mips *c = &current_cpu_data;
1698
1699         probe_pcache();
1700         probe_vcache();
1701         setup_scache();
1702
1703         r4k_blast_dcache_page_setup();
1704         r4k_blast_dcache_page_indexed_setup();
1705         r4k_blast_dcache_setup();
1706         r4k_blast_icache_page_setup();
1707         r4k_blast_icache_page_indexed_setup();
1708         r4k_blast_icache_setup();
1709         r4k_blast_scache_page_setup();
1710         r4k_blast_scache_page_indexed_setup();
1711         r4k_blast_scache_setup();
1712 #ifdef CONFIG_EVA
1713         r4k_blast_dcache_user_page_setup();
1714         r4k_blast_icache_user_page_setup();
1715 #endif
1716
1717         /*
1718          * Some MIPS32 and MIPS64 processors have physically indexed caches.
1719          * This code supports virtually indexed processors and will be
1720          * unnecessarily inefficient on physically indexed processors.
1721          */
1722         if (c->dcache.linesz && cpu_has_dc_aliases)
1723                 shm_align_mask = max_t( unsigned long,
1724                                         c->dcache.sets * c->dcache.linesz - 1,
1725                                         PAGE_SIZE - 1);
1726         else
1727                 shm_align_mask = PAGE_SIZE-1;
1728
1729         __flush_cache_vmap      = r4k__flush_cache_vmap;
1730         __flush_cache_vunmap    = r4k__flush_cache_vunmap;
1731
1732         flush_cache_all         = cache_noop;
1733         __flush_cache_all       = r4k___flush_cache_all;
1734         flush_cache_mm          = r4k_flush_cache_mm;
1735         flush_cache_page        = r4k_flush_cache_page;
1736         flush_cache_range       = r4k_flush_cache_range;
1737
1738         __flush_kernel_vmap_range = r4k_flush_kernel_vmap_range;
1739
1740         flush_cache_sigtramp    = r4k_flush_cache_sigtramp;
1741         flush_icache_all        = r4k_flush_icache_all;
1742         local_flush_data_cache_page     = local_r4k_flush_data_cache_page;
1743         flush_data_cache_page   = r4k_flush_data_cache_page;
1744         flush_icache_range      = r4k_flush_icache_range;
1745         local_flush_icache_range        = local_r4k_flush_icache_range;
1746
1747 #if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT)
1748         if (coherentio) {
1749                 _dma_cache_wback_inv    = (void *)cache_noop;
1750                 _dma_cache_wback        = (void *)cache_noop;
1751                 _dma_cache_inv          = (void *)cache_noop;
1752         } else {
1753                 _dma_cache_wback_inv    = r4k_dma_cache_wback_inv;
1754                 _dma_cache_wback        = r4k_dma_cache_wback_inv;
1755                 _dma_cache_inv          = r4k_dma_cache_inv;
1756         }
1757 #endif
1758
1759         build_clear_page();
1760         build_copy_page();
1761
1762         /*
1763          * We want to run CMP kernels on core with and without coherent
1764          * caches. Therefore, do not use CONFIG_MIPS_CMP to decide whether
1765          * or not to flush caches.
1766          */
1767         local_r4k___flush_cache_all(NULL);
1768
1769         coherency_setup();
1770         board_cache_error_setup = r4k_cache_error_setup;
1771
1772         /*
1773          * Per-CPU overrides
1774          */
1775         switch (current_cpu_type()) {
1776         case CPU_BMIPS4350:
1777         case CPU_BMIPS4380:
1778                 /* No IPI is needed because all CPUs share the same D$ */
1779                 flush_data_cache_page = r4k_blast_dcache_page;
1780                 break;
1781         case CPU_BMIPS5000:
1782                 /* We lose our superpowers if L2 is disabled */
1783                 if (c->scache.flags & MIPS_CACHE_NOT_PRESENT)
1784                         break;
1785
1786                 /* I$ fills from D$ just by emptying the write buffers */
1787                 flush_cache_page = (void *)b5k_instruction_hazard;
1788                 flush_cache_range = (void *)b5k_instruction_hazard;
1789                 flush_cache_sigtramp = (void *)b5k_instruction_hazard;
1790                 local_flush_data_cache_page = (void *)b5k_instruction_hazard;
1791                 flush_data_cache_page = (void *)b5k_instruction_hazard;
1792                 flush_icache_range = (void *)b5k_instruction_hazard;
1793                 local_flush_icache_range = (void *)b5k_instruction_hazard;
1794
1795
1796                 /* Optimization: an L2 flush implicitly flushes the L1 */
1797                 current_cpu_data.options |= MIPS_CPU_INCLUSIVE_CACHES;
1798                 break;
1799         case CPU_LOONGSON3:
1800                 /* Loongson-3 maintains cache coherency by hardware */
1801                 __flush_cache_all       = cache_noop;
1802                 __flush_cache_vmap      = cache_noop;
1803                 __flush_cache_vunmap    = cache_noop;
1804                 __flush_kernel_vmap_range = (void *)cache_noop;
1805                 flush_cache_mm          = (void *)cache_noop;
1806                 flush_cache_page        = (void *)cache_noop;
1807                 flush_cache_range       = (void *)cache_noop;
1808                 flush_cache_sigtramp    = (void *)cache_noop;
1809                 flush_icache_all        = (void *)cache_noop;
1810                 flush_data_cache_page   = (void *)cache_noop;
1811                 local_flush_data_cache_page     = (void *)cache_noop;
1812                 break;
1813         }
1814 }
1815
1816 static int r4k_cache_pm_notifier(struct notifier_block *self, unsigned long cmd,
1817                                void *v)
1818 {
1819         switch (cmd) {
1820         case CPU_PM_ENTER_FAILED:
1821         case CPU_PM_EXIT:
1822                 coherency_setup();
1823                 break;
1824         }
1825
1826         return NOTIFY_OK;
1827 }
1828
1829 static struct notifier_block r4k_cache_pm_notifier_block = {
1830         .notifier_call = r4k_cache_pm_notifier,
1831 };
1832
1833 int __init r4k_cache_init_pm(void)
1834 {
1835         return cpu_pm_register_notifier(&r4k_cache_pm_notifier_block);
1836 }
1837 arch_initcall(r4k_cache_init_pm);