arch/mips/mm/tlb-r4k.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
   7  * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
   8  * Carsten Langgaard, carstenl@mips.com
   9  * Copyright (C) 2002 MIPS Technologies, Inc.  All rights reserved.
  10  */
  11 #include <linux/init.h>
  12 #include <linux/sched.h>
  13 #include <linux/smp.h>
  14 #include <linux/mm.h>
  15 #include <linux/hugetlb.h>
  16
  17 #include <asm/cpu.h>
  18 #include <asm/bootinfo.h>
  19 #include <asm/mmu_context.h>
  20 #include <asm/pgtable.h>
  21 #include <asm/system.h>
  22 #include <asm/tlbmisc.h>
  23
  24 extern void build_tlb_refill_handler(void);
  25
  26 /*
  27  * Make sure all entries differ.  If they're not different
  28  * MIPS32 will take revenge ...
  29  */
  30 #define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
  31
  32 /* Atomicity and interruptability */
  33 #ifdef CONFIG_MIPS_MT_SMTC
  34
  35 #include <asm/smtc.h>
  36 #include <asm/mipsmtregs.h>
  37
  38 #define ENTER_CRITICAL(flags) \
  39         { \
  40         unsigned int mvpflags; \
  41         local_irq_save(flags);\
  42         mvpflags = dvpe()
  43 #define EXIT_CRITICAL(flags) \
  44         evpe(mvpflags); \
  45         local_irq_restore(flags); \
  46         }
  47 #else
  48
  49 #define ENTER_CRITICAL(flags) local_irq_save(flags)
  50 #define EXIT_CRITICAL(flags) local_irq_restore(flags)
  51
  52 #endif /* CONFIG_MIPS_MT_SMTC */
  53
  54 #if defined(CONFIG_CPU_LOONGSON2)
  55 /*
  56  * LOONGSON2 has a 4 entry itlb which is a subset of dtlb,
  57  * unfortrunately, itlb is not totally transparent to software.
  58  */
  59 #define FLUSH_ITLB write_c0_diag(4);
  60
  61 #define FLUSH_ITLB_VM(vma) { if ((vma)->vm_flags & VM_EXEC)  write_c0_diag(4); }
  62
  63 #else
  64
  65 #define FLUSH_ITLB
  66 #define FLUSH_ITLB_VM(vma)
  67
  68 #endif
  69
  70 void local_flush_tlb_all(void)
  71 {
  72         unsigned long flags;
  73         unsigned long old_ctx;
  74         int entry;
  75
  76         ENTER_CRITICAL(flags);
  77         /* Save old context and create impossible VPN2 value */
  78         old_ctx = read_c0_entryhi();
  79         write_c0_entrylo0(0);
  80         write_c0_entrylo1(0);
  81
  82         entry = read_c0_wired();
  83
  84         /* Blast 'em all away. */
  85         while (entry < current_cpu_data.tlbsize) {
  86                 /* Make sure all entries differ. */
  87                 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
  88                 write_c0_index(entry);
  89                 mtc0_tlbw_hazard();
  90                 tlb_write_indexed();
  91                 entry++;
  92         }
  93         tlbw_use_hazard();
  94         write_c0_entryhi(old_ctx);
  95         FLUSH_ITLB;
  96         EXIT_CRITICAL(flags);
  97 }
  98
  99 /* All entries common to a mm share an asid.  To effectively flush
 100    these entries, we just bump the asid. */
 101 void local_flush_tlb_mm(struct mm_struct *mm)
 102 {
 103         int cpu;
 104
 105         preempt_disable();
 106
 107         cpu = smp_processor_id();
 108
 109         if (cpu_context(cpu, mm) != 0) {
 110                 drop_mmu_context(mm, cpu);
 111         }
 112
 113         preempt_enable();
 114 }
 115
 116 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 117         unsigned long end)
 118 {
 119         struct mm_struct *mm = vma->vm_mm;
 120         int cpu = smp_processor_id();
 121
 122         if (cpu_context(cpu, mm) != 0) {
 123                 unsigned long size, flags;
 124                 int huge = is_vm_hugetlb_page(vma);
 125
 126                 ENTER_CRITICAL(flags);
 127                 if (huge) {
 128                         start = round_down(start, HPAGE_SIZE);
 129                         end = round_up(end, HPAGE_SIZE);
 130                         size = (end - start) >> HPAGE_SHIFT;
 131                 } else {
 132                         start = round_down(start, PAGE_SIZE << 1);
 133                         end = round_up(end, PAGE_SIZE << 1);
 134                         size = (end - start) >> (PAGE_SHIFT + 1);
 135                 }
 136                 if (size <= current_cpu_data.tlbsize/2) {
 137                         int oldpid = read_c0_entryhi();
 138                         int newpid = cpu_asid(cpu, mm);
 139
 140                         while (start < end) {
 141                                 int idx;
 142
 143                                 write_c0_entryhi(start | newpid);
 144                                 if (huge)
 145                                         start += HPAGE_SIZE;
 146                                 else
 147                                         start += (PAGE_SIZE << 1);
 148                                 mtc0_tlbw_hazard();
 149                                 tlb_probe();
 150                                 tlb_probe_hazard();
 151                                 idx = read_c0_index();
 152                                 write_c0_entrylo0(0);
 153                                 write_c0_entrylo1(0);
 154                                 if (idx < 0)
 155                                         continue;
 156                                 /* Make sure all entries differ. */
 157                                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 158                                 mtc0_tlbw_hazard();
 159                                 tlb_write_indexed();
 160                         }
 161                         tlbw_use_hazard();
 162                         write_c0_entryhi(oldpid);
 163                 } else {
 164                         drop_mmu_context(mm, cpu);
 165                 }
 166                 FLUSH_ITLB;
 167                 EXIT_CRITICAL(flags);
 168         }
 169 }
 170
 171 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 172 {
 173         unsigned long size, flags;
 174
 175         ENTER_CRITICAL(flags);
 176         size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
 177         size = (size + 1) >> 1;
 178         if (size <= current_cpu_data.tlbsize / 2) {
 179                 int pid = read_c0_entryhi();
 180
 181                 start &= (PAGE_MASK << 1);
 182                 end += ((PAGE_SIZE << 1) - 1);
 183                 end &= (PAGE_MASK << 1);
 184
 185                 while (start < end) {
 186                         int idx;
 187
 188                         write_c0_entryhi(start);
 189                         start += (PAGE_SIZE << 1);
 190                         mtc0_tlbw_hazard();
 191                         tlb_probe();
 192                         tlb_probe_hazard();
 193                         idx = read_c0_index();
 194                         write_c0_entrylo0(0);
 195                         write_c0_entrylo1(0);
 196                         if (idx < 0)
 197                                 continue;
 198                         /* Make sure all entries differ. */
 199                         write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 200                         mtc0_tlbw_hazard();
 201                         tlb_write_indexed();
 202                 }
 203                 tlbw_use_hazard();
 204                 write_c0_entryhi(pid);
 205         } else {
 206                 local_flush_tlb_all();
 207         }
 208         FLUSH_ITLB;
 209         EXIT_CRITICAL(flags);
 210 }
 211
 212 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 213 {
 214         int cpu = smp_processor_id();
 215
 216         if (cpu_context(cpu, vma->vm_mm) != 0) {
 217                 unsigned long flags;
 218                 int oldpid, newpid, idx;
 219
 220                 newpid = cpu_asid(cpu, vma->vm_mm);
 221                 page &= (PAGE_MASK << 1);
 222                 ENTER_CRITICAL(flags);
 223                 oldpid = read_c0_entryhi();
 224                 write_c0_entryhi(page | newpid);
 225                 mtc0_tlbw_hazard();
 226                 tlb_probe();
 227                 tlb_probe_hazard();
 228                 idx = read_c0_index();
 229                 write_c0_entrylo0(0);
 230                 write_c0_entrylo1(0);
 231                 if (idx < 0)
 232                         goto finish;
 233                 /* Make sure all entries differ. */
 234                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 235                 mtc0_tlbw_hazard();
 236                 tlb_write_indexed();
 237                 tlbw_use_hazard();
 238
 239         finish:
 240                 write_c0_entryhi(oldpid);
 241                 FLUSH_ITLB_VM(vma);
 242                 EXIT_CRITICAL(flags);
 243         }
 244 }
 245
 246 /*
 247  * This one is only used for pages with the global bit set so we don't care
 248  * much about the ASID.
 249  */
 250 void local_flush_tlb_one(unsigned long page)
 251 {
 252         unsigned long flags;
 253         int oldpid, idx;
 254
 255         ENTER_CRITICAL(flags);
 256         oldpid = read_c0_entryhi();
 257         page &= (PAGE_MASK << 1);
 258         write_c0_entryhi(page);
 259         mtc0_tlbw_hazard();
 260         tlb_probe();
 261         tlb_probe_hazard();
 262         idx = read_c0_index();
 263         write_c0_entrylo0(0);
 264         write_c0_entrylo1(0);
 265         if (idx >= 0) {
 266                 /* Make sure all entries differ. */
 267                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 268                 mtc0_tlbw_hazard();
 269                 tlb_write_indexed();
 270                 tlbw_use_hazard();
 271         }
 272         write_c0_entryhi(oldpid);
 273         FLUSH_ITLB;
 274         EXIT_CRITICAL(flags);
 275 }
 276
 277 /*
 278  * We will need multiple versions of update_mmu_cache(), one that just
 279  * updates the TLB with the new pte(s), and another which also checks
 280  * for the R4k "end of page" hardware bug and does the needy.
 281  */
 282 void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
 283 {
 284         unsigned long flags;
 285         pgd_t *pgdp;
 286         pud_t *pudp;
 287         pmd_t *pmdp;
 288         pte_t *ptep;
 289         int idx, pid;
 290
 291         /*
 292          * Handle debugger faulting in for debugee.
 293          */
 294         if (current->active_mm != vma->vm_mm)
 295                 return;
 296
 297         ENTER_CRITICAL(flags);
 298
 299         pid = read_c0_entryhi() & ASID_MASK;
 300         address &= (PAGE_MASK << 1);
 301         write_c0_entryhi(address | pid);
 302         pgdp = pgd_offset(vma->vm_mm, address);
 303         mtc0_tlbw_hazard();
 304         tlb_probe();
 305         tlb_probe_hazard();
 306         pudp = pud_offset(pgdp, address);
 307         pmdp = pmd_offset(pudp, address);
 308         idx = read_c0_index();
 309 #ifdef CONFIG_HUGETLB_PAGE
 310         /* this could be a huge page  */
 311         if (pmd_huge(*pmdp)) {
 312                 unsigned long lo;
 313                 write_c0_pagemask(PM_HUGE_MASK);
 314                 ptep = (pte_t *)pmdp;
 315                 lo = pte_to_entrylo(pte_val(*ptep));
 316                 write_c0_entrylo0(lo);
 317                 write_c0_entrylo1(lo + (HPAGE_SIZE >> 7));
 318
 319                 mtc0_tlbw_hazard();
 320                 if (idx < 0)
 321                         tlb_write_random();
 322                 else
 323                         tlb_write_indexed();
 324                 write_c0_pagemask(PM_DEFAULT_MASK);
 325         } else
 326 #endif
 327         {
 328                 ptep = pte_offset_map(pmdp, address);
 329
 330 #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)
 331                 write_c0_entrylo0(ptep->pte_high);
 332                 ptep++;
 333                 write_c0_entrylo1(ptep->pte_high);
 334 #else
 335                 write_c0_entrylo0(pte_to_entrylo(pte_val(*ptep++)));
 336                 write_c0_entrylo1(pte_to_entrylo(pte_val(*ptep)));
 337 #endif
 338                 mtc0_tlbw_hazard();
 339                 if (idx < 0)
 340                         tlb_write_random();
 341                 else
 342                         tlb_write_indexed();
 343         }
 344         tlbw_use_hazard();
 345         FLUSH_ITLB_VM(vma);
 346         EXIT_CRITICAL(flags);
 347 }
 348
 349 void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
 350                      unsigned long entryhi, unsigned long pagemask)
 351 {
 352         unsigned long flags;
 353         unsigned long wired;
 354         unsigned long old_pagemask;
 355         unsigned long old_ctx;
 356
 357         ENTER_CRITICAL(flags);
 358         /* Save old context and create impossible VPN2 value */
 359         old_ctx = read_c0_entryhi();
 360         old_pagemask = read_c0_pagemask();
 361         wired = read_c0_wired();
 362         write_c0_wired(wired + 1);
 363         write_c0_index(wired);
 364         tlbw_use_hazard();      /* What is the hazard here? */
 365         write_c0_pagemask(pagemask);
 366         write_c0_entryhi(entryhi);
 367         write_c0_entrylo0(entrylo0);
 368         write_c0_entrylo1(entrylo1);
 369         mtc0_tlbw_hazard();
 370         tlb_write_indexed();
 371         tlbw_use_hazard();
 372
 373         write_c0_entryhi(old_ctx);
 374         tlbw_use_hazard();      /* What is the hazard here? */
 375         write_c0_pagemask(old_pagemask);
 376         local_flush_tlb_all();
 377         EXIT_CRITICAL(flags);
 378 }
 379
 380 static int __cpuinitdata ntlb;
 381 static int __init set_ntlb(char *str)
 382 {
 383         get_option(&str, &ntlb);
 384         return 1;
 385 }
 386
 387 __setup("ntlb=", set_ntlb);
 388
 389 void __cpuinit tlb_init(void)
 390 {
 391         /*
 392          * You should never change this register:
 393          *   - On R4600 1.7 the tlbp never hits for pages smaller than
 394          *     the value in the c0_pagemask register.
 395          *   - The entire mm handling assumes the c0_pagemask register to
 396          *     be set to fixed-size pages.
 397          */
 398         write_c0_pagemask(PM_DEFAULT_MASK);
 399         write_c0_wired(0);
 400         if (current_cpu_type() == CPU_R10000 ||
 401             current_cpu_type() == CPU_R12000 ||
 402             current_cpu_type() == CPU_R14000)
 403                 write_c0_framemask(0);
 404
 405         if (kernel_uses_smartmips_rixi) {
 406                 /*
 407                  * Enable the no read, no exec bits, and enable large virtual
 408                  * address.
 409                  */
 410                 u32 pg = PG_RIE | PG_XIE;
 411 #ifdef CONFIG_64BIT
 412                 pg |= PG_ELPA;
 413 #endif
 414                 write_c0_pagegrain(pg);
 415         }
 416
 417         /* From this point on the ARC firmware is dead.  */
 418         local_flush_tlb_all();
 419
 420         /* Did I tell you that ARC SUCKS?  */
 421
 422         if (ntlb) {
 423                 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) {
 424                         int wired = current_cpu_data.tlbsize - ntlb;
 425                         write_c0_wired(wired);
 426                         write_c0_index(wired-1);
 427                         printk("Restricting TLB to %d entries\n", ntlb);
 428                 } else
 429                         printk("Ignoring invalid argument ntlb=%d\n", ntlb);
 430         }
 431
 432         build_tlb_refill_handler();
 433 }