2 * Based on arch/arm/mm/mmu.c
4 * Copyright (C) 1995-2005 Russell King
5 * Copyright (C) 2012 ARM Ltd.
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 #include <linux/export.h>
21 #include <linux/kernel.h>
22 #include <linux/errno.h>
23 #include <linux/init.h>
24 #include <linux/mman.h>
25 #include <linux/nodemask.h>
26 #include <linux/memblock.h>
30 #include <asm/cputype.h>
31 #include <asm/fixmap.h>
32 #include <asm/sections.h>
33 #include <asm/setup.h>
34 #include <asm/sizes.h>
36 #include <asm/memblock.h>
37 #include <asm/mmu_context.h>
42 * Empty_zero_page is a special page that is used for zero-initialized data
45 struct page *empty_zero_page;
46 EXPORT_SYMBOL(empty_zero_page);
48 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
49 unsigned long size, pgprot_t vma_prot)
52 return pgprot_noncached(vma_prot);
53 else if (file->f_flags & O_SYNC)
54 return pgprot_writecombine(vma_prot);
57 EXPORT_SYMBOL(phys_mem_access_prot);
59 static void __init *early_alloc(unsigned long sz)
61 void *ptr = __va(memblock_alloc(sz, sz));
66 static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
67 unsigned long end, unsigned long pfn,
73 pte = early_alloc(PTRS_PER_PTE * sizeof(pte_t));
74 __pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE);
76 BUG_ON(pmd_bad(*pmd));
78 pte = pte_offset_kernel(pmd, addr);
80 set_pte(pte, pfn_pte(pfn, prot));
82 } while (pte++, addr += PAGE_SIZE, addr != end);
85 static void __init alloc_init_pmd(struct mm_struct *mm, pud_t *pud,
86 unsigned long addr, unsigned long end,
87 phys_addr_t phys, pgprot_t prot)
93 * Check for initial section mappings in the pgd/pud and remove them.
95 if (pud_none(*pud) || pud_bad(*pud)) {
96 pmd = early_alloc(PTRS_PER_PMD * sizeof(pmd_t));
97 pud_populate(mm, pud, pmd);
100 pmd = pmd_offset(pud, addr);
102 next = pmd_addr_end(addr, end);
103 /* try section mapping first */
104 if (((addr | next | phys) & ~SECTION_MASK) == 0) {
106 set_pmd(pmd, __pmd(phys |
107 pgprot_val(mk_sect_prot(prot))));
109 * Check for previous table entries created during
110 * boot (__create_page_tables) and flush them.
112 if (!pmd_none(old_pmd))
115 alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys),
119 } while (pmd++, addr = next, addr != end);
122 static void __init alloc_init_pud(struct mm_struct *mm, pgd_t *pgd,
123 unsigned long addr, unsigned long end,
124 phys_addr_t phys, pgprot_t prot)
129 if (pgd_none(*pgd)) {
130 pud = early_alloc(PTRS_PER_PUD * sizeof(pud_t));
131 pgd_populate(mm, pgd, pud);
133 BUG_ON(pgd_bad(*pgd));
135 pud = pud_offset(pgd, addr);
137 next = pud_addr_end(addr, end);
140 * For 4K granule only, attempt to put down a 1GB block
142 if ((PAGE_SHIFT == 12) &&
143 ((addr | next | phys) & ~PUD_MASK) == 0) {
144 pud_t old_pud = *pud;
145 set_pud(pud, __pud(phys |
146 pgprot_val(mk_sect_prot(prot))));
149 * If we have an old value for a pud, it will
150 * be pointing to a pmd table that we no longer
151 * need (from swapper_pg_dir).
153 * Look up the old pmd table and free it.
155 if (!pud_none(old_pud)) {
156 phys_addr_t table = __pa(pmd_offset(&old_pud, 0));
157 memblock_free(table, PAGE_SIZE);
161 alloc_init_pmd(mm, pud, addr, next, phys, prot);
164 } while (pud++, addr = next, addr != end);
168 * Create the page directory entries and any necessary page tables for the
169 * mapping specified by 'md'.
171 static void __init __create_mapping(struct mm_struct *mm, pgd_t *pgd,
172 phys_addr_t phys, unsigned long virt,
173 phys_addr_t size, pgprot_t prot)
175 unsigned long addr, length, end, next;
177 addr = virt & PAGE_MASK;
178 length = PAGE_ALIGN(size + (virt & ~PAGE_MASK));
182 next = pgd_addr_end(addr, end);
183 alloc_init_pud(mm, pgd, addr, next, phys, prot);
185 } while (pgd++, addr = next, addr != end);
188 static void __init create_mapping(phys_addr_t phys, unsigned long virt,
191 if (virt < VMALLOC_START) {
192 pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
196 __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), phys, virt,
197 size, PAGE_KERNEL_EXEC);
200 void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
201 unsigned long virt, phys_addr_t size,
204 __create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot);
207 static void __init map_mem(void)
209 struct memblock_region *reg;
213 * Temporarily limit the memblock range. We need to do this as
214 * create_mapping requires puds, pmds and ptes to be allocated from
215 * memory addressable from the initial direct kernel mapping.
217 * The initial direct kernel mapping, located at swapper_pg_dir, gives
218 * us PUD_SIZE (4K pages) or PMD_SIZE (64K pages) memory starting from
219 * PHYS_OFFSET (which must be aligned to 2MB as per
220 * Documentation/arm64/booting.txt).
222 if (IS_ENABLED(CONFIG_ARM64_64K_PAGES))
223 limit = PHYS_OFFSET + PMD_SIZE;
225 limit = PHYS_OFFSET + PUD_SIZE;
226 memblock_set_current_limit(limit);
228 /* map all the memory banks */
229 for_each_memblock(memory, reg) {
230 phys_addr_t start = reg->base;
231 phys_addr_t end = start + reg->size;
236 #ifndef CONFIG_ARM64_64K_PAGES
238 * For the first memory bank align the start address and
239 * current memblock limit to prevent create_mapping() from
240 * allocating pte page tables from unmapped memory.
241 * When 64K pages are enabled, the pte page table for the
242 * first PGDIR_SIZE is already present in swapper_pg_dir.
245 start = ALIGN(start, PMD_SIZE);
247 limit = end & PMD_MASK;
248 memblock_set_current_limit(limit);
252 create_mapping(start, __phys_to_virt(start), end - start);
255 /* Limit no longer required. */
256 memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
260 * paging_init() sets up the page tables, initialises the zone memory
261 * maps and sets up the zero page.
263 void __init paging_init(void)
270 * Finally flush the caches and tlb to ensure that we're in a
276 /* allocate the zero page. */
277 zero_page = early_alloc(PAGE_SIZE);
281 empty_zero_page = virt_to_page(zero_page);
284 * TTBR0 is only used for the identity mapping at this stage. Make it
285 * point to zero page to avoid speculatively fetching new entries.
287 cpu_set_reserved_ttbr0();
292 * Enable the identity mapping to allow the MMU disabling.
294 void setup_mm_for_reboot(void)
296 cpu_switch_mm(idmap_pg_dir, &init_mm);
301 * Check whether a kernel address is valid (derived from arch/x86/).
303 int kern_addr_valid(unsigned long addr)
310 if ((((long)addr) >> VA_BITS) != -1UL)
313 pgd = pgd_offset_k(addr);
317 pud = pud_offset(pgd, addr);
322 return pfn_valid(pud_pfn(*pud));
324 pmd = pmd_offset(pud, addr);
329 return pfn_valid(pmd_pfn(*pmd));
331 pte = pte_offset_kernel(pmd, addr);
335 return pfn_valid(pte_pfn(*pte));
337 #ifdef CONFIG_SPARSEMEM_VMEMMAP
338 #ifdef CONFIG_ARM64_64K_PAGES
339 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
341 return vmemmap_populate_basepages(start, end, node);
343 #else /* !CONFIG_ARM64_64K_PAGES */
344 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
346 unsigned long addr = start;
353 next = pmd_addr_end(addr, end);
355 pgd = vmemmap_pgd_populate(addr, node);
359 pud = vmemmap_pud_populate(pgd, addr, node);
363 pmd = pmd_offset(pud, addr);
364 if (pmd_none(*pmd)) {
367 p = vmemmap_alloc_block_buf(PMD_SIZE, node);
371 set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL));
373 vmemmap_verify((pte_t *)pmd, node, addr, next);
374 } while (addr = next, addr != end);
378 #endif /* CONFIG_ARM64_64K_PAGES */
379 void vmemmap_free(unsigned long start, unsigned long end)
382 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
384 static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss;
385 #if CONFIG_ARM64_PGTABLE_LEVELS > 2
386 static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss;
388 #if CONFIG_ARM64_PGTABLE_LEVELS > 3
389 static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss;
392 static inline pud_t * fixmap_pud(unsigned long addr)
394 pgd_t *pgd = pgd_offset_k(addr);
396 BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd));
398 return pud_offset(pgd, addr);
401 static inline pmd_t * fixmap_pmd(unsigned long addr)
403 pud_t *pud = fixmap_pud(addr);
405 BUG_ON(pud_none(*pud) || pud_bad(*pud));
407 return pmd_offset(pud, addr);
410 static inline pte_t * fixmap_pte(unsigned long addr)
412 pmd_t *pmd = fixmap_pmd(addr);
414 BUG_ON(pmd_none(*pmd) || pmd_bad(*pmd));
416 return pte_offset_kernel(pmd, addr);
419 void __init early_fixmap_init(void)
424 unsigned long addr = FIXADDR_START;
426 pgd = pgd_offset_k(addr);
427 pgd_populate(&init_mm, pgd, bm_pud);
428 pud = pud_offset(pgd, addr);
429 pud_populate(&init_mm, pud, bm_pmd);
430 pmd = pmd_offset(pud, addr);
431 pmd_populate_kernel(&init_mm, pmd, bm_pte);
434 * The boot-ioremap range spans multiple pmds, for which
435 * we are not preparted:
437 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
438 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
440 if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)))
441 || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) {
443 pr_warn("pmd %p != %p, %p\n",
444 pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)),
445 fixmap_pmd(fix_to_virt(FIX_BTMAP_END)));
446 pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
447 fix_to_virt(FIX_BTMAP_BEGIN));
448 pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n",
449 fix_to_virt(FIX_BTMAP_END));
451 pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
452 pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN);
456 void __set_fixmap(enum fixed_addresses idx,
457 phys_addr_t phys, pgprot_t flags)
459 unsigned long addr = __fix_to_virt(idx);
462 if (idx >= __end_of_fixed_addresses) {
467 pte = fixmap_pte(addr);
469 if (pgprot_val(flags)) {
470 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
472 pte_clear(&init_mm, addr, pte);
473 flush_tlb_kernel_range(addr, addr+PAGE_SIZE);