arch/s390/mm/vmem.c

   1 /*
   2  *    Copyright IBM Corp. 2006
   3  *    Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
   4  */
   5
   6 #include <linux/bootmem.h>
   7 #include <linux/pfn.h>
   8 #include <linux/mm.h>
   9 #include <linux/module.h>
  10 #include <linux/list.h>
  11 #include <linux/hugetlb.h>
  12 #include <linux/slab.h>
  13 #include <linux/memblock.h>
  14 #include <asm/pgalloc.h>
  15 #include <asm/pgtable.h>
  16 #include <asm/setup.h>
  17 #include <asm/tlbflush.h>
  18 #include <asm/sections.h>
  19
  20 static DEFINE_MUTEX(vmem_mutex);
  21
  22 struct memory_segment {
  23         struct list_head list;
  24         unsigned long start;
  25         unsigned long size;
  26 };
  27
  28 static LIST_HEAD(mem_segs);
  29
  30 static void __ref *vmem_alloc_pages(unsigned int order)
  31 {
  32         if (slab_is_available())
  33                 return (void *)__get_free_pages(GFP_KERNEL, order);
  34         return alloc_bootmem_pages((1 << order) * PAGE_SIZE);
  35 }
  36
  37 static inline pud_t *vmem_pud_alloc(void)
  38 {
  39         pud_t *pud = NULL;
  40
  41         pud = vmem_alloc_pages(2);
  42         if (!pud)
  43                 return NULL;
  44         clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4);
  45         return pud;
  46 }
  47
  48 static inline pmd_t *vmem_pmd_alloc(void)
  49 {
  50         pmd_t *pmd = NULL;
  51
  52         pmd = vmem_alloc_pages(2);
  53         if (!pmd)
  54                 return NULL;
  55         clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4);
  56         return pmd;
  57 }
  58
  59 static pte_t __ref *vmem_pte_alloc(void)
  60 {
  61         pte_t *pte;
  62
  63         if (slab_is_available())
  64                 pte = (pte_t *) page_table_alloc(&init_mm);
  65         else
  66                 pte = alloc_bootmem_align(PTRS_PER_PTE * sizeof(pte_t),
  67                                           PTRS_PER_PTE * sizeof(pte_t));
  68         if (!pte)
  69                 return NULL;
  70         clear_table((unsigned long *) pte, _PAGE_INVALID,
  71                     PTRS_PER_PTE * sizeof(pte_t));
  72         return pte;
  73 }
  74
  75 /*
  76  * Add a physical memory range to the 1:1 mapping.
  77  */
  78 static int vmem_add_mem(unsigned long start, unsigned long size, int ro)
  79 {
  80         unsigned long end = start + size;
  81         unsigned long address = start;
  82         pgd_t *pg_dir;
  83         pud_t *pu_dir;
  84         pmd_t *pm_dir;
  85         pte_t *pt_dir;
  86         int ret = -ENOMEM;
  87
  88         while (address < end) {
  89                 pg_dir = pgd_offset_k(address);
  90                 if (pgd_none(*pg_dir)) {
  91                         pu_dir = vmem_pud_alloc();
  92                         if (!pu_dir)
  93                                 goto out;
  94                         pgd_populate(&init_mm, pg_dir, pu_dir);
  95                 }
  96                 pu_dir = pud_offset(pg_dir, address);
  97                 if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address &&
  98                     !(address & ~PUD_MASK) && (address + PUD_SIZE <= end) &&
  99                      !debug_pagealloc_enabled()) {
 100                         pud_val(*pu_dir) = __pa(address) |
 101                                 _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE |
 102                                 (ro ? _REGION_ENTRY_PROTECT : 0);
 103                         address += PUD_SIZE;
 104                         continue;
 105                 }
 106                 if (pud_none(*pu_dir)) {
 107                         pm_dir = vmem_pmd_alloc();
 108                         if (!pm_dir)
 109                                 goto out;
 110                         pud_populate(&init_mm, pu_dir, pm_dir);
 111                 }
 112                 pm_dir = pmd_offset(pu_dir, address);
 113                 if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address &&
 114                     !(address & ~PMD_MASK) && (address + PMD_SIZE <= end) &&
 115                     !debug_pagealloc_enabled()) {
 116                         pmd_val(*pm_dir) = __pa(address) |
 117                                 _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE |
 118                                 _SEGMENT_ENTRY_YOUNG |
 119                                 (ro ? _SEGMENT_ENTRY_PROTECT : 0);
 120                         address += PMD_SIZE;
 121                         continue;
 122                 }
 123                 if (pmd_none(*pm_dir)) {
 124                         pt_dir = vmem_pte_alloc();
 125                         if (!pt_dir)
 126                                 goto out;
 127                         pmd_populate(&init_mm, pm_dir, pt_dir);
 128                 }
 129
 130                 pt_dir = pte_offset_kernel(pm_dir, address);
 131                 pte_val(*pt_dir) = __pa(address) |
 132                         pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL);
 133                 address += PAGE_SIZE;
 134         }
 135         ret = 0;
 136 out:
 137         return ret;
 138 }
 139
 140 /*
 141  * Remove a physical memory range from the 1:1 mapping.
 142  * Currently only invalidates page table entries.
 143  */
 144 static void vmem_remove_range(unsigned long start, unsigned long size)
 145 {
 146         unsigned long end = start + size;
 147         unsigned long address = start;
 148         pgd_t *pg_dir;
 149         pud_t *pu_dir;
 150         pmd_t *pm_dir;
 151         pte_t *pt_dir;
 152         pte_t  pte;
 153
 154         pte_val(pte) = _PAGE_INVALID;
 155         while (address < end) {
 156                 pg_dir = pgd_offset_k(address);
 157                 if (pgd_none(*pg_dir)) {
 158                         address += PGDIR_SIZE;
 159                         continue;
 160                 }
 161                 pu_dir = pud_offset(pg_dir, address);
 162                 if (pud_none(*pu_dir)) {
 163                         address += PUD_SIZE;
 164                         continue;
 165                 }
 166                 if (pud_large(*pu_dir)) {
 167                         pud_clear(pu_dir);
 168                         address += PUD_SIZE;
 169                         continue;
 170                 }
 171                 pm_dir = pmd_offset(pu_dir, address);
 172                 if (pmd_none(*pm_dir)) {
 173                         address += PMD_SIZE;
 174                         continue;
 175                 }
 176                 if (pmd_large(*pm_dir)) {
 177                         pmd_clear(pm_dir);
 178                         address += PMD_SIZE;
 179                         continue;
 180                 }
 181                 pt_dir = pte_offset_kernel(pm_dir, address);
 182                 *pt_dir = pte;
 183                 address += PAGE_SIZE;
 184         }
 185         flush_tlb_kernel_range(start, end);
 186 }
 187
 188 /*
 189  * Add a backed mem_map array to the virtual mem_map array.
 190  */
 191 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
 192 {
 193         unsigned long address = start;
 194         pgd_t *pg_dir;
 195         pud_t *pu_dir;
 196         pmd_t *pm_dir;
 197         pte_t *pt_dir;
 198         int ret = -ENOMEM;
 199
 200         for (address = start; address < end;) {
 201                 pg_dir = pgd_offset_k(address);
 202                 if (pgd_none(*pg_dir)) {
 203                         pu_dir = vmem_pud_alloc();
 204                         if (!pu_dir)
 205                                 goto out;
 206                         pgd_populate(&init_mm, pg_dir, pu_dir);
 207                 }
 208
 209                 pu_dir = pud_offset(pg_dir, address);
 210                 if (pud_none(*pu_dir)) {
 211                         pm_dir = vmem_pmd_alloc();
 212                         if (!pm_dir)
 213                                 goto out;
 214                         pud_populate(&init_mm, pu_dir, pm_dir);
 215                 }
 216
 217                 pm_dir = pmd_offset(pu_dir, address);
 218                 if (pmd_none(*pm_dir)) {
 219                         /* Use 1MB frames for vmemmap if available. We always
 220                          * use large frames even if they are only partially
 221                          * used.
 222                          * Otherwise we would have also page tables since
 223                          * vmemmap_populate gets called for each section
 224                          * separately. */
 225                         if (MACHINE_HAS_EDAT1) {
 226                                 void *new_page;
 227
 228                                 new_page = vmemmap_alloc_block(PMD_SIZE, node);
 229                                 if (!new_page)
 230                                         goto out;
 231                                 pmd_val(*pm_dir) = __pa(new_page) |
 232                                         _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE;
 233                                 address = (address + PMD_SIZE) & PMD_MASK;
 234                                 continue;
 235                         }
 236                         pt_dir = vmem_pte_alloc();
 237                         if (!pt_dir)
 238                                 goto out;
 239                         pmd_populate(&init_mm, pm_dir, pt_dir);
 240                 } else if (pmd_large(*pm_dir)) {
 241                         address = (address + PMD_SIZE) & PMD_MASK;
 242                         continue;
 243                 }
 244
 245                 pt_dir = pte_offset_kernel(pm_dir, address);
 246                 if (pte_none(*pt_dir)) {
 247                         void *new_page;
 248
 249                         new_page = vmemmap_alloc_block(PAGE_SIZE, node);
 250                         if (!new_page)
 251                                 goto out;
 252                         pte_val(*pt_dir) =
 253                                 __pa(new_page) | pgprot_val(PAGE_KERNEL);
 254                 }
 255                 address += PAGE_SIZE;
 256         }
 257         ret = 0;
 258 out:
 259         return ret;
 260 }
 261
 262 void vmemmap_free(unsigned long start, unsigned long end)
 263 {
 264 }
 265
 266 /*
 267  * Add memory segment to the segment list if it doesn't overlap with
 268  * an already present segment.
 269  */
 270 static int insert_memory_segment(struct memory_segment *seg)
 271 {
 272         struct memory_segment *tmp;
 273
 274         if (seg->start + seg->size > VMEM_MAX_PHYS ||
 275             seg->start + seg->size < seg->start)
 276                 return -ERANGE;
 277
 278         list_for_each_entry(tmp, &mem_segs, list) {
 279                 if (seg->start >= tmp->start + tmp->size)
 280                         continue;
 281                 if (seg->start + seg->size <= tmp->start)
 282                         continue;
 283                 return -ENOSPC;
 284         }
 285         list_add(&seg->list, &mem_segs);
 286         return 0;
 287 }
 288
 289 /*
 290  * Remove memory segment from the segment list.
 291  */
 292 static void remove_memory_segment(struct memory_segment *seg)
 293 {
 294         list_del(&seg->list);
 295 }
 296
 297 static void __remove_shared_memory(struct memory_segment *seg)
 298 {
 299         remove_memory_segment(seg);
 300         vmem_remove_range(seg->start, seg->size);
 301 }
 302
 303 int vmem_remove_mapping(unsigned long start, unsigned long size)
 304 {
 305         struct memory_segment *seg;
 306         int ret;
 307
 308         mutex_lock(&vmem_mutex);
 309
 310         ret = -ENOENT;
 311         list_for_each_entry(seg, &mem_segs, list) {
 312                 if (seg->start == start && seg->size == size)
 313                         break;
 314         }
 315
 316         if (seg->start != start || seg->size != size)
 317                 goto out;
 318
 319         ret = 0;
 320         __remove_shared_memory(seg);
 321         kfree(seg);
 322 out:
 323         mutex_unlock(&vmem_mutex);
 324         return ret;
 325 }
 326
 327 int vmem_add_mapping(unsigned long start, unsigned long size)
 328 {
 329         struct memory_segment *seg;
 330         int ret;
 331
 332         mutex_lock(&vmem_mutex);
 333         ret = -ENOMEM;
 334         seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 335         if (!seg)
 336                 goto out;
 337         seg->start = start;
 338         seg->size = size;
 339
 340         ret = insert_memory_segment(seg);
 341         if (ret)
 342                 goto out_free;
 343
 344         ret = vmem_add_mem(start, size, 0);
 345         if (ret)
 346                 goto out_remove;
 347         goto out;
 348
 349 out_remove:
 350         __remove_shared_memory(seg);
 351 out_free:
 352         kfree(seg);
 353 out:
 354         mutex_unlock(&vmem_mutex);
 355         return ret;
 356 }
 357
 358 /*
 359  * map whole physical memory to virtual memory (identity mapping)
 360  * we reserve enough space in the vmalloc area for vmemmap to hotplug
 361  * additional memory segments.
 362  */
 363 void __init vmem_map_init(void)
 364 {
 365         unsigned long ro_start, ro_end;
 366         struct memblock_region *reg;
 367         phys_addr_t start, end;
 368
 369         ro_start = PFN_ALIGN((unsigned long)&_stext);
 370         ro_end = (unsigned long)&_eshared & PAGE_MASK;
 371         for_each_memblock(memory, reg) {
 372                 start = reg->base;
 373                 end = reg->base + reg->size;
 374                 if (start >= ro_end || end <= ro_start)
 375                         vmem_add_mem(start, end - start, 0);
 376                 else if (start >= ro_start && end <= ro_end)
 377                         vmem_add_mem(start, end - start, 1);
 378                 else if (start >= ro_start) {
 379                         vmem_add_mem(start, ro_end - start, 1);
 380                         vmem_add_mem(ro_end, end - ro_end, 0);
 381                 } else if (end < ro_end) {
 382                         vmem_add_mem(start, ro_start - start, 0);
 383                         vmem_add_mem(ro_start, end - ro_start, 1);
 384                 } else {
 385                         vmem_add_mem(start, ro_start - start, 0);
 386                         vmem_add_mem(ro_start, ro_end - ro_start, 1);
 387                         vmem_add_mem(ro_end, end - ro_end, 0);
 388                 }
 389         }
 390 }
 391
 392 /*
 393  * Convert memblock.memory  to a memory segment list so there is a single
 394  * list that contains all memory segments.
 395  */
 396 static int __init vmem_convert_memory_chunk(void)
 397 {
 398         struct memblock_region *reg;
 399         struct memory_segment *seg;
 400
 401         mutex_lock(&vmem_mutex);
 402         for_each_memblock(memory, reg) {
 403                 seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 404                 if (!seg)
 405                         panic("Out of memory...\n");
 406                 seg->start = reg->base;
 407                 seg->size = reg->size;
 408                 insert_memory_segment(seg);
 409         }
 410         mutex_unlock(&vmem_mutex);
 411         return 0;
 412 }
 413
 414 core_initcall(vmem_convert_memory_chunk);