arch/x86/platform/efi/efi.c

   1 /*
   2  * Common EFI (Extensible Firmware Interface) support functions
   3  * Based on Extensible Firmware Interface Specification version 1.0
   4  *
   5  * Copyright (C) 1999 VA Linux Systems
   6  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
   7  * Copyright (C) 1999-2002 Hewlett-Packard Co.
   8  *      David Mosberger-Tang <davidm@hpl.hp.com>
   9  *      Stephane Eranian <eranian@hpl.hp.com>
  10  * Copyright (C) 2005-2008 Intel Co.
  11  *      Fenghua Yu <fenghua.yu@intel.com>
  12  *      Bibo Mao <bibo.mao@intel.com>
  13  *      Chandramouli Narayanan <mouli@linux.intel.com>
  14  *      Huang Ying <ying.huang@intel.com>
  15  * Copyright (C) 2013 SuSE Labs
  16  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
  17  *
  18  * Copied from efi_32.c to eliminate the duplicated code between EFI
  19  * 32/64 support code. --ying 2007-10-26
  20  *
  21  * All EFI Runtime Services are not implemented yet as EFI only
  22  * supports physical mode addressing on SoftSDV. This is to be fixed
  23  * in a future version.  --drummond 1999-07-20
  24  *
  25  * Implemented EFI runtime services and virtual mode calls.  --davidm
  26  *
  27  * Goutham Rao: <goutham.rao@intel.com>
  28  *      Skip non-WB memory and ignore empty memory ranges.
  29  */
  30
  31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  32
  33 #include <linux/kernel.h>
  34 #include <linux/init.h>
  35 #include <linux/efi.h>
  36 #include <linux/efi-bgrt.h>
  37 #include <linux/export.h>
  38 #include <linux/bootmem.h>
  39 #include <linux/slab.h>
  40 #include <linux/memblock.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/time.h>
  44 #include <linux/io.h>
  45 #include <linux/reboot.h>
  46 #include <linux/bcd.h>
  47
  48 #include <asm/setup.h>
  49 #include <asm/efi.h>
  50 #include <asm/time.h>
  51 #include <asm/cacheflush.h>
  52 #include <asm/tlbflush.h>
  53 #include <asm/x86_init.h>
  54 #include <asm/uv/uv.h>
  55
  56 static struct efi efi_phys __initdata;
  57 static efi_system_table_t efi_systab __initdata;
  58
  59 static efi_config_table_type_t arch_tables[] __initdata = {
  60 #ifdef CONFIG_X86_UV
  61         {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
  62 #endif
  63         {NULL_GUID, NULL, NULL},
  64 };
  65
  66 u64 efi_setup;          /* efi setup_data physical address */
  67
  68 static int add_efi_memmap __initdata;
  69 static int __init setup_add_efi_memmap(char *arg)
  70 {
  71         add_efi_memmap = 1;
  72         return 0;
  73 }
  74 early_param("add_efi_memmap", setup_add_efi_memmap);
  75
  76 static efi_status_t __init phys_efi_set_virtual_address_map(
  77         unsigned long memory_map_size,
  78         unsigned long descriptor_size,
  79         u32 descriptor_version,
  80         efi_memory_desc_t *virtual_map)
  81 {
  82         efi_status_t status;
  83         unsigned long flags;
  84         pgd_t *save_pgd;
  85
  86         save_pgd = efi_call_phys_prolog();
  87
  88         /* Disable interrupts around EFI calls: */
  89         local_irq_save(flags);
  90         status = efi_call_phys(efi_phys.set_virtual_address_map,
  91                                memory_map_size, descriptor_size,
  92                                descriptor_version, virtual_map);
  93         local_irq_restore(flags);
  94
  95         efi_call_phys_epilog(save_pgd);
  96
  97         return status;
  98 }
  99
 100 void efi_get_time(struct timespec *now)
 101 {
 102         efi_status_t status;
 103         efi_time_t eft;
 104         efi_time_cap_t cap;
 105
 106         status = efi.get_time(&eft, &cap);
 107         if (status != EFI_SUCCESS)
 108                 pr_err("Oops: efitime: can't read time!\n");
 109
 110         now->tv_sec = mktime(eft.year, eft.month, eft.day, eft.hour,
 111                              eft.minute, eft.second);
 112         now->tv_nsec = 0;
 113 }
 114
 115 void __init efi_find_mirror(void)
 116 {
 117         efi_memory_desc_t *md;
 118         u64 mirror_size = 0, total_size = 0;
 119
 120         for_each_efi_memory_desc(md) {
 121                 unsigned long long start = md->phys_addr;
 122                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 123
 124                 total_size += size;
 125                 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
 126                         memblock_mark_mirror(start, size);
 127                         mirror_size += size;
 128                 }
 129         }
 130         if (mirror_size)
 131                 pr_info("Memory: %lldM/%lldM mirrored memory\n",
 132                         mirror_size>>20, total_size>>20);
 133 }
 134
 135 /*
 136  * Tell the kernel about the EFI memory map.  This might include
 137  * more than the max 128 entries that can fit in the e820 legacy
 138  * (zeropage) memory map.
 139  */
 140
 141 static void __init do_add_efi_memmap(void)
 142 {
 143         efi_memory_desc_t *md;
 144
 145         for_each_efi_memory_desc(md) {
 146                 unsigned long long start = md->phys_addr;
 147                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 148                 int e820_type;
 149
 150                 switch (md->type) {
 151                 case EFI_LOADER_CODE:
 152                 case EFI_LOADER_DATA:
 153                 case EFI_BOOT_SERVICES_CODE:
 154                 case EFI_BOOT_SERVICES_DATA:
 155                 case EFI_CONVENTIONAL_MEMORY:
 156                         if (md->attribute & EFI_MEMORY_WB)
 157                                 e820_type = E820_RAM;
 158                         else
 159                                 e820_type = E820_RESERVED;
 160                         break;
 161                 case EFI_ACPI_RECLAIM_MEMORY:
 162                         e820_type = E820_ACPI;
 163                         break;
 164                 case EFI_ACPI_MEMORY_NVS:
 165                         e820_type = E820_NVS;
 166                         break;
 167                 case EFI_UNUSABLE_MEMORY:
 168                         e820_type = E820_UNUSABLE;
 169                         break;
 170                 case EFI_PERSISTENT_MEMORY:
 171                         e820_type = E820_PMEM;
 172                         break;
 173                 default:
 174                         /*
 175                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
 176                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
 177                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
 178                          */
 179                         e820_type = E820_RESERVED;
 180                         break;
 181                 }
 182                 e820_add_region(start, size, e820_type);
 183         }
 184         sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
 185 }
 186
 187 int __init efi_memblock_x86_reserve_range(void)
 188 {
 189         struct efi_info *e = &boot_params.efi_info;
 190         phys_addr_t pmap;
 191
 192         if (efi_enabled(EFI_PARAVIRT))
 193                 return 0;
 194
 195 #ifdef CONFIG_X86_32
 196         /* Can't handle data above 4GB at this time */
 197         if (e->efi_memmap_hi) {
 198                 pr_err("Memory map is above 4GB, disabling EFI.\n");
 199                 return -EINVAL;
 200         }
 201         pmap =  e->efi_memmap;
 202 #else
 203         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
 204 #endif
 205         efi.memmap.phys_map     = pmap;
 206         efi.memmap.nr_map       = e->efi_memmap_size /
 207                                   e->efi_memdesc_size;
 208         efi.memmap.desc_size    = e->efi_memdesc_size;
 209         efi.memmap.desc_version = e->efi_memdesc_version;
 210
 211         WARN(efi.memmap.desc_version != 1,
 212              "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
 213              efi.memmap.desc_version);
 214
 215         memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size);
 216
 217         return 0;
 218 }
 219
 220 void __init efi_print_memmap(void)
 221 {
 222         efi_memory_desc_t *md;
 223         int i = 0;
 224
 225         for_each_efi_memory_desc(md) {
 226                 char buf[64];
 227
 228                 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n",
 229                         i++, efi_md_typeattr_format(buf, sizeof(buf), md),
 230                         md->phys_addr,
 231                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1,
 232                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
 233         }
 234 }
 235
 236 void __init efi_unmap_memmap(void)
 237 {
 238         unsigned long size;
 239
 240         clear_bit(EFI_MEMMAP, &efi.flags);
 241
 242         size = efi.memmap.nr_map * efi.memmap.desc_size;
 243         if (efi.memmap.map) {
 244                 early_memunmap(efi.memmap.map, size);
 245                 efi.memmap.map = NULL;
 246         }
 247 }
 248
 249 static int __init efi_systab_init(void *phys)
 250 {
 251         if (efi_enabled(EFI_64BIT)) {
 252                 efi_system_table_64_t *systab64;
 253                 struct efi_setup_data *data = NULL;
 254                 u64 tmp = 0;
 255
 256                 if (efi_setup) {
 257                         data = early_memremap(efi_setup, sizeof(*data));
 258                         if (!data)
 259                                 return -ENOMEM;
 260                 }
 261                 systab64 = early_memremap((unsigned long)phys,
 262                                          sizeof(*systab64));
 263                 if (systab64 == NULL) {
 264                         pr_err("Couldn't map the system table!\n");
 265                         if (data)
 266                                 early_memunmap(data, sizeof(*data));
 267                         return -ENOMEM;
 268                 }
 269
 270                 efi_systab.hdr = systab64->hdr;
 271                 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
 272                                               systab64->fw_vendor;
 273                 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
 274                 efi_systab.fw_revision = systab64->fw_revision;
 275                 efi_systab.con_in_handle = systab64->con_in_handle;
 276                 tmp |= systab64->con_in_handle;
 277                 efi_systab.con_in = systab64->con_in;
 278                 tmp |= systab64->con_in;
 279                 efi_systab.con_out_handle = systab64->con_out_handle;
 280                 tmp |= systab64->con_out_handle;
 281                 efi_systab.con_out = systab64->con_out;
 282                 tmp |= systab64->con_out;
 283                 efi_systab.stderr_handle = systab64->stderr_handle;
 284                 tmp |= systab64->stderr_handle;
 285                 efi_systab.stderr = systab64->stderr;
 286                 tmp |= systab64->stderr;
 287                 efi_systab.runtime = data ?
 288                                      (void *)(unsigned long)data->runtime :
 289                                      (void *)(unsigned long)systab64->runtime;
 290                 tmp |= data ? data->runtime : systab64->runtime;
 291                 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
 292                 tmp |= systab64->boottime;
 293                 efi_systab.nr_tables = systab64->nr_tables;
 294                 efi_systab.tables = data ? (unsigned long)data->tables :
 295                                            systab64->tables;
 296                 tmp |= data ? data->tables : systab64->tables;
 297
 298                 early_memunmap(systab64, sizeof(*systab64));
 299                 if (data)
 300                         early_memunmap(data, sizeof(*data));
 301 #ifdef CONFIG_X86_32
 302                 if (tmp >> 32) {
 303                         pr_err("EFI data located above 4GB, disabling EFI.\n");
 304                         return -EINVAL;
 305                 }
 306 #endif
 307         } else {
 308                 efi_system_table_32_t *systab32;
 309
 310                 systab32 = early_memremap((unsigned long)phys,
 311                                          sizeof(*systab32));
 312                 if (systab32 == NULL) {
 313                         pr_err("Couldn't map the system table!\n");
 314                         return -ENOMEM;
 315                 }
 316
 317                 efi_systab.hdr = systab32->hdr;
 318                 efi_systab.fw_vendor = systab32->fw_vendor;
 319                 efi_systab.fw_revision = systab32->fw_revision;
 320                 efi_systab.con_in_handle = systab32->con_in_handle;
 321                 efi_systab.con_in = systab32->con_in;
 322                 efi_systab.con_out_handle = systab32->con_out_handle;
 323                 efi_systab.con_out = systab32->con_out;
 324                 efi_systab.stderr_handle = systab32->stderr_handle;
 325                 efi_systab.stderr = systab32->stderr;
 326                 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
 327                 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
 328                 efi_systab.nr_tables = systab32->nr_tables;
 329                 efi_systab.tables = systab32->tables;
 330
 331                 early_memunmap(systab32, sizeof(*systab32));
 332         }
 333
 334         efi.systab = &efi_systab;
 335
 336         /*
 337          * Verify the EFI Table
 338          */
 339         if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 340                 pr_err("System table signature incorrect!\n");
 341                 return -EINVAL;
 342         }
 343         if ((efi.systab->hdr.revision >> 16) == 0)
 344                 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
 345                        efi.systab->hdr.revision >> 16,
 346                        efi.systab->hdr.revision & 0xffff);
 347
 348         return 0;
 349 }
 350
 351 static int __init efi_runtime_init32(void)
 352 {
 353         efi_runtime_services_32_t *runtime;
 354
 355         runtime = early_memremap((unsigned long)efi.systab->runtime,
 356                         sizeof(efi_runtime_services_32_t));
 357         if (!runtime) {
 358                 pr_err("Could not map the runtime service table!\n");
 359                 return -ENOMEM;
 360         }
 361
 362         /*
 363          * We will only need *early* access to the SetVirtualAddressMap
 364          * EFI runtime service. All other runtime services will be called
 365          * via the virtual mapping.
 366          */
 367         efi_phys.set_virtual_address_map =
 368                         (efi_set_virtual_address_map_t *)
 369                         (unsigned long)runtime->set_virtual_address_map;
 370         early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
 371
 372         return 0;
 373 }
 374
 375 static int __init efi_runtime_init64(void)
 376 {
 377         efi_runtime_services_64_t *runtime;
 378
 379         runtime = early_memremap((unsigned long)efi.systab->runtime,
 380                         sizeof(efi_runtime_services_64_t));
 381         if (!runtime) {
 382                 pr_err("Could not map the runtime service table!\n");
 383                 return -ENOMEM;
 384         }
 385
 386         /*
 387          * We will only need *early* access to the SetVirtualAddressMap
 388          * EFI runtime service. All other runtime services will be called
 389          * via the virtual mapping.
 390          */
 391         efi_phys.set_virtual_address_map =
 392                         (efi_set_virtual_address_map_t *)
 393                         (unsigned long)runtime->set_virtual_address_map;
 394         early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
 395
 396         return 0;
 397 }
 398
 399 static int __init efi_runtime_init(void)
 400 {
 401         int rv;
 402
 403         /*
 404          * Check out the runtime services table. We need to map
 405          * the runtime services table so that we can grab the physical
 406          * address of several of the EFI runtime functions, needed to
 407          * set the firmware into virtual mode.
 408          *
 409          * When EFI_PARAVIRT is in force then we could not map runtime
 410          * service memory region because we do not have direct access to it.
 411          * However, runtime services are available through proxy functions
 412          * (e.g. in case of Xen dom0 EFI implementation they call special
 413          * hypercall which executes relevant EFI functions) and that is why
 414          * they are always enabled.
 415          */
 416
 417         if (!efi_enabled(EFI_PARAVIRT)) {
 418                 if (efi_enabled(EFI_64BIT))
 419                         rv = efi_runtime_init64();
 420                 else
 421                         rv = efi_runtime_init32();
 422
 423                 if (rv)
 424                         return rv;
 425         }
 426
 427         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 428
 429         return 0;
 430 }
 431
 432 static int __init efi_memmap_init(void)
 433 {
 434         unsigned long addr, size;
 435
 436         if (efi_enabled(EFI_PARAVIRT))
 437                 return 0;
 438
 439         /* Map the EFI memory map */
 440         size = efi.memmap.nr_map * efi.memmap.desc_size;
 441         addr = (unsigned long)efi.memmap.phys_map;
 442
 443         efi.memmap.map = early_memremap(addr, size);
 444         if (efi.memmap.map == NULL) {
 445                 pr_err("Could not map the memory map!\n");
 446                 return -ENOMEM;
 447         }
 448
 449         efi.memmap.map_end = efi.memmap.map + size;
 450
 451         if (add_efi_memmap)
 452                 do_add_efi_memmap();
 453
 454         set_bit(EFI_MEMMAP, &efi.flags);
 455
 456         return 0;
 457 }
 458
 459 void __init efi_init(void)
 460 {
 461         efi_char16_t *c16;
 462         char vendor[100] = "unknown";
 463         int i = 0;
 464         void *tmp;
 465
 466 #ifdef CONFIG_X86_32
 467         if (boot_params.efi_info.efi_systab_hi ||
 468             boot_params.efi_info.efi_memmap_hi) {
 469                 pr_info("Table located above 4GB, disabling EFI.\n");
 470                 return;
 471         }
 472         efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
 473 #else
 474         efi_phys.systab = (efi_system_table_t *)
 475                           (boot_params.efi_info.efi_systab |
 476                           ((__u64)boot_params.efi_info.efi_systab_hi<<32));
 477 #endif
 478
 479         if (efi_systab_init(efi_phys.systab))
 480                 return;
 481
 482         efi.config_table = (unsigned long)efi.systab->tables;
 483         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
 484         efi.runtime      = (unsigned long)efi.systab->runtime;
 485
 486         /*
 487          * Show what we know for posterity
 488          */
 489         c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
 490         if (c16) {
 491                 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
 492                         vendor[i] = *c16++;
 493                 vendor[i] = '\0';
 494         } else
 495                 pr_err("Could not map the firmware vendor!\n");
 496         early_memunmap(tmp, 2);
 497
 498         pr_info("EFI v%u.%.02u by %s\n",
 499                 efi.systab->hdr.revision >> 16,
 500                 efi.systab->hdr.revision & 0xffff, vendor);
 501
 502         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
 503                 return;
 504
 505         if (efi_config_init(arch_tables))
 506                 return;
 507
 508         /*
 509          * Note: We currently don't support runtime services on an EFI
 510          * that doesn't match the kernel 32/64-bit mode.
 511          */
 512
 513         if (!efi_runtime_supported())
 514                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
 515         else {
 516                 if (efi_runtime_disabled() || efi_runtime_init())
 517                         return;
 518         }
 519         if (efi_memmap_init())
 520                 return;
 521
 522         if (efi_enabled(EFI_DBG))
 523                 efi_print_memmap();
 524
 525         efi_esrt_init();
 526 }
 527
 528 void __init efi_late_init(void)
 529 {
 530         efi_bgrt_init();
 531 }
 532
 533 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
 534 {
 535         u64 addr, npages;
 536
 537         addr = md->virt_addr;
 538         npages = md->num_pages;
 539
 540         memrange_efi_to_native(&addr, &npages);
 541
 542         if (executable)
 543                 set_memory_x(addr, npages);
 544         else
 545                 set_memory_nx(addr, npages);
 546 }
 547
 548 void __init runtime_code_page_mkexec(void)
 549 {
 550         efi_memory_desc_t *md;
 551
 552         /* Make EFI runtime service code area executable */
 553         for_each_efi_memory_desc(md) {
 554                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
 555                         continue;
 556
 557                 efi_set_executable(md, true);
 558         }
 559 }
 560
 561 void __init efi_memory_uc(u64 addr, unsigned long size)
 562 {
 563         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
 564         u64 npages;
 565
 566         npages = round_up(size, page_shift) / page_shift;
 567         memrange_efi_to_native(&addr, &npages);
 568         set_memory_uc(addr, npages);
 569 }
 570
 571 void __init old_map_region(efi_memory_desc_t *md)
 572 {
 573         u64 start_pfn, end_pfn, end;
 574         unsigned long size;
 575         void *va;
 576
 577         start_pfn = PFN_DOWN(md->phys_addr);
 578         size      = md->num_pages << PAGE_SHIFT;
 579         end       = md->phys_addr + size;
 580         end_pfn   = PFN_UP(end);
 581
 582         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
 583                 va = __va(md->phys_addr);
 584
 585                 if (!(md->attribute & EFI_MEMORY_WB))
 586                         efi_memory_uc((u64)(unsigned long)va, size);
 587         } else
 588                 va = efi_ioremap(md->phys_addr, size,
 589                                  md->type, md->attribute);
 590
 591         md->virt_addr = (u64) (unsigned long) va;
 592         if (!va)
 593                 pr_err("ioremap of 0x%llX failed!\n",
 594                        (unsigned long long)md->phys_addr);
 595 }
 596
 597 /* Merge contiguous regions of the same type and attribute */
 598 static void __init efi_merge_regions(void)
 599 {
 600         efi_memory_desc_t *md, *prev_md = NULL;
 601
 602         for_each_efi_memory_desc(md) {
 603                 u64 prev_size;
 604
 605                 if (!prev_md) {
 606                         prev_md = md;
 607                         continue;
 608                 }
 609
 610                 if (prev_md->type != md->type ||
 611                     prev_md->attribute != md->attribute) {
 612                         prev_md = md;
 613                         continue;
 614                 }
 615
 616                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
 617
 618                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
 619                         prev_md->num_pages += md->num_pages;
 620                         md->type = EFI_RESERVED_TYPE;
 621                         md->attribute = 0;
 622                         continue;
 623                 }
 624                 prev_md = md;
 625         }
 626 }
 627
 628 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
 629 {
 630         unsigned long size;
 631         u64 end, systab;
 632
 633         size = md->num_pages << EFI_PAGE_SHIFT;
 634         end = md->phys_addr + size;
 635         systab = (u64)(unsigned long)efi_phys.systab;
 636         if (md->phys_addr <= systab && systab < end) {
 637                 systab += md->virt_addr - md->phys_addr;
 638                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
 639         }
 640 }
 641
 642 static void __init save_runtime_map(void)
 643 {
 644 #ifdef CONFIG_KEXEC_CORE
 645         unsigned long desc_size;
 646         efi_memory_desc_t *md;
 647         void *tmp, *q = NULL;
 648         int count = 0;
 649
 650         if (efi_enabled(EFI_OLD_MEMMAP))
 651                 return;
 652
 653         desc_size = efi.memmap.desc_size;
 654
 655         for_each_efi_memory_desc(md) {
 656                 if (!(md->attribute & EFI_MEMORY_RUNTIME) ||
 657                     (md->type == EFI_BOOT_SERVICES_CODE) ||
 658                     (md->type == EFI_BOOT_SERVICES_DATA))
 659                         continue;
 660                 tmp = krealloc(q, (count + 1) * desc_size, GFP_KERNEL);
 661                 if (!tmp)
 662                         goto out;
 663                 q = tmp;
 664
 665                 memcpy(q + count * desc_size, md, desc_size);
 666                 count++;
 667         }
 668
 669         efi_runtime_map_setup(q, count, desc_size);
 670         return;
 671
 672 out:
 673         kfree(q);
 674         pr_err("Error saving runtime map, efi runtime on kexec non-functional!!\n");
 675 #endif
 676 }
 677
 678 static void *realloc_pages(void *old_memmap, int old_shift)
 679 {
 680         void *ret;
 681
 682         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
 683         if (!ret)
 684                 goto out;
 685
 686         /*
 687          * A first-time allocation doesn't have anything to copy.
 688          */
 689         if (!old_memmap)
 690                 return ret;
 691
 692         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
 693
 694 out:
 695         free_pages((unsigned long)old_memmap, old_shift);
 696         return ret;
 697 }
 698
 699 /*
 700  * Iterate the EFI memory map in reverse order because the regions
 701  * will be mapped top-down. The end result is the same as if we had
 702  * mapped things forward, but doesn't require us to change the
 703  * existing implementation of efi_map_region().
 704  */
 705 static inline void *efi_map_next_entry_reverse(void *entry)
 706 {
 707         /* Initial call */
 708         if (!entry)
 709                 return efi.memmap.map_end - efi.memmap.desc_size;
 710
 711         entry -= efi.memmap.desc_size;
 712         if (entry < efi.memmap.map)
 713                 return NULL;
 714
 715         return entry;
 716 }
 717
 718 /*
 719  * efi_map_next_entry - Return the next EFI memory map descriptor
 720  * @entry: Previous EFI memory map descriptor
 721  *
 722  * This is a helper function to iterate over the EFI memory map, which
 723  * we do in different orders depending on the current configuration.
 724  *
 725  * To begin traversing the memory map @entry must be %NULL.
 726  *
 727  * Returns %NULL when we reach the end of the memory map.
 728  */
 729 static void *efi_map_next_entry(void *entry)
 730 {
 731         if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) {
 732                 /*
 733                  * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE
 734                  * config table feature requires us to map all entries
 735                  * in the same order as they appear in the EFI memory
 736                  * map. That is to say, entry N must have a lower
 737                  * virtual address than entry N+1. This is because the
 738                  * firmware toolchain leaves relative references in
 739                  * the code/data sections, which are split and become
 740                  * separate EFI memory regions. Mapping things
 741                  * out-of-order leads to the firmware accessing
 742                  * unmapped addresses.
 743                  *
 744                  * Since we need to map things this way whether or not
 745                  * the kernel actually makes use of
 746                  * EFI_PROPERTIES_TABLE, let's just switch to this
 747                  * scheme by default for 64-bit.
 748                  */
 749                 return efi_map_next_entry_reverse(entry);
 750         }
 751
 752         /* Initial call */
 753         if (!entry)
 754                 return efi.memmap.map;
 755
 756         entry += efi.memmap.desc_size;
 757         if (entry >= efi.memmap.map_end)
 758                 return NULL;
 759
 760         return entry;
 761 }
 762
 763 /*
 764  * Map the efi memory ranges of the runtime services and update new_mmap with
 765  * virtual addresses.
 766  */
 767 static void * __init efi_map_regions(int *count, int *pg_shift)
 768 {
 769         void *p, *new_memmap = NULL;
 770         unsigned long left = 0;
 771         unsigned long desc_size;
 772         efi_memory_desc_t *md;
 773
 774         desc_size = efi.memmap.desc_size;
 775
 776         p = NULL;
 777         while ((p = efi_map_next_entry(p))) {
 778                 md = p;
 779                 if (!(md->attribute & EFI_MEMORY_RUNTIME)) {
 780 #ifdef CONFIG_X86_64
 781                         if (md->type != EFI_BOOT_SERVICES_CODE &&
 782                             md->type != EFI_BOOT_SERVICES_DATA)
 783 #endif
 784                                 continue;
 785                 }
 786
 787                 efi_map_region(md);
 788                 get_systab_virt_addr(md);
 789
 790                 if (left < desc_size) {
 791                         new_memmap = realloc_pages(new_memmap, *pg_shift);
 792                         if (!new_memmap)
 793                                 return NULL;
 794
 795                         left += PAGE_SIZE << *pg_shift;
 796                         (*pg_shift)++;
 797                 }
 798
 799                 memcpy(new_memmap + (*count * desc_size), md, desc_size);
 800
 801                 left -= desc_size;
 802                 (*count)++;
 803         }
 804
 805         return new_memmap;
 806 }
 807
 808 static void __init kexec_enter_virtual_mode(void)
 809 {
 810 #ifdef CONFIG_KEXEC_CORE
 811         efi_memory_desc_t *md;
 812         unsigned int num_pages;
 813
 814         efi.systab = NULL;
 815
 816         /*
 817          * We don't do virtual mode, since we don't do runtime services, on
 818          * non-native EFI
 819          */
 820         if (!efi_is_native()) {
 821                 efi_unmap_memmap();
 822                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 823                 return;
 824         }
 825
 826         if (efi_alloc_page_tables()) {
 827                 pr_err("Failed to allocate EFI page tables\n");
 828                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 829                 return;
 830         }
 831
 832         /*
 833         * Map efi regions which were passed via setup_data. The virt_addr is a
 834         * fixed addr which was used in first kernel of a kexec boot.
 835         */
 836         for_each_efi_memory_desc(md) {
 837                 efi_map_region_fixed(md); /* FIXME: add error handling */
 838                 get_systab_virt_addr(md);
 839         }
 840
 841         save_runtime_map();
 842
 843         BUG_ON(!efi.systab);
 844
 845         num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE);
 846         num_pages >>= PAGE_SHIFT;
 847
 848         if (efi_setup_page_tables(efi.memmap.phys_map, num_pages)) {
 849                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 850                 return;
 851         }
 852
 853         efi_sync_low_kernel_mappings();
 854
 855         /*
 856          * Now that EFI is in virtual mode, update the function
 857          * pointers in the runtime service table to the new virtual addresses.
 858          *
 859          * Call EFI services through wrapper functions.
 860          */
 861         efi.runtime_version = efi_systab.hdr.revision;
 862
 863         efi_native_runtime_setup();
 864
 865         efi.set_virtual_address_map = NULL;
 866
 867         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
 868                 runtime_code_page_mkexec();
 869
 870         /* clean DUMMY object */
 871         efi_delete_dummy_variable();
 872 #endif
 873 }
 874
 875 /*
 876  * This function will switch the EFI runtime services to virtual mode.
 877  * Essentially, we look through the EFI memmap and map every region that
 878  * has the runtime attribute bit set in its memory descriptor into the
 879  * efi_pgd page table.
 880  *
 881  * The old method which used to update that memory descriptor with the
 882  * virtual address obtained from ioremap() is still supported when the
 883  * kernel is booted with efi=old_map on its command line. Same old
 884  * method enabled the runtime services to be called without having to
 885  * thunk back into physical mode for every invocation.
 886  *
 887  * The new method does a pagetable switch in a preemption-safe manner
 888  * so that we're in a different address space when calling a runtime
 889  * function. For function arguments passing we do copy the PUDs of the
 890  * kernel page table into efi_pgd prior to each call.
 891  *
 892  * Specially for kexec boot, efi runtime maps in previous kernel should
 893  * be passed in via setup_data. In that case runtime ranges will be mapped
 894  * to the same virtual addresses as the first kernel, see
 895  * kexec_enter_virtual_mode().
 896  */
 897 static void __init __efi_enter_virtual_mode(void)
 898 {
 899         int count = 0, pg_shift = 0;
 900         void *new_memmap = NULL;
 901         efi_status_t status;
 902
 903         efi.systab = NULL;
 904
 905         if (efi_alloc_page_tables()) {
 906                 pr_err("Failed to allocate EFI page tables\n");
 907                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 908                 return;
 909         }
 910
 911         efi_merge_regions();
 912         new_memmap = efi_map_regions(&count, &pg_shift);
 913         if (!new_memmap) {
 914                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
 915                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 916                 return;
 917         }
 918
 919         save_runtime_map();
 920
 921         BUG_ON(!efi.systab);
 922
 923         if (efi_setup_page_tables(__pa(new_memmap), 1 << pg_shift)) {
 924                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 925                 return;
 926         }
 927
 928         efi_sync_low_kernel_mappings();
 929
 930         if (efi_is_native()) {
 931                 status = phys_efi_set_virtual_address_map(
 932                                 efi.memmap.desc_size * count,
 933                                 efi.memmap.desc_size,
 934                                 efi.memmap.desc_version,
 935                                 (efi_memory_desc_t *)__pa(new_memmap));
 936         } else {
 937                 status = efi_thunk_set_virtual_address_map(
 938                                 efi_phys.set_virtual_address_map,
 939                                 efi.memmap.desc_size * count,
 940                                 efi.memmap.desc_size,
 941                                 efi.memmap.desc_version,
 942                                 (efi_memory_desc_t *)__pa(new_memmap));
 943         }
 944
 945         if (status != EFI_SUCCESS) {
 946                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
 947                          status);
 948                 panic("EFI call to SetVirtualAddressMap() failed!");
 949         }
 950
 951         /*
 952          * Now that EFI is in virtual mode, update the function
 953          * pointers in the runtime service table to the new virtual addresses.
 954          *
 955          * Call EFI services through wrapper functions.
 956          */
 957         efi.runtime_version = efi_systab.hdr.revision;
 958
 959         if (efi_is_native())
 960                 efi_native_runtime_setup();
 961         else
 962                 efi_thunk_runtime_setup();
 963
 964         efi.set_virtual_address_map = NULL;
 965
 966         /*
 967          * Apply more restrictive page table mapping attributes now that
 968          * SVAM() has been called and the firmware has performed all
 969          * necessary relocation fixups for the new virtual addresses.
 970          */
 971         efi_runtime_update_mappings();
 972         efi_dump_pagetable();
 973
 974         /*
 975          * We mapped the descriptor array into the EFI pagetable above
 976          * but we're not unmapping it here because if we're running in
 977          * EFI mixed mode we need all of memory to be accessible when
 978          * we pass parameters to the EFI runtime services in the
 979          * thunking code.
 980          *
 981          * efi_cleanup_page_tables(__pa(new_memmap), 1 << pg_shift);
 982          */
 983         free_pages((unsigned long)new_memmap, pg_shift);
 984
 985         /* clean DUMMY object */
 986         efi_delete_dummy_variable();
 987 }
 988
 989 void __init efi_enter_virtual_mode(void)
 990 {
 991         if (efi_enabled(EFI_PARAVIRT))
 992                 return;
 993
 994         if (efi_setup)
 995                 kexec_enter_virtual_mode();
 996         else
 997                 __efi_enter_virtual_mode();
 998 }
 999
1000 /*
1001  * Convenience functions to obtain memory types and attributes
1002  */
1003 u32 efi_mem_type(unsigned long phys_addr)
1004 {
1005         efi_memory_desc_t *md;
1006
1007         if (!efi_enabled(EFI_MEMMAP))
1008                 return 0;
1009
1010         for_each_efi_memory_desc(md) {
1011                 if ((md->phys_addr <= phys_addr) &&
1012                     (phys_addr < (md->phys_addr +
1013                                   (md->num_pages << EFI_PAGE_SHIFT))))
1014                         return md->type;
1015         }
1016         return 0;
1017 }
1018
1019 static int __init arch_parse_efi_cmdline(char *str)
1020 {
1021         if (!str) {
1022                 pr_warn("need at least one option\n");
1023                 return -EINVAL;
1024         }
1025
1026         if (parse_option_str(str, "old_map"))
1027                 set_bit(EFI_OLD_MEMMAP, &efi.flags);
1028
1029         return 0;
1030 }
1031 early_param("efi", arch_parse_efi_cmdline);