arch/x86/platform/efi/efi.c

   1 /*
   2  * Common EFI (Extensible Firmware Interface) support functions
   3  * Based on Extensible Firmware Interface Specification version 1.0
   4  *
   5  * Copyright (C) 1999 VA Linux Systems
   6  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
   7  * Copyright (C) 1999-2002 Hewlett-Packard Co.
   8  *      David Mosberger-Tang <davidm@hpl.hp.com>
   9  *      Stephane Eranian <eranian@hpl.hp.com>
  10  * Copyright (C) 2005-2008 Intel Co.
  11  *      Fenghua Yu <fenghua.yu@intel.com>
  12  *      Bibo Mao <bibo.mao@intel.com>
  13  *      Chandramouli Narayanan <mouli@linux.intel.com>
  14  *      Huang Ying <ying.huang@intel.com>
  15  * Copyright (C) 2013 SuSE Labs
  16  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
  17  *
  18  * Copied from efi_32.c to eliminate the duplicated code between EFI
  19  * 32/64 support code. --ying 2007-10-26
  20  *
  21  * All EFI Runtime Services are not implemented yet as EFI only
  22  * supports physical mode addressing on SoftSDV. This is to be fixed
  23  * in a future version.  --drummond 1999-07-20
  24  *
  25  * Implemented EFI runtime services and virtual mode calls.  --davidm
  26  *
  27  * Goutham Rao: <goutham.rao@intel.com>
  28  *      Skip non-WB memory and ignore empty memory ranges.
  29  */
  30
  31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  32
  33 #include <linux/kernel.h>
  34 #include <linux/init.h>
  35 #include <linux/efi.h>
  36 #include <linux/efi-bgrt.h>
  37 #include <linux/export.h>
  38 #include <linux/bootmem.h>
  39 #include <linux/slab.h>
  40 #include <linux/memblock.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/time.h>
  44 #include <linux/io.h>
  45 #include <linux/reboot.h>
  46 #include <linux/bcd.h>
  47
  48 #include <asm/setup.h>
  49 #include <asm/efi.h>
  50 #include <asm/time.h>
  51 #include <asm/cacheflush.h>
  52 #include <asm/tlbflush.h>
  53 #include <asm/x86_init.h>
  54 #include <asm/rtc.h>
  55 #include <asm/uv/uv.h>
  56
  57 #define EFI_DEBUG
  58
  59 struct efi_memory_map memmap;
  60
  61 static struct efi efi_phys __initdata;
  62 static efi_system_table_t efi_systab __initdata;
  63
  64 static efi_config_table_type_t arch_tables[] __initdata = {
  65 #ifdef CONFIG_X86_UV
  66         {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
  67 #endif
  68         {NULL_GUID, NULL, NULL},
  69 };
  70
  71 u64 efi_setup;          /* efi setup_data physical address */
  72
  73 static int add_efi_memmap __initdata;
  74 static int __init setup_add_efi_memmap(char *arg)
  75 {
  76         add_efi_memmap = 1;
  77         return 0;
  78 }
  79 early_param("add_efi_memmap", setup_add_efi_memmap);
  80
  81 static efi_status_t __init phys_efi_set_virtual_address_map(
  82         unsigned long memory_map_size,
  83         unsigned long descriptor_size,
  84         u32 descriptor_version,
  85         efi_memory_desc_t *virtual_map)
  86 {
  87         efi_status_t status;
  88         unsigned long flags;
  89         pgd_t *save_pgd;
  90
  91         save_pgd = efi_call_phys_prolog();
  92
  93         /* Disable interrupts around EFI calls: */
  94         local_irq_save(flags);
  95         status = efi_call_phys(efi_phys.set_virtual_address_map,
  96                                memory_map_size, descriptor_size,
  97                                descriptor_version, virtual_map);
  98         local_irq_restore(flags);
  99
 100         efi_call_phys_epilog(save_pgd);
 101
 102         return status;
 103 }
 104
 105 void efi_get_time(struct timespec *now)
 106 {
 107         efi_status_t status;
 108         efi_time_t eft;
 109         efi_time_cap_t cap;
 110
 111         status = efi.get_time(&eft, &cap);
 112         if (status != EFI_SUCCESS)
 113                 pr_err("Oops: efitime: can't read time!\n");
 114
 115         now->tv_sec = mktime(eft.year, eft.month, eft.day, eft.hour,
 116                              eft.minute, eft.second);
 117         now->tv_nsec = 0;
 118 }
 119
 120 void __init efi_find_mirror(void)
 121 {
 122         void *p;
 123         u64 mirror_size = 0, total_size = 0;
 124
 125         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 126                 efi_memory_desc_t *md = p;
 127                 unsigned long long start = md->phys_addr;
 128                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 129
 130                 total_size += size;
 131                 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
 132                         memblock_mark_mirror(start, size);
 133                         mirror_size += size;
 134                 }
 135         }
 136         if (mirror_size)
 137                 pr_info("Memory: %lldM/%lldM mirrored memory\n",
 138                         mirror_size>>20, total_size>>20);
 139 }
 140
 141 /*
 142  * Tell the kernel about the EFI memory map.  This might include
 143  * more than the max 128 entries that can fit in the e820 legacy
 144  * (zeropage) memory map.
 145  */
 146
 147 static void __init do_add_efi_memmap(void)
 148 {
 149         void *p;
 150
 151         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 152                 efi_memory_desc_t *md = p;
 153                 unsigned long long start = md->phys_addr;
 154                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 155                 int e820_type;
 156
 157                 switch (md->type) {
 158                 case EFI_LOADER_CODE:
 159                 case EFI_LOADER_DATA:
 160                 case EFI_BOOT_SERVICES_CODE:
 161                 case EFI_BOOT_SERVICES_DATA:
 162                 case EFI_CONVENTIONAL_MEMORY:
 163                         if (md->attribute & EFI_MEMORY_WB)
 164                                 e820_type = E820_RAM;
 165                         else
 166                                 e820_type = E820_RESERVED;
 167                         break;
 168                 case EFI_ACPI_RECLAIM_MEMORY:
 169                         e820_type = E820_ACPI;
 170                         break;
 171                 case EFI_ACPI_MEMORY_NVS:
 172                         e820_type = E820_NVS;
 173                         break;
 174                 case EFI_UNUSABLE_MEMORY:
 175                         e820_type = E820_UNUSABLE;
 176                         break;
 177                 case EFI_PERSISTENT_MEMORY:
 178                         e820_type = E820_PMEM;
 179                         break;
 180                 default:
 181                         /*
 182                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
 183                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
 184                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
 185                          */
 186                         e820_type = E820_RESERVED;
 187                         break;
 188                 }
 189                 e820_add_region(start, size, e820_type);
 190         }
 191         sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
 192 }
 193
 194 int __init efi_memblock_x86_reserve_range(void)
 195 {
 196         struct efi_info *e = &boot_params.efi_info;
 197         phys_addr_t pmap;
 198
 199         if (efi_enabled(EFI_PARAVIRT))
 200                 return 0;
 201
 202 #ifdef CONFIG_X86_32
 203         /* Can't handle data above 4GB at this time */
 204         if (e->efi_memmap_hi) {
 205                 pr_err("Memory map is above 4GB, disabling EFI.\n");
 206                 return -EINVAL;
 207         }
 208         pmap =  e->efi_memmap;
 209 #else
 210         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
 211 #endif
 212         memmap.phys_map         = pmap;
 213         memmap.nr_map           = e->efi_memmap_size /
 214                                   e->efi_memdesc_size;
 215         memmap.desc_size        = e->efi_memdesc_size;
 216         memmap.desc_version     = e->efi_memdesc_version;
 217
 218         memblock_reserve(pmap, memmap.nr_map * memmap.desc_size);
 219
 220         efi.memmap = &memmap;
 221
 222         return 0;
 223 }
 224
 225 void __init efi_print_memmap(void)
 226 {
 227 #ifdef EFI_DEBUG
 228         efi_memory_desc_t *md;
 229         void *p;
 230         int i;
 231
 232         for (p = memmap.map, i = 0;
 233              p < memmap.map_end;
 234              p += memmap.desc_size, i++) {
 235                 char buf[64];
 236
 237                 md = p;
 238                 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n",
 239                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 240                         md->phys_addr,
 241                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1,
 242                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
 243         }
 244 #endif  /*  EFI_DEBUG  */
 245 }
 246
 247 void __init efi_unmap_memmap(void)
 248 {
 249         clear_bit(EFI_MEMMAP, &efi.flags);
 250         if (memmap.map) {
 251                 early_memunmap(memmap.map, memmap.nr_map * memmap.desc_size);
 252                 memmap.map = NULL;
 253         }
 254 }
 255
 256 static int __init efi_systab_init(void *phys)
 257 {
 258         if (efi_enabled(EFI_64BIT)) {
 259                 efi_system_table_64_t *systab64;
 260                 struct efi_setup_data *data = NULL;
 261                 u64 tmp = 0;
 262
 263                 if (efi_setup) {
 264                         data = early_memremap(efi_setup, sizeof(*data));
 265                         if (!data)
 266                                 return -ENOMEM;
 267                 }
 268                 systab64 = early_memremap((unsigned long)phys,
 269                                          sizeof(*systab64));
 270                 if (systab64 == NULL) {
 271                         pr_err("Couldn't map the system table!\n");
 272                         if (data)
 273                                 early_memunmap(data, sizeof(*data));
 274                         return -ENOMEM;
 275                 }
 276
 277                 efi_systab.hdr = systab64->hdr;
 278                 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
 279                                               systab64->fw_vendor;
 280                 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
 281                 efi_systab.fw_revision = systab64->fw_revision;
 282                 efi_systab.con_in_handle = systab64->con_in_handle;
 283                 tmp |= systab64->con_in_handle;
 284                 efi_systab.con_in = systab64->con_in;
 285                 tmp |= systab64->con_in;
 286                 efi_systab.con_out_handle = systab64->con_out_handle;
 287                 tmp |= systab64->con_out_handle;
 288                 efi_systab.con_out = systab64->con_out;
 289                 tmp |= systab64->con_out;
 290                 efi_systab.stderr_handle = systab64->stderr_handle;
 291                 tmp |= systab64->stderr_handle;
 292                 efi_systab.stderr = systab64->stderr;
 293                 tmp |= systab64->stderr;
 294                 efi_systab.runtime = data ?
 295                                      (void *)(unsigned long)data->runtime :
 296                                      (void *)(unsigned long)systab64->runtime;
 297                 tmp |= data ? data->runtime : systab64->runtime;
 298                 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
 299                 tmp |= systab64->boottime;
 300                 efi_systab.nr_tables = systab64->nr_tables;
 301                 efi_systab.tables = data ? (unsigned long)data->tables :
 302                                            systab64->tables;
 303                 tmp |= data ? data->tables : systab64->tables;
 304
 305                 early_memunmap(systab64, sizeof(*systab64));
 306                 if (data)
 307                         early_memunmap(data, sizeof(*data));
 308 #ifdef CONFIG_X86_32
 309                 if (tmp >> 32) {
 310                         pr_err("EFI data located above 4GB, disabling EFI.\n");
 311                         return -EINVAL;
 312                 }
 313 #endif
 314         } else {
 315                 efi_system_table_32_t *systab32;
 316
 317                 systab32 = early_memremap((unsigned long)phys,
 318                                          sizeof(*systab32));
 319                 if (systab32 == NULL) {
 320                         pr_err("Couldn't map the system table!\n");
 321                         return -ENOMEM;
 322                 }
 323
 324                 efi_systab.hdr = systab32->hdr;
 325                 efi_systab.fw_vendor = systab32->fw_vendor;
 326                 efi_systab.fw_revision = systab32->fw_revision;
 327                 efi_systab.con_in_handle = systab32->con_in_handle;
 328                 efi_systab.con_in = systab32->con_in;
 329                 efi_systab.con_out_handle = systab32->con_out_handle;
 330                 efi_systab.con_out = systab32->con_out;
 331                 efi_systab.stderr_handle = systab32->stderr_handle;
 332                 efi_systab.stderr = systab32->stderr;
 333                 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
 334                 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
 335                 efi_systab.nr_tables = systab32->nr_tables;
 336                 efi_systab.tables = systab32->tables;
 337
 338                 early_memunmap(systab32, sizeof(*systab32));
 339         }
 340
 341         efi.systab = &efi_systab;
 342
 343         /*
 344          * Verify the EFI Table
 345          */
 346         if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 347                 pr_err("System table signature incorrect!\n");
 348                 return -EINVAL;
 349         }
 350         if ((efi.systab->hdr.revision >> 16) == 0)
 351                 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
 352                        efi.systab->hdr.revision >> 16,
 353                        efi.systab->hdr.revision & 0xffff);
 354
 355         return 0;
 356 }
 357
 358 static int __init efi_runtime_init32(void)
 359 {
 360         efi_runtime_services_32_t *runtime;
 361
 362         runtime = early_memremap((unsigned long)efi.systab->runtime,
 363                         sizeof(efi_runtime_services_32_t));
 364         if (!runtime) {
 365                 pr_err("Could not map the runtime service table!\n");
 366                 return -ENOMEM;
 367         }
 368
 369         /*
 370          * We will only need *early* access to the SetVirtualAddressMap
 371          * EFI runtime service. All other runtime services will be called
 372          * via the virtual mapping.
 373          */
 374         efi_phys.set_virtual_address_map =
 375                         (efi_set_virtual_address_map_t *)
 376                         (unsigned long)runtime->set_virtual_address_map;
 377         early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
 378
 379         return 0;
 380 }
 381
 382 static int __init efi_runtime_init64(void)
 383 {
 384         efi_runtime_services_64_t *runtime;
 385
 386         runtime = early_memremap((unsigned long)efi.systab->runtime,
 387                         sizeof(efi_runtime_services_64_t));
 388         if (!runtime) {
 389                 pr_err("Could not map the runtime service table!\n");
 390                 return -ENOMEM;
 391         }
 392
 393         /*
 394          * We will only need *early* access to the SetVirtualAddressMap
 395          * EFI runtime service. All other runtime services will be called
 396          * via the virtual mapping.
 397          */
 398         efi_phys.set_virtual_address_map =
 399                         (efi_set_virtual_address_map_t *)
 400                         (unsigned long)runtime->set_virtual_address_map;
 401         early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
 402
 403         return 0;
 404 }
 405
 406 static int __init efi_runtime_init(void)
 407 {
 408         int rv;
 409
 410         /*
 411          * Check out the runtime services table. We need to map
 412          * the runtime services table so that we can grab the physical
 413          * address of several of the EFI runtime functions, needed to
 414          * set the firmware into virtual mode.
 415          *
 416          * When EFI_PARAVIRT is in force then we could not map runtime
 417          * service memory region because we do not have direct access to it.
 418          * However, runtime services are available through proxy functions
 419          * (e.g. in case of Xen dom0 EFI implementation they call special
 420          * hypercall which executes relevant EFI functions) and that is why
 421          * they are always enabled.
 422          */
 423
 424         if (!efi_enabled(EFI_PARAVIRT)) {
 425                 if (efi_enabled(EFI_64BIT))
 426                         rv = efi_runtime_init64();
 427                 else
 428                         rv = efi_runtime_init32();
 429
 430                 if (rv)
 431                         return rv;
 432         }
 433
 434         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 435
 436         return 0;
 437 }
 438
 439 static int __init efi_memmap_init(void)
 440 {
 441         if (efi_enabled(EFI_PARAVIRT))
 442                 return 0;
 443
 444         /* Map the EFI memory map */
 445         memmap.map = early_memremap((unsigned long)memmap.phys_map,
 446                                    memmap.nr_map * memmap.desc_size);
 447         if (memmap.map == NULL) {
 448                 pr_err("Could not map the memory map!\n");
 449                 return -ENOMEM;
 450         }
 451         memmap.map_end = memmap.map + (memmap.nr_map * memmap.desc_size);
 452
 453         if (add_efi_memmap)
 454                 do_add_efi_memmap();
 455
 456         set_bit(EFI_MEMMAP, &efi.flags);
 457
 458         return 0;
 459 }
 460
 461 void __init efi_init(void)
 462 {
 463         efi_char16_t *c16;
 464         char vendor[100] = "unknown";
 465         int i = 0;
 466         void *tmp;
 467
 468 #ifdef CONFIG_X86_32
 469         if (boot_params.efi_info.efi_systab_hi ||
 470             boot_params.efi_info.efi_memmap_hi) {
 471                 pr_info("Table located above 4GB, disabling EFI.\n");
 472                 return;
 473         }
 474         efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
 475 #else
 476         efi_phys.systab = (efi_system_table_t *)
 477                           (boot_params.efi_info.efi_systab |
 478                           ((__u64)boot_params.efi_info.efi_systab_hi<<32));
 479 #endif
 480
 481         if (efi_systab_init(efi_phys.systab))
 482                 return;
 483
 484         efi.config_table = (unsigned long)efi.systab->tables;
 485         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
 486         efi.runtime      = (unsigned long)efi.systab->runtime;
 487
 488         /*
 489          * Show what we know for posterity
 490          */
 491         c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
 492         if (c16) {
 493                 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
 494                         vendor[i] = *c16++;
 495                 vendor[i] = '\0';
 496         } else
 497                 pr_err("Could not map the firmware vendor!\n");
 498         early_memunmap(tmp, 2);
 499
 500         pr_info("EFI v%u.%.02u by %s\n",
 501                 efi.systab->hdr.revision >> 16,
 502                 efi.systab->hdr.revision & 0xffff, vendor);
 503
 504         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
 505                 return;
 506
 507         if (efi_config_init(arch_tables))
 508                 return;
 509
 510         /*
 511          * Note: We currently don't support runtime services on an EFI
 512          * that doesn't match the kernel 32/64-bit mode.
 513          */
 514
 515         if (!efi_runtime_supported())
 516                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
 517         else {
 518                 if (efi_runtime_disabled() || efi_runtime_init())
 519                         return;
 520         }
 521         if (efi_memmap_init())
 522                 return;
 523
 524         if (efi_enabled(EFI_DBG))
 525                 efi_print_memmap();
 526
 527         efi_esrt_init();
 528 }
 529
 530 void __init efi_late_init(void)
 531 {
 532         efi_bgrt_init();
 533 }
 534
 535 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
 536 {
 537         u64 addr, npages;
 538
 539         addr = md->virt_addr;
 540         npages = md->num_pages;
 541
 542         memrange_efi_to_native(&addr, &npages);
 543
 544         if (executable)
 545                 set_memory_x(addr, npages);
 546         else
 547                 set_memory_nx(addr, npages);
 548 }
 549
 550 void __init runtime_code_page_mkexec(void)
 551 {
 552         efi_memory_desc_t *md;
 553         void *p;
 554
 555         /* Make EFI runtime service code area executable */
 556         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 557                 md = p;
 558
 559                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
 560                         continue;
 561
 562                 efi_set_executable(md, true);
 563         }
 564 }
 565
 566 void __init efi_memory_uc(u64 addr, unsigned long size)
 567 {
 568         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
 569         u64 npages;
 570
 571         npages = round_up(size, page_shift) / page_shift;
 572         memrange_efi_to_native(&addr, &npages);
 573         set_memory_uc(addr, npages);
 574 }
 575
 576 void __init old_map_region(efi_memory_desc_t *md)
 577 {
 578         u64 start_pfn, end_pfn, end;
 579         unsigned long size;
 580         void *va;
 581
 582         start_pfn = PFN_DOWN(md->phys_addr);
 583         size      = md->num_pages << PAGE_SHIFT;
 584         end       = md->phys_addr + size;
 585         end_pfn   = PFN_UP(end);
 586
 587         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
 588                 va = __va(md->phys_addr);
 589
 590                 if (!(md->attribute & EFI_MEMORY_WB))
 591                         efi_memory_uc((u64)(unsigned long)va, size);
 592         } else
 593                 va = efi_ioremap(md->phys_addr, size,
 594                                  md->type, md->attribute);
 595
 596         md->virt_addr = (u64) (unsigned long) va;
 597         if (!va)
 598                 pr_err("ioremap of 0x%llX failed!\n",
 599                        (unsigned long long)md->phys_addr);
 600 }
 601
 602 /* Merge contiguous regions of the same type and attribute */
 603 static void __init efi_merge_regions(void)
 604 {
 605         void *p;
 606         efi_memory_desc_t *md, *prev_md = NULL;
 607
 608         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 609                 u64 prev_size;
 610                 md = p;
 611
 612                 if (!prev_md) {
 613                         prev_md = md;
 614                         continue;
 615                 }
 616
 617                 if (prev_md->type != md->type ||
 618                     prev_md->attribute != md->attribute) {
 619                         prev_md = md;
 620                         continue;
 621                 }
 622
 623                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
 624
 625                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
 626                         prev_md->num_pages += md->num_pages;
 627                         md->type = EFI_RESERVED_TYPE;
 628                         md->attribute = 0;
 629                         continue;
 630                 }
 631                 prev_md = md;
 632         }
 633 }
 634
 635 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
 636 {
 637         unsigned long size;
 638         u64 end, systab;
 639
 640         size = md->num_pages << EFI_PAGE_SHIFT;
 641         end = md->phys_addr + size;
 642         systab = (u64)(unsigned long)efi_phys.systab;
 643         if (md->phys_addr <= systab && systab < end) {
 644                 systab += md->virt_addr - md->phys_addr;
 645                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
 646         }
 647 }
 648
 649 static void __init save_runtime_map(void)
 650 {
 651 #ifdef CONFIG_KEXEC_CORE
 652         efi_memory_desc_t *md;
 653         void *tmp, *p, *q = NULL;
 654         int count = 0;
 655
 656         if (efi_enabled(EFI_OLD_MEMMAP))
 657                 return;
 658
 659         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 660                 md = p;
 661
 662                 if (!(md->attribute & EFI_MEMORY_RUNTIME) ||
 663                     (md->type == EFI_BOOT_SERVICES_CODE) ||
 664                     (md->type == EFI_BOOT_SERVICES_DATA))
 665                         continue;
 666                 tmp = krealloc(q, (count + 1) * memmap.desc_size, GFP_KERNEL);
 667                 if (!tmp)
 668                         goto out;
 669                 q = tmp;
 670
 671                 memcpy(q + count * memmap.desc_size, md, memmap.desc_size);
 672                 count++;
 673         }
 674
 675         efi_runtime_map_setup(q, count, memmap.desc_size);
 676         return;
 677
 678 out:
 679         kfree(q);
 680         pr_err("Error saving runtime map, efi runtime on kexec non-functional!!\n");
 681 #endif
 682 }
 683
 684 static void *realloc_pages(void *old_memmap, int old_shift)
 685 {
 686         void *ret;
 687
 688         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
 689         if (!ret)
 690                 goto out;
 691
 692         /*
 693          * A first-time allocation doesn't have anything to copy.
 694          */
 695         if (!old_memmap)
 696                 return ret;
 697
 698         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
 699
 700 out:
 701         free_pages((unsigned long)old_memmap, old_shift);
 702         return ret;
 703 }
 704
 705 /*
 706  * Iterate the EFI memory map in reverse order because the regions
 707  * will be mapped top-down. The end result is the same as if we had
 708  * mapped things forward, but doesn't require us to change the
 709  * existing implementation of efi_map_region().
 710  */
 711 static inline void *efi_map_next_entry_reverse(void *entry)
 712 {
 713         /* Initial call */
 714         if (!entry)
 715                 return memmap.map_end - memmap.desc_size;
 716
 717         entry -= memmap.desc_size;
 718         if (entry < memmap.map)
 719                 return NULL;
 720
 721         return entry;
 722 }
 723
 724 /*
 725  * efi_map_next_entry - Return the next EFI memory map descriptor
 726  * @entry: Previous EFI memory map descriptor
 727  *
 728  * This is a helper function to iterate over the EFI memory map, which
 729  * we do in different orders depending on the current configuration.
 730  *
 731  * To begin traversing the memory map @entry must be %NULL.
 732  *
 733  * Returns %NULL when we reach the end of the memory map.
 734  */
 735 static void *efi_map_next_entry(void *entry)
 736 {
 737         if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) {
 738                 /*
 739                  * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE
 740                  * config table feature requires us to map all entries
 741                  * in the same order as they appear in the EFI memory
 742                  * map. That is to say, entry N must have a lower
 743                  * virtual address than entry N+1. This is because the
 744                  * firmware toolchain leaves relative references in
 745                  * the code/data sections, which are split and become
 746                  * separate EFI memory regions. Mapping things
 747                  * out-of-order leads to the firmware accessing
 748                  * unmapped addresses.
 749                  *
 750                  * Since we need to map things this way whether or not
 751                  * the kernel actually makes use of
 752                  * EFI_PROPERTIES_TABLE, let's just switch to this
 753                  * scheme by default for 64-bit.
 754                  */
 755                 return efi_map_next_entry_reverse(entry);
 756         }
 757
 758         /* Initial call */
 759         if (!entry)
 760                 return memmap.map;
 761
 762         entry += memmap.desc_size;
 763         if (entry >= memmap.map_end)
 764                 return NULL;
 765
 766         return entry;
 767 }
 768
 769 /*
 770  * Map the efi memory ranges of the runtime services and update new_mmap with
 771  * virtual addresses.
 772  */
 773 static void * __init efi_map_regions(int *count, int *pg_shift)
 774 {
 775         void *p, *new_memmap = NULL;
 776         unsigned long left = 0;
 777         efi_memory_desc_t *md;
 778
 779         p = NULL;
 780         while ((p = efi_map_next_entry(p))) {
 781                 md = p;
 782                 if (!(md->attribute & EFI_MEMORY_RUNTIME)) {
 783 #ifdef CONFIG_X86_64
 784                         if (md->type != EFI_BOOT_SERVICES_CODE &&
 785                             md->type != EFI_BOOT_SERVICES_DATA)
 786 #endif
 787                                 continue;
 788                 }
 789
 790                 efi_map_region(md);
 791                 get_systab_virt_addr(md);
 792
 793                 if (left < memmap.desc_size) {
 794                         new_memmap = realloc_pages(new_memmap, *pg_shift);
 795                         if (!new_memmap)
 796                                 return NULL;
 797
 798                         left += PAGE_SIZE << *pg_shift;
 799                         (*pg_shift)++;
 800                 }
 801
 802                 memcpy(new_memmap + (*count * memmap.desc_size), md,
 803                        memmap.desc_size);
 804
 805                 left -= memmap.desc_size;
 806                 (*count)++;
 807         }
 808
 809         return new_memmap;
 810 }
 811
 812 static void __init kexec_enter_virtual_mode(void)
 813 {
 814 #ifdef CONFIG_KEXEC_CORE
 815         efi_memory_desc_t *md;
 816         unsigned int num_pages;
 817         void *p;
 818
 819         efi.systab = NULL;
 820
 821         /*
 822          * We don't do virtual mode, since we don't do runtime services, on
 823          * non-native EFI
 824          */
 825         if (!efi_is_native()) {
 826                 efi_unmap_memmap();
 827                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 828                 return;
 829         }
 830
 831         if (efi_alloc_page_tables()) {
 832                 pr_err("Failed to allocate EFI page tables\n");
 833                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 834                 return;
 835         }
 836
 837         /*
 838         * Map efi regions which were passed via setup_data. The virt_addr is a
 839         * fixed addr which was used in first kernel of a kexec boot.
 840         */
 841         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 842                 md = p;
 843                 efi_map_region_fixed(md); /* FIXME: add error handling */
 844                 get_systab_virt_addr(md);
 845         }
 846
 847         save_runtime_map();
 848
 849         BUG_ON(!efi.systab);
 850
 851         num_pages = ALIGN(memmap.nr_map * memmap.desc_size, PAGE_SIZE);
 852         num_pages >>= PAGE_SHIFT;
 853
 854         if (efi_setup_page_tables(memmap.phys_map, num_pages)) {
 855                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 856                 return;
 857         }
 858
 859         efi_sync_low_kernel_mappings();
 860
 861         /*
 862          * Now that EFI is in virtual mode, update the function
 863          * pointers in the runtime service table to the new virtual addresses.
 864          *
 865          * Call EFI services through wrapper functions.
 866          */
 867         efi.runtime_version = efi_systab.hdr.revision;
 868
 869         efi_native_runtime_setup();
 870
 871         efi.set_virtual_address_map = NULL;
 872
 873         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
 874                 runtime_code_page_mkexec();
 875
 876         /* clean DUMMY object */
 877         efi_delete_dummy_variable();
 878 #endif
 879 }
 880
 881 /*
 882  * This function will switch the EFI runtime services to virtual mode.
 883  * Essentially, we look through the EFI memmap and map every region that
 884  * has the runtime attribute bit set in its memory descriptor into the
 885  * efi_pgd page table.
 886  *
 887  * The old method which used to update that memory descriptor with the
 888  * virtual address obtained from ioremap() is still supported when the
 889  * kernel is booted with efi=old_map on its command line. Same old
 890  * method enabled the runtime services to be called without having to
 891  * thunk back into physical mode for every invocation.
 892  *
 893  * The new method does a pagetable switch in a preemption-safe manner
 894  * so that we're in a different address space when calling a runtime
 895  * function. For function arguments passing we do copy the PUDs of the
 896  * kernel page table into efi_pgd prior to each call.
 897  *
 898  * Specially for kexec boot, efi runtime maps in previous kernel should
 899  * be passed in via setup_data. In that case runtime ranges will be mapped
 900  * to the same virtual addresses as the first kernel, see
 901  * kexec_enter_virtual_mode().
 902  */
 903 static void __init __efi_enter_virtual_mode(void)
 904 {
 905         int count = 0, pg_shift = 0;
 906         void *new_memmap = NULL;
 907         efi_status_t status;
 908
 909         efi.systab = NULL;
 910
 911         if (efi_alloc_page_tables()) {
 912                 pr_err("Failed to allocate EFI page tables\n");
 913                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 914                 return;
 915         }
 916
 917         efi_merge_regions();
 918         new_memmap = efi_map_regions(&count, &pg_shift);
 919         if (!new_memmap) {
 920                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
 921                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 922                 return;
 923         }
 924
 925         save_runtime_map();
 926
 927         BUG_ON(!efi.systab);
 928
 929         if (efi_setup_page_tables(__pa(new_memmap), 1 << pg_shift)) {
 930                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 931                 return;
 932         }
 933
 934         efi_sync_low_kernel_mappings();
 935
 936         if (efi_is_native()) {
 937                 status = phys_efi_set_virtual_address_map(
 938                                 memmap.desc_size * count,
 939                                 memmap.desc_size,
 940                                 memmap.desc_version,
 941                                 (efi_memory_desc_t *)__pa(new_memmap));
 942         } else {
 943                 status = efi_thunk_set_virtual_address_map(
 944                                 efi_phys.set_virtual_address_map,
 945                                 memmap.desc_size * count,
 946                                 memmap.desc_size,
 947                                 memmap.desc_version,
 948                                 (efi_memory_desc_t *)__pa(new_memmap));
 949         }
 950
 951         if (status != EFI_SUCCESS) {
 952                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
 953                          status);
 954                 panic("EFI call to SetVirtualAddressMap() failed!");
 955         }
 956
 957         /*
 958          * Now that EFI is in virtual mode, update the function
 959          * pointers in the runtime service table to the new virtual addresses.
 960          *
 961          * Call EFI services through wrapper functions.
 962          */
 963         efi.runtime_version = efi_systab.hdr.revision;
 964
 965         if (efi_is_native())
 966                 efi_native_runtime_setup();
 967         else
 968                 efi_thunk_runtime_setup();
 969
 970         efi.set_virtual_address_map = NULL;
 971
 972         /*
 973          * Apply more restrictive page table mapping attributes now that
 974          * SVAM() has been called and the firmware has performed all
 975          * necessary relocation fixups for the new virtual addresses.
 976          */
 977         efi_runtime_update_mappings();
 978         efi_dump_pagetable();
 979
 980         /*
 981          * We mapped the descriptor array into the EFI pagetable above
 982          * but we're not unmapping it here because if we're running in
 983          * EFI mixed mode we need all of memory to be accessible when
 984          * we pass parameters to the EFI runtime services in the
 985          * thunking code.
 986          *
 987          * efi_cleanup_page_tables(__pa(new_memmap), 1 << pg_shift);
 988          */
 989         free_pages((unsigned long)new_memmap, pg_shift);
 990
 991         /* clean DUMMY object */
 992         efi_delete_dummy_variable();
 993 }
 994
 995 void __init efi_enter_virtual_mode(void)
 996 {
 997         if (efi_enabled(EFI_PARAVIRT))
 998                 return;
 999
1000         if (efi_setup)
1001                 kexec_enter_virtual_mode();
1002         else
1003                 __efi_enter_virtual_mode();
1004 }
1005
1006 /*
1007  * Convenience functions to obtain memory types and attributes
1008  */
1009 u32 efi_mem_type(unsigned long phys_addr)
1010 {
1011         efi_memory_desc_t *md;
1012         void *p;
1013
1014         if (!efi_enabled(EFI_MEMMAP))
1015                 return 0;
1016
1017         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
1018                 md = p;
1019                 if ((md->phys_addr <= phys_addr) &&
1020                     (phys_addr < (md->phys_addr +
1021                                   (md->num_pages << EFI_PAGE_SHIFT))))
1022                         return md->type;
1023         }
1024         return 0;
1025 }
1026
1027 static int __init arch_parse_efi_cmdline(char *str)
1028 {
1029         if (!str) {
1030                 pr_warn("need at least one option\n");
1031                 return -EINVAL;
1032         }
1033
1034         if (parse_option_str(str, "old_map"))
1035                 set_bit(EFI_OLD_MEMMAP, &efi.flags);
1036
1037         return 0;
1038 }
1039 early_param("efi", arch_parse_efi_cmdline);