arch/x86/include/asm/paravirt.h

   1 #ifndef _ASM_X86_PARAVIRT_H
   2 #define _ASM_X86_PARAVIRT_H
   3 /* Various instructions on x86 need to be replaced for
   4  * para-virtualization: those hooks are defined here. */
   5
   6 #ifdef CONFIG_PARAVIRT
   7 #include <asm/pgtable_types.h>
   8 #include <asm/asm.h>
   9
  10 #include <asm/paravirt_types.h>
  11
  12 #ifndef __ASSEMBLY__
  13 #include <linux/bug.h>
  14 #include <linux/types.h>
  15 #include <linux/cpumask.h>
  16 #include <asm/frame.h>
  17
  18 static inline int paravirt_enabled(void)
  19 {
  20         return pv_info.paravirt_enabled;
  21 }
  22
  23 static inline int paravirt_has_feature(unsigned int feature)
  24 {
  25         WARN_ON_ONCE(!pv_info.paravirt_enabled);
  26         return (pv_info.features & feature);
  27 }
  28
  29 static inline void load_sp0(struct tss_struct *tss,
  30                              struct thread_struct *thread)
  31 {
  32         PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread);
  33 }
  34
  35 /* The paravirtualized CPUID instruction. */
  36 static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
  37                            unsigned int *ecx, unsigned int *edx)
  38 {
  39         PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
  40 }
  41
  42 /*
  43  * These special macros can be used to get or set a debugging register
  44  */
  45 static inline unsigned long paravirt_get_debugreg(int reg)
  46 {
  47         return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
  48 }
  49 #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
  50 static inline void set_debugreg(unsigned long val, int reg)
  51 {
  52         PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
  53 }
  54
  55 static inline void clts(void)
  56 {
  57         PVOP_VCALL0(pv_cpu_ops.clts);
  58 }
  59
  60 static inline unsigned long read_cr0(void)
  61 {
  62         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
  63 }
  64
  65 static inline void write_cr0(unsigned long x)
  66 {
  67         PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
  68 }
  69
  70 static inline unsigned long read_cr2(void)
  71 {
  72         return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2);
  73 }
  74
  75 static inline void write_cr2(unsigned long x)
  76 {
  77         PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
  78 }
  79
  80 static inline unsigned long read_cr3(void)
  81 {
  82         return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
  83 }
  84
  85 static inline void write_cr3(unsigned long x)
  86 {
  87         PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
  88 }
  89
  90 static inline unsigned long __read_cr4(void)
  91 {
  92         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
  93 }
  94 static inline unsigned long __read_cr4_safe(void)
  95 {
  96         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
  97 }
  98
  99 static inline void __write_cr4(unsigned long x)
 100 {
 101         PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
 102 }
 103
 104 #ifdef CONFIG_X86_64
 105 static inline unsigned long read_cr8(void)
 106 {
 107         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8);
 108 }
 109
 110 static inline void write_cr8(unsigned long x)
 111 {
 112         PVOP_VCALL1(pv_cpu_ops.write_cr8, x);
 113 }
 114 #endif
 115
 116 static inline void arch_safe_halt(void)
 117 {
 118         PVOP_VCALL0(pv_irq_ops.safe_halt);
 119 }
 120
 121 static inline void halt(void)
 122 {
 123         PVOP_VCALL0(pv_irq_ops.halt);
 124 }
 125
 126 static inline void wbinvd(void)
 127 {
 128         PVOP_VCALL0(pv_cpu_ops.wbinvd);
 129 }
 130
 131 #define get_kernel_rpl()  (pv_info.kernel_rpl)
 132
 133 static inline u64 paravirt_read_msr(unsigned msr)
 134 {
 135         return PVOP_CALL1(u64, pv_cpu_ops.read_msr, msr);
 136 }
 137
 138 static inline void paravirt_write_msr(unsigned msr,
 139                                       unsigned low, unsigned high)
 140 {
 141         return PVOP_VCALL3(pv_cpu_ops.write_msr, msr, low, high);
 142 }
 143
 144 static inline u64 paravirt_read_msr_safe(unsigned msr, int *err)
 145 {
 146         return PVOP_CALL2(u64, pv_cpu_ops.read_msr_safe, msr, err);
 147 }
 148
 149 static inline int paravirt_write_msr_safe(unsigned msr,
 150                                           unsigned low, unsigned high)
 151 {
 152         return PVOP_CALL3(int, pv_cpu_ops.write_msr_safe, msr, low, high);
 153 }
 154
 155 #define rdmsr(msr, val1, val2)                  \
 156 do {                                            \
 157         u64 _l = paravirt_read_msr(msr);        \
 158         val1 = (u32)_l;                         \
 159         val2 = _l >> 32;                        \
 160 } while (0)
 161
 162 #define wrmsr(msr, val1, val2)                  \
 163 do {                                            \
 164         paravirt_write_msr(msr, val1, val2);    \
 165 } while (0)
 166
 167 #define rdmsrl(msr, val)                        \
 168 do {                                            \
 169         val = paravirt_read_msr(msr);           \
 170 } while (0)
 171
 172 static inline void wrmsrl(unsigned msr, u64 val)
 173 {
 174         wrmsr(msr, (u32)val, (u32)(val>>32));
 175 }
 176
 177 #define wrmsr_safe(msr, a, b)   paravirt_write_msr_safe(msr, a, b)
 178
 179 /* rdmsr with exception handling */
 180 #define rdmsr_safe(msr, a, b)                           \
 181 ({                                                      \
 182         int _err;                                       \
 183         u64 _l = paravirt_read_msr_safe(msr, &_err);    \
 184         (*a) = (u32)_l;                                 \
 185         (*b) = _l >> 32;                                \
 186         _err;                                           \
 187 })
 188
 189 static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
 190 {
 191         int err;
 192
 193         *p = paravirt_read_msr_safe(msr, &err);
 194         return err;
 195 }
 196
 197 static inline unsigned long long paravirt_sched_clock(void)
 198 {
 199         return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
 200 }
 201
 202 struct static_key;
 203 extern struct static_key paravirt_steal_enabled;
 204 extern struct static_key paravirt_steal_rq_enabled;
 205
 206 static inline u64 paravirt_steal_clock(int cpu)
 207 {
 208         return PVOP_CALL1(u64, pv_time_ops.steal_clock, cpu);
 209 }
 210
 211 static inline unsigned long long paravirt_read_pmc(int counter)
 212 {
 213         return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
 214 }
 215
 216 #define rdpmc(counter, low, high)               \
 217 do {                                            \
 218         u64 _l = paravirt_read_pmc(counter);    \
 219         low = (u32)_l;                          \
 220         high = _l >> 32;                        \
 221 } while (0)
 222
 223 #define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter))
 224
 225 static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
 226 {
 227         PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries);
 228 }
 229
 230 static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
 231 {
 232         PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries);
 233 }
 234
 235 static inline void load_TR_desc(void)
 236 {
 237         PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
 238 }
 239 static inline void load_gdt(const struct desc_ptr *dtr)
 240 {
 241         PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
 242 }
 243 static inline void load_idt(const struct desc_ptr *dtr)
 244 {
 245         PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
 246 }
 247 static inline void set_ldt(const void *addr, unsigned entries)
 248 {
 249         PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
 250 }
 251 static inline void store_idt(struct desc_ptr *dtr)
 252 {
 253         PVOP_VCALL1(pv_cpu_ops.store_idt, dtr);
 254 }
 255 static inline unsigned long paravirt_store_tr(void)
 256 {
 257         return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
 258 }
 259 #define store_tr(tr)    ((tr) = paravirt_store_tr())
 260 static inline void load_TLS(struct thread_struct *t, unsigned cpu)
 261 {
 262         PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
 263 }
 264
 265 #ifdef CONFIG_X86_64
 266 static inline void load_gs_index(unsigned int gs)
 267 {
 268         PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs);
 269 }
 270 #endif
 271
 272 static inline void write_ldt_entry(struct desc_struct *dt, int entry,
 273                                    const void *desc)
 274 {
 275         PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc);
 276 }
 277
 278 static inline void write_gdt_entry(struct desc_struct *dt, int entry,
 279                                    void *desc, int type)
 280 {
 281         PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type);
 282 }
 283
 284 static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
 285 {
 286         PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g);
 287 }
 288 static inline void set_iopl_mask(unsigned mask)
 289 {
 290         PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
 291 }
 292
 293 /* The paravirtualized I/O functions */
 294 static inline void slow_down_io(void)
 295 {
 296         pv_cpu_ops.io_delay();
 297 #ifdef REALLY_SLOW_IO
 298         pv_cpu_ops.io_delay();
 299         pv_cpu_ops.io_delay();
 300         pv_cpu_ops.io_delay();
 301 #endif
 302 }
 303
 304 static inline void paravirt_activate_mm(struct mm_struct *prev,
 305                                         struct mm_struct *next)
 306 {
 307         PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
 308 }
 309
 310 static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
 311                                           struct mm_struct *mm)
 312 {
 313         PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
 314 }
 315
 316 static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
 317 {
 318         PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
 319 }
 320
 321 static inline void __flush_tlb(void)
 322 {
 323         PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
 324 }
 325 static inline void __flush_tlb_global(void)
 326 {
 327         PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
 328 }
 329 static inline void __flush_tlb_single(unsigned long addr)
 330 {
 331         PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
 332 }
 333
 334 static inline void flush_tlb_others(const struct cpumask *cpumask,
 335                                     struct mm_struct *mm,
 336                                     unsigned long start,
 337                                     unsigned long end)
 338 {
 339         PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end);
 340 }
 341
 342 static inline int paravirt_pgd_alloc(struct mm_struct *mm)
 343 {
 344         return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
 345 }
 346
 347 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
 348 {
 349         PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd);
 350 }
 351
 352 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)
 353 {
 354         PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn);
 355 }
 356 static inline void paravirt_release_pte(unsigned long pfn)
 357 {
 358         PVOP_VCALL1(pv_mmu_ops.release_pte, pfn);
 359 }
 360
 361 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
 362 {
 363         PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn);
 364 }
 365
 366 static inline void paravirt_release_pmd(unsigned long pfn)
 367 {
 368         PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn);
 369 }
 370
 371 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)
 372 {
 373         PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn);
 374 }
 375 static inline void paravirt_release_pud(unsigned long pfn)
 376 {
 377         PVOP_VCALL1(pv_mmu_ops.release_pud, pfn);
 378 }
 379
 380 static inline void pte_update(struct mm_struct *mm, unsigned long addr,
 381                               pte_t *ptep)
 382 {
 383         PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
 384 }
 385
 386 static inline pte_t __pte(pteval_t val)
 387 {
 388         pteval_t ret;
 389
 390         if (sizeof(pteval_t) > sizeof(long))
 391                 ret = PVOP_CALLEE2(pteval_t,
 392                                    pv_mmu_ops.make_pte,
 393                                    val, (u64)val >> 32);
 394         else
 395                 ret = PVOP_CALLEE1(pteval_t,
 396                                    pv_mmu_ops.make_pte,
 397                                    val);
 398
 399         return (pte_t) { .pte = ret };
 400 }
 401
 402 static inline pteval_t pte_val(pte_t pte)
 403 {
 404         pteval_t ret;
 405
 406         if (sizeof(pteval_t) > sizeof(long))
 407                 ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
 408                                    pte.pte, (u64)pte.pte >> 32);
 409         else
 410                 ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val,
 411                                    pte.pte);
 412
 413         return ret;
 414 }
 415
 416 static inline pgd_t __pgd(pgdval_t val)
 417 {
 418         pgdval_t ret;
 419
 420         if (sizeof(pgdval_t) > sizeof(long))
 421                 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
 422                                    val, (u64)val >> 32);
 423         else
 424                 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
 425                                    val);
 426
 427         return (pgd_t) { ret };
 428 }
 429
 430 static inline pgdval_t pgd_val(pgd_t pgd)
 431 {
 432         pgdval_t ret;
 433
 434         if (sizeof(pgdval_t) > sizeof(long))
 435                 ret =  PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
 436                                     pgd.pgd, (u64)pgd.pgd >> 32);
 437         else
 438                 ret =  PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val,
 439                                     pgd.pgd);
 440
 441         return ret;
 442 }
 443
 444 #define  __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
 445 static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
 446                                            pte_t *ptep)
 447 {
 448         pteval_t ret;
 449
 450         ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start,
 451                          mm, addr, ptep);
 452
 453         return (pte_t) { .pte = ret };
 454 }
 455
 456 static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
 457                                            pte_t *ptep, pte_t pte)
 458 {
 459         if (sizeof(pteval_t) > sizeof(long))
 460                 /* 5 arg words */
 461                 pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte);
 462         else
 463                 PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit,
 464                             mm, addr, ptep, pte.pte);
 465 }
 466
 467 static inline void set_pte(pte_t *ptep, pte_t pte)
 468 {
 469         if (sizeof(pteval_t) > sizeof(long))
 470                 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep,
 471                             pte.pte, (u64)pte.pte >> 32);
 472         else
 473                 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep,
 474                             pte.pte);
 475 }
 476
 477 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
 478                               pte_t *ptep, pte_t pte)
 479 {
 480         if (sizeof(pteval_t) > sizeof(long))
 481                 /* 5 arg words */
 482                 pv_mmu_ops.set_pte_at(mm, addr, ptep, pte);
 483         else
 484                 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte);
 485 }
 486
 487 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
 488                               pmd_t *pmdp, pmd_t pmd)
 489 {
 490         if (sizeof(pmdval_t) > sizeof(long))
 491                 /* 5 arg words */
 492                 pv_mmu_ops.set_pmd_at(mm, addr, pmdp, pmd);
 493         else
 494                 PVOP_VCALL4(pv_mmu_ops.set_pmd_at, mm, addr, pmdp,
 495                             native_pmd_val(pmd));
 496 }
 497
 498 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
 499 {
 500         pmdval_t val = native_pmd_val(pmd);
 501
 502         if (sizeof(pmdval_t) > sizeof(long))
 503                 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32);
 504         else
 505                 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val);
 506 }
 507
 508 #if CONFIG_PGTABLE_LEVELS >= 3
 509 static inline pmd_t __pmd(pmdval_t val)
 510 {
 511         pmdval_t ret;
 512
 513         if (sizeof(pmdval_t) > sizeof(long))
 514                 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
 515                                    val, (u64)val >> 32);
 516         else
 517                 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd,
 518                                    val);
 519
 520         return (pmd_t) { ret };
 521 }
 522
 523 static inline pmdval_t pmd_val(pmd_t pmd)
 524 {
 525         pmdval_t ret;
 526
 527         if (sizeof(pmdval_t) > sizeof(long))
 528                 ret =  PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
 529                                     pmd.pmd, (u64)pmd.pmd >> 32);
 530         else
 531                 ret =  PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val,
 532                                     pmd.pmd);
 533
 534         return ret;
 535 }
 536
 537 static inline void set_pud(pud_t *pudp, pud_t pud)
 538 {
 539         pudval_t val = native_pud_val(pud);
 540
 541         if (sizeof(pudval_t) > sizeof(long))
 542                 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
 543                             val, (u64)val >> 32);
 544         else
 545                 PVOP_VCALL2(pv_mmu_ops.set_pud, pudp,
 546                             val);
 547 }
 548 #if CONFIG_PGTABLE_LEVELS == 4
 549 static inline pud_t __pud(pudval_t val)
 550 {
 551         pudval_t ret;
 552
 553         if (sizeof(pudval_t) > sizeof(long))
 554                 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
 555                                    val, (u64)val >> 32);
 556         else
 557                 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
 558                                    val);
 559
 560         return (pud_t) { ret };
 561 }
 562
 563 static inline pudval_t pud_val(pud_t pud)
 564 {
 565         pudval_t ret;
 566
 567         if (sizeof(pudval_t) > sizeof(long))
 568                 ret =  PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
 569                                     pud.pud, (u64)pud.pud >> 32);
 570         else
 571                 ret =  PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val,
 572                                     pud.pud);
 573
 574         return ret;
 575 }
 576
 577 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
 578 {
 579         pgdval_t val = native_pgd_val(pgd);
 580
 581         if (sizeof(pgdval_t) > sizeof(long))
 582                 PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp,
 583                             val, (u64)val >> 32);
 584         else
 585                 PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp,
 586                             val);
 587 }
 588
 589 static inline void pgd_clear(pgd_t *pgdp)
 590 {
 591         set_pgd(pgdp, __pgd(0));
 592 }
 593
 594 static inline void pud_clear(pud_t *pudp)
 595 {
 596         set_pud(pudp, __pud(0));
 597 }
 598
 599 #endif  /* CONFIG_PGTABLE_LEVELS == 4 */
 600
 601 #endif  /* CONFIG_PGTABLE_LEVELS >= 3 */
 602
 603 #ifdef CONFIG_X86_PAE
 604 /* Special-case pte-setting operations for PAE, which can't update a
 605    64-bit pte atomically */
 606 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
 607 {
 608         PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
 609                     pte.pte, pte.pte >> 32);
 610 }
 611
 612 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
 613                              pte_t *ptep)
 614 {
 615         PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
 616 }
 617
 618 static inline void pmd_clear(pmd_t *pmdp)
 619 {
 620         PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
 621 }
 622 #else  /* !CONFIG_X86_PAE */
 623 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
 624 {
 625         set_pte(ptep, pte);
 626 }
 627
 628 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
 629                              pte_t *ptep)
 630 {
 631         set_pte_at(mm, addr, ptep, __pte(0));
 632 }
 633
 634 static inline void pmd_clear(pmd_t *pmdp)
 635 {
 636         set_pmd(pmdp, __pmd(0));
 637 }
 638 #endif  /* CONFIG_X86_PAE */
 639
 640 #define  __HAVE_ARCH_START_CONTEXT_SWITCH
 641 static inline void arch_start_context_switch(struct task_struct *prev)
 642 {
 643         PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev);
 644 }
 645
 646 static inline void arch_end_context_switch(struct task_struct *next)
 647 {
 648         PVOP_VCALL1(pv_cpu_ops.end_context_switch, next);
 649 }
 650
 651 #define  __HAVE_ARCH_ENTER_LAZY_MMU_MODE
 652 static inline void arch_enter_lazy_mmu_mode(void)
 653 {
 654         PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter);
 655 }
 656
 657 static inline void arch_leave_lazy_mmu_mode(void)
 658 {
 659         PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave);
 660 }
 661
 662 static inline void arch_flush_lazy_mmu_mode(void)
 663 {
 664         PVOP_VCALL0(pv_mmu_ops.lazy_mode.flush);
 665 }
 666
 667 static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
 668                                 phys_addr_t phys, pgprot_t flags)
 669 {
 670         pv_mmu_ops.set_fixmap(idx, phys, flags);
 671 }
 672
 673 #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
 674
 675 #ifdef CONFIG_QUEUED_SPINLOCKS
 676
 677 static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock,
 678                                                         u32 val)
 679 {
 680         PVOP_VCALL2(pv_lock_ops.queued_spin_lock_slowpath, lock, val);
 681 }
 682
 683 static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock)
 684 {
 685         PVOP_VCALLEE1(pv_lock_ops.queued_spin_unlock, lock);
 686 }
 687
 688 static __always_inline void pv_wait(u8 *ptr, u8 val)
 689 {
 690         PVOP_VCALL2(pv_lock_ops.wait, ptr, val);
 691 }
 692
 693 static __always_inline void pv_kick(int cpu)
 694 {
 695         PVOP_VCALL1(pv_lock_ops.kick, cpu);
 696 }
 697
 698 #else /* !CONFIG_QUEUED_SPINLOCKS */
 699
 700 static __always_inline void __ticket_lock_spinning(struct arch_spinlock *lock,
 701                                                         __ticket_t ticket)
 702 {
 703         PVOP_VCALLEE2(pv_lock_ops.lock_spinning, lock, ticket);
 704 }
 705
 706 static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock,
 707                                                         __ticket_t ticket)
 708 {
 709         PVOP_VCALL2(pv_lock_ops.unlock_kick, lock, ticket);
 710 }
 711
 712 #endif /* CONFIG_QUEUED_SPINLOCKS */
 713
 714 #endif /* SMP && PARAVIRT_SPINLOCKS */
 715
 716 #ifdef CONFIG_X86_32
 717 #define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
 718 #define PV_RESTORE_REGS "popl %edx; popl %ecx;"
 719
 720 /* save and restore all caller-save registers, except return value */
 721 #define PV_SAVE_ALL_CALLER_REGS         "pushl %ecx;"
 722 #define PV_RESTORE_ALL_CALLER_REGS      "popl  %ecx;"
 723
 724 #define PV_FLAGS_ARG "0"
 725 #define PV_EXTRA_CLOBBERS
 726 #define PV_VEXTRA_CLOBBERS
 727 #else
 728 /* save and restore all caller-save registers, except return value */
 729 #define PV_SAVE_ALL_CALLER_REGS                                         \
 730         "push %rcx;"                                                    \
 731         "push %rdx;"                                                    \
 732         "push %rsi;"                                                    \
 733         "push %rdi;"                                                    \
 734         "push %r8;"                                                     \
 735         "push %r9;"                                                     \
 736         "push %r10;"                                                    \
 737         "push %r11;"
 738 #define PV_RESTORE_ALL_CALLER_REGS                                      \
 739         "pop %r11;"                                                     \
 740         "pop %r10;"                                                     \
 741         "pop %r9;"                                                      \
 742         "pop %r8;"                                                      \
 743         "pop %rdi;"                                                     \
 744         "pop %rsi;"                                                     \
 745         "pop %rdx;"                                                     \
 746         "pop %rcx;"
 747
 748 /* We save some registers, but all of them, that's too much. We clobber all
 749  * caller saved registers but the argument parameter */
 750 #define PV_SAVE_REGS "pushq %%rdi;"
 751 #define PV_RESTORE_REGS "popq %%rdi;"
 752 #define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi"
 753 #define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi"
 754 #define PV_FLAGS_ARG "D"
 755 #endif
 756
 757 /*
 758  * Generate a thunk around a function which saves all caller-save
 759  * registers except for the return value.  This allows C functions to
 760  * be called from assembler code where fewer than normal registers are
 761  * available.  It may also help code generation around calls from C
 762  * code if the common case doesn't use many registers.
 763  *
 764  * When a callee is wrapped in a thunk, the caller can assume that all
 765  * arg regs and all scratch registers are preserved across the
 766  * call. The return value in rax/eax will not be saved, even for void
 767  * functions.
 768  */
 769 #define PV_THUNK_NAME(func) "__raw_callee_save_" #func
 770 #define PV_CALLEE_SAVE_REGS_THUNK(func)                                 \
 771         extern typeof(func) __raw_callee_save_##func;                   \
 772                                                                         \
 773         asm(".pushsection .text;"                                       \
 774             ".globl " PV_THUNK_NAME(func) ";"                           \
 775             ".type " PV_THUNK_NAME(func) ", @function;"                 \
 776             PV_THUNK_NAME(func) ":"                                     \
 777             FRAME_BEGIN                                                 \
 778             PV_SAVE_ALL_CALLER_REGS                                     \
 779             "call " #func ";"                                           \
 780             PV_RESTORE_ALL_CALLER_REGS                                  \
 781             FRAME_END                                                   \
 782             "ret;"                                                      \
 783             ".popsection")
 784
 785 /* Get a reference to a callee-save function */
 786 #define PV_CALLEE_SAVE(func)                                            \
 787         ((struct paravirt_callee_save) { __raw_callee_save_##func })
 788
 789 /* Promise that "func" already uses the right calling convention */
 790 #define __PV_IS_CALLEE_SAVE(func)                       \
 791         ((struct paravirt_callee_save) { func })
 792
 793 static inline notrace unsigned long arch_local_save_flags(void)
 794 {
 795         return PVOP_CALLEE0(unsigned long, pv_irq_ops.save_fl);
 796 }
 797
 798 static inline notrace void arch_local_irq_restore(unsigned long f)
 799 {
 800         PVOP_VCALLEE1(pv_irq_ops.restore_fl, f);
 801 }
 802
 803 static inline notrace void arch_local_irq_disable(void)
 804 {
 805         PVOP_VCALLEE0(pv_irq_ops.irq_disable);
 806 }
 807
 808 static inline notrace void arch_local_irq_enable(void)
 809 {
 810         PVOP_VCALLEE0(pv_irq_ops.irq_enable);
 811 }
 812
 813 static inline notrace unsigned long arch_local_irq_save(void)
 814 {
 815         unsigned long f;
 816
 817         f = arch_local_save_flags();
 818         arch_local_irq_disable();
 819         return f;
 820 }
 821
 822
 823 /* Make sure as little as possible of this mess escapes. */
 824 #undef PARAVIRT_CALL
 825 #undef __PVOP_CALL
 826 #undef __PVOP_VCALL
 827 #undef PVOP_VCALL0
 828 #undef PVOP_CALL0
 829 #undef PVOP_VCALL1
 830 #undef PVOP_CALL1
 831 #undef PVOP_VCALL2
 832 #undef PVOP_CALL2
 833 #undef PVOP_VCALL3
 834 #undef PVOP_CALL3
 835 #undef PVOP_VCALL4
 836 #undef PVOP_CALL4
 837
 838 extern void default_banner(void);
 839
 840 #else  /* __ASSEMBLY__ */
 841
 842 #define _PVSITE(ptype, clobbers, ops, word, algn)       \
 843 771:;                                           \
 844         ops;                                    \
 845 772:;                                           \
 846         .pushsection .parainstructions,"a";     \
 847          .align algn;                           \
 848          word 771b;                             \
 849          .byte ptype;                           \
 850          .byte 772b-771b;                       \
 851          .short clobbers;                       \
 852         .popsection
 853
 854
 855 #define COND_PUSH(set, mask, reg)                       \
 856         .if ((~(set)) & mask); push %reg; .endif
 857 #define COND_POP(set, mask, reg)                        \
 858         .if ((~(set)) & mask); pop %reg; .endif
 859
 860 #ifdef CONFIG_X86_64
 861
 862 #define PV_SAVE_REGS(set)                       \
 863         COND_PUSH(set, CLBR_RAX, rax);          \
 864         COND_PUSH(set, CLBR_RCX, rcx);          \
 865         COND_PUSH(set, CLBR_RDX, rdx);          \
 866         COND_PUSH(set, CLBR_RSI, rsi);          \
 867         COND_PUSH(set, CLBR_RDI, rdi);          \
 868         COND_PUSH(set, CLBR_R8, r8);            \
 869         COND_PUSH(set, CLBR_R9, r9);            \
 870         COND_PUSH(set, CLBR_R10, r10);          \
 871         COND_PUSH(set, CLBR_R11, r11)
 872 #define PV_RESTORE_REGS(set)                    \
 873         COND_POP(set, CLBR_R11, r11);           \
 874         COND_POP(set, CLBR_R10, r10);           \
 875         COND_POP(set, CLBR_R9, r9);             \
 876         COND_POP(set, CLBR_R8, r8);             \
 877         COND_POP(set, CLBR_RDI, rdi);           \
 878         COND_POP(set, CLBR_RSI, rsi);           \
 879         COND_POP(set, CLBR_RDX, rdx);           \
 880         COND_POP(set, CLBR_RCX, rcx);           \
 881         COND_POP(set, CLBR_RAX, rax)
 882
 883 #define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 8)
 884 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
 885 #define PARA_INDIRECT(addr)     *addr(%rip)
 886 #else
 887 #define PV_SAVE_REGS(set)                       \
 888         COND_PUSH(set, CLBR_EAX, eax);          \
 889         COND_PUSH(set, CLBR_EDI, edi);          \
 890         COND_PUSH(set, CLBR_ECX, ecx);          \
 891         COND_PUSH(set, CLBR_EDX, edx)
 892 #define PV_RESTORE_REGS(set)                    \
 893         COND_POP(set, CLBR_EDX, edx);           \
 894         COND_POP(set, CLBR_ECX, ecx);           \
 895         COND_POP(set, CLBR_EDI, edi);           \
 896         COND_POP(set, CLBR_EAX, eax)
 897
 898 #define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 4)
 899 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
 900 #define PARA_INDIRECT(addr)     *%cs:addr
 901 #endif
 902
 903 #define INTERRUPT_RETURN                                                \
 904         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE,       \
 905                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret))
 906
 907 #define DISABLE_INTERRUPTS(clobbers)                                    \
 908         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
 909                   PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);            \
 910                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable);    \
 911                   PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
 912
 913 #define ENABLE_INTERRUPTS(clobbers)                                     \
 914         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers,  \
 915                   PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);            \
 916                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable);     \
 917                   PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
 918
 919 #ifdef CONFIG_X86_32
 920 #define GET_CR0_INTO_EAX                                \
 921         push %ecx; push %edx;                           \
 922         call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \
 923         pop %edx; pop %ecx
 924 #else   /* !CONFIG_X86_32 */
 925
 926 /*
 927  * If swapgs is used while the userspace stack is still current,
 928  * there's no way to call a pvop.  The PV replacement *must* be
 929  * inlined, or the swapgs instruction must be trapped and emulated.
 930  */
 931 #define SWAPGS_UNSAFE_STACK                                             \
 932         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,     \
 933                   swapgs)
 934
 935 /*
 936  * Note: swapgs is very special, and in practise is either going to be
 937  * implemented with a single "swapgs" instruction or something very
 938  * special.  Either way, we don't need to save any registers for
 939  * it.
 940  */
 941 #define SWAPGS                                                          \
 942         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,     \
 943                   call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs)          \
 944                  )
 945
 946 #define GET_CR2_INTO_RAX                                \
 947         call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2)
 948
 949 #define PARAVIRT_ADJUST_EXCEPTION_FRAME                                 \
 950         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \
 951                   CLBR_NONE,                                            \
 952                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame))
 953
 954 #define USERGS_SYSRET64                                                 \
 955         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64),       \
 956                   CLBR_NONE,                                            \
 957                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64))
 958 #endif  /* CONFIG_X86_32 */
 959
 960 #endif /* __ASSEMBLY__ */
 961 #else  /* CONFIG_PARAVIRT */
 962 # define default_banner x86_init_noop
 963 #ifndef __ASSEMBLY__
 964 static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
 965                                           struct mm_struct *mm)
 966 {
 967 }
 968
 969 static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
 970 {
 971 }
 972 #endif /* __ASSEMBLY__ */
 973 #endif /* !CONFIG_PARAVIRT */
 974 #endif /* _ASM_X86_PARAVIRT_H */