arch/x86/kernel/cpu/perf_counter.c

   1 /*
   2  * Performance counter x86 architecture code
   3  *
   4  *  Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
   6  *  Copyright(C) 2009 Jaswinder Singh Rajput
   7  *  Copyright(C) 2009 Advanced Micro Devices, Inc., Robert Richter
   8  *
   9  *  For licencing details see kernel-base/COPYING
  10  */
  11
  12 #include <linux/perf_counter.h>
  13 #include <linux/capability.h>
  14 #include <linux/notifier.h>
  15 #include <linux/hardirq.h>
  16 #include <linux/kprobes.h>
  17 #include <linux/module.h>
  18 #include <linux/kdebug.h>
  19 #include <linux/sched.h>
  20 #include <linux/uaccess.h>
  21
  22 #include <asm/apic.h>
  23 #include <asm/stacktrace.h>
  24 #include <asm/nmi.h>
  25
  26 static bool perf_counters_initialized __read_mostly;
  27
  28 /*
  29  * Number of (generic) HW counters:
  30  */
  31 static int nr_counters_generic __read_mostly;
  32 static u64 perf_counter_mask __read_mostly;
  33 static u64 counter_value_mask __read_mostly;
  34 static int counter_value_bits __read_mostly;
  35
  36 static int nr_counters_fixed __read_mostly;
  37
  38 struct cpu_hw_counters {
  39         struct perf_counter     *counters[X86_PMC_IDX_MAX];
  40         unsigned long           used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  41         unsigned long           interrupts;
  42         u64                     throttle_ctrl;
  43         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  44         int                     enabled;
  45 };
  46
  47 /*
  48  * struct x86_pmu - generic x86 pmu
  49  */
  50 struct x86_pmu {
  51         int             (*handle_irq)(struct pt_regs *, int);
  52         u64             (*save_disable_all)(void);
  53         void            (*restore_all)(u64);
  54         void            (*enable)(int, u64);
  55         void            (*disable)(int, u64);
  56         unsigned        eventsel;
  57         unsigned        perfctr;
  58         u64             (*event_map)(int);
  59         u64             (*raw_event)(u64);
  60         int             max_events;
  61 };
  62
  63 static struct x86_pmu x86_pmu __read_mostly;
  64
  65 static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
  66         .enabled = 1,
  67 };
  68
  69 static __read_mostly int intel_perfmon_version;
  70
  71 /*
  72  * Intel PerfMon v3. Used on Core2 and later.
  73  */
  74 static const u64 intel_perfmon_event_map[] =
  75 {
  76   [PERF_COUNT_CPU_CYCLES]               = 0x003c,
  77   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
  78   [PERF_COUNT_CACHE_REFERENCES]         = 0x4f2e,
  79   [PERF_COUNT_CACHE_MISSES]             = 0x412e,
  80   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
  81   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
  82   [PERF_COUNT_BUS_CYCLES]               = 0x013c,
  83 };
  84
  85 static u64 intel_pmu_event_map(int event)
  86 {
  87         return intel_perfmon_event_map[event];
  88 }
  89
  90 static u64 intel_pmu_raw_event(u64 event)
  91 {
  92 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
  93 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
  94 #define CORE_EVNTSEL_COUNTER_MASK       0xFF000000ULL
  95
  96 #define CORE_EVNTSEL_MASK               \
  97         (CORE_EVNTSEL_EVENT_MASK |      \
  98          CORE_EVNTSEL_UNIT_MASK  |      \
  99          CORE_EVNTSEL_COUNTER_MASK)
 100
 101         return event & CORE_EVNTSEL_MASK;
 102 }
 103
 104 /*
 105  * AMD Performance Monitor K7 and later.
 106  */
 107 static const u64 amd_perfmon_event_map[] =
 108 {
 109   [PERF_COUNT_CPU_CYCLES]               = 0x0076,
 110   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
 111   [PERF_COUNT_CACHE_REFERENCES]         = 0x0080,
 112   [PERF_COUNT_CACHE_MISSES]             = 0x0081,
 113   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
 114   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
 115 };
 116
 117 static u64 amd_pmu_event_map(int event)
 118 {
 119         return amd_perfmon_event_map[event];
 120 }
 121
 122 static u64 amd_pmu_raw_event(u64 event)
 123 {
 124 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
 125 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
 126 #define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
 127
 128 #define K7_EVNTSEL_MASK                 \
 129         (K7_EVNTSEL_EVENT_MASK |        \
 130          K7_EVNTSEL_UNIT_MASK  |        \
 131          K7_EVNTSEL_COUNTER_MASK)
 132
 133         return event & K7_EVNTSEL_MASK;
 134 }
 135
 136 /*
 137  * Propagate counter elapsed time into the generic counter.
 138  * Can only be executed on the CPU where the counter is active.
 139  * Returns the delta events processed.
 140  */
 141 static void
 142 x86_perf_counter_update(struct perf_counter *counter,
 143                         struct hw_perf_counter *hwc, int idx)
 144 {
 145         u64 prev_raw_count, new_raw_count, delta;
 146
 147         /*
 148          * Careful: an NMI might modify the previous counter value.
 149          *
 150          * Our tactic to handle this is to first atomically read and
 151          * exchange a new raw count - then add that new-prev delta
 152          * count to the generic counter atomically:
 153          */
 154 again:
 155         prev_raw_count = atomic64_read(&hwc->prev_count);
 156         rdmsrl(hwc->counter_base + idx, new_raw_count);
 157
 158         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 159                                         new_raw_count) != prev_raw_count)
 160                 goto again;
 161
 162         /*
 163          * Now we have the new raw value and have updated the prev
 164          * timestamp already. We can now calculate the elapsed delta
 165          * (counter-)time and add that to the generic counter.
 166          *
 167          * Careful, not all hw sign-extends above the physical width
 168          * of the count, so we do that by clipping the delta to 32 bits:
 169          */
 170         delta = (u64)(u32)((s32)new_raw_count - (s32)prev_raw_count);
 171
 172         atomic64_add(delta, &counter->count);
 173         atomic64_sub(delta, &hwc->period_left);
 174 }
 175
 176 static atomic_t num_counters;
 177 static DEFINE_MUTEX(pmc_reserve_mutex);
 178
 179 static bool reserve_pmc_hardware(void)
 180 {
 181         int i;
 182
 183         if (nmi_watchdog == NMI_LOCAL_APIC)
 184                 disable_lapic_nmi_watchdog();
 185
 186         for (i = 0; i < nr_counters_generic; i++) {
 187                 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
 188                         goto perfctr_fail;
 189         }
 190
 191         for (i = 0; i < nr_counters_generic; i++) {
 192                 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
 193                         goto eventsel_fail;
 194         }
 195
 196         return true;
 197
 198 eventsel_fail:
 199         for (i--; i >= 0; i--)
 200                 release_evntsel_nmi(x86_pmu.eventsel + i);
 201
 202         i = nr_counters_generic;
 203
 204 perfctr_fail:
 205         for (i--; i >= 0; i--)
 206                 release_perfctr_nmi(x86_pmu.perfctr + i);
 207
 208         if (nmi_watchdog == NMI_LOCAL_APIC)
 209                 enable_lapic_nmi_watchdog();
 210
 211         return false;
 212 }
 213
 214 static void release_pmc_hardware(void)
 215 {
 216         int i;
 217
 218         for (i = 0; i < nr_counters_generic; i++) {
 219                 release_perfctr_nmi(x86_pmu.perfctr + i);
 220                 release_evntsel_nmi(x86_pmu.eventsel + i);
 221         }
 222
 223         if (nmi_watchdog == NMI_LOCAL_APIC)
 224                 enable_lapic_nmi_watchdog();
 225 }
 226
 227 static void hw_perf_counter_destroy(struct perf_counter *counter)
 228 {
 229         if (atomic_dec_and_mutex_lock(&num_counters, &pmc_reserve_mutex)) {
 230                 release_pmc_hardware();
 231                 mutex_unlock(&pmc_reserve_mutex);
 232         }
 233 }
 234
 235 /*
 236  * Setup the hardware configuration for a given hw_event_type
 237  */
 238 static int __hw_perf_counter_init(struct perf_counter *counter)
 239 {
 240         struct perf_counter_hw_event *hw_event = &counter->hw_event;
 241         struct hw_perf_counter *hwc = &counter->hw;
 242         int err;
 243
 244         /* disable temporarily */
 245         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
 246                 return -ENOSYS;
 247
 248         if (unlikely(!perf_counters_initialized))
 249                 return -EINVAL;
 250
 251         err = 0;
 252         if (atomic_inc_not_zero(&num_counters)) {
 253                 mutex_lock(&pmc_reserve_mutex);
 254                 if (atomic_read(&num_counters) == 0 && !reserve_pmc_hardware())
 255                         err = -EBUSY;
 256                 else
 257                         atomic_inc(&num_counters);
 258                 mutex_unlock(&pmc_reserve_mutex);
 259         }
 260         if (err)
 261                 return err;
 262
 263         /*
 264          * Generate PMC IRQs:
 265          * (keep 'enabled' bit clear for now)
 266          */
 267         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 268
 269         /*
 270          * Count user and OS events unless requested not to.
 271          */
 272         if (!hw_event->exclude_user)
 273                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
 274         if (!hw_event->exclude_kernel)
 275                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
 276
 277         /*
 278          * If privileged enough, allow NMI events:
 279          */
 280         hwc->nmi = 0;
 281         if (capable(CAP_SYS_ADMIN) && hw_event->nmi)
 282                 hwc->nmi = 1;
 283
 284         hwc->irq_period         = hw_event->irq_period;
 285         /*
 286          * Intel PMCs cannot be accessed sanely above 32 bit width,
 287          * so we install an artificial 1<<31 period regardless of
 288          * the generic counter period:
 289          */
 290         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
 291                 if ((s64)hwc->irq_period <= 0 || hwc->irq_period > 0x7FFFFFFF)
 292                         hwc->irq_period = 0x7FFFFFFF;
 293
 294         atomic64_set(&hwc->period_left, hwc->irq_period);
 295
 296         /*
 297          * Raw event type provide the config in the event structure
 298          */
 299         if (perf_event_raw(hw_event)) {
 300                 hwc->config |= x86_pmu.raw_event(perf_event_config(hw_event));
 301         } else {
 302                 if (perf_event_id(hw_event) >= x86_pmu.max_events)
 303                         return -EINVAL;
 304                 /*
 305                  * The generic map:
 306                  */
 307                 hwc->config |= x86_pmu.event_map(perf_event_id(hw_event));
 308         }
 309
 310         counter->destroy = hw_perf_counter_destroy;
 311
 312         return 0;
 313 }
 314
 315 static u64 intel_pmu_save_disable_all(void)
 316 {
 317         u64 ctrl;
 318
 319         rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 320         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 321
 322         return ctrl;
 323 }
 324
 325 static u64 amd_pmu_save_disable_all(void)
 326 {
 327         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 328         int enabled, idx;
 329
 330         enabled = cpuc->enabled;
 331         cpuc->enabled = 0;
 332         /*
 333          * ensure we write the disable before we start disabling the
 334          * counters proper, so that amd_pmu_enable_counter() does the
 335          * right thing.
 336          */
 337         barrier();
 338
 339         for (idx = 0; idx < nr_counters_generic; idx++) {
 340                 u64 val;
 341
 342                 if (!test_bit(idx, cpuc->active_mask))
 343                         continue;
 344                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 345                 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
 346                         continue;
 347                 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
 348                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 349         }
 350
 351         return enabled;
 352 }
 353
 354 u64 hw_perf_save_disable(void)
 355 {
 356         if (unlikely(!perf_counters_initialized))
 357                 return 0;
 358
 359         return x86_pmu.save_disable_all();
 360 }
 361 /*
 362  * Exported because of ACPI idle
 363  */
 364 EXPORT_SYMBOL_GPL(hw_perf_save_disable);
 365
 366 static void intel_pmu_restore_all(u64 ctrl)
 367 {
 368         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 369 }
 370
 371 static void amd_pmu_restore_all(u64 ctrl)
 372 {
 373         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 374         int idx;
 375
 376         cpuc->enabled = ctrl;
 377         barrier();
 378         if (!ctrl)
 379                 return;
 380
 381         for (idx = 0; idx < nr_counters_generic; idx++) {
 382                 u64 val;
 383
 384                 if (!test_bit(idx, cpuc->active_mask))
 385                         continue;
 386                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 387                 if (val & ARCH_PERFMON_EVENTSEL0_ENABLE)
 388                         continue;
 389                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 390                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 391         }
 392 }
 393
 394 void hw_perf_restore(u64 ctrl)
 395 {
 396         if (unlikely(!perf_counters_initialized))
 397                 return;
 398
 399         x86_pmu.restore_all(ctrl);
 400 }
 401 /*
 402  * Exported because of ACPI idle
 403  */
 404 EXPORT_SYMBOL_GPL(hw_perf_restore);
 405
 406 static inline u64 intel_pmu_get_status(u64 mask)
 407 {
 408         u64 status;
 409
 410         if (unlikely(!perf_counters_initialized))
 411                 return 0;
 412         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 413
 414         return status;
 415 }
 416
 417 static inline void intel_pmu_ack_status(u64 ack)
 418 {
 419         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
 420 }
 421
 422 static void intel_pmu_enable_counter(int idx, u64 config)
 423 {
 424         wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx,
 425                         config | ARCH_PERFMON_EVENTSEL0_ENABLE);
 426 }
 427
 428 static void amd_pmu_enable_counter(int idx, u64 config)
 429 {
 430         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 431
 432         set_bit(idx, cpuc->active_mask);
 433         if (cpuc->enabled)
 434                 config |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 435
 436         wrmsrl(MSR_K7_EVNTSEL0 + idx, config);
 437 }
 438
 439 static void hw_perf_enable(int idx, u64 config)
 440 {
 441         if (unlikely(!perf_counters_initialized))
 442                 return;
 443
 444         x86_pmu.enable(idx, config);
 445 }
 446
 447 static void intel_pmu_disable_counter(int idx, u64 config)
 448 {
 449         wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx, config);
 450 }
 451
 452 static void amd_pmu_disable_counter(int idx, u64 config)
 453 {
 454         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 455
 456         clear_bit(idx, cpuc->active_mask);
 457         wrmsrl(MSR_K7_EVNTSEL0 + idx, config);
 458
 459 }
 460
 461 static void hw_perf_disable(int idx, u64 config)
 462 {
 463         if (unlikely(!perf_counters_initialized))
 464                 return;
 465
 466         x86_pmu.disable(idx, config);
 467 }
 468
 469 static inline void
 470 __pmc_fixed_disable(struct perf_counter *counter,
 471                     struct hw_perf_counter *hwc, unsigned int __idx)
 472 {
 473         int idx = __idx - X86_PMC_IDX_FIXED;
 474         u64 ctrl_val, mask;
 475         int err;
 476
 477         mask = 0xfULL << (idx * 4);
 478
 479         rdmsrl(hwc->config_base, ctrl_val);
 480         ctrl_val &= ~mask;
 481         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 482 }
 483
 484 static inline void
 485 __x86_pmu_disable(struct perf_counter *counter,
 486                   struct hw_perf_counter *hwc, unsigned int idx)
 487 {
 488         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
 489                 __pmc_fixed_disable(counter, hwc, idx);
 490         else
 491                 hw_perf_disable(idx, hwc->config);
 492 }
 493
 494 static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
 495
 496 /*
 497  * Set the next IRQ period, based on the hwc->period_left value.
 498  * To be called with the counter disabled in hw:
 499  */
 500 static void
 501 x86_perf_counter_set_period(struct perf_counter *counter,
 502                              struct hw_perf_counter *hwc, int idx)
 503 {
 504         s64 left = atomic64_read(&hwc->period_left);
 505         s64 period = hwc->irq_period;
 506         int err;
 507
 508         /*
 509          * If we are way outside a reasoable range then just skip forward:
 510          */
 511         if (unlikely(left <= -period)) {
 512                 left = period;
 513                 atomic64_set(&hwc->period_left, left);
 514         }
 515
 516         if (unlikely(left <= 0)) {
 517                 left += period;
 518                 atomic64_set(&hwc->period_left, left);
 519         }
 520
 521         per_cpu(prev_left[idx], smp_processor_id()) = left;
 522
 523         /*
 524          * The hw counter starts counting from this counter offset,
 525          * mark it to be able to extra future deltas:
 526          */
 527         atomic64_set(&hwc->prev_count, (u64)-left);
 528
 529         err = checking_wrmsrl(hwc->counter_base + idx,
 530                              (u64)(-left) & counter_value_mask);
 531 }
 532
 533 static inline void
 534 __pmc_fixed_enable(struct perf_counter *counter,
 535                    struct hw_perf_counter *hwc, unsigned int __idx)
 536 {
 537         int idx = __idx - X86_PMC_IDX_FIXED;
 538         u64 ctrl_val, bits, mask;
 539         int err;
 540
 541         /*
 542          * Enable IRQ generation (0x8),
 543          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
 544          * if requested:
 545          */
 546         bits = 0x8ULL;
 547         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
 548                 bits |= 0x2;
 549         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
 550                 bits |= 0x1;
 551         bits <<= (idx * 4);
 552         mask = 0xfULL << (idx * 4);
 553
 554         rdmsrl(hwc->config_base, ctrl_val);
 555         ctrl_val &= ~mask;
 556         ctrl_val |= bits;
 557         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 558 }
 559
 560 static void
 561 __x86_pmu_enable(struct perf_counter *counter,
 562                  struct hw_perf_counter *hwc, int idx)
 563 {
 564         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
 565                 __pmc_fixed_enable(counter, hwc, idx);
 566         else
 567                 hw_perf_enable(idx, hwc->config);
 568 }
 569
 570 static int
 571 fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
 572 {
 573         unsigned int event;
 574
 575         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
 576                 return -1;
 577
 578         if (unlikely(hwc->nmi))
 579                 return -1;
 580
 581         event = hwc->config & ARCH_PERFMON_EVENT_MASK;
 582
 583         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_INSTRUCTIONS)))
 584                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
 585         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_CPU_CYCLES)))
 586                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
 587         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_BUS_CYCLES)))
 588                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
 589
 590         return -1;
 591 }
 592
 593 /*
 594  * Find a PMC slot for the freshly enabled / scheduled in counter:
 595  */
 596 static int x86_pmu_enable(struct perf_counter *counter)
 597 {
 598         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 599         struct hw_perf_counter *hwc = &counter->hw;
 600         int idx;
 601
 602         idx = fixed_mode_idx(counter, hwc);
 603         if (idx >= 0) {
 604                 /*
 605                  * Try to get the fixed counter, if that is already taken
 606                  * then try to get a generic counter:
 607                  */
 608                 if (test_and_set_bit(idx, cpuc->used))
 609                         goto try_generic;
 610
 611                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
 612                 /*
 613                  * We set it so that counter_base + idx in wrmsr/rdmsr maps to
 614                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
 615                  */
 616                 hwc->counter_base =
 617                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
 618                 hwc->idx = idx;
 619         } else {
 620                 idx = hwc->idx;
 621                 /* Try to get the previous generic counter again */
 622                 if (test_and_set_bit(idx, cpuc->used)) {
 623 try_generic:
 624                         idx = find_first_zero_bit(cpuc->used, nr_counters_generic);
 625                         if (idx == nr_counters_generic)
 626                                 return -EAGAIN;
 627
 628                         set_bit(idx, cpuc->used);
 629                         hwc->idx = idx;
 630                 }
 631                 hwc->config_base  = x86_pmu.eventsel;
 632                 hwc->counter_base = x86_pmu.perfctr;
 633         }
 634
 635         perf_counters_lapic_init(hwc->nmi);
 636
 637         __x86_pmu_disable(counter, hwc, idx);
 638
 639         cpuc->counters[idx] = counter;
 640         /*
 641          * Make it visible before enabling the hw:
 642          */
 643         barrier();
 644
 645         x86_perf_counter_set_period(counter, hwc, idx);
 646         __x86_pmu_enable(counter, hwc, idx);
 647
 648         return 0;
 649 }
 650
 651 void perf_counter_print_debug(void)
 652 {
 653         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
 654         struct cpu_hw_counters *cpuc;
 655         int cpu, idx;
 656
 657         if (!nr_counters_generic)
 658                 return;
 659
 660         local_irq_disable();
 661
 662         cpu = smp_processor_id();
 663         cpuc = &per_cpu(cpu_hw_counters, cpu);
 664
 665         if (intel_perfmon_version >= 2) {
 666                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 667                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 668                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
 669                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
 670
 671                 pr_info("\n");
 672                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
 673                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
 674                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
 675                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
 676         }
 677         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used);
 678
 679         for (idx = 0; idx < nr_counters_generic; idx++) {
 680                 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
 681                 rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
 682
 683                 prev_left = per_cpu(prev_left[idx], cpu);
 684
 685                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
 686                         cpu, idx, pmc_ctrl);
 687                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
 688                         cpu, idx, pmc_count);
 689                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
 690                         cpu, idx, prev_left);
 691         }
 692         for (idx = 0; idx < nr_counters_fixed; idx++) {
 693                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
 694
 695                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
 696                         cpu, idx, pmc_count);
 697         }
 698         local_irq_enable();
 699 }
 700
 701 static void x86_pmu_disable(struct perf_counter *counter)
 702 {
 703         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 704         struct hw_perf_counter *hwc = &counter->hw;
 705         unsigned int idx = hwc->idx;
 706
 707         __x86_pmu_disable(counter, hwc, idx);
 708
 709         clear_bit(idx, cpuc->used);
 710         cpuc->counters[idx] = NULL;
 711         /*
 712          * Make sure the cleared pointer becomes visible before we
 713          * (potentially) free the counter:
 714          */
 715         barrier();
 716
 717         /*
 718          * Drain the remaining delta count out of a counter
 719          * that we are disabling:
 720          */
 721         x86_perf_counter_update(counter, hwc, idx);
 722 }
 723
 724 /*
 725  * Save and restart an expired counter. Called by NMI contexts,
 726  * so it has to be careful about preempting normal counter ops:
 727  */
 728 static void intel_pmu_save_and_restart(struct perf_counter *counter)
 729 {
 730         struct hw_perf_counter *hwc = &counter->hw;
 731         int idx = hwc->idx;
 732
 733         x86_perf_counter_update(counter, hwc, idx);
 734         x86_perf_counter_set_period(counter, hwc, idx);
 735
 736         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
 737                 __x86_pmu_enable(counter, hwc, idx);
 738 }
 739
 740 /*
 741  * Maximum interrupt frequency of 100KHz per CPU
 742  */
 743 #define PERFMON_MAX_INTERRUPTS (100000/HZ)
 744
 745 /*
 746  * This handler is triggered by the local APIC, so the APIC IRQ handling
 747  * rules apply:
 748  */
 749 static int intel_pmu_handle_irq(struct pt_regs *regs, int nmi)
 750 {
 751         int bit, cpu = smp_processor_id();
 752         u64 ack, status;
 753         struct cpu_hw_counters *cpuc = &per_cpu(cpu_hw_counters, cpu);
 754         int ret = 0;
 755
 756         cpuc->throttle_ctrl = intel_pmu_save_disable_all();
 757
 758         status = intel_pmu_get_status(cpuc->throttle_ctrl);
 759         if (!status)
 760                 goto out;
 761
 762         ret = 1;
 763 again:
 764         inc_irq_stat(apic_perf_irqs);
 765         ack = status;
 766         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
 767                 struct perf_counter *counter = cpuc->counters[bit];
 768
 769                 clear_bit(bit, (unsigned long *) &status);
 770                 if (!counter)
 771                         continue;
 772
 773                 intel_pmu_save_and_restart(counter);
 774                 if (perf_counter_overflow(counter, nmi, regs, 0))
 775                         __x86_pmu_disable(counter, &counter->hw, bit);
 776         }
 777
 778         intel_pmu_ack_status(ack);
 779
 780         /*
 781          * Repeat if there is more work to be done:
 782          */
 783         status = intel_pmu_get_status(cpuc->throttle_ctrl);
 784         if (status)
 785                 goto again;
 786 out:
 787         /*
 788          * Restore - do not reenable when global enable is off or throttled:
 789          */
 790         if (++cpuc->interrupts < PERFMON_MAX_INTERRUPTS)
 791                 intel_pmu_restore_all(cpuc->throttle_ctrl);
 792
 793         return ret;
 794 }
 795
 796 static int amd_pmu_handle_irq(struct pt_regs *regs, int nmi) { return 0; }
 797
 798 void perf_counter_unthrottle(void)
 799 {
 800         struct cpu_hw_counters *cpuc;
 801
 802         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
 803                 return;
 804
 805         if (unlikely(!perf_counters_initialized))
 806                 return;
 807
 808         cpuc = &__get_cpu_var(cpu_hw_counters);
 809         if (cpuc->interrupts >= PERFMON_MAX_INTERRUPTS) {
 810                 if (printk_ratelimit())
 811                         printk(KERN_WARNING "PERFMON: max interrupts exceeded!\n");
 812                 hw_perf_restore(cpuc->throttle_ctrl);
 813         }
 814         cpuc->interrupts = 0;
 815 }
 816
 817 void smp_perf_counter_interrupt(struct pt_regs *regs)
 818 {
 819         irq_enter();
 820         apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
 821         ack_APIC_irq();
 822         x86_pmu.handle_irq(regs, 0);
 823         irq_exit();
 824 }
 825
 826 void smp_perf_pending_interrupt(struct pt_regs *regs)
 827 {
 828         irq_enter();
 829         ack_APIC_irq();
 830         inc_irq_stat(apic_pending_irqs);
 831         perf_counter_do_pending();
 832         irq_exit();
 833 }
 834
 835 void set_perf_counter_pending(void)
 836 {
 837         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
 838 }
 839
 840 void perf_counters_lapic_init(int nmi)
 841 {
 842         u32 apic_val;
 843
 844         if (!perf_counters_initialized)
 845                 return;
 846         /*
 847          * Enable the performance counter vector in the APIC LVT:
 848          */
 849         apic_val = apic_read(APIC_LVTERR);
 850
 851         apic_write(APIC_LVTERR, apic_val | APIC_LVT_MASKED);
 852         if (nmi)
 853                 apic_write(APIC_LVTPC, APIC_DM_NMI);
 854         else
 855                 apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
 856         apic_write(APIC_LVTERR, apic_val);
 857 }
 858
 859 static int __kprobes
 860 perf_counter_nmi_handler(struct notifier_block *self,
 861                          unsigned long cmd, void *__args)
 862 {
 863         struct die_args *args = __args;
 864         struct pt_regs *regs;
 865         int ret;
 866
 867         switch (cmd) {
 868         case DIE_NMI:
 869         case DIE_NMI_IPI:
 870                 break;
 871
 872         default:
 873                 return NOTIFY_DONE;
 874         }
 875
 876         regs = args->regs;
 877
 878         apic_write(APIC_LVTPC, APIC_DM_NMI);
 879         ret = x86_pmu.handle_irq(regs, 1);
 880
 881         return ret ? NOTIFY_STOP : NOTIFY_OK;
 882 }
 883
 884 static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
 885         .notifier_call          = perf_counter_nmi_handler,
 886         .next                   = NULL,
 887         .priority               = 1
 888 };
 889
 890 static struct x86_pmu intel_pmu = {
 891         .handle_irq             = intel_pmu_handle_irq,
 892         .save_disable_all       = intel_pmu_save_disable_all,
 893         .restore_all            = intel_pmu_restore_all,
 894         .enable                 = intel_pmu_enable_counter,
 895         .disable                = intel_pmu_disable_counter,
 896         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
 897         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
 898         .event_map              = intel_pmu_event_map,
 899         .raw_event              = intel_pmu_raw_event,
 900         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
 901 };
 902
 903 static struct x86_pmu amd_pmu = {
 904         .handle_irq             = amd_pmu_handle_irq,
 905         .save_disable_all       = amd_pmu_save_disable_all,
 906         .restore_all            = amd_pmu_restore_all,
 907         .enable                 = amd_pmu_enable_counter,
 908         .disable                = amd_pmu_disable_counter,
 909         .eventsel               = MSR_K7_EVNTSEL0,
 910         .perfctr                = MSR_K7_PERFCTR0,
 911         .event_map              = amd_pmu_event_map,
 912         .raw_event              = amd_pmu_raw_event,
 913         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
 914 };
 915
 916 static int intel_pmu_init(void)
 917 {
 918         union cpuid10_edx edx;
 919         union cpuid10_eax eax;
 920         unsigned int unused;
 921         unsigned int ebx;
 922
 923         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
 924                 return -ENODEV;
 925
 926         /*
 927          * Check whether the Architectural PerfMon supports
 928          * Branch Misses Retired Event or not.
 929          */
 930         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
 931         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
 932                 return -ENODEV;
 933
 934         intel_perfmon_version = eax.split.version_id;
 935         if (intel_perfmon_version < 2)
 936                 return -ENODEV;
 937
 938         pr_info("Intel Performance Monitoring support detected.\n");
 939         pr_info("... version:         %d\n", intel_perfmon_version);
 940         pr_info("... bit width:       %d\n", eax.split.bit_width);
 941         pr_info("... mask length:     %d\n", eax.split.mask_length);
 942
 943         x86_pmu = intel_pmu;
 944
 945         nr_counters_generic = eax.split.num_counters;
 946         nr_counters_fixed = edx.split.num_counters_fixed;
 947         counter_value_mask = (1ULL << eax.split.bit_width) - 1;
 948
 949         return 0;
 950 }
 951
 952 static int amd_pmu_init(void)
 953 {
 954         x86_pmu = amd_pmu;
 955
 956         nr_counters_generic = 4;
 957         nr_counters_fixed = 0;
 958         counter_value_mask = 0x0000FFFFFFFFFFFFULL;
 959         counter_value_bits = 48;
 960
 961         pr_info("AMD Performance Monitoring support detected.\n");
 962         return 0;
 963 }
 964
 965 void __init init_hw_perf_counters(void)
 966 {
 967         int err;
 968
 969         switch (boot_cpu_data.x86_vendor) {
 970         case X86_VENDOR_INTEL:
 971                 err = intel_pmu_init();
 972                 break;
 973         case X86_VENDOR_AMD:
 974                 err = amd_pmu_init();
 975                 break;
 976         default:
 977                 return;
 978         }
 979         if (err != 0)
 980                 return;
 981
 982         pr_info("... num counters:    %d\n", nr_counters_generic);
 983         if (nr_counters_generic > X86_PMC_MAX_GENERIC) {
 984                 nr_counters_generic = X86_PMC_MAX_GENERIC;
 985                 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
 986                         nr_counters_generic, X86_PMC_MAX_GENERIC);
 987         }
 988         perf_counter_mask = (1 << nr_counters_generic) - 1;
 989         perf_max_counters = nr_counters_generic;
 990
 991         pr_info("... value mask:      %016Lx\n", counter_value_mask);
 992
 993         if (nr_counters_fixed > X86_PMC_MAX_FIXED) {
 994                 nr_counters_fixed = X86_PMC_MAX_FIXED;
 995                 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
 996                         nr_counters_fixed, X86_PMC_MAX_FIXED);
 997         }
 998         pr_info("... fixed counters:  %d\n", nr_counters_fixed);
 999
1000         perf_counter_mask |= ((1LL << nr_counters_fixed)-1) << X86_PMC_IDX_FIXED;
1001
1002         pr_info("... counter mask:    %016Lx\n", perf_counter_mask);
1003         perf_counters_initialized = true;
1004
1005         perf_counters_lapic_init(0);
1006         register_die_notifier(&perf_counter_nmi_notifier);
1007 }
1008
1009 static void x86_pmu_read(struct perf_counter *counter)
1010 {
1011         x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
1012 }
1013
1014 static const struct pmu pmu = {
1015         .enable         = x86_pmu_enable,
1016         .disable        = x86_pmu_disable,
1017         .read           = x86_pmu_read,
1018 };
1019
1020 const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
1021 {
1022         int err;
1023
1024         err = __hw_perf_counter_init(counter);
1025         if (err)
1026                 return ERR_PTR(err);
1027
1028         return &pmu;
1029 }
1030
1031 /*
1032  * callchain support
1033  */
1034
1035 static inline
1036 void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
1037 {
1038         if (entry->nr < MAX_STACK_DEPTH)
1039                 entry->ip[entry->nr++] = ip;
1040 }
1041
1042 static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
1043 static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
1044
1045
1046 static void
1047 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
1048 {
1049         /* Ignore warnings */
1050 }
1051
1052 static void backtrace_warning(void *data, char *msg)
1053 {
1054         /* Ignore warnings */
1055 }
1056
1057 static int backtrace_stack(void *data, char *name)
1058 {
1059         /* Don't bother with IRQ stacks for now */
1060         return -1;
1061 }
1062
1063 static void backtrace_address(void *data, unsigned long addr, int reliable)
1064 {
1065         struct perf_callchain_entry *entry = data;
1066
1067         if (reliable)
1068                 callchain_store(entry, addr);
1069 }
1070
1071 static const struct stacktrace_ops backtrace_ops = {
1072         .warning                = backtrace_warning,
1073         .warning_symbol         = backtrace_warning_symbol,
1074         .stack                  = backtrace_stack,
1075         .address                = backtrace_address,
1076 };
1077
1078 static void
1079 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
1080 {
1081         unsigned long bp;
1082         char *stack;
1083         int nr = entry->nr;
1084
1085         callchain_store(entry, instruction_pointer(regs));
1086
1087         stack = ((char *)regs + sizeof(struct pt_regs));
1088 #ifdef CONFIG_FRAME_POINTER
1089         bp = frame_pointer(regs);
1090 #else
1091         bp = 0;
1092 #endif
1093
1094         dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
1095
1096         entry->kernel = entry->nr - nr;
1097 }
1098
1099
1100 struct stack_frame {
1101         const void __user       *next_fp;
1102         unsigned long           return_address;
1103 };
1104
1105 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
1106 {
1107         int ret;
1108
1109         if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
1110                 return 0;
1111
1112         ret = 1;
1113         pagefault_disable();
1114         if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
1115                 ret = 0;
1116         pagefault_enable();
1117
1118         return ret;
1119 }
1120
1121 static void
1122 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
1123 {
1124         struct stack_frame frame;
1125         const void __user *fp;
1126         int nr = entry->nr;
1127
1128         regs = (struct pt_regs *)current->thread.sp0 - 1;
1129         fp   = (void __user *)regs->bp;
1130
1131         callchain_store(entry, regs->ip);
1132
1133         while (entry->nr < MAX_STACK_DEPTH) {
1134                 frame.next_fp        = NULL;
1135                 frame.return_address = 0;
1136
1137                 if (!copy_stack_frame(fp, &frame))
1138                         break;
1139
1140                 if ((unsigned long)fp < user_stack_pointer(regs))
1141                         break;
1142
1143                 callchain_store(entry, frame.return_address);
1144                 fp = frame.next_fp;
1145         }
1146
1147         entry->user = entry->nr - nr;
1148 }
1149
1150 static void
1151 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
1152 {
1153         int is_user;
1154
1155         if (!regs)
1156                 return;
1157
1158         is_user = user_mode(regs);
1159
1160         if (!current || current->pid == 0)
1161                 return;
1162
1163         if (is_user && current->state != TASK_RUNNING)
1164                 return;
1165
1166         if (!is_user)
1167                 perf_callchain_kernel(regs, entry);
1168
1169         if (current->mm)
1170                 perf_callchain_user(regs, entry);
1171 }
1172
1173 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1174 {
1175         struct perf_callchain_entry *entry;
1176
1177         if (in_nmi())
1178                 entry = &__get_cpu_var(nmi_entry);
1179         else
1180                 entry = &__get_cpu_var(irq_entry);
1181
1182         entry->nr = 0;
1183         entry->hv = 0;
1184         entry->kernel = 0;
1185         entry->user = 0;
1186
1187         perf_do_callchain(regs, entry);
1188
1189         return entry;
1190 }