arch/x86/kernel/cpu/perf_counter.c

   1 /*
   2  * Performance counter x86 architecture code
   3  *
   4  *  Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
   6  *
   7  *  For licencing details see kernel-base/COPYING
   8  */
   9
  10 #include <linux/perf_counter.h>
  11 #include <linux/capability.h>
  12 #include <linux/notifier.h>
  13 #include <linux/hardirq.h>
  14 #include <linux/kprobes.h>
  15 #include <linux/module.h>
  16 #include <linux/kdebug.h>
  17 #include <linux/sched.h>
  18
  19 #include <asm/perf_counter.h>
  20 #include <asm/apic.h>
  21
  22 static bool perf_counters_initialized __read_mostly;
  23
  24 /*
  25  * Number of (generic) HW counters:
  26  */
  27 static int nr_counters_generic __read_mostly;
  28 static u64 perf_counter_mask __read_mostly;
  29 static u64 counter_value_mask __read_mostly;
  30
  31 static int nr_counters_fixed __read_mostly;
  32
  33 struct cpu_hw_counters {
  34         struct perf_counter     *counters[X86_PMC_IDX_MAX];
  35         unsigned long           used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  36 };
  37
  38 /*
  39  * Intel PerfMon v3. Used on Core2 and later.
  40  */
  41 static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters);
  42
  43 static const int intel_perfmon_event_map[] =
  44 {
  45   [PERF_COUNT_CPU_CYCLES]               = 0x003c,
  46   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
  47   [PERF_COUNT_CACHE_REFERENCES]         = 0x4f2e,
  48   [PERF_COUNT_CACHE_MISSES]             = 0x412e,
  49   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
  50   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
  51   [PERF_COUNT_BUS_CYCLES]               = 0x013c,
  52 };
  53
  54 static const int max_intel_perfmon_events = ARRAY_SIZE(intel_perfmon_event_map);
  55
  56 /*
  57  * Propagate counter elapsed time into the generic counter.
  58  * Can only be executed on the CPU where the counter is active.
  59  * Returns the delta events processed.
  60  */
  61 static void
  62 x86_perf_counter_update(struct perf_counter *counter,
  63                         struct hw_perf_counter *hwc, int idx)
  64 {
  65         u64 prev_raw_count, new_raw_count, delta;
  66
  67         /*
  68          * Careful: an NMI might modify the previous counter value.
  69          *
  70          * Our tactic to handle this is to first atomically read and
  71          * exchange a new raw count - then add that new-prev delta
  72          * count to the generic counter atomically:
  73          */
  74 again:
  75         prev_raw_count = atomic64_read(&hwc->prev_count);
  76         rdmsrl(hwc->counter_base + idx, new_raw_count);
  77
  78         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
  79                                         new_raw_count) != prev_raw_count)
  80                 goto again;
  81
  82         /*
  83          * Now we have the new raw value and have updated the prev
  84          * timestamp already. We can now calculate the elapsed delta
  85          * (counter-)time and add that to the generic counter.
  86          *
  87          * Careful, not all hw sign-extends above the physical width
  88          * of the count, so we do that by clipping the delta to 32 bits:
  89          */
  90         delta = (u64)(u32)((s32)new_raw_count - (s32)prev_raw_count);
  91
  92         atomic64_add(delta, &counter->count);
  93         atomic64_sub(delta, &hwc->period_left);
  94 }
  95
  96 /*
  97  * Setup the hardware configuration for a given hw_event_type
  98  */
  99 static int __hw_perf_counter_init(struct perf_counter *counter)
 100 {
 101         struct perf_counter_hw_event *hw_event = &counter->hw_event;
 102         struct hw_perf_counter *hwc = &counter->hw;
 103
 104         if (unlikely(!perf_counters_initialized))
 105                 return -EINVAL;
 106
 107         /*
 108          * Count user events, and generate PMC IRQs:
 109          * (keep 'enabled' bit clear for now)
 110          */
 111         hwc->config = ARCH_PERFMON_EVENTSEL_USR | ARCH_PERFMON_EVENTSEL_INT;
 112
 113         /*
 114          * If privileged enough, count OS events too, and allow
 115          * NMI events as well:
 116          */
 117         hwc->nmi = 0;
 118         if (capable(CAP_SYS_ADMIN)) {
 119                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
 120                 if (hw_event->nmi)
 121                         hwc->nmi = 1;
 122         }
 123
 124         hwc->irq_period         = hw_event->irq_period;
 125         /*
 126          * Intel PMCs cannot be accessed sanely above 32 bit width,
 127          * so we install an artificial 1<<31 period regardless of
 128          * the generic counter period:
 129          */
 130         if ((s64)hwc->irq_period <= 0 || hwc->irq_period > 0x7FFFFFFF)
 131                 hwc->irq_period = 0x7FFFFFFF;
 132
 133         atomic64_set(&hwc->period_left, hwc->irq_period);
 134
 135         /*
 136          * Raw event type provide the config in the event structure
 137          */
 138         if (hw_event->raw) {
 139                 hwc->config |= hw_event->type;
 140         } else {
 141                 if (hw_event->type >= max_intel_perfmon_events)
 142                         return -EINVAL;
 143                 /*
 144                  * The generic map:
 145                  */
 146                 hwc->config |= intel_perfmon_event_map[hw_event->type];
 147         }
 148         counter->wakeup_pending = 0;
 149
 150         return 0;
 151 }
 152
 153 u64 hw_perf_save_disable(void)
 154 {
 155         u64 ctrl;
 156
 157         if (unlikely(!perf_counters_initialized))
 158                 return 0;
 159
 160         rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 161         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 162
 163         return ctrl;
 164 }
 165 EXPORT_SYMBOL_GPL(hw_perf_save_disable);
 166
 167 void hw_perf_restore(u64 ctrl)
 168 {
 169         if (unlikely(!perf_counters_initialized))
 170                 return;
 171
 172         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 173 }
 174 EXPORT_SYMBOL_GPL(hw_perf_restore);
 175
 176 static inline void
 177 __pmc_fixed_disable(struct perf_counter *counter,
 178                     struct hw_perf_counter *hwc, unsigned int __idx)
 179 {
 180         int idx = __idx - X86_PMC_IDX_FIXED;
 181         u64 ctrl_val, mask;
 182         int err;
 183
 184         mask = 0xfULL << (idx * 4);
 185
 186         rdmsrl(hwc->config_base, ctrl_val);
 187         ctrl_val &= ~mask;
 188         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 189 }
 190
 191 static inline void
 192 __pmc_generic_disable(struct perf_counter *counter,
 193                            struct hw_perf_counter *hwc, unsigned int idx)
 194 {
 195         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
 196                 __pmc_fixed_disable(counter, hwc, idx);
 197         else
 198                 wrmsr_safe(hwc->config_base + idx, hwc->config, 0);
 199 }
 200
 201 static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
 202
 203 /*
 204  * Set the next IRQ period, based on the hwc->period_left value.
 205  * To be called with the counter disabled in hw:
 206  */
 207 static void
 208 __hw_perf_counter_set_period(struct perf_counter *counter,
 209                              struct hw_perf_counter *hwc, int idx)
 210 {
 211         s64 left = atomic64_read(&hwc->period_left);
 212         s32 period = hwc->irq_period;
 213         int err;
 214
 215         /*
 216          * If we are way outside a reasoable range then just skip forward:
 217          */
 218         if (unlikely(left <= -period)) {
 219                 left = period;
 220                 atomic64_set(&hwc->period_left, left);
 221         }
 222
 223         if (unlikely(left <= 0)) {
 224                 left += period;
 225                 atomic64_set(&hwc->period_left, left);
 226         }
 227
 228         per_cpu(prev_left[idx], smp_processor_id()) = left;
 229
 230         /*
 231          * The hw counter starts counting from this counter offset,
 232          * mark it to be able to extra future deltas:
 233          */
 234         atomic64_set(&hwc->prev_count, (u64)-left);
 235
 236         err = checking_wrmsrl(hwc->counter_base + idx,
 237                              (u64)(-left) & counter_value_mask);
 238 }
 239
 240 static inline void
 241 __pmc_fixed_enable(struct perf_counter *counter,
 242                    struct hw_perf_counter *hwc, unsigned int __idx)
 243 {
 244         int idx = __idx - X86_PMC_IDX_FIXED;
 245         u64 ctrl_val, bits, mask;
 246         int err;
 247
 248         /*
 249          * Enable IRQ generation (0x8) and ring-3 counting (0x2),
 250          * and enable ring-0 counting if allowed:
 251          */
 252         bits = 0x8ULL | 0x2ULL;
 253         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
 254                 bits |= 0x1;
 255         bits <<= (idx * 4);
 256         mask = 0xfULL << (idx * 4);
 257
 258         rdmsrl(hwc->config_base, ctrl_val);
 259         ctrl_val &= ~mask;
 260         ctrl_val |= bits;
 261         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 262 }
 263
 264 static void
 265 __pmc_generic_enable(struct perf_counter *counter,
 266                           struct hw_perf_counter *hwc, int idx)
 267 {
 268         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
 269                 __pmc_fixed_enable(counter, hwc, idx);
 270         else
 271                 wrmsr(hwc->config_base + idx,
 272                       hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE, 0);
 273 }
 274
 275 static int
 276 fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
 277 {
 278         unsigned int event;
 279
 280         if (unlikely(hwc->nmi))
 281                 return -1;
 282
 283         event = hwc->config & ARCH_PERFMON_EVENT_MASK;
 284
 285         if (unlikely(event == intel_perfmon_event_map[PERF_COUNT_INSTRUCTIONS]))
 286                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
 287         if (unlikely(event == intel_perfmon_event_map[PERF_COUNT_CPU_CYCLES]))
 288                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
 289         if (unlikely(event == intel_perfmon_event_map[PERF_COUNT_BUS_CYCLES]))
 290                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
 291
 292         return -1;
 293 }
 294
 295 /*
 296  * Find a PMC slot for the freshly enabled / scheduled in counter:
 297  */
 298 static int pmc_generic_enable(struct perf_counter *counter)
 299 {
 300         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 301         struct hw_perf_counter *hwc = &counter->hw;
 302         int idx;
 303
 304         idx = fixed_mode_idx(counter, hwc);
 305         if (idx >= 0) {
 306                 /*
 307                  * Try to get the fixed counter, if that is already taken
 308                  * then try to get a generic counter:
 309                  */
 310                 if (test_and_set_bit(idx, cpuc->used))
 311                         goto try_generic;
 312
 313                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
 314                 /*
 315                  * We set it so that counter_base + idx in wrmsr/rdmsr maps to
 316                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
 317                  */
 318                 hwc->counter_base =
 319                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
 320                 hwc->idx = idx;
 321         } else {
 322                 idx = hwc->idx;
 323                 /* Try to get the previous generic counter again */
 324                 if (test_and_set_bit(idx, cpuc->used)) {
 325 try_generic:
 326                         idx = find_first_zero_bit(cpuc->used, nr_counters_generic);
 327                         if (idx == nr_counters_generic)
 328                                 return -EAGAIN;
 329
 330                         set_bit(idx, cpuc->used);
 331                         hwc->idx = idx;
 332                 }
 333                 hwc->config_base  = MSR_ARCH_PERFMON_EVENTSEL0;
 334                 hwc->counter_base = MSR_ARCH_PERFMON_PERFCTR0;
 335         }
 336
 337         perf_counters_lapic_init(hwc->nmi);
 338
 339         __pmc_generic_disable(counter, hwc, idx);
 340
 341         cpuc->counters[idx] = counter;
 342         /*
 343          * Make it visible before enabling the hw:
 344          */
 345         smp_wmb();
 346
 347         __hw_perf_counter_set_period(counter, hwc, idx);
 348         __pmc_generic_enable(counter, hwc, idx);
 349
 350         return 0;
 351 }
 352
 353 void perf_counter_print_debug(void)
 354 {
 355         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
 356         struct cpu_hw_counters *cpuc;
 357         int cpu, idx;
 358
 359         if (!nr_counters_generic)
 360                 return;
 361
 362         local_irq_disable();
 363
 364         cpu = smp_processor_id();
 365         cpuc = &per_cpu(cpu_hw_counters, cpu);
 366
 367         rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 368         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 369         rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
 370         rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
 371
 372         printk(KERN_INFO "\n");
 373         printk(KERN_INFO "CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
 374         printk(KERN_INFO "CPU#%d: status:     %016llx\n", cpu, status);
 375         printk(KERN_INFO "CPU#%d: overflow:   %016llx\n", cpu, overflow);
 376         printk(KERN_INFO "CPU#%d: fixed:      %016llx\n", cpu, fixed);
 377         printk(KERN_INFO "CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used);
 378
 379         for (idx = 0; idx < nr_counters_generic; idx++) {
 380                 rdmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx, pmc_ctrl);
 381                 rdmsrl(MSR_ARCH_PERFMON_PERFCTR0  + idx, pmc_count);
 382
 383                 prev_left = per_cpu(prev_left[idx], cpu);
 384
 385                 printk(KERN_INFO "CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
 386                         cpu, idx, pmc_ctrl);
 387                 printk(KERN_INFO "CPU#%d:   gen-PMC%d count: %016llx\n",
 388                         cpu, idx, pmc_count);
 389                 printk(KERN_INFO "CPU#%d:   gen-PMC%d left:  %016llx\n",
 390                         cpu, idx, prev_left);
 391         }
 392         for (idx = 0; idx < nr_counters_fixed; idx++) {
 393                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
 394
 395                 printk(KERN_INFO "CPU#%d: fixed-PMC%d count: %016llx\n",
 396                         cpu, idx, pmc_count);
 397         }
 398         local_irq_enable();
 399 }
 400
 401 static void pmc_generic_disable(struct perf_counter *counter)
 402 {
 403         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 404         struct hw_perf_counter *hwc = &counter->hw;
 405         unsigned int idx = hwc->idx;
 406
 407         __pmc_generic_disable(counter, hwc, idx);
 408
 409         clear_bit(idx, cpuc->used);
 410         cpuc->counters[idx] = NULL;
 411         /*
 412          * Make sure the cleared pointer becomes visible before we
 413          * (potentially) free the counter:
 414          */
 415         smp_wmb();
 416
 417         /*
 418          * Drain the remaining delta count out of a counter
 419          * that we are disabling:
 420          */
 421         x86_perf_counter_update(counter, hwc, idx);
 422 }
 423
 424 static void perf_store_irq_data(struct perf_counter *counter, u64 data)
 425 {
 426         struct perf_data *irqdata = counter->irqdata;
 427
 428         if (irqdata->len > PERF_DATA_BUFLEN - sizeof(u64)) {
 429                 irqdata->overrun++;
 430         } else {
 431                 u64 *p = (u64 *) &irqdata->data[irqdata->len];
 432
 433                 *p = data;
 434                 irqdata->len += sizeof(u64);
 435         }
 436 }
 437
 438 /*
 439  * Save and restart an expired counter. Called by NMI contexts,
 440  * so it has to be careful about preempting normal counter ops:
 441  */
 442 static void perf_save_and_restart(struct perf_counter *counter)
 443 {
 444         struct hw_perf_counter *hwc = &counter->hw;
 445         int idx = hwc->idx;
 446
 447         x86_perf_counter_update(counter, hwc, idx);
 448         __hw_perf_counter_set_period(counter, hwc, idx);
 449
 450         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
 451                 __pmc_generic_enable(counter, hwc, idx);
 452 }
 453
 454 static void
 455 perf_handle_group(struct perf_counter *sibling, u64 *status, u64 *overflown)
 456 {
 457         struct perf_counter *counter, *group_leader = sibling->group_leader;
 458
 459         /*
 460          * Store sibling timestamps (if any):
 461          */
 462         list_for_each_entry(counter, &group_leader->sibling_list, list_entry) {
 463
 464                 x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
 465                 perf_store_irq_data(sibling, counter->hw_event.type);
 466                 perf_store_irq_data(sibling, atomic64_read(&counter->count));
 467         }
 468 }
 469
 470 /*
 471  * This handler is triggered by the local APIC, so the APIC IRQ handling
 472  * rules apply:
 473  */
 474 static void __smp_perf_counter_interrupt(struct pt_regs *regs, int nmi)
 475 {
 476         int bit, cpu = smp_processor_id();
 477         u64 ack, status, saved_global;
 478         struct cpu_hw_counters *cpuc;
 479
 480         rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, saved_global);
 481
 482         /* Disable counters globally */
 483         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 484         ack_APIC_irq();
 485
 486         cpuc = &per_cpu(cpu_hw_counters, cpu);
 487
 488         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 489         if (!status)
 490                 goto out;
 491
 492 again:
 493         ack = status;
 494         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
 495                 struct perf_counter *counter = cpuc->counters[bit];
 496
 497                 clear_bit(bit, (unsigned long *) &status);
 498                 if (!counter)
 499                         continue;
 500
 501                 perf_save_and_restart(counter);
 502
 503                 switch (counter->hw_event.record_type) {
 504                 case PERF_RECORD_SIMPLE:
 505                         continue;
 506                 case PERF_RECORD_IRQ:
 507                         perf_store_irq_data(counter, instruction_pointer(regs));
 508                         break;
 509                 case PERF_RECORD_GROUP:
 510                         perf_handle_group(counter, &status, &ack);
 511                         break;
 512                 }
 513                 /*
 514                  * From NMI context we cannot call into the scheduler to
 515                  * do a task wakeup - but we mark these generic as
 516                  * wakeup_pending and initate a wakeup callback:
 517                  */
 518                 if (nmi) {
 519                         counter->wakeup_pending = 1;
 520                         set_tsk_thread_flag(current, TIF_PERF_COUNTERS);
 521                 } else {
 522                         wake_up(&counter->waitq);
 523                 }
 524         }
 525
 526         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
 527
 528         /*
 529          * Repeat if there is more work to be done:
 530          */
 531         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 532         if (status)
 533                 goto again;
 534 out:
 535         /*
 536          * Restore - do not reenable when global enable is off:
 537          */
 538         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, saved_global);
 539 }
 540
 541 void smp_perf_counter_interrupt(struct pt_regs *regs)
 542 {
 543         irq_enter();
 544         inc_irq_stat(apic_perf_irqs);
 545         apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
 546         __smp_perf_counter_interrupt(regs, 0);
 547
 548         irq_exit();
 549 }
 550
 551 /*
 552  * This handler is triggered by NMI contexts:
 553  */
 554 void perf_counter_notify(struct pt_regs *regs)
 555 {
 556         struct cpu_hw_counters *cpuc;
 557         unsigned long flags;
 558         int bit, cpu;
 559
 560         local_irq_save(flags);
 561         cpu = smp_processor_id();
 562         cpuc = &per_cpu(cpu_hw_counters, cpu);
 563
 564         for_each_bit(bit, cpuc->used, X86_PMC_IDX_MAX) {
 565                 struct perf_counter *counter = cpuc->counters[bit];
 566
 567                 if (!counter)
 568                         continue;
 569
 570                 if (counter->wakeup_pending) {
 571                         counter->wakeup_pending = 0;
 572                         wake_up(&counter->waitq);
 573                 }
 574         }
 575
 576         local_irq_restore(flags);
 577 }
 578
 579 void __cpuinit perf_counters_lapic_init(int nmi)
 580 {
 581         u32 apic_val;
 582
 583         if (!perf_counters_initialized)
 584                 return;
 585         /*
 586          * Enable the performance counter vector in the APIC LVT:
 587          */
 588         apic_val = apic_read(APIC_LVTERR);
 589
 590         apic_write(APIC_LVTERR, apic_val | APIC_LVT_MASKED);
 591         if (nmi)
 592                 apic_write(APIC_LVTPC, APIC_DM_NMI);
 593         else
 594                 apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
 595         apic_write(APIC_LVTERR, apic_val);
 596 }
 597
 598 static int __kprobes
 599 perf_counter_nmi_handler(struct notifier_block *self,
 600                          unsigned long cmd, void *__args)
 601 {
 602         struct die_args *args = __args;
 603         struct pt_regs *regs;
 604
 605         if (likely(cmd != DIE_NMI_IPI))
 606                 return NOTIFY_DONE;
 607
 608         regs = args->regs;
 609
 610         apic_write(APIC_LVTPC, APIC_DM_NMI);
 611         __smp_perf_counter_interrupt(regs, 1);
 612
 613         return NOTIFY_STOP;
 614 }
 615
 616 static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
 617         .notifier_call          = perf_counter_nmi_handler
 618 };
 619
 620 void __init init_hw_perf_counters(void)
 621 {
 622         union cpuid10_eax eax;
 623         unsigned int ebx;
 624         unsigned int unused;
 625         union cpuid10_edx edx;
 626
 627         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
 628                 return;
 629
 630         /*
 631          * Check whether the Architectural PerfMon supports
 632          * Branch Misses Retired Event or not.
 633          */
 634         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
 635         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
 636                 return;
 637
 638         printk(KERN_INFO "Intel Performance Monitoring support detected.\n");
 639
 640         printk(KERN_INFO "... version:         %d\n", eax.split.version_id);
 641         printk(KERN_INFO "... num counters:    %d\n", eax.split.num_counters);
 642         nr_counters_generic = eax.split.num_counters;
 643         if (nr_counters_generic > X86_PMC_MAX_GENERIC) {
 644                 nr_counters_generic = X86_PMC_MAX_GENERIC;
 645                 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
 646                         nr_counters_generic, X86_PMC_MAX_GENERIC);
 647         }
 648         perf_counter_mask = (1 << nr_counters_generic) - 1;
 649         perf_max_counters = nr_counters_generic;
 650
 651         printk(KERN_INFO "... bit width:       %d\n", eax.split.bit_width);
 652         counter_value_mask = (1ULL << eax.split.bit_width) - 1;
 653         printk(KERN_INFO "... value mask:      %016Lx\n", counter_value_mask);
 654
 655         printk(KERN_INFO "... mask length:     %d\n", eax.split.mask_length);
 656
 657         nr_counters_fixed = edx.split.num_counters_fixed;
 658         if (nr_counters_fixed > X86_PMC_MAX_FIXED) {
 659                 nr_counters_fixed = X86_PMC_MAX_FIXED;
 660                 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
 661                         nr_counters_fixed, X86_PMC_MAX_FIXED);
 662         }
 663         printk(KERN_INFO "... fixed counters:  %d\n", nr_counters_fixed);
 664
 665         perf_counter_mask |= ((1LL << nr_counters_fixed)-1) << X86_PMC_IDX_FIXED;
 666
 667         printk(KERN_INFO "... counter mask:    %016Lx\n", perf_counter_mask);
 668         perf_counters_initialized = true;
 669
 670         perf_counters_lapic_init(0);
 671         register_die_notifier(&perf_counter_nmi_notifier);
 672 }
 673
 674 static void pmc_generic_read(struct perf_counter *counter)
 675 {
 676         x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
 677 }
 678
 679 static const struct hw_perf_counter_ops x86_perf_counter_ops = {
 680         .enable         = pmc_generic_enable,
 681         .disable        = pmc_generic_disable,
 682         .read           = pmc_generic_read,
 683 };
 684
 685 const struct hw_perf_counter_ops *
 686 hw_perf_counter_init(struct perf_counter *counter)
 687 {
 688         int err;
 689
 690         err = __hw_perf_counter_init(counter);
 691         if (err)
 692                 return NULL;
 693
 694         return &x86_perf_counter_ops;
 695 }