include/linux/percpu.h

   1 #ifndef __LINUX_PERCPU_H
   2 #define __LINUX_PERCPU_H
   3
   4 #include <linux/preempt.h>
   5 #include <linux/slab.h> /* For kmalloc() */
   6 #include <linux/smp.h>
   7 #include <linux/cpumask.h>
   8
   9 #include <asm/percpu.h>
  10
  11 #ifdef CONFIG_SMP
  12 #define PER_CPU_BASE_SECTION ".data.percpu"
  13
  14 #ifdef MODULE
  15 #define PER_CPU_SHARED_ALIGNED_SECTION ""
  16 #else
  17 #define PER_CPU_SHARED_ALIGNED_SECTION ".shared_aligned"
  18 #endif
  19 #define PER_CPU_FIRST_SECTION ".first"
  20
  21 #else
  22
  23 #define PER_CPU_BASE_SECTION ".data"
  24 #define PER_CPU_SHARED_ALIGNED_SECTION ""
  25 #define PER_CPU_FIRST_SECTION ""
  26
  27 #endif
  28
  29 #define DEFINE_PER_CPU_SECTION(type, name, section)                     \
  30         __attribute__((__section__(PER_CPU_BASE_SECTION section)))      \
  31         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
  32
  33 #define DEFINE_PER_CPU(type, name)                                      \
  34         DEFINE_PER_CPU_SECTION(type, name, "")
  35
  36 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                       \
  37         DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \
  38         ____cacheline_aligned_in_smp
  39
  40 #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)                         \
  41         DEFINE_PER_CPU_SECTION(type, name, ".page_aligned")
  42
  43 #define DEFINE_PER_CPU_FIRST(type, name)                                \
  44         DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)
  45
  46 #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
  47 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
  48
  49 /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */
  50 #ifndef PERCPU_ENOUGH_ROOM
  51 #ifdef CONFIG_MODULES
  52 #define PERCPU_MODULE_RESERVE   8192
  53 #else
  54 #define PERCPU_MODULE_RESERVE   0
  55 #endif
  56
  57 #define PERCPU_ENOUGH_ROOM                                              \
  58         (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE)
  59 #endif  /* PERCPU_ENOUGH_ROOM */
  60
  61 /*
  62  * Must be an lvalue. Since @var must be a simple identifier,
  63  * we force a syntax error here if it isn't.
  64  */
  65 #define get_cpu_var(var) (*({                           \
  66         extern int simple_identifier_##var(void);       \
  67         preempt_disable();                              \
  68         &__get_cpu_var(var); }))
  69 #define put_cpu_var(var) preempt_enable()
  70
  71 #ifdef CONFIG_SMP
  72
  73 struct percpu_data {
  74         void *ptrs[1];
  75 };
  76
  77 #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata)
  78 /*
  79  * Use this to get to a cpu's version of the per-cpu object dynamically
  80  * allocated. Non-atomic access to the current CPU's version should
  81  * probably be combined with get_cpu()/put_cpu().
  82  */
  83 #define percpu_ptr(ptr, cpu)                              \
  84 ({                                                        \
  85         struct percpu_data *__p = __percpu_disguise(ptr); \
  86         (__typeof__(ptr))__p->ptrs[(cpu)];                \
  87 })
  88
  89 extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask);
  90 extern void percpu_free(void *__pdata);
  91
  92 #else /* CONFIG_SMP */
  93
  94 #define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
  95
  96 static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask)
  97 {
  98         return kzalloc(size, gfp);
  99 }
 100
 101 static inline void percpu_free(void *__pdata)
 102 {
 103         kfree(__pdata);
 104 }
 105
 106 #endif /* CONFIG_SMP */
 107
 108 #define percpu_alloc_mask(size, gfp, mask) \
 109         __percpu_alloc_mask((size), (gfp), &(mask))
 110
 111 #define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map)
 112
 113 /* (legacy) interface for use without CPU hotplug handling */
 114
 115 #define __alloc_percpu(size)    percpu_alloc_mask((size), GFP_KERNEL, \
 116                                                   cpu_possible_map)
 117 #define alloc_percpu(type)      (type *)__alloc_percpu(sizeof(type))
 118 #define free_percpu(ptr)        percpu_free((ptr))
 119 #define per_cpu_ptr(ptr, cpu)   percpu_ptr((ptr), (cpu))
 120
 121 #endif /* __LINUX_PERCPU_H */