1 #ifndef __LINUX_PERCPU_H
2 #define __LINUX_PERCPU_H
4 #include <linux/preempt.h>
5 #include <linux/slab.h> /* For kmalloc() */
7 #include <linux/cpumask.h>
9 #include <asm/percpu.h>
12 #define PER_CPU_BASE_SECTION ".data.percpu"
15 #define PER_CPU_SHARED_ALIGNED_SECTION ""
17 #define PER_CPU_SHARED_ALIGNED_SECTION ".shared_aligned"
19 #define PER_CPU_FIRST_SECTION ".first"
23 #define PER_CPU_BASE_SECTION ".data"
24 #define PER_CPU_SHARED_ALIGNED_SECTION ""
25 #define PER_CPU_FIRST_SECTION ""
29 #define DEFINE_PER_CPU_SECTION(type, name, section) \
30 __attribute__((__section__(PER_CPU_BASE_SECTION section))) \
31 PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
33 #define DEFINE_PER_CPU(type, name) \
34 DEFINE_PER_CPU_SECTION(type, name, "")
36 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \
37 DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \
38 ____cacheline_aligned_in_smp
40 #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \
41 DEFINE_PER_CPU_SECTION(type, name, ".page_aligned")
43 #define DEFINE_PER_CPU_FIRST(type, name) \
44 DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)
46 #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
47 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
49 /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */
50 #ifndef PERCPU_ENOUGH_ROOM
52 #define PERCPU_MODULE_RESERVE 8192
54 #define PERCPU_MODULE_RESERVE 0
57 #define PERCPU_ENOUGH_ROOM \
58 (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE)
59 #endif /* PERCPU_ENOUGH_ROOM */
62 * Must be an lvalue. Since @var must be a simple identifier,
63 * we force a syntax error here if it isn't.
65 #define get_cpu_var(var) (*({ \
66 extern int simple_identifier_##var(void); \
68 &__get_cpu_var(var); }))
69 #define put_cpu_var(var) preempt_enable()
77 #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata)
79 * Use this to get to a cpu's version of the per-cpu object dynamically
80 * allocated. Non-atomic access to the current CPU's version should
81 * probably be combined with get_cpu()/put_cpu().
83 #define percpu_ptr(ptr, cpu) \
85 struct percpu_data *__p = __percpu_disguise(ptr); \
86 (__typeof__(ptr))__p->ptrs[(cpu)]; \
89 extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask);
90 extern void percpu_free(void *__pdata);
92 #else /* CONFIG_SMP */
94 #define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
96 static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask)
98 return kzalloc(size, gfp);
101 static inline void percpu_free(void *__pdata)
106 #endif /* CONFIG_SMP */
108 #define percpu_alloc_mask(size, gfp, mask) \
109 __percpu_alloc_mask((size), (gfp), &(mask))
111 #define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map)
113 /* (legacy) interface for use without CPU hotplug handling */
115 #define __alloc_percpu(size) percpu_alloc_mask((size), GFP_KERNEL, \
117 #define alloc_percpu(type) (type *)__alloc_percpu(sizeof(type))
118 #define free_percpu(ptr) percpu_free((ptr))
119 #define per_cpu_ptr(ptr, cpu) percpu_ptr((ptr), (cpu))
121 #endif /* __LINUX_PERCPU_H */