2 * linux/kernel/irq/ipi.c
4 * Copyright (C) 2015 Imagination Technologies Ltd
5 * Author: Qais Yousef <qais.yousef@imgtec.com>
7 * This file contains driver APIs to the IPI subsystem.
10 #define pr_fmt(fmt) "genirq/ipi: " fmt
12 #include <linux/irqdomain.h>
13 #include <linux/irq.h>
16 * irq_reserve_ipi() - Setup an IPI to destination cpumask
18 * @dest: cpumask of cpus which can receive the IPI
20 * Allocate a virq that can be used to send IPI to any CPU in dest mask.
22 * On success it'll return linux irq number and 0 on failure
24 unsigned int irq_reserve_ipi(struct irq_domain *domain,
25 const struct cpumask *dest)
27 unsigned int nr_irqs, offset;
28 struct irq_data *data;
31 if (!domain ||!irq_domain_is_ipi(domain)) {
32 pr_warn("Reservation on a non IPI domain\n");
36 if (!cpumask_subset(dest, cpu_possible_mask)) {
37 pr_warn("Reservation is not in possible_cpu_mask\n");
41 nr_irqs = cpumask_weight(dest);
43 pr_warn("Reservation for empty destination mask\n");
47 if (irq_domain_is_ipi_single(domain)) {
49 * If the underlying implementation uses a single HW irq on
50 * all cpus then we only need a single Linux irq number for
51 * it. We have no restrictions vs. the destination mask. The
52 * underlying implementation can deal with holes nicely.
60 * The IPI requires a seperate HW irq on each CPU. We require
61 * that the destination mask is consecutive. If an
62 * implementation needs to support holes, it can reserve
65 offset = cpumask_first(dest);
67 * Find a hole and if found look for another set bit after the
68 * hole. For now we don't support this scenario.
70 next = cpumask_next_zero(offset, dest);
71 if (next < nr_cpu_ids)
72 next = cpumask_next(next, dest);
73 if (next < nr_cpu_ids) {
74 pr_warn("Destination mask has holes\n");
79 virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE);
81 pr_warn("Can't reserve IPI, failed to alloc descs\n");
85 virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE,
89 pr_warn("Can't reserve IPI, failed to alloc hw irqs\n");
93 for (i = 0; i < nr_irqs; i++) {
94 data = irq_get_irq_data(virq + i);
95 cpumask_copy(data->common->affinity, dest);
96 data->common->ipi_offset = offset;
97 irq_set_status_flags(virq + i, IRQ_NO_BALANCING);
102 irq_free_descs(virq, nr_irqs);
107 * irq_destroy_ipi() - unreserve an IPI that was previously allocated
108 * @irq: linux irq number to be destroyed
109 * @dest: cpumask of cpus which should have the IPI removed
111 * Return the IPIs allocated with irq_reserve_ipi() to the system destroying
112 * all virqs associated with them.
114 void irq_destroy_ipi(unsigned int irq, const struct cpumask *dest)
116 struct irq_data *data = irq_get_irq_data(irq);
117 struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL;
118 struct irq_domain *domain;
119 unsigned int nr_irqs;
121 if (!irq || !data || !ipimask)
124 domain = data->domain;
125 if (WARN_ON(domain == NULL))
128 if (!irq_domain_is_ipi(domain)) {
129 pr_warn("Trying to destroy a non IPI domain!\n");
133 if (WARN_ON(!cpumask_subset(dest, ipimask)))
135 * Must be destroying a subset of CPUs to which this IPI
136 * was set up to target
140 if (irq_domain_is_ipi_per_cpu(domain)) {
141 irq = irq + cpumask_first(dest) - data->common->ipi_offset;
142 nr_irqs = cpumask_weight(dest);
147 irq_domain_free_irqs(irq, nr_irqs);
151 * ipi_get_hwirq - Get the hwirq associated with an IPI to a cpu
152 * @irq: linux irq number
153 * @cpu: the target cpu
155 * When dealing with coprocessors IPI, we need to inform the coprocessor of
156 * the hwirq it needs to use to receive and send IPIs.
158 * Returns hwirq value on success and INVALID_HWIRQ on failure.
160 irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu)
162 struct irq_data *data = irq_get_irq_data(irq);
163 struct cpumask *ipimask = data ? irq_data_get_affinity_mask(data) : NULL;
165 if (!data || !ipimask || cpu > nr_cpu_ids)
166 return INVALID_HWIRQ;
168 if (!cpumask_test_cpu(cpu, ipimask))
169 return INVALID_HWIRQ;
172 * Get the real hardware irq number if the underlying implementation
173 * uses a seperate irq per cpu. If the underlying implementation uses
174 * a single hardware irq for all cpus then the IPI send mechanism
175 * needs to take care of the cpu destinations.
177 if (irq_domain_is_ipi_per_cpu(data->domain))
178 data = irq_get_irq_data(irq + cpu - data->common->ipi_offset);
180 return data ? irqd_to_hwirq(data) : INVALID_HWIRQ;
182 EXPORT_SYMBOL_GPL(ipi_get_hwirq);
184 static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data,
185 const struct cpumask *dest, unsigned int cpu)
187 struct cpumask *ipimask = irq_data_get_affinity_mask(data);
189 if (!chip || !ipimask)
192 if (!chip->ipi_send_single && !chip->ipi_send_mask)
195 if (cpu > nr_cpu_ids)
199 if (!cpumask_subset(dest, ipimask))
202 if (!cpumask_test_cpu(cpu, ipimask))
209 * __ipi_send_single - send an IPI to a target Linux SMP CPU
210 * @desc: pointer to irq_desc of the IRQ
211 * @cpu: destination CPU, must in the destination mask passed to
214 * This function is for architecture or core code to speed up IPI sending. Not
215 * usable from driver code.
217 * Returns zero on success and negative error number on failure.
219 int __ipi_send_single(struct irq_desc *desc, unsigned int cpu)
221 struct irq_data *data = irq_desc_get_irq_data(desc);
222 struct irq_chip *chip = irq_data_get_irq_chip(data);
226 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
227 * Since the callers should be arch or core code which is generally
228 * trusted, only check for errors when debugging.
230 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
233 if (!chip->ipi_send_single) {
234 chip->ipi_send_mask(data, cpumask_of(cpu));
238 /* FIXME: Store this information in irqdata flags */
239 if (irq_domain_is_ipi_per_cpu(data->domain) &&
240 cpu != data->common->ipi_offset) {
241 /* use the correct data for that cpu */
242 unsigned irq = data->irq + cpu - data->common->ipi_offset;
244 data = irq_get_irq_data(irq);
246 chip->ipi_send_single(data, cpu);
251 * ipi_send_mask - send an IPI to target Linux SMP CPU(s)
252 * @desc: pointer to irq_desc of the IRQ
253 * @dest: dest CPU(s), must be a subset of the mask passed to
256 * This function is for architecture or core code to speed up IPI sending. Not
257 * usable from driver code.
259 * Returns zero on success and negative error number on failure.
261 int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest)
263 struct irq_data *data = irq_desc_get_irq_data(desc);
264 struct irq_chip *chip = irq_data_get_irq_chip(data);
269 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
270 * Since the callers should be arch or core code which is generally
271 * trusted, only check for errors when debugging.
273 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
276 if (chip->ipi_send_mask) {
277 chip->ipi_send_mask(data, dest);
281 if (irq_domain_is_ipi_per_cpu(data->domain)) {
282 unsigned int base = data->irq;
284 for_each_cpu(cpu, dest) {
285 unsigned irq = base + cpu - data->common->ipi_offset;
287 data = irq_get_irq_data(irq);
288 chip->ipi_send_single(data, cpu);
291 for_each_cpu(cpu, dest)
292 chip->ipi_send_single(data, cpu);
298 * ipi_send_single - Send an IPI to a single CPU
299 * @virq: linux irq number from irq_reserve_ipi()
300 * @cpu: destination CPU, must in the destination mask passed to
303 * Returns zero on success and negative error number on failure.
305 int ipi_send_single(unsigned int virq, unsigned int cpu)
307 struct irq_desc *desc = irq_to_desc(virq);
308 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
309 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
311 if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
314 return __ipi_send_single(desc, cpu);
316 EXPORT_SYMBOL_GPL(ipi_send_single);
319 * ipi_send_mask - Send an IPI to target CPU(s)
320 * @virq: linux irq number from irq_reserve_ipi()
321 * @dest: dest CPU(s), must be a subset of the mask passed to
324 * Returns zero on success and negative error number on failure.
326 int ipi_send_mask(unsigned int virq, const struct cpumask *dest)
328 struct irq_desc *desc = irq_to_desc(virq);
329 struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
330 struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;
332 if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
335 return __ipi_send_mask(desc, dest);
337 EXPORT_SYMBOL_GPL(ipi_send_mask);