2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2015 Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * The full GNU General Public License is included in this distribution in
15 * the file called "COPYING".
20 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
24 #include <linux/init.h>
25 #include <linux/module.h>
26 #include <linux/slab.h>
27 #include <linux/pci.h>
28 #include <linux/interrupt.h>
29 #include <linux/dmaengine.h>
30 #include <linux/delay.h>
31 #include <linux/dma-mapping.h>
32 #include <linux/workqueue.h>
33 #include <linux/prefetch.h>
35 #include "registers.h"
38 #include "../dmaengine.h"
40 int ioat_pending_level = 4;
41 module_param(ioat_pending_level, int, 0644);
42 MODULE_PARM_DESC(ioat_pending_level,
43 "high-water mark for pushing ioat descriptors (default: 4)");
46 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
48 * @data: interrupt data
50 static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
52 struct ioatdma_device *instance = data;
53 struct ioatdma_chan *ioat_chan;
54 unsigned long attnstatus;
58 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
60 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
63 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
64 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
68 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
69 for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) {
70 ioat_chan = ioat_chan_by_index(instance, bit);
71 if (test_bit(IOAT_RUN, &ioat_chan->state))
72 tasklet_schedule(&ioat_chan->cleanup_task);
75 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
80 * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
82 * @data: interrupt data
84 static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
86 struct ioatdma_chan *ioat_chan = data;
88 if (test_bit(IOAT_RUN, &ioat_chan->state))
89 tasklet_schedule(&ioat_chan->cleanup_task);
94 /* common channel initialization */
96 ioat_init_channel(struct ioatdma_device *ioat_dma,
97 struct ioatdma_chan *ioat_chan, int idx)
99 struct dma_device *dma = &ioat_dma->dma_dev;
100 struct dma_chan *c = &ioat_chan->dma_chan;
101 unsigned long data = (unsigned long) c;
103 ioat_chan->ioat_dma = ioat_dma;
104 ioat_chan->reg_base = ioat_dma->reg_base + (0x80 * (idx + 1));
105 spin_lock_init(&ioat_chan->cleanup_lock);
106 ioat_chan->dma_chan.device = dma;
107 dma_cookie_init(&ioat_chan->dma_chan);
108 list_add_tail(&ioat_chan->dma_chan.device_node, &dma->channels);
109 ioat_dma->idx[idx] = ioat_chan;
110 init_timer(&ioat_chan->timer);
111 ioat_chan->timer.function = ioat_dma->timer_fn;
112 ioat_chan->timer.data = data;
113 tasklet_init(&ioat_chan->cleanup_task, ioat_dma->cleanup_fn, data);
116 void ioat_stop(struct ioatdma_chan *ioat_chan)
118 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
119 struct pci_dev *pdev = ioat_dma->pdev;
120 int chan_id = chan_num(ioat_chan);
121 struct msix_entry *msix;
123 /* 1/ stop irq from firing tasklets
124 * 2/ stop the tasklet from re-arming irqs
126 clear_bit(IOAT_RUN, &ioat_chan->state);
128 /* flush inflight interrupts */
129 switch (ioat_dma->irq_mode) {
131 msix = &ioat_dma->msix_entries[chan_id];
132 synchronize_irq(msix->vector);
136 synchronize_irq(pdev->irq);
142 /* flush inflight timers */
143 del_timer_sync(&ioat_chan->timer);
145 /* flush inflight tasklet runs */
146 tasklet_kill(&ioat_chan->cleanup_task);
148 /* final cleanup now that everything is quiesced and can't re-arm */
149 ioat_dma->cleanup_fn((unsigned long)&ioat_chan->dma_chan);
152 dma_addr_t ioat_get_current_completion(struct ioatdma_chan *ioat_chan)
154 dma_addr_t phys_complete;
157 completion = *ioat_chan->completion;
158 phys_complete = ioat_chansts_to_addr(completion);
160 dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__,
161 (unsigned long long) phys_complete);
163 if (is_ioat_halted(completion)) {
164 u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
166 dev_err(to_dev(ioat_chan), "Channel halted, chanerr = %x\n",
169 /* TODO do something to salvage the situation */
172 return phys_complete;
175 bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan,
176 dma_addr_t *phys_complete)
178 *phys_complete = ioat_get_current_completion(ioat_chan);
179 if (*phys_complete == ioat_chan->last_completion)
181 clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
182 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
188 ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
189 struct dma_tx_state *txstate)
191 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
192 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
195 ret = dma_cookie_status(c, cookie, txstate);
196 if (ret == DMA_COMPLETE)
199 ioat_dma->cleanup_fn((unsigned long) c);
201 return dma_cookie_status(c, cookie, txstate);
205 * Perform a IOAT transaction to verify the HW works.
207 #define IOAT_TEST_SIZE 2000
209 static void ioat_dma_test_callback(void *dma_async_param)
211 struct completion *cmp = dma_async_param;
217 * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
218 * @ioat_dma: dma device to be tested
220 int ioat_dma_self_test(struct ioatdma_device *ioat_dma)
225 struct dma_device *dma = &ioat_dma->dma_dev;
226 struct device *dev = &ioat_dma->pdev->dev;
227 struct dma_chan *dma_chan;
228 struct dma_async_tx_descriptor *tx;
229 dma_addr_t dma_dest, dma_src;
232 struct completion cmp;
236 src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
239 dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
245 /* Fill in src buffer */
246 for (i = 0; i < IOAT_TEST_SIZE; i++)
249 /* Start copy, using first DMA channel */
250 dma_chan = container_of(dma->channels.next, struct dma_chan,
252 if (dma->device_alloc_chan_resources(dma_chan) < 1) {
253 dev_err(dev, "selftest cannot allocate chan resource\n");
258 dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
259 if (dma_mapping_error(dev, dma_src)) {
260 dev_err(dev, "mapping src buffer failed\n");
263 dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
264 if (dma_mapping_error(dev, dma_dest)) {
265 dev_err(dev, "mapping dest buffer failed\n");
268 flags = DMA_PREP_INTERRUPT;
269 tx = ioat_dma->dma_dev.device_prep_dma_memcpy(dma_chan, dma_dest,
270 dma_src, IOAT_TEST_SIZE,
273 dev_err(dev, "Self-test prep failed, disabling\n");
279 init_completion(&cmp);
280 tx->callback = ioat_dma_test_callback;
281 tx->callback_param = &cmp;
282 cookie = tx->tx_submit(tx);
284 dev_err(dev, "Self-test setup failed, disabling\n");
288 dma->device_issue_pending(dma_chan);
290 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
293 dma->device_tx_status(dma_chan, cookie, NULL)
295 dev_err(dev, "Self-test copy timed out, disabling\n");
299 if (memcmp(src, dest, IOAT_TEST_SIZE)) {
300 dev_err(dev, "Self-test copy failed compare, disabling\n");
306 dma_unmap_single(dev, dma_dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
308 dma_unmap_single(dev, dma_src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
310 dma->device_free_chan_resources(dma_chan);
317 static char ioat_interrupt_style[32] = "msix";
318 module_param_string(ioat_interrupt_style, ioat_interrupt_style,
319 sizeof(ioat_interrupt_style), 0644);
320 MODULE_PARM_DESC(ioat_interrupt_style,
321 "set ioat interrupt style: msix (default), msi, intx");
324 * ioat_dma_setup_interrupts - setup interrupt handler
325 * @ioat_dma: ioat dma device
327 int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma)
329 struct ioatdma_chan *ioat_chan;
330 struct pci_dev *pdev = ioat_dma->pdev;
331 struct device *dev = &pdev->dev;
332 struct msix_entry *msix;
337 if (!strcmp(ioat_interrupt_style, "msix"))
339 if (!strcmp(ioat_interrupt_style, "msi"))
341 if (!strcmp(ioat_interrupt_style, "intx"))
343 dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
347 /* The number of MSI-X vectors should equal the number of channels */
348 msixcnt = ioat_dma->dma_dev.chancnt;
349 for (i = 0; i < msixcnt; i++)
350 ioat_dma->msix_entries[i].entry = i;
352 err = pci_enable_msix_exact(pdev, ioat_dma->msix_entries, msixcnt);
356 for (i = 0; i < msixcnt; i++) {
357 msix = &ioat_dma->msix_entries[i];
358 ioat_chan = ioat_chan_by_index(ioat_dma, i);
359 err = devm_request_irq(dev, msix->vector,
360 ioat_dma_do_interrupt_msix, 0,
361 "ioat-msix", ioat_chan);
363 for (j = 0; j < i; j++) {
364 msix = &ioat_dma->msix_entries[j];
365 ioat_chan = ioat_chan_by_index(ioat_dma, j);
366 devm_free_irq(dev, msix->vector, ioat_chan);
371 intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
372 ioat_dma->irq_mode = IOAT_MSIX;
376 err = pci_enable_msi(pdev);
380 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
381 "ioat-msi", ioat_dma);
383 pci_disable_msi(pdev);
386 ioat_dma->irq_mode = IOAT_MSI;
390 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
391 IRQF_SHARED, "ioat-intx", ioat_dma);
395 ioat_dma->irq_mode = IOAT_INTX;
397 if (ioat_dma->intr_quirk)
398 ioat_dma->intr_quirk(ioat_dma);
399 intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
400 writeb(intrctrl, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
404 /* Disable all interrupt generation */
405 writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
406 ioat_dma->irq_mode = IOAT_NOIRQ;
407 dev_err(dev, "no usable interrupts\n");
410 EXPORT_SYMBOL(ioat_dma_setup_interrupts);
412 static void ioat_disable_interrupts(struct ioatdma_device *ioat_dma)
414 /* Disable all interrupt generation */
415 writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
418 int ioat_probe(struct ioatdma_device *ioat_dma)
421 struct dma_device *dma = &ioat_dma->dma_dev;
422 struct pci_dev *pdev = ioat_dma->pdev;
423 struct device *dev = &pdev->dev;
425 /* DMA coherent memory pool for DMA descriptor allocations */
426 ioat_dma->dma_pool = pci_pool_create("dma_desc_pool", pdev,
427 sizeof(struct ioat_dma_descriptor),
429 if (!ioat_dma->dma_pool) {
434 ioat_dma->completion_pool = pci_pool_create("completion_pool", pdev,
439 if (!ioat_dma->completion_pool) {
441 goto err_completion_pool;
444 ioat_dma->enumerate_channels(ioat_dma);
446 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
447 dma->dev = &pdev->dev;
450 dev_err(dev, "channel enumeration error\n");
451 goto err_setup_interrupts;
454 err = ioat_dma_setup_interrupts(ioat_dma);
456 goto err_setup_interrupts;
458 err = ioat_dma->self_test(ioat_dma);
465 ioat_disable_interrupts(ioat_dma);
466 err_setup_interrupts:
467 pci_pool_destroy(ioat_dma->completion_pool);
469 pci_pool_destroy(ioat_dma->dma_pool);
474 int ioat_register(struct ioatdma_device *ioat_dma)
476 int err = dma_async_device_register(&ioat_dma->dma_dev);
479 ioat_disable_interrupts(ioat_dma);
480 pci_pool_destroy(ioat_dma->completion_pool);
481 pci_pool_destroy(ioat_dma->dma_pool);
487 static ssize_t cap_show(struct dma_chan *c, char *page)
489 struct dma_device *dma = c->device;
491 return sprintf(page, "copy%s%s%s%s%s\n",
492 dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "",
493 dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "",
494 dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "",
495 dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "",
496 dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : "");
499 struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap);
501 static ssize_t version_show(struct dma_chan *c, char *page)
503 struct dma_device *dma = c->device;
504 struct ioatdma_device *ioat_dma = to_ioatdma_device(dma);
506 return sprintf(page, "%d.%d\n",
507 ioat_dma->version >> 4, ioat_dma->version & 0xf);
509 struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version);
512 ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
514 struct ioat_sysfs_entry *entry;
515 struct ioatdma_chan *ioat_chan;
517 entry = container_of(attr, struct ioat_sysfs_entry, attr);
518 ioat_chan = container_of(kobj, struct ioatdma_chan, kobj);
522 return entry->show(&ioat_chan->dma_chan, page);
525 const struct sysfs_ops ioat_sysfs_ops = {
526 .show = ioat_attr_show,
529 void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type)
531 struct dma_device *dma = &ioat_dma->dma_dev;
534 list_for_each_entry(c, &dma->channels, device_node) {
535 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
536 struct kobject *parent = &c->dev->device.kobj;
539 err = kobject_init_and_add(&ioat_chan->kobj, type,
540 parent, "quickdata");
542 dev_warn(to_dev(ioat_chan),
543 "sysfs init error (%d), continuing...\n", err);
544 kobject_put(&ioat_chan->kobj);
545 set_bit(IOAT_KOBJ_INIT_FAIL, &ioat_chan->state);
550 void ioat_kobject_del(struct ioatdma_device *ioat_dma)
552 struct dma_device *dma = &ioat_dma->dma_dev;
555 list_for_each_entry(c, &dma->channels, device_node) {
556 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
558 if (!test_bit(IOAT_KOBJ_INIT_FAIL, &ioat_chan->state)) {
559 kobject_del(&ioat_chan->kobj);
560 kobject_put(&ioat_chan->kobj);
565 void ioat_dma_remove(struct ioatdma_device *ioat_dma)
567 struct dma_device *dma = &ioat_dma->dma_dev;
569 ioat_disable_interrupts(ioat_dma);
571 ioat_kobject_del(ioat_dma);
573 dma_async_device_unregister(dma);
575 pci_pool_destroy(ioat_dma->dma_pool);
576 pci_pool_destroy(ioat_dma->completion_pool);
578 INIT_LIST_HEAD(&dma->channels);