drivers/rapidio/devices/tsi721_dma.c

   1 /*
   2  * DMA Engine support for Tsi721 PCIExpress-to-SRIO bridge
   3  *
   4  * Copyright 2011 Integrated Device Technology, Inc.
   5  * Alexandre Bounine <alexandre.bounine@idt.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License as published by the Free
   9  * Software Foundation; either version 2 of the License, or (at your option)
  10  * any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful, but WITHOUT
  13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15  * more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along with
  18  * this program; if not, write to the Free Software Foundation, Inc., 59
  19  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  20  */
  21
  22 #include <linux/io.h>
  23 #include <linux/errno.h>
  24 #include <linux/init.h>
  25 #include <linux/ioport.h>
  26 #include <linux/kernel.h>
  27 #include <linux/module.h>
  28 #include <linux/pci.h>
  29 #include <linux/rio.h>
  30 #include <linux/rio_drv.h>
  31 #include <linux/dma-mapping.h>
  32 #include <linux/interrupt.h>
  33 #include <linux/kfifo.h>
  34 #include <linux/delay.h>
  35
  36 #include "tsi721.h"
  37
  38 static inline struct tsi721_bdma_chan *to_tsi721_chan(struct dma_chan *chan)
  39 {
  40         return container_of(chan, struct tsi721_bdma_chan, dchan);
  41 }
  42
  43 static inline struct tsi721_device *to_tsi721(struct dma_device *ddev)
  44 {
  45         return container_of(ddev, struct rio_mport, dma)->priv;
  46 }
  47
  48 static inline
  49 struct tsi721_tx_desc *to_tsi721_desc(struct dma_async_tx_descriptor *txd)
  50 {
  51         return container_of(txd, struct tsi721_tx_desc, txd);
  52 }
  53
  54 static inline
  55 struct tsi721_tx_desc *tsi721_dma_first_active(
  56                                 struct tsi721_bdma_chan *bdma_chan)
  57 {
  58         return list_first_entry(&bdma_chan->active_list,
  59                                 struct tsi721_tx_desc, desc_node);
  60 }
  61
  62 static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan)
  63 {
  64         struct tsi721_dma_desc *bd_ptr;
  65         struct device *dev = bdma_chan->dchan.device->dev;
  66         u64             *sts_ptr;
  67         dma_addr_t      bd_phys;
  68         dma_addr_t      sts_phys;
  69         int             sts_size;
  70         int             bd_num = bdma_chan->bd_num;
  71
  72         dev_dbg(dev, "Init Block DMA Engine, CH%d\n", bdma_chan->id);
  73
  74         /* Allocate space for DMA descriptors */
  75         bd_ptr = dma_zalloc_coherent(dev,
  76                                 bd_num * sizeof(struct tsi721_dma_desc),
  77                                 &bd_phys, GFP_KERNEL);
  78         if (!bd_ptr)
  79                 return -ENOMEM;
  80
  81         bdma_chan->bd_phys = bd_phys;
  82         bdma_chan->bd_base = bd_ptr;
  83
  84         dev_dbg(dev, "DMA descriptors @ %p (phys = %llx)\n",
  85                 bd_ptr, (unsigned long long)bd_phys);
  86
  87         /* Allocate space for descriptor status FIFO */
  88         sts_size = (bd_num >= TSI721_DMA_MINSTSSZ) ?
  89                                         bd_num : TSI721_DMA_MINSTSSZ;
  90         sts_size = roundup_pow_of_two(sts_size);
  91         sts_ptr = dma_zalloc_coherent(dev,
  92                                      sts_size * sizeof(struct tsi721_dma_sts),
  93                                      &sts_phys, GFP_KERNEL);
  94         if (!sts_ptr) {
  95                 /* Free space allocated for DMA descriptors */
  96                 dma_free_coherent(dev,
  97                                   bd_num * sizeof(struct tsi721_dma_desc),
  98                                   bd_ptr, bd_phys);
  99                 bdma_chan->bd_base = NULL;
 100                 return -ENOMEM;
 101         }
 102
 103         bdma_chan->sts_phys = sts_phys;
 104         bdma_chan->sts_base = sts_ptr;
 105         bdma_chan->sts_size = sts_size;
 106
 107         dev_dbg(dev,
 108                 "desc status FIFO @ %p (phys = %llx) size=0x%x\n",
 109                 sts_ptr, (unsigned long long)sts_phys, sts_size);
 110
 111         /* Initialize DMA descriptors ring */
 112         bd_ptr[bd_num - 1].type_id = cpu_to_le32(DTYPE3 << 29);
 113         bd_ptr[bd_num - 1].next_lo = cpu_to_le32((u64)bd_phys &
 114                                                  TSI721_DMAC_DPTRL_MASK);
 115         bd_ptr[bd_num - 1].next_hi = cpu_to_le32((u64)bd_phys >> 32);
 116
 117         /* Setup DMA descriptor pointers */
 118         iowrite32(((u64)bd_phys >> 32),
 119                 bdma_chan->regs + TSI721_DMAC_DPTRH);
 120         iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK),
 121                 bdma_chan->regs + TSI721_DMAC_DPTRL);
 122
 123         /* Setup descriptor status FIFO */
 124         iowrite32(((u64)sts_phys >> 32),
 125                 bdma_chan->regs + TSI721_DMAC_DSBH);
 126         iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK),
 127                 bdma_chan->regs + TSI721_DMAC_DSBL);
 128         iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size),
 129                 bdma_chan->regs + TSI721_DMAC_DSSZ);
 130
 131         /* Clear interrupt bits */
 132         iowrite32(TSI721_DMAC_INT_ALL,
 133                 bdma_chan->regs + TSI721_DMAC_INT);
 134
 135         ioread32(bdma_chan->regs + TSI721_DMAC_INT);
 136
 137         /* Toggle DMA channel initialization */
 138         iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL);
 139         ioread32(bdma_chan->regs + TSI721_DMAC_CTL);
 140         bdma_chan->wr_count = bdma_chan->wr_count_next = 0;
 141         bdma_chan->sts_rdptr = 0;
 142         udelay(10);
 143
 144         return 0;
 145 }
 146
 147 static int tsi721_bdma_ch_free(struct tsi721_bdma_chan *bdma_chan)
 148 {
 149         u32 ch_stat;
 150
 151         if (bdma_chan->bd_base == NULL)
 152                 return 0;
 153
 154         /* Check if DMA channel still running */
 155         ch_stat = ioread32(bdma_chan->regs + TSI721_DMAC_STS);
 156         if (ch_stat & TSI721_DMAC_STS_RUN)
 157                 return -EFAULT;
 158
 159         /* Put DMA channel into init state */
 160         iowrite32(TSI721_DMAC_CTL_INIT, bdma_chan->regs + TSI721_DMAC_CTL);
 161
 162         /* Free space allocated for DMA descriptors */
 163         dma_free_coherent(bdma_chan->dchan.device->dev,
 164                 bdma_chan->bd_num * sizeof(struct tsi721_dma_desc),
 165                 bdma_chan->bd_base, bdma_chan->bd_phys);
 166         bdma_chan->bd_base = NULL;
 167
 168         /* Free space allocated for status FIFO */
 169         dma_free_coherent(bdma_chan->dchan.device->dev,
 170                 bdma_chan->sts_size * sizeof(struct tsi721_dma_sts),
 171                 bdma_chan->sts_base, bdma_chan->sts_phys);
 172         bdma_chan->sts_base = NULL;
 173         return 0;
 174 }
 175
 176 static void
 177 tsi721_bdma_interrupt_enable(struct tsi721_bdma_chan *bdma_chan, int enable)
 178 {
 179         if (enable) {
 180                 /* Clear pending BDMA channel interrupts */
 181                 iowrite32(TSI721_DMAC_INT_ALL,
 182                         bdma_chan->regs + TSI721_DMAC_INT);
 183                 ioread32(bdma_chan->regs + TSI721_DMAC_INT);
 184                 /* Enable BDMA channel interrupts */
 185                 iowrite32(TSI721_DMAC_INT_ALL,
 186                         bdma_chan->regs + TSI721_DMAC_INTE);
 187         } else {
 188                 /* Disable BDMA channel interrupts */
 189                 iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE);
 190                 /* Clear pending BDMA channel interrupts */
 191                 iowrite32(TSI721_DMAC_INT_ALL,
 192                         bdma_chan->regs + TSI721_DMAC_INT);
 193         }
 194
 195 }
 196
 197 static bool tsi721_dma_is_idle(struct tsi721_bdma_chan *bdma_chan)
 198 {
 199         u32 sts;
 200
 201         sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS);
 202         return ((sts & TSI721_DMAC_STS_RUN) == 0);
 203 }
 204
 205 void tsi721_bdma_handler(struct tsi721_bdma_chan *bdma_chan)
 206 {
 207         /* Disable BDMA channel interrupts */
 208         iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE);
 209         if (bdma_chan->active)
 210                 tasklet_schedule(&bdma_chan->tasklet);
 211 }
 212
 213 #ifdef CONFIG_PCI_MSI
 214 /**
 215  * tsi721_omsg_msix - MSI-X interrupt handler for BDMA channels
 216  * @irq: Linux interrupt number
 217  * @ptr: Pointer to interrupt-specific data (BDMA channel structure)
 218  *
 219  * Handles BDMA channel interrupts signaled using MSI-X.
 220  */
 221 static irqreturn_t tsi721_bdma_msix(int irq, void *ptr)
 222 {
 223         struct tsi721_bdma_chan *bdma_chan = ptr;
 224
 225         tsi721_bdma_handler(bdma_chan);
 226         return IRQ_HANDLED;
 227 }
 228 #endif /* CONFIG_PCI_MSI */
 229
 230 /* Must be called with the spinlock held */
 231 static void tsi721_start_dma(struct tsi721_bdma_chan *bdma_chan)
 232 {
 233         if (!tsi721_dma_is_idle(bdma_chan)) {
 234                 dev_err(bdma_chan->dchan.device->dev,
 235                         "BUG: Attempt to start non-idle channel\n");
 236                 return;
 237         }
 238
 239         if (bdma_chan->wr_count == bdma_chan->wr_count_next) {
 240                 dev_err(bdma_chan->dchan.device->dev,
 241                         "BUG: Attempt to start DMA with no BDs ready\n");
 242                 return;
 243         }
 244
 245         dev_dbg(bdma_chan->dchan.device->dev,
 246                 "tx_chan: %p, chan: %d, regs: %p\n",
 247                 bdma_chan, bdma_chan->dchan.chan_id, bdma_chan->regs);
 248
 249         iowrite32(bdma_chan->wr_count_next,
 250                 bdma_chan->regs + TSI721_DMAC_DWRCNT);
 251         ioread32(bdma_chan->regs + TSI721_DMAC_DWRCNT);
 252
 253         bdma_chan->wr_count = bdma_chan->wr_count_next;
 254 }
 255
 256 static void tsi721_desc_put(struct tsi721_bdma_chan *bdma_chan,
 257                             struct tsi721_tx_desc *desc)
 258 {
 259         dev_dbg(bdma_chan->dchan.device->dev,
 260                 "Put desc: %p into free list\n", desc);
 261
 262         if (desc) {
 263                 spin_lock_bh(&bdma_chan->lock);
 264                 list_splice_init(&desc->tx_list, &bdma_chan->free_list);
 265                 list_add(&desc->desc_node, &bdma_chan->free_list);
 266                 bdma_chan->wr_count_next = bdma_chan->wr_count;
 267                 spin_unlock_bh(&bdma_chan->lock);
 268         }
 269 }
 270
 271 static
 272 struct tsi721_tx_desc *tsi721_desc_get(struct tsi721_bdma_chan *bdma_chan)
 273 {
 274         struct tsi721_tx_desc *tx_desc, *_tx_desc;
 275         struct tsi721_tx_desc *ret = NULL;
 276         int i;
 277
 278         spin_lock_bh(&bdma_chan->lock);
 279         list_for_each_entry_safe(tx_desc, _tx_desc,
 280                                  &bdma_chan->free_list, desc_node) {
 281                 if (async_tx_test_ack(&tx_desc->txd)) {
 282                         list_del(&tx_desc->desc_node);
 283                         ret = tx_desc;
 284                         break;
 285                 }
 286                 dev_dbg(bdma_chan->dchan.device->dev,
 287                         "desc %p not ACKed\n", tx_desc);
 288         }
 289
 290         if (ret == NULL) {
 291                 dev_dbg(bdma_chan->dchan.device->dev,
 292                         "%s: unable to obtain tx descriptor\n", __func__);
 293                 goto err_out;
 294         }
 295
 296         i = bdma_chan->wr_count_next % bdma_chan->bd_num;
 297         if (i == bdma_chan->bd_num - 1) {
 298                 i = 0;
 299                 bdma_chan->wr_count_next++; /* skip link descriptor */
 300         }
 301
 302         bdma_chan->wr_count_next++;
 303         tx_desc->txd.phys = bdma_chan->bd_phys +
 304                                 i * sizeof(struct tsi721_dma_desc);
 305         tx_desc->hw_desc = &((struct tsi721_dma_desc *)bdma_chan->bd_base)[i];
 306 err_out:
 307         spin_unlock_bh(&bdma_chan->lock);
 308
 309         return ret;
 310 }
 311
 312 static int
 313 tsi721_desc_fill_init(struct tsi721_tx_desc *desc, struct scatterlist *sg,
 314         enum dma_rtype rtype, u32 sys_size)
 315 {
 316         struct tsi721_dma_desc *bd_ptr = desc->hw_desc;
 317         u64 rio_addr;
 318
 319         /* Initialize DMA descriptor */
 320         bd_ptr->type_id = cpu_to_le32((DTYPE1 << 29) |
 321                                         (rtype << 19) | desc->destid);
 322         bd_ptr->bcount = cpu_to_le32(((desc->rio_addr & 0x3) << 30) |
 323                                      (sys_size << 26));
 324         rio_addr = (desc->rio_addr >> 2) |
 325                                 ((u64)(desc->rio_addr_u & 0x3) << 62);
 326         bd_ptr->raddr_lo = cpu_to_le32(rio_addr & 0xffffffff);
 327         bd_ptr->raddr_hi = cpu_to_le32(rio_addr >> 32);
 328         bd_ptr->t1.bufptr_lo = cpu_to_le32(
 329                                         (u64)sg_dma_address(sg) & 0xffffffff);
 330         bd_ptr->t1.bufptr_hi = cpu_to_le32((u64)sg_dma_address(sg) >> 32);
 331         bd_ptr->t1.s_dist = 0;
 332         bd_ptr->t1.s_size = 0;
 333
 334         return 0;
 335 }
 336
 337 static int
 338 tsi721_desc_fill_end(struct tsi721_tx_desc *desc)
 339 {
 340         struct tsi721_dma_desc *bd_ptr = desc->hw_desc;
 341
 342         /* Update DMA descriptor */
 343         if (desc->interrupt)
 344                 bd_ptr->type_id |= cpu_to_le32(TSI721_DMAD_IOF);
 345         bd_ptr->bcount |= cpu_to_le32(desc->bcount & TSI721_DMAD_BCOUNT1);
 346
 347         return 0;
 348 }
 349
 350
 351 static void tsi721_dma_chain_complete(struct tsi721_bdma_chan *bdma_chan,
 352                                       struct tsi721_tx_desc *desc)
 353 {
 354         struct dma_async_tx_descriptor *txd = &desc->txd;
 355         dma_async_tx_callback callback = txd->callback;
 356         void *param = txd->callback_param;
 357
 358         list_splice_init(&desc->tx_list, &bdma_chan->free_list);
 359         list_move(&desc->desc_node, &bdma_chan->free_list);
 360         bdma_chan->completed_cookie = txd->cookie;
 361
 362         if (callback)
 363                 callback(param);
 364 }
 365
 366 static void tsi721_dma_complete_all(struct tsi721_bdma_chan *bdma_chan)
 367 {
 368         struct tsi721_tx_desc *desc, *_d;
 369         LIST_HEAD(list);
 370
 371         BUG_ON(!tsi721_dma_is_idle(bdma_chan));
 372
 373         if (!list_empty(&bdma_chan->queue))
 374                 tsi721_start_dma(bdma_chan);
 375
 376         list_splice_init(&bdma_chan->active_list, &list);
 377         list_splice_init(&bdma_chan->queue, &bdma_chan->active_list);
 378
 379         list_for_each_entry_safe(desc, _d, &list, desc_node)
 380                 tsi721_dma_chain_complete(bdma_chan, desc);
 381 }
 382
 383 static void tsi721_clr_stat(struct tsi721_bdma_chan *bdma_chan)
 384 {
 385         u32 srd_ptr;
 386         u64 *sts_ptr;
 387         int i, j;
 388
 389         /* Check and clear descriptor status FIFO entries */
 390         srd_ptr = bdma_chan->sts_rdptr;
 391         sts_ptr = bdma_chan->sts_base;
 392         j = srd_ptr * 8;
 393         while (sts_ptr[j]) {
 394                 for (i = 0; i < 8 && sts_ptr[j]; i++, j++)
 395                         sts_ptr[j] = 0;
 396
 397                 ++srd_ptr;
 398                 srd_ptr %= bdma_chan->sts_size;
 399                 j = srd_ptr * 8;
 400         }
 401
 402         iowrite32(srd_ptr, bdma_chan->regs + TSI721_DMAC_DSRP);
 403         bdma_chan->sts_rdptr = srd_ptr;
 404 }
 405
 406 static void tsi721_advance_work(struct tsi721_bdma_chan *bdma_chan)
 407 {
 408         if (list_empty(&bdma_chan->active_list) ||
 409                 list_is_singular(&bdma_chan->active_list)) {
 410                 dev_dbg(bdma_chan->dchan.device->dev,
 411                         "%s: Active_list empty\n", __func__);
 412                 tsi721_dma_complete_all(bdma_chan);
 413         } else {
 414                 dev_dbg(bdma_chan->dchan.device->dev,
 415                         "%s: Active_list NOT empty\n", __func__);
 416                 tsi721_dma_chain_complete(bdma_chan,
 417                                         tsi721_dma_first_active(bdma_chan));
 418                 tsi721_start_dma(bdma_chan);
 419         }
 420 }
 421
 422 static void tsi721_dma_tasklet(unsigned long data)
 423 {
 424         struct tsi721_bdma_chan *bdma_chan = (struct tsi721_bdma_chan *)data;
 425         u32 dmac_int, dmac_sts;
 426
 427         dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT);
 428         dev_dbg(bdma_chan->dchan.device->dev, "%s: DMAC%d_INT = 0x%x\n",
 429                 __func__, bdma_chan->id, dmac_int);
 430         /* Clear channel interrupts */
 431         iowrite32(dmac_int, bdma_chan->regs + TSI721_DMAC_INT);
 432
 433         if (dmac_int & TSI721_DMAC_INT_ERR) {
 434                 dmac_sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS);
 435                 dev_err(bdma_chan->dchan.device->dev,
 436                         "%s: DMA ERROR - DMAC%d_STS = 0x%x\n",
 437                         __func__, bdma_chan->id, dmac_sts);
 438         }
 439
 440         if (dmac_int & TSI721_DMAC_INT_STFULL) {
 441                 dev_err(bdma_chan->dchan.device->dev,
 442                         "%s: DMAC%d descriptor status FIFO is full\n",
 443                         __func__, bdma_chan->id);
 444         }
 445
 446         if (dmac_int & (TSI721_DMAC_INT_DONE | TSI721_DMAC_INT_IOFDONE)) {
 447                 tsi721_clr_stat(bdma_chan);
 448                 spin_lock(&bdma_chan->lock);
 449                 tsi721_advance_work(bdma_chan);
 450                 spin_unlock(&bdma_chan->lock);
 451         }
 452
 453         /* Re-Enable BDMA channel interrupts */
 454         iowrite32(TSI721_DMAC_INT_ALL, bdma_chan->regs + TSI721_DMAC_INTE);
 455 }
 456
 457 static dma_cookie_t tsi721_tx_submit(struct dma_async_tx_descriptor *txd)
 458 {
 459         struct tsi721_tx_desc *desc = to_tsi721_desc(txd);
 460         struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(txd->chan);
 461         dma_cookie_t cookie;
 462
 463         spin_lock_bh(&bdma_chan->lock);
 464
 465         cookie = txd->chan->cookie;
 466         if (++cookie < 0)
 467                 cookie = 1;
 468         txd->chan->cookie = cookie;
 469         txd->cookie = cookie;
 470
 471         if (list_empty(&bdma_chan->active_list)) {
 472                 list_add_tail(&desc->desc_node, &bdma_chan->active_list);
 473                 tsi721_start_dma(bdma_chan);
 474         } else {
 475                 list_add_tail(&desc->desc_node, &bdma_chan->queue);
 476         }
 477
 478         spin_unlock_bh(&bdma_chan->lock);
 479         return cookie;
 480 }
 481
 482 static int tsi721_alloc_chan_resources(struct dma_chan *dchan)
 483 {
 484         struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 485 #ifdef CONFIG_PCI_MSI
 486         struct tsi721_device *priv = to_tsi721(dchan->device);
 487 #endif
 488         struct tsi721_tx_desc *desc = NULL;
 489         LIST_HEAD(tmp_list);
 490         int i;
 491         int rc;
 492
 493         if (bdma_chan->bd_base)
 494                 return bdma_chan->bd_num - 1;
 495
 496         /* Initialize BDMA channel */
 497         if (tsi721_bdma_ch_init(bdma_chan)) {
 498                 dev_err(dchan->device->dev, "Unable to initialize data DMA"
 499                         " channel %d, aborting\n", bdma_chan->id);
 500                 return -ENOMEM;
 501         }
 502
 503         /* Alocate matching number of logical descriptors */
 504         desc = kcalloc((bdma_chan->bd_num - 1), sizeof(struct tsi721_tx_desc),
 505                         GFP_KERNEL);
 506         if (!desc) {
 507                 dev_err(dchan->device->dev,
 508                         "Failed to allocate logical descriptors\n");
 509                 rc = -ENOMEM;
 510                 goto err_out;
 511         }
 512
 513         bdma_chan->tx_desc = desc;
 514
 515         for (i = 0; i < bdma_chan->bd_num - 1; i++) {
 516                 dma_async_tx_descriptor_init(&desc[i].txd, dchan);
 517                 desc[i].txd.tx_submit = tsi721_tx_submit;
 518                 desc[i].txd.flags = DMA_CTRL_ACK;
 519                 INIT_LIST_HEAD(&desc[i].tx_list);
 520                 list_add_tail(&desc[i].desc_node, &tmp_list);
 521         }
 522
 523         spin_lock_bh(&bdma_chan->lock);
 524         list_splice(&tmp_list, &bdma_chan->free_list);
 525         bdma_chan->completed_cookie = dchan->cookie = 1;
 526         spin_unlock_bh(&bdma_chan->lock);
 527
 528 #ifdef CONFIG_PCI_MSI
 529         if (priv->flags & TSI721_USING_MSIX) {
 530                 /* Request interrupt service if we are in MSI-X mode */
 531                 rc = request_irq(
 532                         priv->msix[TSI721_VECT_DMA0_DONE +
 533                                    bdma_chan->id].vector,
 534                         tsi721_bdma_msix, 0,
 535                         priv->msix[TSI721_VECT_DMA0_DONE +
 536                                    bdma_chan->id].irq_name,
 537                         (void *)bdma_chan);
 538
 539                 if (rc) {
 540                         dev_dbg(dchan->device->dev,
 541                                 "Unable to allocate MSI-X interrupt for "
 542                                 "BDMA%d-DONE\n", bdma_chan->id);
 543                         goto err_out;
 544                 }
 545
 546                 rc = request_irq(priv->msix[TSI721_VECT_DMA0_INT +
 547                                             bdma_chan->id].vector,
 548                                 tsi721_bdma_msix, 0,
 549                                 priv->msix[TSI721_VECT_DMA0_INT +
 550                                            bdma_chan->id].irq_name,
 551                                 (void *)bdma_chan);
 552
 553                 if (rc) {
 554                         dev_dbg(dchan->device->dev,
 555                                 "Unable to allocate MSI-X interrupt for "
 556                                 "BDMA%d-INT\n", bdma_chan->id);
 557                         free_irq(
 558                                 priv->msix[TSI721_VECT_DMA0_DONE +
 559                                            bdma_chan->id].vector,
 560                                 (void *)bdma_chan);
 561                         rc = -EIO;
 562                         goto err_out;
 563                 }
 564         }
 565 #endif /* CONFIG_PCI_MSI */
 566
 567         bdma_chan->active = true;
 568         tsi721_bdma_interrupt_enable(bdma_chan, 1);
 569
 570         return bdma_chan->bd_num - 1;
 571
 572 err_out:
 573         kfree(desc);
 574         tsi721_bdma_ch_free(bdma_chan);
 575         return rc;
 576 }
 577
 578 static void tsi721_free_chan_resources(struct dma_chan *dchan)
 579 {
 580         struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 581         struct tsi721_device *priv = to_tsi721(dchan->device);
 582         LIST_HEAD(list);
 583
 584         dev_dbg(dchan->device->dev, "%s: Entry\n", __func__);
 585
 586         if (bdma_chan->bd_base == NULL)
 587                 return;
 588
 589         BUG_ON(!list_empty(&bdma_chan->active_list));
 590         BUG_ON(!list_empty(&bdma_chan->queue));
 591
 592         tsi721_bdma_interrupt_enable(bdma_chan, 0);
 593         bdma_chan->active = false;
 594
 595 #ifdef CONFIG_PCI_MSI
 596         if (priv->flags & TSI721_USING_MSIX) {
 597                 synchronize_irq(priv->msix[TSI721_VECT_DMA0_DONE +
 598                                            bdma_chan->id].vector);
 599                 synchronize_irq(priv->msix[TSI721_VECT_DMA0_INT +
 600                                            bdma_chan->id].vector);
 601         } else
 602 #endif
 603         synchronize_irq(priv->pdev->irq);
 604
 605         tasklet_kill(&bdma_chan->tasklet);
 606
 607         spin_lock_bh(&bdma_chan->lock);
 608         list_splice_init(&bdma_chan->free_list, &list);
 609         spin_unlock_bh(&bdma_chan->lock);
 610
 611 #ifdef CONFIG_PCI_MSI
 612         if (priv->flags & TSI721_USING_MSIX) {
 613                 free_irq(priv->msix[TSI721_VECT_DMA0_DONE +
 614                                     bdma_chan->id].vector, (void *)bdma_chan);
 615                 free_irq(priv->msix[TSI721_VECT_DMA0_INT +
 616                                     bdma_chan->id].vector, (void *)bdma_chan);
 617         }
 618 #endif /* CONFIG_PCI_MSI */
 619
 620         tsi721_bdma_ch_free(bdma_chan);
 621         kfree(bdma_chan->tx_desc);
 622 }
 623
 624 static
 625 enum dma_status tsi721_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
 626                                  struct dma_tx_state *txstate)
 627 {
 628         struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 629         dma_cookie_t            last_used;
 630         dma_cookie_t            last_completed;
 631         int                     ret;
 632
 633         spin_lock_bh(&bdma_chan->lock);
 634         last_completed = bdma_chan->completed_cookie;
 635         last_used = dchan->cookie;
 636         spin_unlock_bh(&bdma_chan->lock);
 637
 638         ret = dma_async_is_complete(cookie, last_completed, last_used);
 639
 640         dma_set_tx_state(txstate, last_completed, last_used, 0);
 641
 642         dev_dbg(dchan->device->dev,
 643                 "%s: exit, ret: %d, last_completed: %d, last_used: %d\n",
 644                 __func__, ret, last_completed, last_used);
 645
 646         return ret;
 647 }
 648
 649 static void tsi721_issue_pending(struct dma_chan *dchan)
 650 {
 651         struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 652
 653         dev_dbg(dchan->device->dev, "%s: Entry\n", __func__);
 654
 655         if (tsi721_dma_is_idle(bdma_chan)) {
 656                 spin_lock_bh(&bdma_chan->lock);
 657                 tsi721_advance_work(bdma_chan);
 658                 spin_unlock_bh(&bdma_chan->lock);
 659         } else
 660                 dev_dbg(dchan->device->dev,
 661                         "%s: DMA channel still busy\n", __func__);
 662 }
 663
 664 static
 665 struct dma_async_tx_descriptor *tsi721_prep_rio_sg(struct dma_chan *dchan,
 666                         struct scatterlist *sgl, unsigned int sg_len,
 667                         enum dma_transfer_direction dir, unsigned long flags,
 668                         void *tinfo)
 669 {
 670         struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 671         struct tsi721_tx_desc *desc = NULL;
 672         struct tsi721_tx_desc *first = NULL;
 673         struct scatterlist *sg;
 674         struct rio_dma_ext *rext = tinfo;
 675         u64 rio_addr = rext->rio_addr; /* limited to 64-bit rio_addr for now */
 676         unsigned int i;
 677         u32 sys_size = dma_to_mport(dchan->device)->sys_size;
 678         enum dma_rtype rtype;
 679         dma_addr_t next_addr = -1;
 680
 681         if (!sgl || !sg_len) {
 682                 dev_err(dchan->device->dev, "%s: No SG list\n", __func__);
 683                 return NULL;
 684         }
 685
 686         if (dir == DMA_DEV_TO_MEM)
 687                 rtype = NREAD;
 688         else if (dir == DMA_MEM_TO_DEV) {
 689                 switch (rext->wr_type) {
 690                 case RDW_ALL_NWRITE:
 691                         rtype = ALL_NWRITE;
 692                         break;
 693                 case RDW_ALL_NWRITE_R:
 694                         rtype = ALL_NWRITE_R;
 695                         break;
 696                 case RDW_LAST_NWRITE_R:
 697                 default:
 698                         rtype = LAST_NWRITE_R;
 699                         break;
 700                 }
 701         } else {
 702                 dev_err(dchan->device->dev,
 703                         "%s: Unsupported DMA direction option\n", __func__);
 704                 return NULL;
 705         }
 706
 707         for_each_sg(sgl, sg, sg_len, i) {
 708                 int err;
 709
 710                 if (sg_dma_len(sg) > TSI721_BDMA_MAX_BCOUNT) {
 711                         dev_err(dchan->device->dev,
 712                                 "%s: SG entry %d is too large\n", __func__, i);
 713                         goto err_desc_put;
 714                 }
 715
 716                 /*
 717                  * If this sg entry forms contiguous block with previous one,
 718                  * try to merge it into existing DMA descriptor
 719                  */
 720                 if (desc) {
 721                         if (next_addr == sg_dma_address(sg) &&
 722                             desc->bcount + sg_dma_len(sg) <=
 723                                                 TSI721_BDMA_MAX_BCOUNT) {
 724                                 /* Adjust byte count of the descriptor */
 725                                 desc->bcount += sg_dma_len(sg);
 726                                 goto entry_done;
 727                         }
 728
 729                         /*
 730                          * Finalize this descriptor using total
 731                          * byte count value.
 732                          */
 733                         tsi721_desc_fill_end(desc);
 734                         dev_dbg(dchan->device->dev, "%s: desc final len: %d\n",
 735                                 __func__, desc->bcount);
 736                 }
 737
 738                 /*
 739                  * Obtain and initialize a new descriptor
 740                  */
 741                 desc = tsi721_desc_get(bdma_chan);
 742                 if (!desc) {
 743                         dev_err(dchan->device->dev,
 744                                 "%s: Failed to get new descriptor for SG %d\n",
 745                                 __func__, i);
 746                         goto err_desc_put;
 747                 }
 748
 749                 desc->destid = rext->destid;
 750                 desc->rio_addr = rio_addr;
 751                 desc->rio_addr_u = 0;
 752                 desc->bcount = sg_dma_len(sg);
 753
 754                 dev_dbg(dchan->device->dev,
 755                         "sg%d desc: 0x%llx, addr: 0x%llx len: %d\n",
 756                         i, (u64)desc->txd.phys,
 757                         (unsigned long long)sg_dma_address(sg),
 758                         sg_dma_len(sg));
 759
 760                 dev_dbg(dchan->device->dev,
 761                         "bd_ptr = %p did=%d raddr=0x%llx\n",
 762                         desc->hw_desc, desc->destid, desc->rio_addr);
 763
 764                 err = tsi721_desc_fill_init(desc, sg, rtype, sys_size);
 765                 if (err) {
 766                         dev_err(dchan->device->dev,
 767                                 "Failed to build desc: %d\n", err);
 768                         goto err_desc_put;
 769                 }
 770
 771                 next_addr = sg_dma_address(sg);
 772
 773                 if (!first)
 774                         first = desc;
 775                 else
 776                         list_add_tail(&desc->desc_node, &first->tx_list);
 777
 778 entry_done:
 779                 if (sg_is_last(sg)) {
 780                         desc->interrupt = (flags & DMA_PREP_INTERRUPT) != 0;
 781                         tsi721_desc_fill_end(desc);
 782                         dev_dbg(dchan->device->dev, "%s: desc final len: %d\n",
 783                                 __func__, desc->bcount);
 784                 } else {
 785                         rio_addr += sg_dma_len(sg);
 786                         next_addr += sg_dma_len(sg);
 787                 }
 788         }
 789
 790         first->txd.cookie = -EBUSY;
 791         desc->txd.flags = flags;
 792
 793         return &first->txd;
 794
 795 err_desc_put:
 796         tsi721_desc_put(bdma_chan, first);
 797         return NULL;
 798 }
 799
 800 static int tsi721_device_control(struct dma_chan *dchan, enum dma_ctrl_cmd cmd,
 801                              unsigned long arg)
 802 {
 803         struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 804         struct tsi721_tx_desc *desc, *_d;
 805         LIST_HEAD(list);
 806
 807         dev_dbg(dchan->device->dev, "%s: Entry\n", __func__);
 808
 809         if (cmd != DMA_TERMINATE_ALL)
 810                 return -ENXIO;
 811
 812         spin_lock_bh(&bdma_chan->lock);
 813
 814         /* make sure to stop the transfer */
 815         iowrite32(TSI721_DMAC_CTL_SUSP, bdma_chan->regs + TSI721_DMAC_CTL);
 816
 817         list_splice_init(&bdma_chan->active_list, &list);
 818         list_splice_init(&bdma_chan->queue, &list);
 819
 820         list_for_each_entry_safe(desc, _d, &list, desc_node)
 821                 tsi721_dma_chain_complete(bdma_chan, desc);
 822
 823         spin_unlock_bh(&bdma_chan->lock);
 824
 825         return 0;
 826 }
 827
 828 int tsi721_register_dma(struct tsi721_device *priv)
 829 {
 830         int i;
 831         int nr_channels = TSI721_DMA_MAXCH;
 832         int err;
 833         struct rio_mport *mport = priv->mport;
 834
 835         mport->dma.dev = &priv->pdev->dev;
 836         mport->dma.chancnt = nr_channels;
 837
 838         INIT_LIST_HEAD(&mport->dma.channels);
 839
 840         for (i = 0; i < nr_channels; i++) {
 841                 struct tsi721_bdma_chan *bdma_chan = &priv->bdma[i];
 842
 843                 if (i == TSI721_DMACH_MAINT)
 844                         continue;
 845
 846                 bdma_chan->bd_num = TSI721_BDMA_BD_RING_SZ;
 847                 bdma_chan->regs = priv->regs + TSI721_DMAC_BASE(i);
 848
 849                 bdma_chan->dchan.device = &mport->dma;
 850                 bdma_chan->dchan.cookie = 1;
 851                 bdma_chan->dchan.chan_id = i;
 852                 bdma_chan->id = i;
 853                 bdma_chan->active = false;
 854
 855                 spin_lock_init(&bdma_chan->lock);
 856
 857                 INIT_LIST_HEAD(&bdma_chan->active_list);
 858                 INIT_LIST_HEAD(&bdma_chan->queue);
 859                 INIT_LIST_HEAD(&bdma_chan->free_list);
 860
 861                 tasklet_init(&bdma_chan->tasklet, tsi721_dma_tasklet,
 862                              (unsigned long)bdma_chan);
 863                 list_add_tail(&bdma_chan->dchan.device_node,
 864                               &mport->dma.channels);
 865         }
 866
 867         dma_cap_zero(mport->dma.cap_mask);
 868         dma_cap_set(DMA_PRIVATE, mport->dma.cap_mask);
 869         dma_cap_set(DMA_SLAVE, mport->dma.cap_mask);
 870
 871         mport->dma.device_alloc_chan_resources = tsi721_alloc_chan_resources;
 872         mport->dma.device_free_chan_resources = tsi721_free_chan_resources;
 873         mport->dma.device_tx_status = tsi721_tx_status;
 874         mport->dma.device_issue_pending = tsi721_issue_pending;
 875         mport->dma.device_prep_slave_sg = tsi721_prep_rio_sg;
 876         mport->dma.device_control = tsi721_device_control;
 877
 878         err = dma_async_device_register(&mport->dma);
 879         if (err)
 880                 dev_err(&priv->pdev->dev, "Failed to register DMA device\n");
 881
 882         return err;
 883 }