drivers/staging/xillybus/xillybus_core.c

   1 /*
   2  * linux/drivers/misc/xillybus_core.c
   3  *
   4  * Copyright 2011 Xillybus Ltd, http://xillybus.com
   5  *
   6  * Driver for the Xillybus FPGA/host framework.
   7  *
   8  * This driver interfaces with a special IP core in an FPGA, setting up
   9  * a pipe between a hardware FIFO in the programmable logic and a device
  10  * file in the host. The number of such pipes and their attributes are
  11  * set up on the logic. This driver detects these automatically and
  12  * creates the device files accordingly.
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the smems of the GNU General Public License as published by
  16  * the Free Software Foundation; version 2 of the License.
  17  */
  18
  19 #include <linux/list.h>
  20 #include <linux/device.h>
  21 #include <linux/module.h>
  22 #include <linux/io.h>
  23 #include <linux/dma-mapping.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/sched.h>
  26 #include <linux/fs.h>
  27 #include <linux/cdev.h>
  28 #include <linux/spinlock.h>
  29 #include <linux/mutex.h>
  30 #include <linux/crc32.h>
  31 #include <linux/poll.h>
  32 #include <linux/delay.h>
  33 #include <linux/slab.h>
  34 #include <linux/workqueue.h>
  35 #include "xillybus.h"
  36
  37 MODULE_DESCRIPTION("Xillybus core functions");
  38 MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
  39 MODULE_VERSION("1.07");
  40 MODULE_ALIAS("xillybus_core");
  41 MODULE_LICENSE("GPL v2");
  42
  43 /* General timeout is 100 ms, rx timeout is 10 ms */
  44 #define XILLY_RX_TIMEOUT (10*HZ/1000)
  45 #define XILLY_TIMEOUT (100*HZ/1000)
  46
  47 #define fpga_msg_ctrl_reg 0x0002
  48 #define fpga_dma_control_reg 0x0008
  49 #define fpga_dma_bufno_reg 0x0009
  50 #define fpga_dma_bufaddr_lowaddr_reg 0x000a
  51 #define fpga_dma_bufaddr_highaddr_reg 0x000b
  52 #define fpga_buf_ctrl_reg 0x000c
  53 #define fpga_buf_offset_reg 0x000d
  54 #define fpga_endian_reg 0x0010
  55
  56 #define XILLYMSG_OPCODE_RELEASEBUF 1
  57 #define XILLYMSG_OPCODE_QUIESCEACK 2
  58 #define XILLYMSG_OPCODE_FIFOEOF 3
  59 #define XILLYMSG_OPCODE_FATAL_ERROR 4
  60 #define XILLYMSG_OPCODE_NONEMPTY 5
  61
  62 static const char xillyname[] = "xillybus";
  63
  64 static struct class *xillybus_class;
  65
  66 /*
  67  * ep_list_lock is the last lock to be taken; No other lock requests are
  68  * allowed while holding it. It merely protects list_of_endpoints, and not
  69  * the endpoints listed in it.
  70  */
  71
  72 static LIST_HEAD(list_of_endpoints);
  73 static struct mutex ep_list_lock;
  74 static struct workqueue_struct *xillybus_wq;
  75
  76 /*
  77  * Locking scheme: Mutexes protect invocations of character device methods.
  78  * If both locks are taken, wr_mutex is taken first, rd_mutex second.
  79  *
  80  * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the
  81  * buffers' end_offset fields against changes made by IRQ handler (and in
  82  * theory, other file request handlers, but the mutex handles that). Nothing
  83  * else.
  84  * They are held for short direct memory manipulations. Needless to say,
  85  * no mutex locking is allowed when a spinlock is held.
  86  *
  87  * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset.
  88  *
  89  * register_mutex is endpoint-specific, and is held when non-atomic
  90  * register operations are performed. wr_mutex and rd_mutex may be
  91  * held when register_mutex is taken, but none of the spinlocks. Note that
  92  * register_mutex doesn't protect against sporadic buf_ctrl_reg writes
  93  * which are unrelated to buf_offset_reg, since they are harmless.
  94  *
  95  * Blocking on the wait queues is allowed with mutexes held, but not with
  96  * spinlocks.
  97  *
  98  * Only interruptible blocking is allowed on mutexes and wait queues.
  99  *
 100  * All in all, the locking order goes (with skips allowed, of course):
 101  * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock
 102  */
 103
 104 static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf)
 105 {
 106         int opcode;
 107         int msg_channel, msg_bufno, msg_data, msg_dir;
 108
 109         opcode = (buf[0] >> 24) & 0xff;
 110         msg_dir = buf[0] & 1;
 111         msg_channel = (buf[0] >> 1) & 0x7ff;
 112         msg_bufno = (buf[0] >> 12) & 0x3ff;
 113         msg_data = buf[1] & 0xfffffff;
 114
 115         dev_warn(endpoint->dev,
 116                  "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n",
 117                  opcode, msg_channel, msg_dir, msg_bufno, msg_data);
 118 }
 119
 120 /*
 121  * xillybus_isr assumes the interrupt is allocated exclusively to it,
 122  * which is the natural case MSI and several other hardware-oriented
 123  * interrupts. Sharing is not allowed.
 124  */
 125
 126 irqreturn_t xillybus_isr(int irq, void *data)
 127 {
 128         struct xilly_endpoint *ep = data;
 129         u32 *buf;
 130         unsigned int buf_size;
 131         int i;
 132         int opcode;
 133         unsigned int msg_channel, msg_bufno, msg_data, msg_dir;
 134         struct xilly_channel *channel;
 135
 136         /*
 137          * The endpoint structure is altered during periods when it's
 138          * guaranteed no interrupt will occur, but in theory, the cache
 139          * lines may not be updated. So a memory barrier is issued.
 140          */
 141
 142         smp_rmb();
 143
 144         buf = ep->msgbuf_addr;
 145         buf_size = ep->msg_buf_size/sizeof(u32);
 146
 147
 148         ep->ephw->hw_sync_sgl_for_cpu(ep,
 149                                       ep->msgbuf_dma_addr,
 150                                       ep->msg_buf_size,
 151                                       DMA_FROM_DEVICE);
 152
 153         for (i = 0; i < buf_size; i += 2)
 154                 if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) {
 155                         malformed_message(ep, &buf[i]);
 156                         dev_warn(ep->dev,
 157                                  "Sending a NACK on counter %x (instead of %x) on entry %d\n",
 158                                 ((buf[i+1] >> 28) & 0xf),
 159                                 ep->msg_counter,
 160                                 i/2);
 161
 162                         if (++ep->failed_messages > 10)
 163                                 dev_err(ep->dev,
 164                                         "Lost sync with interrupt messages. Stopping.\n");
 165                         else {
 166                                 ep->ephw->hw_sync_sgl_for_device(
 167                                         ep,
 168                                         ep->msgbuf_dma_addr,
 169                                         ep->msg_buf_size,
 170                                         DMA_FROM_DEVICE);
 171
 172                                 iowrite32(0x01,  /* Message NACK */
 173                                           &ep->registers[fpga_msg_ctrl_reg]);
 174                         }
 175                         return IRQ_HANDLED;
 176                 } else if (buf[i] & (1 << 22)) /* Last message */
 177                         break;
 178
 179         if (i >= buf_size) {
 180                 dev_err(ep->dev, "Bad interrupt message. Stopping.\n");
 181                 return IRQ_HANDLED;
 182         }
 183
 184         buf_size = i;
 185
 186         for (i = 0; i <= buf_size; i += 2) { /* Scan through messages */
 187                 opcode = (buf[i] >> 24) & 0xff;
 188
 189                 msg_dir = buf[i] & 1;
 190                 msg_channel = (buf[i] >> 1) & 0x7ff;
 191                 msg_bufno = (buf[i] >> 12) & 0x3ff;
 192                 msg_data = buf[i+1] & 0xfffffff;
 193
 194                 switch (opcode) {
 195                 case XILLYMSG_OPCODE_RELEASEBUF:
 196
 197                         if ((msg_channel > ep->num_channels) ||
 198                             (msg_channel == 0)) {
 199                                 malformed_message(ep, &buf[i]);
 200                                 break;
 201                         }
 202
 203                         channel = ep->channels[msg_channel];
 204
 205                         if (msg_dir) { /* Write channel */
 206                                 if (msg_bufno >= channel->num_wr_buffers) {
 207                                         malformed_message(ep, &buf[i]);
 208                                         break;
 209                                 }
 210                                 spin_lock(&channel->wr_spinlock);
 211                                 channel->wr_buffers[msg_bufno]->end_offset =
 212                                         msg_data;
 213                                 channel->wr_fpga_buf_idx = msg_bufno;
 214                                 channel->wr_empty = 0;
 215                                 channel->wr_sleepy = 0;
 216                                 spin_unlock(&channel->wr_spinlock);
 217
 218                                 wake_up_interruptible(&channel->wr_wait);
 219
 220                         } else {
 221                                 /* Read channel */
 222
 223                                 if (msg_bufno >= channel->num_rd_buffers) {
 224                                         malformed_message(ep, &buf[i]);
 225                                         break;
 226                                 }
 227
 228                                 spin_lock(&channel->rd_spinlock);
 229                                 channel->rd_fpga_buf_idx = msg_bufno;
 230                                 channel->rd_full = 0;
 231                                 spin_unlock(&channel->rd_spinlock);
 232
 233                                 wake_up_interruptible(&channel->rd_wait);
 234                                 if (!channel->rd_synchronous)
 235                                         queue_delayed_work(
 236                                                 xillybus_wq,
 237                                                 &channel->rd_workitem,
 238                                                 XILLY_RX_TIMEOUT);
 239                         }
 240
 241                         break;
 242                 case XILLYMSG_OPCODE_NONEMPTY:
 243                         if ((msg_channel > ep->num_channels) ||
 244                             (msg_channel == 0) || (!msg_dir) ||
 245                             !ep->channels[msg_channel]->wr_supports_nonempty) {
 246                                 malformed_message(ep, &buf[i]);
 247                                 break;
 248                         }
 249
 250                         channel = ep->channels[msg_channel];
 251
 252                         if (msg_bufno >= channel->num_wr_buffers) {
 253                                 malformed_message(ep, &buf[i]);
 254                                 break;
 255                         }
 256                         spin_lock(&channel->wr_spinlock);
 257                         if (msg_bufno == channel->wr_host_buf_idx)
 258                                 channel->wr_ready = 1;
 259                         spin_unlock(&channel->wr_spinlock);
 260
 261                         wake_up_interruptible(&channel->wr_ready_wait);
 262
 263                         break;
 264                 case XILLYMSG_OPCODE_QUIESCEACK:
 265                         ep->idtlen = msg_data;
 266                         wake_up_interruptible(&ep->ep_wait);
 267
 268                         break;
 269                 case XILLYMSG_OPCODE_FIFOEOF:
 270                         channel = ep->channels[msg_channel];
 271                         spin_lock(&channel->wr_spinlock);
 272                         channel->wr_eof = msg_bufno;
 273                         channel->wr_sleepy = 0;
 274
 275                         channel->wr_hangup = channel->wr_empty &&
 276                                 (channel->wr_host_buf_idx == msg_bufno);
 277
 278                         spin_unlock(&channel->wr_spinlock);
 279
 280                         wake_up_interruptible(&channel->wr_wait);
 281
 282                         break;
 283                 case XILLYMSG_OPCODE_FATAL_ERROR:
 284                         ep->fatal_error = 1;
 285                         wake_up_interruptible(&ep->ep_wait); /* For select() */
 286                         dev_err(ep->dev,
 287                                 "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n");
 288                         break;
 289                 default:
 290                         malformed_message(ep, &buf[i]);
 291                         break;
 292                 }
 293         }
 294
 295         ep->ephw->hw_sync_sgl_for_device(ep,
 296                                          ep->msgbuf_dma_addr,
 297                                          ep->msg_buf_size,
 298                                          DMA_FROM_DEVICE);
 299
 300         ep->msg_counter = (ep->msg_counter + 1) & 0xf;
 301         ep->failed_messages = 0;
 302         iowrite32(0x03, &ep->registers[fpga_msg_ctrl_reg]); /* Message ACK */
 303
 304         return IRQ_HANDLED;
 305 }
 306 EXPORT_SYMBOL(xillybus_isr);
 307
 308 /*
 309  * A few trivial memory management functions.
 310  * NOTE: These functions are used only on probe and remove, and therefore
 311  * no locks are applied!
 312  */
 313
 314 void xillybus_do_cleanup(struct xilly_cleanup *mem,
 315                          struct xilly_endpoint *endpoint)
 316 {
 317         struct list_head *this, *next;
 318
 319         list_for_each_safe(this, next, &mem->to_unmap) {
 320                 struct xilly_dma *entry =
 321                         list_entry(this, struct xilly_dma, node);
 322
 323                 endpoint->ephw->unmap_single(entry);
 324                 kfree(entry);
 325         }
 326
 327         INIT_LIST_HEAD(&mem->to_unmap);
 328
 329         list_for_each_safe(this, next, &mem->to_kfree)
 330                 kfree(this);
 331
 332         INIT_LIST_HEAD(&mem->to_kfree);
 333
 334         list_for_each_safe(this, next, &mem->to_pagefree) {
 335                 struct xilly_page *entry =
 336                         list_entry(this, struct xilly_page, node);
 337
 338                 free_pages(entry->addr, entry->order);
 339                 kfree(entry);
 340         }
 341         INIT_LIST_HEAD(&mem->to_pagefree);
 342 }
 343 EXPORT_SYMBOL(xillybus_do_cleanup);
 344
 345 static void *xilly_malloc(struct xilly_cleanup *mem, size_t size)
 346 {
 347         void *ptr;
 348
 349         ptr = kzalloc(sizeof(struct list_head) + size, GFP_KERNEL);
 350
 351         if (!ptr)
 352                 return ptr;
 353
 354         list_add_tail((struct list_head *) ptr, &mem->to_kfree);
 355
 356         return ptr + sizeof(struct list_head);
 357 }
 358
 359 static unsigned long xilly_pagealloc(struct xilly_cleanup *mem,
 360                                      unsigned long order)
 361 {
 362         unsigned long addr;
 363         struct xilly_page *this;
 364
 365         this = kmalloc(sizeof(struct xilly_page), GFP_KERNEL);
 366         if (!this)
 367                 return 0;
 368
 369         addr =  __get_free_pages(GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO, order);
 370
 371         if (!addr) {
 372                 kfree(this);
 373                 return 0;
 374         }
 375
 376         this->addr = addr;
 377         this->order = order;
 378
 379         list_add_tail(&this->node, &mem->to_pagefree);
 380
 381         return addr;
 382 }
 383
 384
 385 static void xillybus_autoflush(struct work_struct *work);
 386
 387 static int xilly_setupchannels(struct xilly_endpoint *ep,
 388                                struct xilly_cleanup *mem,
 389                                unsigned char *chandesc,
 390                                int entries
 391         )
 392 {
 393         int i, entry, wr_nbuffer, rd_nbuffer;
 394         struct xilly_channel *channel;
 395         int channelnum, bufnum, bufsize, format, is_writebuf;
 396         int bytebufsize;
 397         int synchronous, allowpartial, exclusive_open, seekable;
 398         int supports_nonempty;
 399         void *wr_salami = NULL;
 400         void *rd_salami = NULL;
 401         int left_of_wr_salami = 0;
 402         int left_of_rd_salami = 0;
 403         dma_addr_t dma_addr;
 404         int msg_buf_done = 0;
 405
 406         struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */
 407
 408         channel = xilly_malloc(mem, ep->num_channels *
 409                                sizeof(struct xilly_channel));
 410
 411         if (!channel)
 412                 goto memfail;
 413
 414         ep->channels = xilly_malloc(mem, (ep->num_channels + 1) *
 415                                     sizeof(struct xilly_channel *));
 416
 417         if (!ep->channels)
 418                 goto memfail;
 419
 420         ep->channels[0] = NULL; /* Channel 0 is message buf. */
 421
 422         /* Initialize all channels with defaults */
 423
 424         for (i = 1; i <= ep->num_channels; i++) {
 425                 channel->wr_buffers = NULL;
 426                 channel->rd_buffers = NULL;
 427                 channel->num_wr_buffers = 0;
 428                 channel->num_rd_buffers = 0;
 429                 channel->wr_fpga_buf_idx = -1;
 430                 channel->wr_host_buf_idx = 0;
 431                 channel->wr_host_buf_pos = 0;
 432                 channel->wr_empty = 1;
 433                 channel->wr_ready = 0;
 434                 channel->wr_sleepy = 1;
 435                 channel->rd_fpga_buf_idx = 0;
 436                 channel->rd_host_buf_idx = 0;
 437                 channel->rd_host_buf_pos = 0;
 438                 channel->rd_full = 0;
 439                 channel->wr_ref_count = 0;
 440                 channel->rd_ref_count = 0;
 441
 442                 spin_lock_init(&channel->wr_spinlock);
 443                 spin_lock_init(&channel->rd_spinlock);
 444                 mutex_init(&channel->wr_mutex);
 445                 mutex_init(&channel->rd_mutex);
 446                 init_waitqueue_head(&channel->rd_wait);
 447                 init_waitqueue_head(&channel->wr_wait);
 448                 init_waitqueue_head(&channel->wr_ready_wait);
 449
 450                 INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush);
 451
 452                 channel->endpoint = ep;
 453                 channel->chan_num = i;
 454
 455                 channel->log2_element_size = 0;
 456
 457                 ep->channels[i] = channel++;
 458         }
 459
 460         /*
 461          * The DMA buffer address update is atomic on the FPGA, so even if
 462          * it was in the middle of sending messages to some buffer, changing
 463          * the address is safe, since the data will go to either of the
 464          * buffers. Not that this situation should occur at all anyhow.
 465          */
 466
 467         wr_nbuffer = 1;
 468         rd_nbuffer = 1; /* Buffer zero isn't used at all */
 469
 470         for (entry = 0; entry < entries; entry++, chandesc += 4) {
 471                 is_writebuf = chandesc[0] & 0x01;
 472                 channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7);
 473                 format = (chandesc[1] >> 4) & 0x03;
 474                 allowpartial = (chandesc[1] >> 6) & 0x01;
 475                 synchronous = (chandesc[1] >> 7) & 0x01;
 476                 bufsize = 1 << (chandesc[2] & 0x1f);
 477                 bufnum = 1 << (chandesc[3] & 0x0f);
 478                 exclusive_open = (chandesc[2] >> 7) & 0x01;
 479                 seekable = (chandesc[2] >> 6) & 0x01;
 480                 supports_nonempty = (chandesc[2] >> 5) & 0x01;
 481
 482                 if ((channelnum > ep->num_channels) ||
 483                     ((channelnum == 0) && !is_writebuf)) {
 484                         dev_err(ep->dev,
 485                                 "IDT requests channel out of range. Aborting.\n");
 486                         return -ENODEV;
 487                 }
 488
 489                 channel = ep->channels[channelnum]; /* NULL for msg channel */
 490
 491                 bytebufsize = bufsize << 2; /* Overwritten just below */
 492
 493                 if (!is_writebuf) {
 494                         channel->num_rd_buffers = bufnum;
 495                         channel->log2_element_size = ((format > 2) ?
 496                                                       2 : format);
 497                         bytebufsize = channel->rd_buf_size = bufsize *
 498                                 (1 << channel->log2_element_size);
 499                         channel->rd_allow_partial = allowpartial;
 500                         channel->rd_synchronous = synchronous;
 501                         channel->rd_exclusive_open = exclusive_open;
 502                         channel->seekable = seekable;
 503
 504                         channel->rd_buffers = xilly_malloc(
 505                                 mem,
 506                                 bufnum * sizeof(struct xilly_buffer *));
 507
 508                         if (!channel->rd_buffers)
 509                                 goto memfail;
 510
 511                         this_buffer = xilly_malloc(
 512                                 mem,
 513                                 bufnum * sizeof(struct xilly_buffer));
 514
 515                         if (!this_buffer)
 516                                 goto memfail;
 517                 }
 518
 519                 else if (channelnum > 0) {
 520                         channel->num_wr_buffers = bufnum;
 521                         channel->log2_element_size = ((format > 2) ?
 522                                                       2 : format);
 523                         bytebufsize = channel->wr_buf_size = bufsize *
 524                                 (1 << channel->log2_element_size);
 525
 526                         channel->seekable = seekable;
 527                         channel->wr_supports_nonempty = supports_nonempty;
 528
 529                         channel->wr_allow_partial = allowpartial;
 530                         channel->wr_synchronous = synchronous;
 531                         channel->wr_exclusive_open = exclusive_open;
 532
 533                         channel->wr_buffers = xilly_malloc(
 534                                 mem,
 535                                 bufnum * sizeof(struct xilly_buffer *));
 536
 537                         if (!channel->wr_buffers)
 538                                 goto memfail;
 539
 540                         this_buffer = xilly_malloc(
 541                                 mem,
 542                                 bufnum * sizeof(struct xilly_buffer));
 543
 544                         if (!this_buffer)
 545                                 goto memfail;
 546                 }
 547
 548                 /*
 549                  * Although daunting, we cut the chunks for read buffers
 550                  * from a different salami than the write buffers',
 551                  * possibly improving performance.
 552                  */
 553
 554                 if (is_writebuf)
 555                         for (i = 0; i < bufnum; i++) {
 556                                 /*
 557                                  * Buffers are expected in descending
 558                                  * byte-size order, so there is either
 559                                  * enough for this buffer or none at all.
 560                                  */
 561                                 if ((left_of_wr_salami < bytebufsize) &&
 562                                     (left_of_wr_salami > 0)) {
 563                                         dev_err(ep->dev,
 564                                                 "Corrupt buffer allocation in IDT. Aborting.\n");
 565                                         return -ENODEV;
 566                                 }
 567
 568                                 if (left_of_wr_salami == 0) {
 569                                         int allocorder, allocsize;
 570
 571                                         allocsize = PAGE_SIZE;
 572                                         allocorder = 0;
 573                                         while (bytebufsize > allocsize) {
 574                                                 allocsize *= 2;
 575                                                 allocorder++;
 576                                         }
 577
 578                                         wr_salami = (void *)
 579                                                 xilly_pagealloc(mem,
 580                                                                 allocorder);
 581                                         if (!wr_salami)
 582                                                 goto memfail;
 583                                         left_of_wr_salami = allocsize;
 584                                 }
 585
 586                                 dma_addr = ep->ephw->map_single(
 587                                         mem,
 588                                         ep,
 589                                         wr_salami,
 590                                         bytebufsize,
 591                                         DMA_FROM_DEVICE);
 592
 593                                 if (!dma_addr)
 594                                         goto dmafail;
 595
 596                                 iowrite32(
 597                                         (u32) (dma_addr & 0xffffffff),
 598                                         &ep->registers[
 599                                                 fpga_dma_bufaddr_lowaddr_reg]
 600                                         );
 601                                 iowrite32(
 602                                         ((u32) ((((u64) dma_addr) >> 32)
 603                                                 & 0xffffffff)),
 604                                         &ep->registers[
 605                                                 fpga_dma_bufaddr_highaddr_reg]
 606                                         );
 607                                 mmiowb();
 608
 609                                 if (channelnum > 0) {
 610                                         this_buffer->addr = wr_salami;
 611                                         this_buffer->dma_addr = dma_addr;
 612                                         channel->wr_buffers[i] = this_buffer++;
 613
 614                                         iowrite32(
 615                                                 0x80000000 | wr_nbuffer++,
 616                                                 &ep->registers[
 617                                                         fpga_dma_bufno_reg]);
 618                                 } else {
 619                                         ep->msgbuf_addr = wr_salami;
 620                                         ep->msgbuf_dma_addr = dma_addr;
 621                                         ep->msg_buf_size = bytebufsize;
 622                                         msg_buf_done++;
 623
 624                                         iowrite32(
 625                                                 0x80000000, &ep->registers[
 626                                                         fpga_dma_bufno_reg]);
 627                                 }
 628
 629                                 left_of_wr_salami -= bytebufsize;
 630                                 wr_salami += bytebufsize;
 631                         }
 632                 else /* Read buffers */
 633                         for (i = 0; i < bufnum; i++) {
 634                                 /*
 635                                  * Buffers are expected in descending
 636                                  * byte-size order, so there is either
 637                                  * enough for this buffer or none at all.
 638                                  */
 639                                 if ((left_of_rd_salami < bytebufsize) &&
 640                                     (left_of_rd_salami > 0)) {
 641                                         dev_err(ep->dev,
 642                                                 "Corrupt buffer allocation in IDT. Aborting.\n");
 643                                         return -ENODEV;
 644                                 }
 645
 646                                 if (left_of_rd_salami == 0) {
 647                                         int allocorder, allocsize;
 648
 649                                         allocsize = PAGE_SIZE;
 650                                         allocorder = 0;
 651                                         while (bytebufsize > allocsize) {
 652                                                 allocsize *= 2;
 653                                                 allocorder++;
 654                                         }
 655
 656                                         rd_salami = (void *)
 657                                                 xilly_pagealloc(
 658                                                         mem,
 659                                                         allocorder);
 660
 661                                         if (!rd_salami)
 662                                                 goto memfail;
 663                                         left_of_rd_salami = allocsize;
 664                                 }
 665
 666                                 dma_addr = ep->ephw->map_single(
 667                                         mem,
 668                                         ep,
 669                                         rd_salami,
 670                                         bytebufsize,
 671                                         DMA_TO_DEVICE);
 672
 673                                 if (!dma_addr)
 674                                         goto dmafail;
 675
 676                                 iowrite32(
 677                                         (u32) (dma_addr & 0xffffffff),
 678                                         &ep->registers[
 679                                                 fpga_dma_bufaddr_lowaddr_reg]
 680                                         );
 681                                 iowrite32(
 682                                         ((u32) ((((u64) dma_addr) >> 32)
 683                                                 & 0xffffffff)),
 684                                         &ep->registers[
 685                                                 fpga_dma_bufaddr_highaddr_reg]
 686                                         );
 687                                 mmiowb();
 688
 689                                 this_buffer->addr = rd_salami;
 690                                 this_buffer->dma_addr = dma_addr;
 691                                 channel->rd_buffers[i] = this_buffer++;
 692
 693                                 iowrite32(rd_nbuffer++,
 694                                           &ep->registers[fpga_dma_bufno_reg]);
 695
 696                                 left_of_rd_salami -= bytebufsize;
 697                                 rd_salami += bytebufsize;
 698                         }
 699         }
 700
 701         if (!msg_buf_done) {
 702                 dev_err(ep->dev,
 703                         "Corrupt IDT: No message buffer. Aborting.\n");
 704                 return -ENODEV;
 705         }
 706
 707         return 0;
 708
 709 memfail:
 710         dev_err(ep->dev,
 711                 "Failed to allocate write buffer memory. Aborting.\n");
 712         return -ENOMEM;
 713 dmafail:
 714         dev_err(ep->dev, "Failed to map DMA memory!. Aborting.\n");
 715         return -ENOMEM;
 716 }
 717
 718 static void xilly_scan_idt(struct xilly_endpoint *endpoint,
 719                            struct xilly_idt_handle *idt_handle)
 720 {
 721         int count = 0;
 722         unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr;
 723         unsigned char *end_of_idt = idt + endpoint->idtlen - 4;
 724         unsigned char *scan;
 725         int len;
 726
 727         scan = idt;
 728         idt_handle->idt = idt;
 729
 730         scan++; /* Skip version number */
 731
 732         while ((scan <= end_of_idt) && *scan) {
 733                 while ((scan <= end_of_idt) && *scan++)
 734                         /* Do nothing, just scan thru string */;
 735                 count++;
 736         }
 737
 738         scan++;
 739
 740         if (scan > end_of_idt) {
 741                 dev_err(endpoint->dev,
 742                         "IDT device name list overflow. Aborting.\n");
 743                 idt_handle->chandesc = NULL;
 744                 return;
 745         } else
 746                 idt_handle->chandesc = scan;
 747
 748         len = endpoint->idtlen - (3 + ((int) (scan - idt)));
 749
 750         if (len & 0x03) {
 751                 idt_handle->chandesc = NULL;
 752
 753                 dev_err(endpoint->dev,
 754                         "Corrupt IDT device name list. Aborting.\n");
 755         }
 756
 757         idt_handle->entries = len >> 2;
 758
 759         endpoint->num_channels = count;
 760 }
 761
 762 static int xilly_obtain_idt(struct xilly_endpoint *endpoint)
 763 {
 764         int rc = 0;
 765         struct xilly_channel *channel;
 766         unsigned char *version;
 767
 768         channel = endpoint->channels[1]; /* This should be generated ad-hoc */
 769
 770         channel->wr_sleepy = 1;
 771         wmb(); /* Setting wr_sleepy must come before the command */
 772
 773         iowrite32(1 |
 774                    (3 << 24), /* Opcode 3 for channel 0 = Send IDT */
 775                    &endpoint->registers[fpga_buf_ctrl_reg]);
 776         mmiowb(); /* Just to appear safe */
 777
 778         wait_event_interruptible_timeout(channel->wr_wait,
 779                                          (!channel->wr_sleepy),
 780                                          XILLY_TIMEOUT);
 781
 782         if (channel->wr_sleepy) {
 783                 dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n");
 784
 785                 if (endpoint->fatal_error)
 786                         return -EIO;
 787
 788                 rc = -ENODEV;
 789                 return rc;
 790         }
 791
 792         endpoint->ephw->hw_sync_sgl_for_cpu(
 793                 channel->endpoint,
 794                 channel->wr_buffers[0]->dma_addr,
 795                 channel->wr_buf_size,
 796                 DMA_FROM_DEVICE);
 797
 798         if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) {
 799                 dev_err(endpoint->dev,
 800                         "IDT length mismatch (%d != %d). Aborting.\n",
 801                        channel->wr_buffers[0]->end_offset, endpoint->idtlen);
 802                 rc = -ENODEV;
 803                 return rc;
 804         }
 805
 806         if (crc32_le(~0, channel->wr_buffers[0]->addr,
 807                      endpoint->idtlen+1) != 0) {
 808                 dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n");
 809                 rc = -ENODEV;
 810                 return rc;
 811         }
 812
 813         version = channel->wr_buffers[0]->addr;
 814
 815         /* Check version number. Accept anything below 0x82 for now. */
 816         if (*version > 0x82) {
 817                 dev_err(endpoint->dev,
 818                         "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgarde. Aborting.\n",
 819                        (int) *version);
 820                 rc = -ENODEV;
 821                 return rc;
 822         }
 823
 824         return 0; /* Success */
 825 }
 826
 827 static ssize_t xillybus_read(struct file *filp, char __user *userbuf,
 828                              size_t count, loff_t *f_pos)
 829 {
 830         ssize_t rc;
 831         unsigned long flags;
 832         int bytes_done = 0;
 833         int no_time_left = 0;
 834         long deadline, left_to_sleep;
 835         struct xilly_channel *channel = filp->private_data;
 836
 837         int empty, reached_eof, exhausted, ready;
 838         /* Initializations are there only to silence warnings */
 839
 840         int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
 841         int waiting_bufidx;
 842
 843         if (channel->endpoint->fatal_error)
 844                 return -EIO;
 845
 846         deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
 847
 848         rc = mutex_lock_interruptible(&channel->wr_mutex);
 849
 850         if (rc)
 851                 return rc;
 852
 853         rc = 0; /* Just to be clear about it. Compiler optimizes this out */
 854
 855         while (1) { /* Note that we may drop mutex within this loop */
 856                 int bytes_to_do = count - bytes_done;
 857                 spin_lock_irqsave(&channel->wr_spinlock, flags);
 858
 859                 empty = channel->wr_empty;
 860                 ready = !empty || channel->wr_ready;
 861
 862                 if (!empty) {
 863                         bufidx = channel->wr_host_buf_idx;
 864                         bufpos = channel->wr_host_buf_pos;
 865                         howmany = ((channel->wr_buffers[bufidx]->end_offset
 866                                     + 1) << channel->log2_element_size)
 867                                 - bufpos;
 868
 869                         /* Update wr_host_* to its post-operation state */
 870                         if (howmany > bytes_to_do) {
 871                                 bufferdone = 0;
 872
 873                                 howmany = bytes_to_do;
 874                                 channel->wr_host_buf_pos += howmany;
 875                         } else {
 876                                 bufferdone = 1;
 877
 878                                 channel->wr_host_buf_pos = 0;
 879
 880                                 if (bufidx == channel->wr_fpga_buf_idx) {
 881                                         channel->wr_empty = 1;
 882                                         channel->wr_sleepy = 1;
 883                                         channel->wr_ready = 0;
 884                                 }
 885
 886                                 if (bufidx >= (channel->num_wr_buffers - 1))
 887                                         channel->wr_host_buf_idx = 0;
 888                                 else
 889                                         channel->wr_host_buf_idx++;
 890                         }
 891                 }
 892
 893                 /*
 894                  * Marking our situation after the possible changes above,
 895                  * for use after releasing the spinlock.
 896                  *
 897                  * empty = empty before change
 898                  * exhasted = empty after possible change
 899                  */
 900
 901                 reached_eof = channel->wr_empty &&
 902                         (channel->wr_host_buf_idx == channel->wr_eof);
 903                 channel->wr_hangup = reached_eof;
 904                 exhausted = channel->wr_empty;
 905                 waiting_bufidx = channel->wr_host_buf_idx;
 906
 907                 spin_unlock_irqrestore(&channel->wr_spinlock, flags);
 908
 909                 if (!empty) { /* Go on, now without the spinlock */
 910
 911                         if (bufpos == 0) /* Position zero means it's virgin */
 912                                 channel->endpoint->ephw->hw_sync_sgl_for_cpu(
 913                                         channel->endpoint,
 914                                         channel->wr_buffers[bufidx]->dma_addr,
 915                                         channel->wr_buf_size,
 916                                         DMA_FROM_DEVICE);
 917
 918                         if (copy_to_user(
 919                                     userbuf,
 920                                     channel->wr_buffers[bufidx]->addr
 921                                     + bufpos, howmany))
 922                                 rc = -EFAULT;
 923
 924                         userbuf += howmany;
 925                         bytes_done += howmany;
 926
 927                         if (bufferdone) {
 928                                 channel->endpoint->ephw->
 929                                         hw_sync_sgl_for_device
 930                                         (
 931                                                 channel->endpoint,
 932                                                 channel->wr_buffers[bufidx]->
 933                                                 dma_addr,
 934                                                 channel->wr_buf_size,
 935                                                 DMA_FROM_DEVICE);
 936
 937                                 /*
 938                                  * Tell FPGA the buffer is done with. It's an
 939                                  * atomic operation to the FPGA, so what
 940                                  * happens with other channels doesn't matter,
 941                                  * and the certain channel is protected with
 942                                  * the channel-specific mutex.
 943                                  */
 944
 945                                 iowrite32(1 | (channel->chan_num << 1)
 946                                            | (bufidx << 12),
 947                                            &channel->endpoint->registers[
 948                                                    fpga_buf_ctrl_reg]);
 949                                 mmiowb(); /* Just to appear safe */
 950                         }
 951
 952                         if (rc) {
 953                                 mutex_unlock(&channel->wr_mutex);
 954                                 return rc;
 955                         }
 956                 }
 957
 958                 /* This includes a zero-count return = EOF */
 959                 if ((bytes_done >= count) || reached_eof)
 960                         break;
 961
 962                 if (!exhausted)
 963                         continue; /* More in RAM buffer(s)? Just go on. */
 964
 965                 if ((bytes_done > 0) &&
 966                     (no_time_left ||
 967                      (channel->wr_synchronous && channel->wr_allow_partial)))
 968                         break;
 969
 970                 /*
 971                  * Nonblocking read: The "ready" flag tells us that the FPGA
 972                  * has data to send. In non-blocking mode, if it isn't on,
 973                  * just return. But if there is, we jump directly to the point
 974                  * where we ask for the FPGA to send all it has, and wait
 975                  * until that data arrives. So in a sense, we *do* block in
 976                  * nonblocking mode, but only for a very short time.
 977                  */
 978
 979                 if (!no_time_left && (filp->f_flags & O_NONBLOCK)) {
 980                         if (bytes_done > 0)
 981                                 break;
 982
 983                         if (ready)
 984                                 goto desperate;
 985
 986                         bytes_done = -EAGAIN;
 987                         break;
 988                 }
 989
 990                 if (!no_time_left || (bytes_done > 0)) {
 991                         /*
 992                          * Note that in case of an element-misaligned read
 993                          * request, offsetlimit will include the last element,
 994                          * which will be partially read from.
 995                          */
 996                         int offsetlimit = ((count - bytes_done) - 1) >>
 997                                 channel->log2_element_size;
 998                         int buf_elements = channel->wr_buf_size >>
 999                                 channel->log2_element_size;
1000
1001                         /*
1002                          * In synchronous mode, always send an offset limit.
1003                          * Just don't send a value too big.
1004                          */
1005
1006                         if (channel->wr_synchronous) {
1007                                 /* Don't request more than one buffer */
1008                                 if (channel->wr_allow_partial &&
1009                                     (offsetlimit >= buf_elements))
1010                                         offsetlimit = buf_elements - 1;
1011
1012                                 /* Don't request more than all buffers */
1013                                 if (!channel->wr_allow_partial &&
1014                                     (offsetlimit >=
1015                                      (buf_elements * channel->num_wr_buffers)))
1016                                         offsetlimit = buf_elements *
1017                                                 channel->num_wr_buffers - 1;
1018                         }
1019
1020                         /*
1021                          * In asynchronous mode, force early flush of a buffer
1022                          * only if that will allow returning a full count. The
1023                          * "offsetlimit < ( ... )" rather than "<=" excludes
1024                          * requesting a full buffer, which would obviously
1025                          * cause a buffer transmission anyhow
1026                          */
1027
1028                         if (channel->wr_synchronous ||
1029                             (offsetlimit < (buf_elements - 1))) {
1030
1031                                 mutex_lock(&channel->endpoint->register_mutex);
1032
1033                                 iowrite32(offsetlimit,
1034                                           &channel->endpoint->registers[
1035                                                   fpga_buf_offset_reg]);
1036                                 mmiowb();
1037
1038                                 iowrite32(1 | (channel->chan_num << 1) |
1039                                            (2 << 24) |  /* 2 = offset limit */
1040                                            (waiting_bufidx << 12),
1041                                            &channel->endpoint->registers[
1042                                                    fpga_buf_ctrl_reg]);
1043
1044                                 mmiowb(); /* Just to appear safe */
1045
1046                                 mutex_unlock(&channel->endpoint->
1047                                              register_mutex);
1048                         }
1049
1050                 }
1051
1052                 /*
1053                  * If partial completion is disallowed, there is no point in
1054                  * timeout sleeping. Neither if no_time_left is set and
1055                  * there's no data.
1056                  */
1057
1058                 if (!channel->wr_allow_partial ||
1059                     (no_time_left && (bytes_done == 0))) {
1060
1061                         /*
1062                          * This do-loop will run more than once if another
1063                          * thread reasserted wr_sleepy before we got the mutex
1064                          * back, so we try again.
1065                          */
1066
1067                         do {
1068                                 mutex_unlock(&channel->wr_mutex);
1069
1070                                 if (wait_event_interruptible(
1071                                             channel->wr_wait,
1072                                             (!channel->wr_sleepy)))
1073                                         goto interrupted;
1074
1075                                 if (mutex_lock_interruptible(
1076                                             &channel->wr_mutex))
1077                                         goto interrupted;
1078                         } while (channel->wr_sleepy);
1079
1080                         continue;
1081
1082 interrupted: /* Mutex is not held if got here */
1083                         if (channel->endpoint->fatal_error)
1084                                 return -EIO;
1085                         if (bytes_done)
1086                                 return bytes_done;
1087                         if (filp->f_flags & O_NONBLOCK)
1088                                 return -EAGAIN; /* Don't admit snoozing */
1089                         return -EINTR;
1090                 }
1091
1092                 left_to_sleep = deadline - ((long) jiffies);
1093
1094                 /*
1095                  * If our time is out, skip the waiting. We may miss wr_sleepy
1096                  * being deasserted but hey, almost missing the train is like
1097                  * missing it.
1098                  */
1099
1100                 if (left_to_sleep > 0) {
1101                         left_to_sleep =
1102                                 wait_event_interruptible_timeout(
1103                                         channel->wr_wait,
1104                                         (!channel->wr_sleepy),
1105                                         left_to_sleep);
1106
1107                         if (!channel->wr_sleepy)
1108                                 continue;
1109
1110                         if (left_to_sleep < 0) { /* Interrupt */
1111                                 mutex_unlock(&channel->wr_mutex);
1112                                 if (channel->endpoint->fatal_error)
1113                                         return -EIO;
1114                                 if (bytes_done)
1115                                         return bytes_done;
1116                                 return -EINTR;
1117                         }
1118                 }
1119
1120 desperate:
1121                 no_time_left = 1; /* We're out of sleeping time. Desperate! */
1122
1123                 if (bytes_done == 0) {
1124                         /*
1125                          * Reaching here means that we allow partial return,
1126                          * that we've run out of time, and that we have
1127                          * nothing to return.
1128                          * So tell the FPGA to send anything it has or gets.
1129                          */
1130
1131                         iowrite32(1 | (channel->chan_num << 1) |
1132                                    (3 << 24) |  /* Opcode 3, flush it all! */
1133                                    (waiting_bufidx << 12),
1134                                    &channel->endpoint->registers[
1135                                            fpga_buf_ctrl_reg]);
1136                         mmiowb(); /* Just to appear safe */
1137                 }
1138
1139                 /*
1140                  * Formally speaking, we should block for data at this point.
1141                  * But to keep the code cleaner, we'll just finish the loop,
1142                  * make the unlikely check for data, and then block at the
1143                  * usual place.
1144                  */
1145         }
1146
1147         mutex_unlock(&channel->wr_mutex);
1148
1149         if (channel->endpoint->fatal_error)
1150                 return -EIO;
1151
1152         return bytes_done;
1153 }
1154
1155 /*
1156  * The timeout argument takes values as follows:
1157  *  >0 : Flush with timeout
1158  * ==0 : Flush, and wait idefinitely for the flush to complete
1159  *  <0 : Autoflush: Flush only if there's a single buffer occupied
1160  */
1161
1162 static int xillybus_myflush(struct xilly_channel *channel, long timeout)
1163 {
1164         int rc = 0;
1165         unsigned long flags;
1166
1167         int end_offset_plus1;
1168         int bufidx, bufidx_minus1;
1169         int i;
1170         int empty;
1171         int new_rd_host_buf_pos;
1172
1173         if (channel->endpoint->fatal_error)
1174                 return -EIO;
1175         rc = mutex_lock_interruptible(&channel->rd_mutex);
1176
1177         if (rc)
1178                 return rc;
1179
1180         /*
1181          * Don't flush a closed channel. This can happen when the work queued
1182          * autoflush thread fires off after the file has closed. This is not
1183          * an error, just something to dismiss.
1184          */
1185
1186         if (!channel->rd_ref_count)
1187                 goto done;
1188
1189         bufidx = channel->rd_host_buf_idx;
1190
1191         bufidx_minus1 = (bufidx == 0) ? channel->num_rd_buffers - 1 : bufidx-1;
1192
1193         end_offset_plus1 = channel->rd_host_buf_pos >>
1194                 channel->log2_element_size;
1195
1196         new_rd_host_buf_pos = channel->rd_host_buf_pos -
1197                 (end_offset_plus1 << channel->log2_element_size);
1198
1199         /* Submit the current buffer if it's nonempty */
1200         if (end_offset_plus1) {
1201                 unsigned char *tail = channel->rd_buffers[bufidx]->addr +
1202                         (end_offset_plus1 << channel->log2_element_size);
1203
1204                 /* Copy  unflushed data, so we can put it in next buffer */
1205                 for (i = 0; i < new_rd_host_buf_pos; i++)
1206                         channel->rd_leftovers[i] = *tail++;
1207
1208                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1209
1210                 /* Autoflush only if a single buffer is occupied */
1211
1212                 if ((timeout < 0) &&
1213                     (channel->rd_full ||
1214                      (bufidx_minus1 != channel->rd_fpga_buf_idx))) {
1215                         spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1216                         /*
1217                          * A new work item may be queued by the ISR exactly
1218                          * now, since the execution of a work item allows the
1219                          * queuing of a new one while it's running.
1220                          */
1221                         goto done;
1222                 }
1223
1224                 /* The 4th element is never needed for data, so it's a flag */
1225                 channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0);
1226
1227                 /* Set up rd_full to reflect a certain moment's state */
1228
1229                 if (bufidx == channel->rd_fpga_buf_idx)
1230                         channel->rd_full = 1;
1231                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1232
1233                 if (bufidx >= (channel->num_rd_buffers - 1))
1234                         channel->rd_host_buf_idx = 0;
1235                 else
1236                         channel->rd_host_buf_idx++;
1237
1238                 channel->endpoint->ephw->hw_sync_sgl_for_device(
1239                         channel->endpoint,
1240                         channel->rd_buffers[bufidx]->dma_addr,
1241                         channel->rd_buf_size,
1242                         DMA_TO_DEVICE);
1243
1244                 mutex_lock(&channel->endpoint->register_mutex);
1245
1246                 iowrite32(end_offset_plus1 - 1,
1247                           &channel->endpoint->registers[fpga_buf_offset_reg]);
1248                 mmiowb();
1249
1250                 iowrite32((channel->chan_num << 1) | /* Channel ID */
1251                            (2 << 24) |  /* Opcode 2, submit buffer */
1252                            (bufidx << 12),
1253                            &channel->endpoint->registers[fpga_buf_ctrl_reg]);
1254                 mmiowb(); /* Just to appear safe */
1255
1256                 mutex_unlock(&channel->endpoint->register_mutex);
1257         } else if (bufidx == 0)
1258                 bufidx = channel->num_rd_buffers - 1;
1259         else
1260                 bufidx--;
1261
1262         channel->rd_host_buf_pos = new_rd_host_buf_pos;
1263
1264         if (timeout < 0)
1265                 goto done; /* Autoflush */
1266
1267
1268         /*
1269          * bufidx is now the last buffer written to (or equal to
1270          * rd_fpga_buf_idx if buffer was never written to), and
1271          * channel->rd_host_buf_idx the one after it.
1272          *
1273          * If bufidx == channel->rd_fpga_buf_idx we're either empty or full.
1274          */
1275
1276         rc = 0;
1277
1278         while (1) { /* Loop waiting for draining of buffers */
1279                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1280
1281                 if (bufidx != channel->rd_fpga_buf_idx)
1282                         channel->rd_full = 1; /*
1283                                                * Not really full,
1284                                                * but needs waiting.
1285                                                */
1286
1287                 empty = !channel->rd_full;
1288
1289                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1290
1291                 if (empty)
1292                         break;
1293
1294                 /*
1295                  * Indefinite sleep with mutex taken. With data waiting for
1296                  * flushing user should not be surprised if open() for write
1297                  * sleeps.
1298                  */
1299                 if (timeout == 0)
1300                         wait_event_interruptible(channel->rd_wait,
1301                                                  (!channel->rd_full));
1302
1303                 else if (wait_event_interruptible_timeout(
1304                                  channel->rd_wait,
1305                                  (!channel->rd_full),
1306                                  timeout) == 0) {
1307                         dev_warn(channel->endpoint->dev,
1308                                 "Timed out while flushing. Output data may be lost.\n");
1309
1310                         rc = -ETIMEDOUT;
1311                         break;
1312                 }
1313
1314                 if (channel->rd_full) {
1315                         rc = -EINTR;
1316                         break;
1317                 }
1318         }
1319
1320 done:
1321         mutex_unlock(&channel->rd_mutex);
1322
1323         if (channel->endpoint->fatal_error)
1324                 return -EIO;
1325
1326         return rc;
1327 }
1328
1329 static int xillybus_flush(struct file *filp, fl_owner_t id)
1330 {
1331         if (!(filp->f_mode & FMODE_WRITE))
1332                 return 0;
1333
1334         return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */
1335 }
1336
1337 static void xillybus_autoflush(struct work_struct *work)
1338 {
1339         struct delayed_work *workitem = container_of(
1340                 work, struct delayed_work, work);
1341         struct xilly_channel *channel = container_of(
1342                 workitem, struct xilly_channel, rd_workitem);
1343         int rc;
1344
1345         rc = xillybus_myflush(channel, -1);
1346
1347         if (rc == -EINTR)
1348                 dev_warn(channel->endpoint->dev,
1349                          "Autoflush failed because work queue thread got a signal.\n");
1350         else if (rc)
1351                 dev_err(channel->endpoint->dev,
1352                         "Autoflush failed under weird circumstances.\n");
1353 }
1354
1355 static ssize_t xillybus_write(struct file *filp, const char __user *userbuf,
1356                               size_t count, loff_t *f_pos)
1357 {
1358         ssize_t rc;
1359         unsigned long flags;
1360         int bytes_done = 0;
1361         struct xilly_channel *channel = filp->private_data;
1362
1363         int full, exhausted;
1364         /* Initializations are there only to silence warnings */
1365
1366         int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
1367         int end_offset_plus1 = 0;
1368
1369         if (channel->endpoint->fatal_error)
1370                 return -EIO;
1371
1372         rc = mutex_lock_interruptible(&channel->rd_mutex);
1373
1374         if (rc)
1375                 return rc;
1376
1377         rc = 0; /* Just to be clear about it. Compiler optimizes this out */
1378
1379         while (1) {
1380                 int bytes_to_do = count - bytes_done;
1381
1382                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1383
1384                 full = channel->rd_full;
1385
1386                 if (!full) {
1387                         bufidx = channel->rd_host_buf_idx;
1388                         bufpos = channel->rd_host_buf_pos;
1389                         howmany = channel->rd_buf_size - bufpos;
1390
1391                         /*
1392                          * Update rd_host_* to its state after this operation.
1393                          * count=0 means committing the buffer immediately,
1394                          * which is like flushing, but not necessarily block.
1395                          */
1396
1397                         if ((howmany > bytes_to_do) &&
1398                             (count ||
1399                              ((bufpos >> channel->log2_element_size) == 0))) {
1400                                 bufferdone = 0;
1401
1402                                 howmany = bytes_to_do;
1403                                 channel->rd_host_buf_pos += howmany;
1404                         } else {
1405                                 bufferdone = 1;
1406
1407                                 if (count) {
1408                                         end_offset_plus1 =
1409                                                 channel->rd_buf_size >>
1410                                                 channel->log2_element_size;
1411                                         channel->rd_host_buf_pos = 0;
1412                                 } else {
1413                                         unsigned char *tail;
1414                                         int i;
1415
1416                                         end_offset_plus1 = bufpos >>
1417                                                 channel->log2_element_size;
1418
1419                                         channel->rd_host_buf_pos -=
1420                                                 end_offset_plus1 <<
1421                                                 channel->log2_element_size;
1422
1423                                         tail = channel->
1424                                                 rd_buffers[bufidx]->addr +
1425                                                 (end_offset_plus1 <<
1426                                                  channel->log2_element_size);
1427
1428                                         for (i = 0;
1429                                              i < channel->rd_host_buf_pos;
1430                                              i++)
1431                                                 channel->rd_leftovers[i] =
1432                                                         *tail++;
1433                                 }
1434
1435                                 if (bufidx == channel->rd_fpga_buf_idx)
1436                                         channel->rd_full = 1;
1437
1438                                 if (bufidx >= (channel->num_rd_buffers - 1))
1439                                         channel->rd_host_buf_idx = 0;
1440                                 else
1441                                         channel->rd_host_buf_idx++;
1442                         }
1443                 }
1444
1445                 /*
1446                  * Marking our situation after the possible changes above,
1447                  * for use  after releasing the spinlock.
1448                  *
1449                  * full = full before change
1450                  * exhasted = full after possible change
1451                  */
1452
1453                 exhausted = channel->rd_full;
1454
1455                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1456
1457                 if (!full) { /* Go on, now without the spinlock */
1458                         unsigned char *head =
1459                                 channel->rd_buffers[bufidx]->addr;
1460                         int i;
1461
1462                         if ((bufpos == 0) || /* Zero means it's virgin */
1463                             (channel->rd_leftovers[3] != 0)) {
1464                                 channel->endpoint->ephw->hw_sync_sgl_for_cpu(
1465                                         channel->endpoint,
1466                                         channel->rd_buffers[bufidx]->dma_addr,
1467                                         channel->rd_buf_size,
1468                                         DMA_TO_DEVICE);
1469
1470                                 /* Virgin, but leftovers are due */
1471                                 for (i = 0; i < bufpos; i++)
1472                                         *head++ = channel->rd_leftovers[i];
1473
1474                                 channel->rd_leftovers[3] = 0; /* Clear flag */
1475                         }
1476
1477                         if (copy_from_user(
1478                                     channel->rd_buffers[bufidx]->addr + bufpos,
1479                                     userbuf, howmany))
1480                                 rc = -EFAULT;
1481
1482                         userbuf += howmany;
1483                         bytes_done += howmany;
1484
1485                         if (bufferdone) {
1486                                 channel->endpoint->ephw->
1487                                         hw_sync_sgl_for_device(
1488                                                 channel->endpoint,
1489                                                 channel->rd_buffers[bufidx]->
1490                                                 dma_addr,
1491                                                 channel->rd_buf_size,
1492                                                 DMA_TO_DEVICE);
1493
1494                                 mutex_lock(&channel->endpoint->register_mutex);
1495
1496                                 iowrite32(end_offset_plus1 - 1,
1497                                           &channel->endpoint->registers[
1498                                                   fpga_buf_offset_reg]);
1499                                 mmiowb();
1500                                 iowrite32((channel->chan_num << 1) |
1501                                            (2 << 24) |  /* 2 = submit buffer */
1502                                            (bufidx << 12),
1503                                            &channel->endpoint->registers[
1504                                                    fpga_buf_ctrl_reg]);
1505                                 mmiowb(); /* Just to appear safe */
1506
1507                                 mutex_unlock(&channel->endpoint->
1508                                              register_mutex);
1509
1510                                 channel->rd_leftovers[3] =
1511                                         (channel->rd_host_buf_pos != 0);
1512                         }
1513
1514                         if (rc) {
1515                                 mutex_unlock(&channel->rd_mutex);
1516
1517                                 if (channel->endpoint->fatal_error)
1518                                         return -EIO;
1519
1520                                 if (!channel->rd_synchronous)
1521                                         queue_delayed_work(
1522                                                 xillybus_wq,
1523                                                 &channel->rd_workitem,
1524                                                 XILLY_RX_TIMEOUT);
1525
1526                                 return rc;
1527                         }
1528                 }
1529
1530                 if (bytes_done >= count)
1531                         break;
1532
1533                 if (!exhausted)
1534                         continue; /* If there's more space, just go on */
1535
1536                 if ((bytes_done > 0) && channel->rd_allow_partial)
1537                         break;
1538
1539                 /*
1540                  * Indefinite sleep with mutex taken. With data waiting for
1541                  * flushing, user should not be surprised if open() for write
1542                  * sleeps.
1543                  */
1544
1545                 if (filp->f_flags & O_NONBLOCK) {
1546                         bytes_done = -EAGAIN;
1547                         break;
1548                 }
1549
1550                 wait_event_interruptible(channel->rd_wait,
1551                                          (!channel->rd_full));
1552
1553                 if (channel->rd_full) {
1554                         mutex_unlock(&channel->rd_mutex);
1555
1556                         if (channel->endpoint->fatal_error)
1557                                 return -EIO;
1558
1559                         if (bytes_done)
1560                                 return bytes_done;
1561                         return -EINTR;
1562                 }
1563         }
1564
1565         mutex_unlock(&channel->rd_mutex);
1566
1567         if (!channel->rd_synchronous)
1568                 queue_delayed_work(xillybus_wq,
1569                                    &channel->rd_workitem,
1570                                    XILLY_RX_TIMEOUT);
1571
1572         if ((channel->rd_synchronous) && (bytes_done > 0)) {
1573                 rc = xillybus_myflush(filp->private_data, 0); /* No timeout */
1574
1575                 if (rc && (rc != -EINTR))
1576                         return rc;
1577         }
1578
1579         if (channel->endpoint->fatal_error)
1580                 return -EIO;
1581
1582         return bytes_done;
1583 }
1584
1585 static int xillybus_open(struct inode *inode, struct file *filp)
1586 {
1587         int rc = 0;
1588         unsigned long flags;
1589         int minor = iminor(inode);
1590         int major = imajor(inode);
1591         struct xilly_endpoint *ep_iter, *endpoint = NULL;
1592         struct xilly_channel *channel;
1593
1594         mutex_lock(&ep_list_lock);
1595
1596         list_for_each_entry(ep_iter, &list_of_endpoints, ep_list) {
1597                 if ((ep_iter->major == major) &&
1598                     (minor >= ep_iter->lowest_minor) &&
1599                     (minor < (ep_iter->lowest_minor +
1600                               ep_iter->num_channels))) {
1601                         endpoint = ep_iter;
1602                         break;
1603                 }
1604         }
1605         mutex_unlock(&ep_list_lock);
1606
1607         if (!endpoint) {
1608                 pr_err("xillybus: open() failed to find a device for major=%d and minor=%d\n",
1609                        major, minor);
1610                 return -ENODEV;
1611         }
1612
1613         if (endpoint->fatal_error)
1614                 return -EIO;
1615
1616         channel = endpoint->channels[1 + minor - endpoint->lowest_minor];
1617         filp->private_data = channel;
1618
1619
1620         /*
1621          * It gets complicated because:
1622          * 1. We don't want to take a mutex we don't have to
1623          * 2. We don't want to open one direction if the other will fail.
1624          */
1625
1626         if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers))
1627                 return -ENODEV;
1628
1629         if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers))
1630                 return -ENODEV;
1631
1632         if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) &&
1633             (channel->wr_synchronous || !channel->wr_allow_partial ||
1634              !channel->wr_supports_nonempty)) {
1635                 dev_err(endpoint->dev,
1636                         "open() failed: O_NONBLOCK not allowed for read on this device\n");
1637                 return -ENODEV;
1638         }
1639
1640         if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) &&
1641             (channel->rd_synchronous || !channel->rd_allow_partial)) {
1642                 dev_err(endpoint->dev,
1643                         "open() failed: O_NONBLOCK not allowed for write on this device\n");
1644                 return -ENODEV;
1645         }
1646
1647         /*
1648          * Note: open() may block on getting mutexes despite O_NONBLOCK.
1649          * This shouldn't occur normally, since multiple open of the same
1650          * file descriptor is almost always prohibited anyhow
1651          * (*_exclusive_open is normally set in real-life systems).
1652          */
1653
1654         if (filp->f_mode & FMODE_READ) {
1655                 rc = mutex_lock_interruptible(&channel->wr_mutex);
1656                 if (rc)
1657                         return rc;
1658         }
1659
1660         if (filp->f_mode & FMODE_WRITE) {
1661                 rc = mutex_lock_interruptible(&channel->rd_mutex);
1662                 if (rc)
1663                         goto unlock_wr;
1664         }
1665
1666         if ((filp->f_mode & FMODE_READ) &&
1667             (channel->wr_ref_count != 0) &&
1668             (channel->wr_exclusive_open)) {
1669                 rc = -EBUSY;
1670                 goto unlock;
1671         }
1672
1673         if ((filp->f_mode & FMODE_WRITE) &&
1674             (channel->rd_ref_count != 0) &&
1675             (channel->rd_exclusive_open)) {
1676                 rc = -EBUSY;
1677                 goto unlock;
1678         }
1679
1680
1681         if (filp->f_mode & FMODE_READ) {
1682                 if (channel->wr_ref_count == 0) { /* First open of file */
1683                         /* Move the host to first buffer */
1684                         spin_lock_irqsave(&channel->wr_spinlock, flags);
1685                         channel->wr_host_buf_idx = 0;
1686                         channel->wr_host_buf_pos = 0;
1687                         channel->wr_fpga_buf_idx = -1;
1688                         channel->wr_empty = 1;
1689                         channel->wr_ready = 0;
1690                         channel->wr_sleepy = 1;
1691                         channel->wr_eof = -1;
1692                         channel->wr_hangup = 0;
1693
1694                         spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1695
1696                         iowrite32(1 | (channel->chan_num << 1) |
1697                                   (4 << 24) |  /* Opcode 4, open channel */
1698                                   ((channel->wr_synchronous & 1) << 23),
1699                                   &channel->endpoint->registers[
1700                                           fpga_buf_ctrl_reg]);
1701                         mmiowb(); /* Just to appear safe */
1702                 }
1703
1704                 channel->wr_ref_count++;
1705         }
1706
1707         if (filp->f_mode & FMODE_WRITE) {
1708                 if (channel->rd_ref_count == 0) { /* First open of file */
1709                         /* Move the host to first buffer */
1710                         spin_lock_irqsave(&channel->rd_spinlock, flags);
1711                         channel->rd_host_buf_idx = 0;
1712                         channel->rd_host_buf_pos = 0;
1713                         channel->rd_leftovers[3] = 0; /* No leftovers. */
1714                         channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1;
1715                         channel->rd_full = 0;
1716
1717                         spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1718
1719                         iowrite32((channel->chan_num << 1) |
1720                                   (4 << 24),   /* Opcode 4, open channel */
1721                                   &channel->endpoint->registers[
1722                                           fpga_buf_ctrl_reg]);
1723                         mmiowb(); /* Just to appear safe */
1724                 }
1725
1726                 channel->rd_ref_count++;
1727         }
1728
1729 unlock:
1730         if (filp->f_mode & FMODE_WRITE)
1731                 mutex_unlock(&channel->rd_mutex);
1732 unlock_wr:
1733         if (filp->f_mode & FMODE_READ)
1734                 mutex_unlock(&channel->wr_mutex);
1735
1736         if (!rc && (!channel->seekable))
1737                 return nonseekable_open(inode, filp);
1738
1739         return rc;
1740 }
1741
1742 static int xillybus_release(struct inode *inode, struct file *filp)
1743 {
1744         int rc;
1745         unsigned long flags;
1746         struct xilly_channel *channel = filp->private_data;
1747
1748         int buf_idx;
1749         int eof;
1750
1751         if (channel->endpoint->fatal_error)
1752                 return -EIO;
1753
1754         if (filp->f_mode & FMODE_WRITE) {
1755                 rc = mutex_lock_interruptible(&channel->rd_mutex);
1756
1757                 if (rc) {
1758                         dev_warn(channel->endpoint->dev,
1759                                  "Failed to close file. Hardware left in messy state.\n");
1760                         return rc;
1761                 }
1762
1763                 channel->rd_ref_count--;
1764
1765                 if (channel->rd_ref_count == 0) {
1766
1767                         /*
1768                          * We rely on the kernel calling flush()
1769                          * before we get here.
1770                          */
1771
1772                         iowrite32((channel->chan_num << 1) | /* Channel ID */
1773                                   (5 << 24),  /* Opcode 5, close channel */
1774                                   &channel->endpoint->registers[
1775                                           fpga_buf_ctrl_reg]);
1776                         mmiowb(); /* Just to appear safe */
1777                 }
1778                 mutex_unlock(&channel->rd_mutex);
1779         }
1780
1781         if (filp->f_mode & FMODE_READ) {
1782                 rc = mutex_lock_interruptible(&channel->wr_mutex);
1783                 if (rc) {
1784                         dev_warn(channel->endpoint->dev,
1785                                  "Failed to close file. Hardware left in messy state.\n");
1786                         return rc;
1787                 }
1788
1789                 channel->wr_ref_count--;
1790
1791                 if (channel->wr_ref_count == 0) {
1792
1793                         iowrite32(1 | (channel->chan_num << 1) |
1794                                    (5 << 24),  /* Opcode 5, close channel */
1795                                    &channel->endpoint->registers[
1796                                            fpga_buf_ctrl_reg]);
1797                         mmiowb(); /* Just to appear safe */
1798
1799                         /*
1800                          * This is crazily cautious: We make sure that not
1801                          * only that we got an EOF (be it because we closed
1802                          * the channel or because of a user's EOF), but verify
1803                          * that it's one beyond the last buffer arrived, so
1804                          * we have no leftover buffers pending before wrapping
1805                          * up (which can only happen in asynchronous channels,
1806                          * BTW)
1807                          */
1808
1809                         while (1) {
1810                                 spin_lock_irqsave(&channel->wr_spinlock,
1811                                                   flags);
1812                                 buf_idx = channel->wr_fpga_buf_idx;
1813                                 eof = channel->wr_eof;
1814                                 channel->wr_sleepy = 1;
1815                                 spin_unlock_irqrestore(&channel->wr_spinlock,
1816                                                        flags);
1817
1818                                 /*
1819                                  * Check if eof points at the buffer after
1820                                  * the last one the FPGA submitted. Note that
1821                                  * no EOF is marked by negative eof.
1822                                  */
1823
1824                                 buf_idx++;
1825                                 if (buf_idx == channel->num_wr_buffers)
1826                                         buf_idx = 0;
1827
1828                                 if (buf_idx == eof)
1829                                         break;
1830
1831                                 /*
1832                                  * Steal extra 100 ms if awaken by interrupt.
1833                                  * This is a simple workaround for an
1834                                  * interrupt pending when entering, which would
1835                                  * otherwise result in declaring the hardware
1836                                  * non-responsive.
1837                                  */
1838
1839                                 if (wait_event_interruptible(
1840                                             channel->wr_wait,
1841                                             (!channel->wr_sleepy)))
1842                                         msleep(100);
1843
1844                                 if (channel->wr_sleepy) {
1845                                         mutex_unlock(&channel->wr_mutex);
1846                                         dev_warn(channel->endpoint->dev,
1847                                                  "Hardware failed to respond to close command, therefore left in messy state.\n");
1848                                         return -EINTR;
1849                                 }
1850                         }
1851                 }
1852
1853                 mutex_unlock(&channel->wr_mutex);
1854         }
1855
1856         return 0;
1857 }
1858 static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence)
1859 {
1860         struct xilly_channel *channel = filp->private_data;
1861         loff_t pos = filp->f_pos;
1862         int rc = 0;
1863
1864         /*
1865          * Take both mutexes not allowing interrupts, since it seems like
1866          * common applications don't expect an -EINTR here. Besides, multiple
1867          * access to a single file descriptor on seekable devices is a mess
1868          * anyhow.
1869          */
1870
1871         if (channel->endpoint->fatal_error)
1872                 return -EIO;
1873
1874         mutex_lock(&channel->wr_mutex);
1875         mutex_lock(&channel->rd_mutex);
1876
1877         switch (whence) {
1878         case 0:
1879                 pos = offset;
1880                 break;
1881         case 1:
1882                 pos += offset;
1883                 break;
1884         case 2:
1885                 pos = offset; /* Going to the end => to the beginning */
1886                 break;
1887         default:
1888                 rc = -EINVAL;
1889                 goto end;
1890         }
1891
1892         /* In any case, we must finish on an element boundary */
1893         if (pos & ((1 << channel->log2_element_size) - 1)) {
1894                 rc = -EINVAL;
1895                 goto end;
1896         }
1897
1898         mutex_lock(&channel->endpoint->register_mutex);
1899
1900         iowrite32(pos >> channel->log2_element_size,
1901                   &channel->endpoint->registers[fpga_buf_offset_reg]);
1902         mmiowb();
1903         iowrite32((channel->chan_num << 1) |
1904                   (6 << 24),  /* Opcode 6, set address */
1905                   &channel->endpoint->registers[fpga_buf_ctrl_reg]);
1906         mmiowb(); /* Just to appear safe */
1907
1908         mutex_unlock(&channel->endpoint->register_mutex);
1909
1910 end:
1911         mutex_unlock(&channel->rd_mutex);
1912         mutex_unlock(&channel->wr_mutex);
1913
1914         if (rc) /* Return error after releasing mutexes */
1915                 return rc;
1916
1917         filp->f_pos = pos;
1918
1919         /*
1920          * Since seekable devices are allowed only when the channel is
1921          * synchronous, we assume that there is no data pending in either
1922          * direction (which holds true as long as no concurrent access on the
1923          * file descriptor takes place).
1924          * The only thing we may need to throw away is leftovers from partial
1925          * write() flush.
1926          */
1927
1928         channel->rd_leftovers[3] = 0;
1929
1930         return pos;
1931 }
1932
1933 static unsigned int xillybus_poll(struct file *filp, poll_table *wait)
1934 {
1935         struct xilly_channel *channel = filp->private_data;
1936         unsigned int mask = 0;
1937         unsigned long flags;
1938
1939         poll_wait(filp, &channel->endpoint->ep_wait, wait);
1940
1941         /*
1942          * poll() won't play ball regarding read() channels which
1943          * aren't asynchronous and support the nonempty message. Allowing
1944          * that will create situations where data has been delivered at
1945          * the FPGA, and users expecting select() to wake up, which it may
1946          * not.
1947          */
1948
1949         if (!channel->wr_synchronous && channel->wr_supports_nonempty) {
1950                 poll_wait(filp, &channel->wr_wait, wait);
1951                 poll_wait(filp, &channel->wr_ready_wait, wait);
1952
1953                 spin_lock_irqsave(&channel->wr_spinlock, flags);
1954                 if (!channel->wr_empty || channel->wr_ready)
1955                         mask |= POLLIN | POLLRDNORM;
1956
1957                 if (channel->wr_hangup)
1958                         /*
1959                          * Not POLLHUP, because its behavior is in the
1960                          * mist, and POLLIN does what we want: Wake up
1961                          * the read file descriptor so it sees EOF.
1962                          */
1963                         mask |=  POLLIN | POLLRDNORM;
1964                 spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1965         }
1966
1967         /*
1968          * If partial data write is disallowed on a write() channel,
1969          * it's pointless to ever signal OK to write, because is could
1970          * block despite some space being available.
1971          */
1972
1973         if (channel->rd_allow_partial) {
1974                 poll_wait(filp, &channel->rd_wait, wait);
1975
1976                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1977                 if (!channel->rd_full)
1978                         mask |= POLLOUT | POLLWRNORM;
1979                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1980         }
1981
1982         if (channel->endpoint->fatal_error)
1983                 mask |= POLLERR;
1984
1985         return mask;
1986 }
1987
1988 static const struct file_operations xillybus_fops = {
1989         .owner      = THIS_MODULE,
1990         .read       = xillybus_read,
1991         .write      = xillybus_write,
1992         .open       = xillybus_open,
1993         .flush      = xillybus_flush,
1994         .release    = xillybus_release,
1995         .llseek     = xillybus_llseek,
1996         .poll       = xillybus_poll,
1997 };
1998
1999 static int xillybus_init_chrdev(struct xilly_endpoint *endpoint,
2000                                 const unsigned char *idt)
2001 {
2002         int rc;
2003         dev_t dev;
2004         int devnum, i, minor, major;
2005         char devname[48];
2006         struct device *device;
2007
2008         rc = alloc_chrdev_region(&dev, 0, /* minor start */
2009                                  endpoint->num_channels,
2010                                  xillyname);
2011
2012         if (rc) {
2013                 dev_warn(endpoint->dev, "Failed to obtain major/minors");
2014                 goto error1;
2015         }
2016
2017         endpoint->major = major = MAJOR(dev);
2018         endpoint->lowest_minor = minor = MINOR(dev);
2019
2020         cdev_init(&endpoint->cdev, &xillybus_fops);
2021         endpoint->cdev.owner = endpoint->ephw->owner;
2022         rc = cdev_add(&endpoint->cdev, MKDEV(major, minor),
2023                       endpoint->num_channels);
2024         if (rc) {
2025                 dev_warn(endpoint->dev, "Failed to add cdev. Aborting.\n");
2026                 goto error2;
2027         }
2028
2029         idt++;
2030
2031         for (i = minor, devnum = 0;
2032              devnum < endpoint->num_channels;
2033              devnum++, i++) {
2034                 snprintf(devname, sizeof(devname)-1, "xillybus_%s", idt);
2035
2036                 devname[sizeof(devname)-1] = 0; /* Should never matter */
2037
2038                 while (*idt++)
2039                         /* Skip to next */;
2040
2041                 device = device_create(xillybus_class,
2042                                        NULL,
2043                                        MKDEV(major, i),
2044                                        NULL,
2045                                        "%s", devname);
2046
2047                 if (IS_ERR(device)) {
2048                         dev_warn(endpoint->dev,
2049                                  "Failed to create %s device. Aborting.\n",
2050                                  devname);
2051                         goto error3;
2052                 }
2053         }
2054
2055         dev_info(endpoint->dev, "Created %d device files.\n",
2056                  endpoint->num_channels);
2057         return 0; /* succeed */
2058
2059 error3:
2060         devnum--; i--;
2061         for (; devnum >= 0; devnum--, i--)
2062                 device_destroy(xillybus_class, MKDEV(major, i));
2063
2064         cdev_del(&endpoint->cdev);
2065 error2:
2066         unregister_chrdev_region(MKDEV(major, minor), endpoint->num_channels);
2067 error1:
2068
2069         return rc;
2070 }
2071
2072 static void xillybus_cleanup_chrdev(struct xilly_endpoint *endpoint)
2073 {
2074         int minor;
2075
2076         for (minor = endpoint->lowest_minor;
2077              minor < (endpoint->lowest_minor + endpoint->num_channels);
2078              minor++)
2079                 device_destroy(xillybus_class, MKDEV(endpoint->major, minor));
2080         cdev_del(&endpoint->cdev);
2081         unregister_chrdev_region(MKDEV(endpoint->major,
2082                                        endpoint->lowest_minor),
2083                                  endpoint->num_channels);
2084
2085         dev_info(endpoint->dev, "Removed %d device files.\n",
2086                  endpoint->num_channels);
2087 }
2088
2089
2090 struct xilly_endpoint *xillybus_init_endpoint(struct pci_dev *pdev,
2091                                               struct device *dev,
2092                                               struct xilly_endpoint_hardware
2093                                               *ephw)
2094 {
2095         struct xilly_endpoint *endpoint;
2096
2097         endpoint = kzalloc(sizeof(*endpoint), GFP_KERNEL);
2098         if (!endpoint) {
2099                 dev_err(dev, "Failed to allocate memory. Aborting.\n");
2100                 return NULL;
2101         }
2102
2103         endpoint->pdev = pdev;
2104         endpoint->dev = dev;
2105         endpoint->ephw = ephw;
2106         INIT_LIST_HEAD(&endpoint->cleanup.to_kfree);
2107         INIT_LIST_HEAD(&endpoint->cleanup.to_pagefree);
2108         INIT_LIST_HEAD(&endpoint->cleanup.to_unmap);
2109         endpoint->msg_counter = 0x0b;
2110         endpoint->failed_messages = 0;
2111         endpoint->fatal_error = 0;
2112
2113         init_waitqueue_head(&endpoint->ep_wait);
2114         mutex_init(&endpoint->register_mutex);
2115
2116         return endpoint;
2117 }
2118 EXPORT_SYMBOL(xillybus_init_endpoint);
2119
2120 static int xilly_quiesce(struct xilly_endpoint *endpoint)
2121 {
2122         endpoint->idtlen = -1;
2123         wmb(); /* Make sure idtlen is set before sending command */
2124         iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
2125                   &endpoint->registers[fpga_dma_control_reg]);
2126         mmiowb();
2127
2128         wait_event_interruptible_timeout(endpoint->ep_wait,
2129                                          (endpoint->idtlen >= 0),
2130                                          XILLY_TIMEOUT);
2131
2132         if (endpoint->idtlen < 0) {
2133                 dev_err(endpoint->dev,
2134                         "Failed to quiesce the device on exit. Quitting while leaving a mess.\n");
2135                 return -ENODEV;
2136         }
2137         return 0; /* Success */
2138 }
2139
2140 int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint)
2141 {
2142         int rc = 0;
2143
2144         struct xilly_cleanup tmpmem;
2145         int idtbuffersize = (1 << PAGE_SHIFT);
2146
2147         /*
2148          * The bogus IDT is used during bootstrap for allocating the initial
2149          * message buffer, and then the message buffer and space for the IDT
2150          * itself. The initial message buffer is of a single page's size, but
2151          * it's soon replaced with a more modest one (and memory is freed).
2152          */
2153
2154         unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0,
2155                                        3, 192, PAGE_SHIFT, 0 };
2156         struct xilly_idt_handle idt_handle;
2157
2158         INIT_LIST_HEAD(&tmpmem.to_kfree);
2159         INIT_LIST_HEAD(&tmpmem.to_pagefree);
2160         INIT_LIST_HEAD(&tmpmem.to_unmap);
2161
2162         /*
2163          * Writing the value 0x00000001 to Endianness register signals which
2164          * endianness this processor is using, so the FPGA can swap words as
2165          * necessary.
2166          */
2167
2168         iowrite32(1, &endpoint->registers[fpga_endian_reg]);
2169         mmiowb(); /* Writes below are affected by the one above. */
2170
2171         /* Bootstrap phase I: Allocate temporary message buffer */
2172
2173         endpoint->num_channels = 0;
2174
2175         rc = xilly_setupchannels(endpoint, &tmpmem, bogus_idt, 1);
2176
2177         if (rc)
2178                 goto failed_buffers;
2179
2180         /* Clear the message subsystem (and counter in particular) */
2181         iowrite32(0x04, &endpoint->registers[fpga_msg_ctrl_reg]);
2182         mmiowb();
2183
2184         endpoint->idtlen = -1;
2185
2186         smp_wmb();
2187
2188         /*
2189          * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT
2190          * buffer size.
2191          */
2192         iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
2193                    &endpoint->registers[fpga_dma_control_reg]);
2194         mmiowb();
2195
2196         wait_event_interruptible_timeout(endpoint->ep_wait,
2197                                          (endpoint->idtlen >= 0),
2198                                          XILLY_TIMEOUT);
2199
2200         if (endpoint->idtlen < 0) {
2201                 dev_err(endpoint->dev, "No response from FPGA. Aborting.\n");
2202                 rc = -ENODEV;
2203                 goto failed_quiesce;
2204         }
2205
2206         /* Enable DMA */
2207         iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)),
2208                    &endpoint->registers[fpga_dma_control_reg]);
2209         mmiowb();
2210
2211         /* Bootstrap phase II: Allocate buffer for IDT and obtain it */
2212         while (endpoint->idtlen >= idtbuffersize) {
2213                 idtbuffersize *= 2;
2214                 bogus_idt[6]++;
2215         }
2216
2217         endpoint->num_channels = 1;
2218
2219         rc = xilly_setupchannels(endpoint, &tmpmem, bogus_idt, 2);
2220
2221         if (rc)
2222                 goto failed_idt;
2223
2224         smp_wmb();
2225
2226         rc = xilly_obtain_idt(endpoint);
2227
2228         if (rc)
2229                 goto failed_idt;
2230
2231         xilly_scan_idt(endpoint, &idt_handle);
2232
2233         if (!idt_handle.chandesc) {
2234                 rc = -ENODEV;
2235                 goto failed_idt;
2236         }
2237         /* Bootstrap phase III: Allocate buffers according to IDT */
2238
2239         rc = xilly_setupchannels(endpoint,
2240                                  &endpoint->cleanup,
2241                                  idt_handle.chandesc,
2242                                  idt_handle.entries);
2243
2244         if (rc)
2245                 goto failed_idt;
2246
2247         smp_wmb(); /* mutex_lock below should suffice, but won't hurt.*/
2248
2249         /*
2250          * endpoint is now completely configured. We put it on the list
2251          * available to open() before registering the char device(s)
2252          */
2253
2254         mutex_lock(&ep_list_lock);
2255         list_add_tail(&endpoint->ep_list, &list_of_endpoints);
2256         mutex_unlock(&ep_list_lock);
2257
2258         rc = xillybus_init_chrdev(endpoint, idt_handle.idt);
2259
2260         if (rc)
2261                 goto failed_chrdevs;
2262
2263         xillybus_do_cleanup(&tmpmem, endpoint);
2264
2265         return 0;
2266
2267 failed_chrdevs:
2268         mutex_lock(&ep_list_lock);
2269         list_del(&endpoint->ep_list);
2270         mutex_unlock(&ep_list_lock);
2271
2272 failed_idt:
2273         /* Quiesce the device. Now it's serious to do it */
2274         rc = xilly_quiesce(endpoint);
2275
2276         if (rc)
2277                 return rc; /* FPGA may still DMA, so no release */
2278
2279         flush_workqueue(xillybus_wq);
2280 failed_quiesce:
2281 failed_buffers:
2282         xillybus_do_cleanup(&tmpmem, endpoint);
2283
2284         return rc;
2285 }
2286 EXPORT_SYMBOL(xillybus_endpoint_discovery);
2287
2288 void xillybus_endpoint_remove(struct xilly_endpoint *endpoint)
2289 {
2290         xillybus_cleanup_chrdev(endpoint);
2291
2292         mutex_lock(&ep_list_lock);
2293         list_del(&endpoint->ep_list);
2294         mutex_unlock(&ep_list_lock);
2295
2296         xilly_quiesce(endpoint);
2297
2298         /*
2299          * Flushing is done upon endpoint release to prevent access to memory
2300          * just about to be released. This makes the quiesce complete.
2301          */
2302         flush_workqueue(xillybus_wq);
2303 }
2304 EXPORT_SYMBOL(xillybus_endpoint_remove);
2305
2306 static int __init xillybus_init(void)
2307 {
2308         int rc = 0;
2309
2310         mutex_init(&ep_list_lock);
2311
2312         xillybus_class = class_create(THIS_MODULE, xillyname);
2313         if (IS_ERR(xillybus_class)) {
2314                 rc = PTR_ERR(xillybus_class);
2315                 pr_warn("Failed to register class xillybus\n");
2316
2317                 return rc;
2318         }
2319
2320         xillybus_wq = alloc_workqueue(xillyname, 0, 0);
2321
2322         return 0; /* Success */
2323 }
2324
2325 static void __exit xillybus_exit(void)
2326 {
2327         /* flush_workqueue() was called for each endpoint released */
2328         destroy_workqueue(xillybus_wq);
2329
2330         class_destroy(xillybus_class);
2331 }
2332
2333 module_init(xillybus_init);
2334 module_exit(xillybus_exit);