2 * sx8.c: Driver for Promise SATA SX8 looks-like-I2O hardware
4 * Copyright 2004-2005 Red Hat, Inc.
6 * Author/maintainer: Jeff Garzik <jgarzik@pobox.com>
8 * This file is subject to the terms and conditions of the GNU General Public
9 * License. See the file "COPYING" in the main directory of this archive
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/init.h>
16 #include <linux/pci.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
19 #include <linux/blkdev.h>
20 #include <linux/sched.h>
21 #include <linux/devfs_fs_kernel.h>
22 #include <linux/interrupt.h>
23 #include <linux/compiler.h>
24 #include <linux/workqueue.h>
25 #include <linux/bitops.h>
26 #include <linux/delay.h>
27 #include <linux/time.h>
28 #include <linux/hdreg.h>
29 #include <linux/dma-mapping.h>
31 #include <asm/semaphore.h>
32 #include <asm/uaccess.h>
36 #define CARM_VERBOSE_DEBUG
39 #undef CARM_VERBOSE_DEBUG
43 #define DRV_NAME "sx8"
44 #define DRV_VERSION "1.0"
45 #define PFX DRV_NAME ": "
47 MODULE_AUTHOR("Jeff Garzik");
48 MODULE_LICENSE("GPL");
49 MODULE_DESCRIPTION("Promise SATA SX8 block driver");
50 MODULE_VERSION(DRV_VERSION);
53 * SX8 hardware has a single message queue for all ATA ports.
54 * When this driver was written, the hardware (firmware?) would
55 * corrupt data eventually, if more than one request was outstanding.
56 * As one can imagine, having 8 ports bottlenecking on a single
57 * command hurts performance.
59 * Based on user reports, later versions of the hardware (firmware?)
60 * seem to be able to survive with more than one command queued.
62 * Therefore, we default to the safe option -- 1 command -- but
63 * allow the user to increase this.
65 * SX8 should be able to support up to ~60 queued commands (CARM_MAX_REQ),
66 * but problems seem to occur when you exceed ~30, even on newer hardware.
68 static int max_queue = 1;
69 module_param(max_queue, int, 0444);
70 MODULE_PARM_DESC(max_queue, "Maximum number of queued commands. (min==1, max==30, safe==1)");
73 #define NEXT_RESP(idx) ((idx + 1) % RMSG_Q_LEN)
75 /* 0xf is just arbitrary, non-zero noise; this is sorta like poisoning */
76 #define TAG_ENCODE(tag) (((tag) << 16) | 0xf)
77 #define TAG_DECODE(tag) (((tag) >> 16) & 0x1f)
78 #define TAG_VALID(tag) ((((tag) & 0xf) == 0xf) && (TAG_DECODE(tag) < 32))
80 /* note: prints function name for you */
82 #define DPRINTK(fmt, args...) printk(KERN_ERR "%s: " fmt, __FUNCTION__, ## args)
83 #ifdef CARM_VERBOSE_DEBUG
84 #define VPRINTK(fmt, args...) printk(KERN_ERR "%s: " fmt, __FUNCTION__, ## args)
86 #define VPRINTK(fmt, args...)
87 #endif /* CARM_VERBOSE_DEBUG */
89 #define DPRINTK(fmt, args...)
90 #define VPRINTK(fmt, args...)
91 #endif /* CARM_DEBUG */
96 #define assert(expr) \
97 if(unlikely(!(expr))) { \
98 printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \
99 #expr,__FILE__,__FUNCTION__,__LINE__); \
103 /* defines only for the constants which don't work well as enums */
107 /* adapter-wide limits */
109 CARM_SHM_SIZE = (4096 << 7),
110 CARM_MINORS_PER_MAJOR = 256 / CARM_MAX_PORTS,
111 CARM_MAX_WAIT_Q = CARM_MAX_PORTS + 1,
113 /* command message queue limits */
114 CARM_MAX_REQ = 64, /* max command msgs per host */
115 CARM_MSG_LOW_WATER = (CARM_MAX_REQ / 4), /* refill mark */
117 /* S/G limits, host-wide and per-request */
118 CARM_MAX_REQ_SG = 32, /* max s/g entries per request */
119 CARM_MAX_HOST_SG = 600, /* max s/g entries per host */
120 CARM_SG_LOW_WATER = (CARM_MAX_HOST_SG / 4), /* re-fill mark */
122 /* hardware registers */
124 CARM_INT_STAT = 0x10, /* interrupt status */
125 CARM_INT_MASK = 0x14, /* interrupt mask */
126 CARM_HMUC = 0x18, /* host message unit control */
127 RBUF_ADDR_LO = 0x20, /* response msg DMA buf low 32 bits */
128 RBUF_ADDR_HI = 0x24, /* response msg DMA buf high 32 bits */
130 CARM_RESP_IDX = 0x2c,
131 CARM_CMS0 = 0x30, /* command message size reg 0 */
136 /* bits in CARM_INT_{STAT,MASK} */
137 INT_RESERVED = 0xfffffff0,
138 INT_WATCHDOG = (1 << 3), /* watchdog timer */
139 INT_Q_OVERFLOW = (1 << 2), /* cmd msg q overflow */
140 INT_Q_AVAILABLE = (1 << 1), /* cmd msg q has free space */
141 INT_RESPONSE = (1 << 0), /* response msg available */
142 INT_ACK_MASK = INT_WATCHDOG | INT_Q_OVERFLOW,
143 INT_DEF_MASK = INT_RESERVED | INT_Q_OVERFLOW |
146 /* command messages, and related register bits */
147 CARM_HAVE_RESP = 0x01,
151 CARM_MSG_GET_CAPACITY = 4,
158 CARM_WZBC = (1 << 0),
160 CARM_Q_FULL = (1 << 3),
164 /* CARM_MSG_IOCTL messages */
165 CARM_IOC_SCAN_CHAN = 5, /* scan channels for devices */
166 CARM_IOC_GET_TCQ = 13, /* get tcq/ncq depth */
167 CARM_IOC_SET_TCQ = 14, /* set tcq/ncq depth */
169 IOC_SCAN_CHAN_NODEV = 0x1f,
170 IOC_SCAN_CHAN_OFFSET = 0x40,
172 /* CARM_MSG_ARRAY messages */
175 ARRAY_NO_EXIST = (1 << 31),
177 /* response messages */
178 RMSG_SZ = 8, /* sizeof(struct carm_response) */
179 RMSG_Q_LEN = 48, /* resp. msg list length */
180 RMSG_OK = 1, /* bit indicating msg was successful */
181 /* length of entire resp. msg buffer */
182 RBUF_LEN = RMSG_SZ * RMSG_Q_LEN,
184 PDC_SHM_SIZE = (4096 << 7), /* length of entire h/w buffer */
186 /* CARM_MSG_MISC messages */
191 /* MISC_GET_FW_VER feature bits */
192 FW_VER_4PORT = (1 << 2), /* 1=4 ports, 0=8 ports */
193 FW_VER_NON_RAID = (1 << 1), /* 1=non-RAID firmware, 0=RAID */
194 FW_VER_ZCR = (1 << 0), /* zero channel RAID (whatever that is) */
196 /* carm_host flags */
197 FL_NON_RAID = FW_VER_NON_RAID,
198 FL_4PORT = FW_VER_4PORT,
199 FL_FW_VER_MASK = (FW_VER_NON_RAID | FW_VER_4PORT),
201 FL_DYN_MAJOR = (1 << 17),
205 CARM_SG_BOUNDARY = 0xffffUL, /* s/g segment boundary */
208 enum scatter_gather_types {
214 HST_INVALID, /* invalid state; never used */
215 HST_ALLOC_BUF, /* setting up master SHM area */
216 HST_ERROR, /* we never leave here */
217 HST_PORT_SCAN, /* start dev scan */
218 HST_DEV_SCAN_START, /* start per-device probe */
219 HST_DEV_SCAN, /* continue per-device probe */
220 HST_DEV_ACTIVATE, /* activate devices we found */
221 HST_PROBE_FINISHED, /* probe is complete */
222 HST_PROBE_START, /* initiate probe */
223 HST_SYNC_TIME, /* tell firmware what time it is */
224 HST_GET_FW_VER, /* get firmware version, adapter port cnt */
228 static const char *state_name[] = {
233 "HST_DEV_SCAN_START",
236 "HST_PROBE_FINISHED",
244 unsigned int port_no;
245 unsigned int n_queued;
246 struct gendisk *disk;
247 struct carm_host *host;
249 /* attached device characteristics */
257 struct carm_request {
260 unsigned int msg_type;
261 unsigned int msg_subtype;
262 unsigned int msg_bucket;
264 struct carm_port *port;
265 struct scatterlist sg[CARM_MAX_REQ_SG];
279 struct pci_dev *pdev;
283 request_queue_t *oob_q;
286 unsigned int hw_sg_used;
288 unsigned int resp_idx;
290 unsigned int wait_q_prod;
291 unsigned int wait_q_cons;
292 request_queue_t *wait_q[CARM_MAX_WAIT_Q];
296 struct carm_request req[CARM_MAX_REQ];
301 unsigned long dev_active;
302 unsigned long dev_present;
303 struct carm_port port[CARM_MAX_PORTS];
305 struct work_struct fsm_task;
307 struct semaphore probe_sem;
310 struct carm_response {
313 } __attribute__((packed));
318 } __attribute__((packed));
329 struct carm_msg_sg sg[32];
330 } __attribute__((packed));
332 struct carm_msg_allocbuf {
346 struct carm_msg_sg sg[8];
347 } __attribute__((packed));
349 struct carm_msg_ioctl {
357 } __attribute__((packed));
359 struct carm_msg_sync_time {
366 } __attribute__((packed));
368 struct carm_msg_get_fw_ver {
375 } __attribute__((packed));
382 } __attribute__((packed));
384 struct carm_array_info {
392 __le16 stripe_blk_sz;
406 /* device list continues beyond this point? */
407 } __attribute__((packed));
409 static int carm_init_one (struct pci_dev *pdev, const struct pci_device_id *ent);
410 static void carm_remove_one (struct pci_dev *pdev);
411 static int carm_bdev_ioctl(struct inode *ino, struct file *fil,
412 unsigned int cmd, unsigned long arg);
414 static struct pci_device_id carm_pci_tbl[] = {
415 { PCI_VENDOR_ID_PROMISE, 0x8000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
416 { PCI_VENDOR_ID_PROMISE, 0x8002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
417 { } /* terminate list */
419 MODULE_DEVICE_TABLE(pci, carm_pci_tbl);
421 static struct pci_driver carm_driver = {
423 .id_table = carm_pci_tbl,
424 .probe = carm_init_one,
425 .remove = carm_remove_one,
428 static struct block_device_operations carm_bd_ops = {
429 .owner = THIS_MODULE,
430 .ioctl = carm_bdev_ioctl,
433 static unsigned int carm_host_id;
434 static unsigned long carm_major_alloc;
438 static int carm_bdev_ioctl(struct inode *ino, struct file *fil,
439 unsigned int cmd, unsigned long arg)
441 void __user *usermem = (void __user *) arg;
442 struct carm_port *port = ino->i_bdev->bd_disk->private_data;
443 struct hd_geometry geom;
450 geom.heads = (u8) port->dev_geom_head;
451 geom.sectors = (u8) port->dev_geom_sect;
452 geom.cylinders = port->dev_geom_cyl;
453 geom.start = get_start_sect(ino->i_bdev);
455 if (copy_to_user(usermem, &geom, sizeof(geom)))
466 static const u32 msg_sizes[] = { 32, 64, 128, CARM_MSG_SIZE };
468 static inline int carm_lookup_bucket(u32 msg_size)
472 for (i = 0; i < ARRAY_SIZE(msg_sizes); i++)
473 if (msg_size <= msg_sizes[i])
479 static void carm_init_buckets(void __iomem *mmio)
483 for (i = 0; i < ARRAY_SIZE(msg_sizes); i++)
484 writel(msg_sizes[i], mmio + CARM_CMS0 + (4 * i));
487 static inline void *carm_ref_msg(struct carm_host *host,
488 unsigned int msg_idx)
490 return host->msg_base + (msg_idx * CARM_MSG_SIZE);
493 static inline dma_addr_t carm_ref_msg_dma(struct carm_host *host,
494 unsigned int msg_idx)
496 return host->msg_dma + (msg_idx * CARM_MSG_SIZE);
499 static int carm_send_msg(struct carm_host *host,
500 struct carm_request *crq)
502 void __iomem *mmio = host->mmio;
503 u32 msg = (u32) carm_ref_msg_dma(host, crq->tag);
504 u32 cm_bucket = crq->msg_bucket;
510 tmp = readl(mmio + CARM_HMUC);
511 if (tmp & CARM_Q_FULL) {
513 tmp = readl(mmio + CARM_INT_MASK);
514 tmp |= INT_Q_AVAILABLE;
515 writel(tmp, mmio + CARM_INT_MASK);
516 readl(mmio + CARM_INT_MASK); /* flush */
518 DPRINTK("host msg queue full\n");
521 writel(msg | (cm_bucket << 1), mmio + CARM_IHQP);
522 readl(mmio + CARM_IHQP); /* flush */
528 static struct carm_request *carm_get_request(struct carm_host *host)
532 /* obey global hardware limit on S/G entries */
533 if (host->hw_sg_used >= (CARM_MAX_HOST_SG - CARM_MAX_REQ_SG))
536 for (i = 0; i < max_queue; i++)
537 if ((host->msg_alloc & (1ULL << i)) == 0) {
538 struct carm_request *crq = &host->req[i];
542 host->msg_alloc |= (1ULL << i);
545 assert(host->n_msgs <= CARM_MAX_REQ);
549 DPRINTK("no request available, returning NULL\n");
553 static int carm_put_request(struct carm_host *host, struct carm_request *crq)
555 assert(crq->tag < max_queue);
557 if (unlikely((host->msg_alloc & (1ULL << crq->tag)) == 0))
558 return -EINVAL; /* tried to clear a tag that was not active */
560 assert(host->hw_sg_used >= crq->n_elem);
562 host->msg_alloc &= ~(1ULL << crq->tag);
563 host->hw_sg_used -= crq->n_elem;
569 static struct carm_request *carm_get_special(struct carm_host *host)
572 struct carm_request *crq = NULL;
576 while (tries-- > 0) {
577 spin_lock_irqsave(&host->lock, flags);
578 crq = carm_get_request(host);
579 spin_unlock_irqrestore(&host->lock, flags);
589 rq = blk_get_request(host->oob_q, WRITE /* bogus */, GFP_KERNEL);
591 spin_lock_irqsave(&host->lock, flags);
592 carm_put_request(host, crq);
593 spin_unlock_irqrestore(&host->lock, flags);
601 static int carm_array_info (struct carm_host *host, unsigned int array_idx)
603 struct carm_msg_ioctl *ioc;
607 struct carm_request *crq;
610 crq = carm_get_special(host);
618 ioc = carm_ref_msg(host, idx);
619 msg_dma = carm_ref_msg_dma(host, idx);
620 msg_data = (u32) (msg_dma + sizeof(struct carm_array_info));
622 crq->msg_type = CARM_MSG_ARRAY;
623 crq->msg_subtype = CARM_ARRAY_INFO;
624 rc = carm_lookup_bucket(sizeof(struct carm_msg_ioctl) +
625 sizeof(struct carm_array_info));
627 crq->msg_bucket = (u32) rc;
629 memset(ioc, 0, sizeof(*ioc));
630 ioc->type = CARM_MSG_ARRAY;
631 ioc->subtype = CARM_ARRAY_INFO;
632 ioc->array_id = (u8) array_idx;
633 ioc->handle = cpu_to_le32(TAG_ENCODE(idx));
634 ioc->data_addr = cpu_to_le32(msg_data);
636 spin_lock_irq(&host->lock);
637 assert(host->state == HST_DEV_SCAN_START ||
638 host->state == HST_DEV_SCAN);
639 spin_unlock_irq(&host->lock);
641 DPRINTK("blk_insert_request, tag == %u\n", idx);
642 blk_insert_request(host->oob_q, crq->rq, 1, crq);
647 spin_lock_irq(&host->lock);
648 host->state = HST_ERROR;
649 spin_unlock_irq(&host->lock);
653 typedef unsigned int (*carm_sspc_t)(struct carm_host *, unsigned int, void *);
655 static int carm_send_special (struct carm_host *host, carm_sspc_t func)
657 struct carm_request *crq;
658 struct carm_msg_ioctl *ioc;
660 unsigned int idx, msg_size;
663 crq = carm_get_special(host);
669 mem = carm_ref_msg(host, idx);
671 msg_size = func(host, idx, mem);
674 crq->msg_type = ioc->type;
675 crq->msg_subtype = ioc->subtype;
676 rc = carm_lookup_bucket(msg_size);
678 crq->msg_bucket = (u32) rc;
680 DPRINTK("blk_insert_request, tag == %u\n", idx);
681 blk_insert_request(host->oob_q, crq->rq, 1, crq);
686 static unsigned int carm_fill_sync_time(struct carm_host *host,
687 unsigned int idx, void *mem)
690 struct carm_msg_sync_time *st = mem;
692 do_gettimeofday(&tv);
694 memset(st, 0, sizeof(*st));
695 st->type = CARM_MSG_MISC;
696 st->subtype = MISC_SET_TIME;
697 st->handle = cpu_to_le32(TAG_ENCODE(idx));
698 st->timestamp = cpu_to_le32(tv.tv_sec);
700 return sizeof(struct carm_msg_sync_time);
703 static unsigned int carm_fill_alloc_buf(struct carm_host *host,
704 unsigned int idx, void *mem)
706 struct carm_msg_allocbuf *ab = mem;
708 memset(ab, 0, sizeof(*ab));
709 ab->type = CARM_MSG_MISC;
710 ab->subtype = MISC_ALLOC_MEM;
711 ab->handle = cpu_to_le32(TAG_ENCODE(idx));
713 ab->sg_type = SGT_32BIT;
714 ab->addr = cpu_to_le32(host->shm_dma + (PDC_SHM_SIZE >> 1));
715 ab->len = cpu_to_le32(PDC_SHM_SIZE >> 1);
716 ab->evt_pool = cpu_to_le32(host->shm_dma + (16 * 1024));
717 ab->n_evt = cpu_to_le32(1024);
718 ab->rbuf_pool = cpu_to_le32(host->shm_dma);
719 ab->n_rbuf = cpu_to_le32(RMSG_Q_LEN);
720 ab->msg_pool = cpu_to_le32(host->shm_dma + RBUF_LEN);
721 ab->n_msg = cpu_to_le32(CARM_Q_LEN);
722 ab->sg[0].start = cpu_to_le32(host->shm_dma + (PDC_SHM_SIZE >> 1));
723 ab->sg[0].len = cpu_to_le32(65536);
725 return sizeof(struct carm_msg_allocbuf);
728 static unsigned int carm_fill_scan_channels(struct carm_host *host,
729 unsigned int idx, void *mem)
731 struct carm_msg_ioctl *ioc = mem;
732 u32 msg_data = (u32) (carm_ref_msg_dma(host, idx) +
733 IOC_SCAN_CHAN_OFFSET);
735 memset(ioc, 0, sizeof(*ioc));
736 ioc->type = CARM_MSG_IOCTL;
737 ioc->subtype = CARM_IOC_SCAN_CHAN;
738 ioc->handle = cpu_to_le32(TAG_ENCODE(idx));
739 ioc->data_addr = cpu_to_le32(msg_data);
741 /* fill output data area with "no device" default values */
742 mem += IOC_SCAN_CHAN_OFFSET;
743 memset(mem, IOC_SCAN_CHAN_NODEV, CARM_MAX_PORTS);
745 return IOC_SCAN_CHAN_OFFSET + CARM_MAX_PORTS;
748 static unsigned int carm_fill_get_fw_ver(struct carm_host *host,
749 unsigned int idx, void *mem)
751 struct carm_msg_get_fw_ver *ioc = mem;
752 u32 msg_data = (u32) (carm_ref_msg_dma(host, idx) + sizeof(*ioc));
754 memset(ioc, 0, sizeof(*ioc));
755 ioc->type = CARM_MSG_MISC;
756 ioc->subtype = MISC_GET_FW_VER;
757 ioc->handle = cpu_to_le32(TAG_ENCODE(idx));
758 ioc->data_addr = cpu_to_le32(msg_data);
760 return sizeof(struct carm_msg_get_fw_ver) +
761 sizeof(struct carm_fw_ver);
764 static inline void carm_end_request_queued(struct carm_host *host,
765 struct carm_request *crq,
768 struct request *req = crq->rq;
771 rc = end_that_request_first(req, uptodate, req->hard_nr_sectors);
774 end_that_request_last(req);
776 rc = carm_put_request(host, crq);
780 static inline void carm_push_q (struct carm_host *host, request_queue_t *q)
782 unsigned int idx = host->wait_q_prod % CARM_MAX_WAIT_Q;
785 VPRINTK("STOPPED QUEUE %p\n", q);
787 host->wait_q[idx] = q;
789 BUG_ON(host->wait_q_prod == host->wait_q_cons); /* overrun */
792 static inline request_queue_t *carm_pop_q(struct carm_host *host)
796 if (host->wait_q_prod == host->wait_q_cons)
799 idx = host->wait_q_cons % CARM_MAX_WAIT_Q;
802 return host->wait_q[idx];
805 static inline void carm_round_robin(struct carm_host *host)
807 request_queue_t *q = carm_pop_q(host);
810 VPRINTK("STARTED QUEUE %p\n", q);
814 static inline void carm_end_rq(struct carm_host *host, struct carm_request *crq,
817 carm_end_request_queued(host, crq, is_ok);
819 carm_round_robin(host);
820 else if ((host->n_msgs <= CARM_MSG_LOW_WATER) &&
821 (host->hw_sg_used <= CARM_SG_LOW_WATER)) {
822 carm_round_robin(host);
826 static void carm_oob_rq_fn(request_queue_t *q)
828 struct carm_host *host = q->queuedata;
829 struct carm_request *crq;
834 DPRINTK("get req\n");
835 rq = elv_next_request(q);
839 blkdev_dequeue_request(rq);
843 assert(crq->rq == rq);
847 DPRINTK("send req\n");
848 rc = carm_send_msg(host, crq);
850 blk_requeue_request(q, rq);
851 carm_push_q(host, q);
852 return; /* call us again later, eventually */
857 static void carm_rq_fn(request_queue_t *q)
859 struct carm_port *port = q->queuedata;
860 struct carm_host *host = port->host;
861 struct carm_msg_rw *msg;
862 struct carm_request *crq;
864 struct scatterlist *sg;
865 int writing = 0, pci_dir, i, n_elem, rc;
867 unsigned int msg_size;
870 VPRINTK("get req\n");
871 rq = elv_next_request(q);
875 crq = carm_get_request(host);
877 carm_push_q(host, q);
878 return; /* call us again later, eventually */
882 blkdev_dequeue_request(rq);
884 if (rq_data_dir(rq) == WRITE) {
886 pci_dir = PCI_DMA_TODEVICE;
888 pci_dir = PCI_DMA_FROMDEVICE;
891 /* get scatterlist from block layer */
893 n_elem = blk_rq_map_sg(q, rq, sg);
895 carm_end_rq(host, crq, 0);
896 return; /* request with no s/g entries? */
899 /* map scatterlist to PCI bus addresses */
900 n_elem = pci_map_sg(host->pdev, sg, n_elem, pci_dir);
902 carm_end_rq(host, crq, 0);
903 return; /* request with no s/g entries? */
905 crq->n_elem = n_elem;
907 host->hw_sg_used += n_elem;
910 * build read/write message
913 VPRINTK("build msg\n");
914 msg = (struct carm_msg_rw *) carm_ref_msg(host, crq->tag);
917 msg->type = CARM_MSG_WRITE;
918 crq->msg_type = CARM_MSG_WRITE;
920 msg->type = CARM_MSG_READ;
921 crq->msg_type = CARM_MSG_READ;
924 msg->id = port->port_no;
925 msg->sg_count = n_elem;
926 msg->sg_type = SGT_32BIT;
927 msg->handle = cpu_to_le32(TAG_ENCODE(crq->tag));
928 msg->lba = cpu_to_le32(rq->sector & 0xffffffff);
929 tmp = (rq->sector >> 16) >> 16;
930 msg->lba_high = cpu_to_le16( (u16) tmp );
931 msg->lba_count = cpu_to_le16(rq->nr_sectors);
933 msg_size = sizeof(struct carm_msg_rw) - sizeof(msg->sg);
934 for (i = 0; i < n_elem; i++) {
935 struct carm_msg_sg *carm_sg = &msg->sg[i];
936 carm_sg->start = cpu_to_le32(sg_dma_address(&crq->sg[i]));
937 carm_sg->len = cpu_to_le32(sg_dma_len(&crq->sg[i]));
938 msg_size += sizeof(struct carm_msg_sg);
941 rc = carm_lookup_bucket(msg_size);
943 crq->msg_bucket = (u32) rc;
946 * queue read/write message to hardware
949 VPRINTK("send msg, tag == %u\n", crq->tag);
950 rc = carm_send_msg(host, crq);
952 carm_put_request(host, crq);
953 blk_requeue_request(q, rq);
954 carm_push_q(host, q);
955 return; /* call us again later, eventually */
958 goto queue_one_request;
961 static void carm_handle_array_info(struct carm_host *host,
962 struct carm_request *crq, u8 *mem,
965 struct carm_port *port;
966 u8 *msg_data = mem + sizeof(struct carm_array_info);
967 struct carm_array_info *desc = (struct carm_array_info *) msg_data;
974 carm_end_rq(host, crq, is_ok);
978 if (le32_to_cpu(desc->array_status) & ARRAY_NO_EXIST)
981 cur_port = host->cur_scan_dev;
983 /* should never occur */
984 if ((cur_port < 0) || (cur_port >= CARM_MAX_PORTS)) {
985 printk(KERN_ERR PFX "BUG: cur_scan_dev==%d, array_id==%d\n",
986 cur_port, (int) desc->array_id);
990 port = &host->port[cur_port];
992 lo = (u64) le32_to_cpu(desc->size);
993 hi = (u64) le16_to_cpu(desc->size_hi);
995 port->capacity = lo | (hi << 32);
996 port->dev_geom_head = le16_to_cpu(desc->head);
997 port->dev_geom_sect = le16_to_cpu(desc->sect);
998 port->dev_geom_cyl = le16_to_cpu(desc->cyl);
1000 host->dev_active |= (1 << cur_port);
1002 strncpy(port->name, desc->name, sizeof(port->name));
1003 port->name[sizeof(port->name) - 1] = 0;
1004 slen = strlen(port->name);
1005 while (slen && (port->name[slen - 1] == ' ')) {
1006 port->name[slen - 1] = 0;
1010 printk(KERN_INFO DRV_NAME "(%s): port %u device %Lu sectors\n",
1011 pci_name(host->pdev), port->port_no,
1012 (unsigned long long) port->capacity);
1013 printk(KERN_INFO DRV_NAME "(%s): port %u device \"%s\"\n",
1014 pci_name(host->pdev), port->port_no, port->name);
1017 assert(host->state == HST_DEV_SCAN);
1018 schedule_work(&host->fsm_task);
1021 static void carm_handle_scan_chan(struct carm_host *host,
1022 struct carm_request *crq, u8 *mem,
1025 u8 *msg_data = mem + IOC_SCAN_CHAN_OFFSET;
1026 unsigned int i, dev_count = 0;
1027 int new_state = HST_DEV_SCAN_START;
1031 carm_end_rq(host, crq, is_ok);
1034 new_state = HST_ERROR;
1038 /* TODO: scan and support non-disk devices */
1039 for (i = 0; i < 8; i++)
1040 if (msg_data[i] == 0) { /* direct-access device (disk) */
1041 host->dev_present |= (1 << i);
1045 printk(KERN_INFO DRV_NAME "(%s): found %u interesting devices\n",
1046 pci_name(host->pdev), dev_count);
1049 assert(host->state == HST_PORT_SCAN);
1050 host->state = new_state;
1051 schedule_work(&host->fsm_task);
1054 static void carm_handle_generic(struct carm_host *host,
1055 struct carm_request *crq, int is_ok,
1056 int cur_state, int next_state)
1060 carm_end_rq(host, crq, is_ok);
1062 assert(host->state == cur_state);
1064 host->state = next_state;
1066 host->state = HST_ERROR;
1067 schedule_work(&host->fsm_task);
1070 static inline void carm_handle_rw(struct carm_host *host,
1071 struct carm_request *crq, int is_ok)
1077 if (rq_data_dir(crq->rq) == WRITE)
1078 pci_dir = PCI_DMA_TODEVICE;
1080 pci_dir = PCI_DMA_FROMDEVICE;
1082 pci_unmap_sg(host->pdev, &crq->sg[0], crq->n_elem, pci_dir);
1084 carm_end_rq(host, crq, is_ok);
1087 static inline void carm_handle_resp(struct carm_host *host,
1088 __le32 ret_handle_le, u32 status)
1090 u32 handle = le32_to_cpu(ret_handle_le);
1091 unsigned int msg_idx;
1092 struct carm_request *crq;
1093 int is_ok = (status == RMSG_OK);
1096 VPRINTK("ENTER, handle == 0x%x\n", handle);
1098 if (unlikely(!TAG_VALID(handle))) {
1099 printk(KERN_ERR DRV_NAME "(%s): BUG: invalid tag 0x%x\n",
1100 pci_name(host->pdev), handle);
1104 msg_idx = TAG_DECODE(handle);
1105 VPRINTK("tag == %u\n", msg_idx);
1107 crq = &host->req[msg_idx];
1110 if (likely(crq->msg_type == CARM_MSG_READ ||
1111 crq->msg_type == CARM_MSG_WRITE)) {
1112 carm_handle_rw(host, crq, is_ok);
1116 mem = carm_ref_msg(host, msg_idx);
1118 switch (crq->msg_type) {
1119 case CARM_MSG_IOCTL: {
1120 switch (crq->msg_subtype) {
1121 case CARM_IOC_SCAN_CHAN:
1122 carm_handle_scan_chan(host, crq, mem, is_ok);
1125 /* unknown / invalid response */
1131 case CARM_MSG_MISC: {
1132 switch (crq->msg_subtype) {
1133 case MISC_ALLOC_MEM:
1134 carm_handle_generic(host, crq, is_ok,
1135 HST_ALLOC_BUF, HST_SYNC_TIME);
1138 carm_handle_generic(host, crq, is_ok,
1139 HST_SYNC_TIME, HST_GET_FW_VER);
1141 case MISC_GET_FW_VER: {
1142 struct carm_fw_ver *ver = (struct carm_fw_ver *)
1143 mem + sizeof(struct carm_msg_get_fw_ver);
1145 host->fw_ver = le32_to_cpu(ver->version);
1146 host->flags |= (ver->features & FL_FW_VER_MASK);
1148 carm_handle_generic(host, crq, is_ok,
1149 HST_GET_FW_VER, HST_PORT_SCAN);
1153 /* unknown / invalid response */
1159 case CARM_MSG_ARRAY: {
1160 switch (crq->msg_subtype) {
1161 case CARM_ARRAY_INFO:
1162 carm_handle_array_info(host, crq, mem, is_ok);
1165 /* unknown / invalid response */
1172 /* unknown / invalid response */
1179 printk(KERN_WARNING DRV_NAME "(%s): BUG: unhandled message type %d/%d\n",
1180 pci_name(host->pdev), crq->msg_type, crq->msg_subtype);
1181 carm_end_rq(host, crq, 0);
1184 static inline void carm_handle_responses(struct carm_host *host)
1186 void __iomem *mmio = host->mmio;
1187 struct carm_response *resp = (struct carm_response *) host->shm;
1188 unsigned int work = 0;
1189 unsigned int idx = host->resp_idx % RMSG_Q_LEN;
1192 u32 status = le32_to_cpu(resp[idx].status);
1194 if (status == 0xffffffff) {
1195 VPRINTK("ending response on index %u\n", idx);
1196 writel(idx << 3, mmio + CARM_RESP_IDX);
1200 /* response to a message we sent */
1201 else if ((status & (1 << 31)) == 0) {
1202 VPRINTK("handling msg response on index %u\n", idx);
1203 carm_handle_resp(host, resp[idx].ret_handle, status);
1204 resp[idx].status = cpu_to_le32(0xffffffff);
1207 /* asynchronous events the hardware throws our way */
1208 else if ((status & 0xff000000) == (1 << 31)) {
1209 u8 *evt_type_ptr = (u8 *) &resp[idx];
1210 u8 evt_type = *evt_type_ptr;
1211 printk(KERN_WARNING DRV_NAME "(%s): unhandled event type %d\n",
1212 pci_name(host->pdev), (int) evt_type);
1213 resp[idx].status = cpu_to_le32(0xffffffff);
1216 idx = NEXT_RESP(idx);
1220 VPRINTK("EXIT, work==%u\n", work);
1221 host->resp_idx += work;
1224 static irqreturn_t carm_interrupt(int irq, void *__host, struct pt_regs *regs)
1226 struct carm_host *host = __host;
1230 unsigned long flags;
1233 VPRINTK("no host\n");
1237 spin_lock_irqsave(&host->lock, flags);
1241 /* reading should also clear interrupts */
1242 mask = readl(mmio + CARM_INT_STAT);
1244 if (mask == 0 || mask == 0xffffffff) {
1245 VPRINTK("no work, mask == 0x%x\n", mask);
1249 if (mask & INT_ACK_MASK)
1250 writel(mask, mmio + CARM_INT_STAT);
1252 if (unlikely(host->state == HST_INVALID)) {
1253 VPRINTK("not initialized yet, mask = 0x%x\n", mask);
1257 if (mask & CARM_HAVE_RESP) {
1259 carm_handle_responses(host);
1263 spin_unlock_irqrestore(&host->lock, flags);
1265 return IRQ_RETVAL(handled);
1268 static void carm_fsm_task (void *_data)
1270 struct carm_host *host = _data;
1271 unsigned long flags;
1273 int rc, i, next_dev;
1275 int new_state = HST_INVALID;
1277 spin_lock_irqsave(&host->lock, flags);
1278 state = host->state;
1279 spin_unlock_irqrestore(&host->lock, flags);
1281 DPRINTK("ENTER, state == %s\n", state_name[state]);
1284 case HST_PROBE_START:
1285 new_state = HST_ALLOC_BUF;
1290 rc = carm_send_special(host, carm_fill_alloc_buf);
1292 new_state = HST_ERROR;
1298 rc = carm_send_special(host, carm_fill_sync_time);
1300 new_state = HST_ERROR;
1305 case HST_GET_FW_VER:
1306 rc = carm_send_special(host, carm_fill_get_fw_ver);
1308 new_state = HST_ERROR;
1314 rc = carm_send_special(host, carm_fill_scan_channels);
1316 new_state = HST_ERROR;
1321 case HST_DEV_SCAN_START:
1322 host->cur_scan_dev = -1;
1323 new_state = HST_DEV_SCAN;
1329 for (i = host->cur_scan_dev + 1; i < CARM_MAX_PORTS; i++)
1330 if (host->dev_present & (1 << i)) {
1335 if (next_dev >= 0) {
1336 host->cur_scan_dev = next_dev;
1337 rc = carm_array_info(host, next_dev);
1339 new_state = HST_ERROR;
1343 new_state = HST_DEV_ACTIVATE;
1348 case HST_DEV_ACTIVATE: {
1350 for (i = 0; i < CARM_MAX_PORTS; i++)
1351 if (host->dev_active & (1 << i)) {
1352 struct carm_port *port = &host->port[i];
1353 struct gendisk *disk = port->disk;
1355 set_capacity(disk, port->capacity);
1360 printk(KERN_INFO DRV_NAME "(%s): %d ports activated\n",
1361 pci_name(host->pdev), activated);
1363 new_state = HST_PROBE_FINISHED;
1368 case HST_PROBE_FINISHED:
1369 up(&host->probe_sem);
1377 /* should never occur */
1378 printk(KERN_ERR PFX "BUG: unknown state %d\n", state);
1383 if (new_state != HST_INVALID) {
1384 spin_lock_irqsave(&host->lock, flags);
1385 host->state = new_state;
1386 spin_unlock_irqrestore(&host->lock, flags);
1389 schedule_work(&host->fsm_task);
1392 static int carm_init_wait(void __iomem *mmio, u32 bits, unsigned int test_bit)
1396 for (i = 0; i < 50000; i++) {
1397 u32 tmp = readl(mmio + CARM_LMUC);
1401 if ((tmp & bits) == bits)
1404 if ((tmp & bits) == 0)
1411 printk(KERN_ERR PFX "carm_init_wait timeout, bits == 0x%x, test_bit == %s\n",
1412 bits, test_bit ? "yes" : "no");
1416 static void carm_init_responses(struct carm_host *host)
1418 void __iomem *mmio = host->mmio;
1420 struct carm_response *resp = (struct carm_response *) host->shm;
1422 for (i = 0; i < RMSG_Q_LEN; i++)
1423 resp[i].status = cpu_to_le32(0xffffffff);
1425 writel(0, mmio + CARM_RESP_IDX);
1428 static int carm_init_host(struct carm_host *host)
1430 void __iomem *mmio = host->mmio;
1437 writel(0, mmio + CARM_INT_MASK);
1439 tmp8 = readb(mmio + CARM_INITC);
1442 writeb(tmp8, mmio + CARM_INITC);
1443 readb(mmio + CARM_INITC); /* flush */
1445 DPRINTK("snooze...\n");
1449 tmp = readl(mmio + CARM_HMUC);
1450 if (tmp & CARM_CME) {
1451 DPRINTK("CME bit present, waiting\n");
1452 rc = carm_init_wait(mmio, CARM_CME, 1);
1454 DPRINTK("EXIT, carm_init_wait 1 failed\n");
1458 if (tmp & CARM_RME) {
1459 DPRINTK("RME bit present, waiting\n");
1460 rc = carm_init_wait(mmio, CARM_RME, 1);
1462 DPRINTK("EXIT, carm_init_wait 2 failed\n");
1467 tmp &= ~(CARM_RME | CARM_CME);
1468 writel(tmp, mmio + CARM_HMUC);
1469 readl(mmio + CARM_HMUC); /* flush */
1471 rc = carm_init_wait(mmio, CARM_RME | CARM_CME, 0);
1473 DPRINTK("EXIT, carm_init_wait 3 failed\n");
1477 carm_init_buckets(mmio);
1479 writel(host->shm_dma & 0xffffffff, mmio + RBUF_ADDR_LO);
1480 writel((host->shm_dma >> 16) >> 16, mmio + RBUF_ADDR_HI);
1481 writel(RBUF_LEN, mmio + RBUF_BYTE_SZ);
1483 tmp = readl(mmio + CARM_HMUC);
1484 tmp |= (CARM_RME | CARM_CME | CARM_WZBC);
1485 writel(tmp, mmio + CARM_HMUC);
1486 readl(mmio + CARM_HMUC); /* flush */
1488 rc = carm_init_wait(mmio, CARM_RME | CARM_CME, 1);
1490 DPRINTK("EXIT, carm_init_wait 4 failed\n");
1494 writel(0, mmio + CARM_HMPHA);
1495 writel(INT_DEF_MASK, mmio + CARM_INT_MASK);
1497 carm_init_responses(host);
1499 /* start initialization, probing state machine */
1500 spin_lock_irq(&host->lock);
1501 assert(host->state == HST_INVALID);
1502 host->state = HST_PROBE_START;
1503 spin_unlock_irq(&host->lock);
1504 schedule_work(&host->fsm_task);
1510 static int carm_init_disks(struct carm_host *host)
1515 for (i = 0; i < CARM_MAX_PORTS; i++) {
1516 struct gendisk *disk;
1518 struct carm_port *port;
1520 port = &host->port[i];
1524 disk = alloc_disk(CARM_MINORS_PER_MAJOR);
1531 sprintf(disk->disk_name, DRV_NAME "/%u",
1532 (unsigned int) (host->id * CARM_MAX_PORTS) + i);
1533 sprintf(disk->devfs_name, DRV_NAME "/%u_%u", host->id, i);
1534 disk->major = host->major;
1535 disk->first_minor = i * CARM_MINORS_PER_MAJOR;
1536 disk->fops = &carm_bd_ops;
1537 disk->private_data = port;
1539 q = blk_init_queue(carm_rq_fn, &host->lock);
1545 blk_queue_max_hw_segments(q, CARM_MAX_REQ_SG);
1546 blk_queue_max_phys_segments(q, CARM_MAX_REQ_SG);
1547 blk_queue_segment_boundary(q, CARM_SG_BOUNDARY);
1549 q->queuedata = port;
1555 static void carm_free_disks(struct carm_host *host)
1559 for (i = 0; i < CARM_MAX_PORTS; i++) {
1560 struct gendisk *disk = host->port[i].disk;
1562 request_queue_t *q = disk->queue;
1564 if (disk->flags & GENHD_FL_UP)
1567 blk_cleanup_queue(q);
1573 static int carm_init_shm(struct carm_host *host)
1575 host->shm = pci_alloc_consistent(host->pdev, CARM_SHM_SIZE,
1580 host->msg_base = host->shm + RBUF_LEN;
1581 host->msg_dma = host->shm_dma + RBUF_LEN;
1583 memset(host->shm, 0xff, RBUF_LEN);
1584 memset(host->msg_base, 0, PDC_SHM_SIZE - RBUF_LEN);
1589 static int carm_init_one (struct pci_dev *pdev, const struct pci_device_id *ent)
1591 static unsigned int printed_version;
1592 struct carm_host *host;
1593 unsigned int pci_dac;
1598 if (!printed_version++)
1599 printk(KERN_DEBUG DRV_NAME " version " DRV_VERSION "\n");
1601 rc = pci_enable_device(pdev);
1605 rc = pci_request_regions(pdev, DRV_NAME);
1609 #ifdef IF_64BIT_DMA_IS_POSSIBLE /* grrrr... */
1610 rc = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
1612 rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
1614 printk(KERN_ERR DRV_NAME "(%s): consistent DMA mask failure\n",
1616 goto err_out_regions;
1621 rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
1623 printk(KERN_ERR DRV_NAME "(%s): DMA mask failure\n",
1625 goto err_out_regions;
1628 #ifdef IF_64BIT_DMA_IS_POSSIBLE /* grrrr... */
1632 host = kmalloc(sizeof(*host), GFP_KERNEL);
1634 printk(KERN_ERR DRV_NAME "(%s): memory alloc failure\n",
1637 goto err_out_regions;
1640 memset(host, 0, sizeof(*host));
1642 host->flags = pci_dac ? FL_DAC : 0;
1643 spin_lock_init(&host->lock);
1644 INIT_WORK(&host->fsm_task, carm_fsm_task, host);
1645 init_MUTEX_LOCKED(&host->probe_sem);
1647 for (i = 0; i < ARRAY_SIZE(host->req); i++)
1648 host->req[i].tag = i;
1650 host->mmio = ioremap(pci_resource_start(pdev, 0),
1651 pci_resource_len(pdev, 0));
1653 printk(KERN_ERR DRV_NAME "(%s): MMIO alloc failure\n",
1659 rc = carm_init_shm(host);
1661 printk(KERN_ERR DRV_NAME "(%s): DMA SHM alloc failure\n",
1663 goto err_out_iounmap;
1666 q = blk_init_queue(carm_oob_rq_fn, &host->lock);
1668 printk(KERN_ERR DRV_NAME "(%s): OOB queue alloc failure\n",
1671 goto err_out_pci_free;
1674 q->queuedata = host;
1677 * Figure out which major to use: 160, 161, or dynamic
1679 if (!test_and_set_bit(0, &carm_major_alloc))
1681 else if (!test_and_set_bit(1, &carm_major_alloc))
1684 host->flags |= FL_DYN_MAJOR;
1686 host->id = carm_host_id;
1687 sprintf(host->name, DRV_NAME "%d", carm_host_id);
1689 rc = register_blkdev(host->major, host->name);
1691 goto err_out_free_majors;
1692 if (host->flags & FL_DYN_MAJOR)
1695 devfs_mk_dir(DRV_NAME);
1697 rc = carm_init_disks(host);
1699 goto err_out_blkdev_disks;
1701 pci_set_master(pdev);
1703 rc = request_irq(pdev->irq, carm_interrupt, SA_SHIRQ, DRV_NAME, host);
1705 printk(KERN_ERR DRV_NAME "(%s): irq alloc failure\n",
1707 goto err_out_blkdev_disks;
1710 rc = carm_init_host(host);
1712 goto err_out_free_irq;
1714 DPRINTK("waiting for probe_sem\n");
1715 down(&host->probe_sem);
1717 printk(KERN_INFO "%s: pci %s, ports %d, io %lx, irq %u, major %d\n",
1718 host->name, pci_name(pdev), (int) CARM_MAX_PORTS,
1719 pci_resource_start(pdev, 0), pdev->irq, host->major);
1722 pci_set_drvdata(pdev, host);
1726 free_irq(pdev->irq, host);
1727 err_out_blkdev_disks:
1728 carm_free_disks(host);
1729 unregister_blkdev(host->major, host->name);
1730 err_out_free_majors:
1731 if (host->major == 160)
1732 clear_bit(0, &carm_major_alloc);
1733 else if (host->major == 161)
1734 clear_bit(1, &carm_major_alloc);
1735 blk_cleanup_queue(host->oob_q);
1737 pci_free_consistent(pdev, CARM_SHM_SIZE, host->shm, host->shm_dma);
1739 iounmap(host->mmio);
1743 pci_release_regions(pdev);
1745 pci_disable_device(pdev);
1749 static void carm_remove_one (struct pci_dev *pdev)
1751 struct carm_host *host = pci_get_drvdata(pdev);
1754 printk(KERN_ERR PFX "BUG: no host data for PCI(%s)\n",
1759 free_irq(pdev->irq, host);
1760 carm_free_disks(host);
1761 devfs_remove(DRV_NAME);
1762 unregister_blkdev(host->major, host->name);
1763 if (host->major == 160)
1764 clear_bit(0, &carm_major_alloc);
1765 else if (host->major == 161)
1766 clear_bit(1, &carm_major_alloc);
1767 blk_cleanup_queue(host->oob_q);
1768 pci_free_consistent(pdev, CARM_SHM_SIZE, host->shm, host->shm_dma);
1769 iounmap(host->mmio);
1771 pci_release_regions(pdev);
1772 pci_disable_device(pdev);
1773 pci_set_drvdata(pdev, NULL);
1776 static int __init carm_init(void)
1778 return pci_module_init(&carm_driver);
1781 static void __exit carm_exit(void)
1783 pci_unregister_driver(&carm_driver);
1786 module_init(carm_init);
1787 module_exit(carm_exit);