2 * libcxgbi.c: Chelsio common library for T3/T4 iSCSI driver.
4 * Copyright (c) 2010 Chelsio Communications, Inc.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
10 * Written by: Karen Xie (kxie@chelsio.com)
11 * Written by: Rakesh Ranjan (rranjan@chelsio.com)
14 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
16 #include <linux/skbuff.h>
17 #include <linux/crypto.h>
18 #include <linux/scatterlist.h>
19 #include <linux/pci.h>
20 #include <scsi/scsi.h>
21 #include <scsi/scsi_cmnd.h>
22 #include <scsi/scsi_host.h>
23 #include <linux/if_vlan.h>
24 #include <linux/inet.h>
26 #include <net/route.h>
28 #include <net/ip6_route.h>
29 #include <net/addrconf.h>
31 #include <linux/inetdevice.h> /* ip_dev_find */
32 #include <linux/module.h>
35 static unsigned int dbg_level;
39 #define DRV_MODULE_NAME "libcxgbi"
40 #define DRV_MODULE_DESC "Chelsio iSCSI driver library"
41 #define DRV_MODULE_VERSION "0.9.0"
42 #define DRV_MODULE_RELDATE "Jun. 2010"
44 MODULE_AUTHOR("Chelsio Communications, Inc.");
45 MODULE_DESCRIPTION(DRV_MODULE_DESC);
46 MODULE_VERSION(DRV_MODULE_VERSION);
47 MODULE_LICENSE("GPL");
49 module_param(dbg_level, uint, 0644);
50 MODULE_PARM_DESC(dbg_level, "libiscsi debug level (default=0)");
54 * cxgbi device management
55 * maintains a list of the cxgbi devices
57 static LIST_HEAD(cdev_list);
58 static DEFINE_MUTEX(cdev_mutex);
60 static LIST_HEAD(cdev_rcu_list);
61 static DEFINE_SPINLOCK(cdev_rcu_lock);
63 int cxgbi_device_portmap_create(struct cxgbi_device *cdev, unsigned int base,
64 unsigned int max_conn)
66 struct cxgbi_ports_map *pmap = &cdev->pmap;
68 pmap->port_csk = cxgbi_alloc_big_mem(max_conn *
69 sizeof(struct cxgbi_sock *),
71 if (!pmap->port_csk) {
72 pr_warn("cdev 0x%p, portmap OOM %u.\n", cdev, max_conn);
76 pmap->max_connect = max_conn;
77 pmap->sport_base = base;
78 spin_lock_init(&pmap->lock);
81 EXPORT_SYMBOL_GPL(cxgbi_device_portmap_create);
83 void cxgbi_device_portmap_cleanup(struct cxgbi_device *cdev)
85 struct cxgbi_ports_map *pmap = &cdev->pmap;
86 struct cxgbi_sock *csk;
89 for (i = 0; i < pmap->max_connect; i++) {
90 if (pmap->port_csk[i]) {
91 csk = pmap->port_csk[i];
92 pmap->port_csk[i] = NULL;
93 log_debug(1 << CXGBI_DBG_SOCK,
94 "csk 0x%p, cdev 0x%p, offload down.\n",
96 spin_lock_bh(&csk->lock);
97 cxgbi_sock_set_flag(csk, CTPF_OFFLOAD_DOWN);
98 cxgbi_sock_closed(csk);
99 spin_unlock_bh(&csk->lock);
104 EXPORT_SYMBOL_GPL(cxgbi_device_portmap_cleanup);
106 static inline void cxgbi_device_destroy(struct cxgbi_device *cdev)
108 log_debug(1 << CXGBI_DBG_DEV,
109 "cdev 0x%p, p# %u.\n", cdev, cdev->nports);
110 cxgbi_hbas_remove(cdev);
111 cxgbi_device_portmap_cleanup(cdev);
112 if (cdev->dev_ddp_cleanup)
113 cdev->dev_ddp_cleanup(cdev);
115 cxgbi_ddp_cleanup(cdev);
117 cxgbi_ddp_cleanup(cdev);
118 if (cdev->pmap.max_connect)
119 cxgbi_free_big_mem(cdev->pmap.port_csk);
123 struct cxgbi_device *cxgbi_device_register(unsigned int extra,
126 struct cxgbi_device *cdev;
128 cdev = kzalloc(sizeof(*cdev) + extra + nports *
129 (sizeof(struct cxgbi_hba *) +
130 sizeof(struct net_device *)),
133 pr_warn("nport %d, OOM.\n", nports);
136 cdev->ports = (struct net_device **)(cdev + 1);
137 cdev->hbas = (struct cxgbi_hba **)(((char*)cdev->ports) + nports *
138 sizeof(struct net_device *));
140 cdev->dd_data = ((char *)cdev->hbas) +
141 nports * sizeof(struct cxgbi_hba *);
142 spin_lock_init(&cdev->pmap.lock);
144 mutex_lock(&cdev_mutex);
145 list_add_tail(&cdev->list_head, &cdev_list);
146 mutex_unlock(&cdev_mutex);
148 spin_lock(&cdev_rcu_lock);
149 list_add_tail_rcu(&cdev->rcu_node, &cdev_rcu_list);
150 spin_unlock(&cdev_rcu_lock);
152 log_debug(1 << CXGBI_DBG_DEV,
153 "cdev 0x%p, p# %u.\n", cdev, nports);
156 EXPORT_SYMBOL_GPL(cxgbi_device_register);
158 void cxgbi_device_unregister(struct cxgbi_device *cdev)
160 log_debug(1 << CXGBI_DBG_DEV,
161 "cdev 0x%p, p# %u,%s.\n",
162 cdev, cdev->nports, cdev->nports ? cdev->ports[0]->name : "");
164 mutex_lock(&cdev_mutex);
165 list_del(&cdev->list_head);
166 mutex_unlock(&cdev_mutex);
168 spin_lock(&cdev_rcu_lock);
169 list_del_rcu(&cdev->rcu_node);
170 spin_unlock(&cdev_rcu_lock);
173 cxgbi_device_destroy(cdev);
175 EXPORT_SYMBOL_GPL(cxgbi_device_unregister);
177 void cxgbi_device_unregister_all(unsigned int flag)
179 struct cxgbi_device *cdev, *tmp;
181 mutex_lock(&cdev_mutex);
182 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
183 if ((cdev->flags & flag) == flag) {
184 mutex_unlock(&cdev_mutex);
185 cxgbi_device_unregister(cdev);
186 mutex_lock(&cdev_mutex);
189 mutex_unlock(&cdev_mutex);
191 EXPORT_SYMBOL_GPL(cxgbi_device_unregister_all);
193 struct cxgbi_device *cxgbi_device_find_by_lldev(void *lldev)
195 struct cxgbi_device *cdev, *tmp;
197 mutex_lock(&cdev_mutex);
198 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
199 if (cdev->lldev == lldev) {
200 mutex_unlock(&cdev_mutex);
204 mutex_unlock(&cdev_mutex);
206 log_debug(1 << CXGBI_DBG_DEV,
207 "lldev 0x%p, NO match found.\n", lldev);
210 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_lldev);
212 struct cxgbi_device *cxgbi_device_find_by_netdev(struct net_device *ndev,
215 struct net_device *vdev = NULL;
216 struct cxgbi_device *cdev, *tmp;
219 if (ndev->priv_flags & IFF_802_1Q_VLAN) {
221 ndev = vlan_dev_real_dev(ndev);
222 log_debug(1 << CXGBI_DBG_DEV,
223 "vlan dev %s -> %s.\n", vdev->name, ndev->name);
226 mutex_lock(&cdev_mutex);
227 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
228 for (i = 0; i < cdev->nports; i++) {
229 if (ndev == cdev->ports[i]) {
230 cdev->hbas[i]->vdev = vdev;
231 mutex_unlock(&cdev_mutex);
238 mutex_unlock(&cdev_mutex);
239 log_debug(1 << CXGBI_DBG_DEV,
240 "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name);
243 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev);
245 struct cxgbi_device *cxgbi_device_find_by_netdev_rcu(struct net_device *ndev,
248 struct net_device *vdev = NULL;
249 struct cxgbi_device *cdev;
252 if (ndev->priv_flags & IFF_802_1Q_VLAN) {
254 ndev = vlan_dev_real_dev(ndev);
255 pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name);
259 list_for_each_entry_rcu(cdev, &cdev_rcu_list, rcu_node) {
260 for (i = 0; i < cdev->nports; i++) {
261 if (ndev == cdev->ports[i]) {
262 cdev->hbas[i]->vdev = vdev;
272 log_debug(1 << CXGBI_DBG_DEV,
273 "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name);
276 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev_rcu);
278 static struct cxgbi_device *cxgbi_device_find_by_mac(struct net_device *ndev,
281 struct net_device *vdev = NULL;
282 struct cxgbi_device *cdev, *tmp;
285 if (ndev->priv_flags & IFF_802_1Q_VLAN) {
287 ndev = vlan_dev_real_dev(ndev);
288 pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name);
291 mutex_lock(&cdev_mutex);
292 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) {
293 for (i = 0; i < cdev->nports; i++) {
294 if (!memcmp(ndev->dev_addr, cdev->ports[i]->dev_addr,
296 cdev->hbas[i]->vdev = vdev;
297 mutex_unlock(&cdev_mutex);
304 mutex_unlock(&cdev_mutex);
305 log_debug(1 << CXGBI_DBG_DEV,
306 "ndev 0x%p, %s, NO match mac found.\n",
311 void cxgbi_hbas_remove(struct cxgbi_device *cdev)
314 struct cxgbi_hba *chba;
316 log_debug(1 << CXGBI_DBG_DEV,
317 "cdev 0x%p, p#%u.\n", cdev, cdev->nports);
319 for (i = 0; i < cdev->nports; i++) {
320 chba = cdev->hbas[i];
322 cdev->hbas[i] = NULL;
323 iscsi_host_remove(chba->shost);
324 pci_dev_put(cdev->pdev);
325 iscsi_host_free(chba->shost);
329 EXPORT_SYMBOL_GPL(cxgbi_hbas_remove);
331 int cxgbi_hbas_add(struct cxgbi_device *cdev, u64 max_lun,
332 unsigned int max_id, struct scsi_host_template *sht,
333 struct scsi_transport_template *stt)
335 struct cxgbi_hba *chba;
336 struct Scsi_Host *shost;
339 log_debug(1 << CXGBI_DBG_DEV, "cdev 0x%p, p#%u.\n", cdev, cdev->nports);
341 for (i = 0; i < cdev->nports; i++) {
342 shost = iscsi_host_alloc(sht, sizeof(*chba), 1);
344 pr_info("0x%p, p%d, %s, host alloc failed.\n",
345 cdev, i, cdev->ports[i]->name);
350 shost->transportt = stt;
351 shost->max_lun = max_lun;
352 shost->max_id = max_id;
353 shost->max_channel = 0;
354 shost->max_cmd_len = 16;
356 chba = iscsi_host_priv(shost);
358 chba->ndev = cdev->ports[i];
361 log_debug(1 << CXGBI_DBG_DEV,
362 "cdev 0x%p, p#%d %s: chba 0x%p.\n",
363 cdev, i, cdev->ports[i]->name, chba);
365 pci_dev_get(cdev->pdev);
366 err = iscsi_host_add(shost, &cdev->pdev->dev);
368 pr_info("cdev 0x%p, p#%d %s, host add failed.\n",
369 cdev, i, cdev->ports[i]->name);
370 pci_dev_put(cdev->pdev);
371 scsi_host_put(shost);
375 cdev->hbas[i] = chba;
381 cxgbi_hbas_remove(cdev);
384 EXPORT_SYMBOL_GPL(cxgbi_hbas_add);
389 * - source port management
390 * To find a free source port in the port allocation map we use a very simple
391 * rotor scheme to look for the next free port.
393 * If a source port has been specified make sure that it doesn't collide with
394 * our normal source port allocation map. If it's outside the range of our
395 * allocation/deallocation scheme just let them use it.
397 * If the source port is outside our allocation range, the caller is
398 * responsible for keeping track of their port usage.
400 static int sock_get_port(struct cxgbi_sock *csk)
402 struct cxgbi_device *cdev = csk->cdev;
403 struct cxgbi_ports_map *pmap = &cdev->pmap;
408 if (!pmap->max_connect) {
409 pr_err("cdev 0x%p, p#%u %s, NO port map.\n",
410 cdev, csk->port_id, cdev->ports[csk->port_id]->name);
411 return -EADDRNOTAVAIL;
414 if (csk->csk_family == AF_INET)
415 port = &csk->saddr.sin_port;
417 port = &csk->saddr6.sin6_port;
420 pr_err("source port NON-ZERO %u.\n",
425 spin_lock_bh(&pmap->lock);
426 if (pmap->used >= pmap->max_connect) {
427 spin_unlock_bh(&pmap->lock);
428 pr_info("cdev 0x%p, p#%u %s, ALL ports used.\n",
429 cdev, csk->port_id, cdev->ports[csk->port_id]->name);
430 return -EADDRNOTAVAIL;
433 start = idx = pmap->next;
435 if (++idx >= pmap->max_connect)
437 if (!pmap->port_csk[idx]) {
439 *port = htons(pmap->sport_base + idx);
441 pmap->port_csk[idx] = csk;
442 spin_unlock_bh(&pmap->lock);
444 log_debug(1 << CXGBI_DBG_SOCK,
445 "cdev 0x%p, p#%u %s, p %u, %u.\n",
447 cdev->ports[csk->port_id]->name,
448 pmap->sport_base + idx, pmap->next);
451 } while (idx != start);
452 spin_unlock_bh(&pmap->lock);
454 /* should not happen */
455 pr_warn("cdev 0x%p, p#%u %s, next %u?\n",
456 cdev, csk->port_id, cdev->ports[csk->port_id]->name,
458 return -EADDRNOTAVAIL;
461 static void sock_put_port(struct cxgbi_sock *csk)
463 struct cxgbi_device *cdev = csk->cdev;
464 struct cxgbi_ports_map *pmap = &cdev->pmap;
467 if (csk->csk_family == AF_INET)
468 port = &csk->saddr.sin_port;
470 port = &csk->saddr6.sin6_port;
473 int idx = ntohs(*port) - pmap->sport_base;
476 if (idx < 0 || idx >= pmap->max_connect) {
477 pr_err("cdev 0x%p, p#%u %s, port %u OOR.\n",
479 cdev->ports[csk->port_id]->name,
484 spin_lock_bh(&pmap->lock);
485 pmap->port_csk[idx] = NULL;
487 spin_unlock_bh(&pmap->lock);
489 log_debug(1 << CXGBI_DBG_SOCK,
490 "cdev 0x%p, p#%u %s, release %u.\n",
491 cdev, csk->port_id, cdev->ports[csk->port_id]->name,
492 pmap->sport_base + idx);
499 * iscsi tcp connection
501 void cxgbi_sock_free_cpl_skbs(struct cxgbi_sock *csk)
503 if (csk->cpl_close) {
504 kfree_skb(csk->cpl_close);
505 csk->cpl_close = NULL;
507 if (csk->cpl_abort_req) {
508 kfree_skb(csk->cpl_abort_req);
509 csk->cpl_abort_req = NULL;
511 if (csk->cpl_abort_rpl) {
512 kfree_skb(csk->cpl_abort_rpl);
513 csk->cpl_abort_rpl = NULL;
516 EXPORT_SYMBOL_GPL(cxgbi_sock_free_cpl_skbs);
518 static struct cxgbi_sock *cxgbi_sock_create(struct cxgbi_device *cdev)
520 struct cxgbi_sock *csk = kzalloc(sizeof(*csk), GFP_NOIO);
523 pr_info("alloc csk %zu failed.\n", sizeof(*csk));
527 if (cdev->csk_alloc_cpls(csk) < 0) {
528 pr_info("csk 0x%p, alloc cpls failed.\n", csk);
533 spin_lock_init(&csk->lock);
534 kref_init(&csk->refcnt);
535 skb_queue_head_init(&csk->receive_queue);
536 skb_queue_head_init(&csk->write_queue);
537 setup_timer(&csk->retry_timer, NULL, (unsigned long)csk);
538 rwlock_init(&csk->callback_lock);
541 cxgbi_sock_set_state(csk, CTP_CLOSED);
543 log_debug(1 << CXGBI_DBG_SOCK, "cdev 0x%p, new csk 0x%p.\n", cdev, csk);
548 static struct rtable *find_route_ipv4(struct flowi4 *fl4,
549 __be32 saddr, __be32 daddr,
550 __be16 sport, __be16 dport, u8 tos)
554 rt = ip_route_output_ports(&init_net, fl4, NULL, daddr, saddr,
555 dport, sport, IPPROTO_TCP, tos, 0);
562 static struct cxgbi_sock *cxgbi_check_route(struct sockaddr *dst_addr)
564 struct sockaddr_in *daddr = (struct sockaddr_in *)dst_addr;
565 struct dst_entry *dst;
566 struct net_device *ndev;
567 struct cxgbi_device *cdev;
568 struct rtable *rt = NULL;
571 struct cxgbi_sock *csk = NULL;
572 unsigned int mtu = 0;
576 rt = find_route_ipv4(&fl4, 0, daddr->sin_addr.s_addr, 0, daddr->sin_port, 0);
578 pr_info("no route to ipv4 0x%x, port %u.\n",
579 be32_to_cpu(daddr->sin_addr.s_addr),
580 be16_to_cpu(daddr->sin_port));
585 n = dst_neigh_lookup(dst, &daddr->sin_addr.s_addr);
592 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
593 pr_info("multi-cast route %pI4, port %u, dev %s.\n",
594 &daddr->sin_addr.s_addr, ntohs(daddr->sin_port),
600 if (ndev->flags & IFF_LOOPBACK) {
601 ndev = ip_dev_find(&init_net, daddr->sin_addr.s_addr);
603 pr_info("rt dev %s, loopback -> %s, mtu %u.\n",
604 n->dev->name, ndev->name, mtu);
607 cdev = cxgbi_device_find_by_netdev(ndev, &port);
609 pr_info("dst %pI4, %s, NOT cxgbi device.\n",
610 &daddr->sin_addr.s_addr, ndev->name);
614 log_debug(1 << CXGBI_DBG_SOCK,
615 "route to %pI4 :%u, ndev p#%d,%s, cdev 0x%p.\n",
616 &daddr->sin_addr.s_addr, ntohs(daddr->sin_port),
617 port, ndev->name, cdev);
619 csk = cxgbi_sock_create(cdev);
629 csk->csk_family = AF_INET;
630 csk->daddr.sin_addr.s_addr = daddr->sin_addr.s_addr;
631 csk->daddr.sin_port = daddr->sin_port;
632 csk->daddr.sin_family = daddr->sin_family;
633 csk->saddr.sin_family = daddr->sin_family;
634 csk->saddr.sin_addr.s_addr = fl4.saddr;
645 cxgbi_sock_closed(csk);
650 #if IS_ENABLED(CONFIG_IPV6)
651 static struct rt6_info *find_route_ipv6(const struct in6_addr *saddr,
652 const struct in6_addr *daddr)
657 memcpy(&fl.saddr, saddr, sizeof(struct in6_addr));
659 memcpy(&fl.daddr, daddr, sizeof(struct in6_addr));
660 return (struct rt6_info *)ip6_route_output(&init_net, NULL, &fl);
663 static struct cxgbi_sock *cxgbi_check_route6(struct sockaddr *dst_addr)
665 struct sockaddr_in6 *daddr6 = (struct sockaddr_in6 *)dst_addr;
666 struct dst_entry *dst;
667 struct net_device *ndev;
668 struct cxgbi_device *cdev;
669 struct rt6_info *rt = NULL;
671 struct in6_addr pref_saddr;
672 struct cxgbi_sock *csk = NULL;
673 unsigned int mtu = 0;
677 rt = find_route_ipv6(NULL, &daddr6->sin6_addr);
680 pr_info("no route to ipv6 %pI6 port %u\n",
681 daddr6->sin6_addr.s6_addr,
682 be16_to_cpu(daddr6->sin6_port));
689 n = dst_neigh_lookup(dst, &daddr6->sin6_addr);
692 pr_info("%pI6, port %u, dst no neighbour.\n",
693 daddr6->sin6_addr.s6_addr,
694 be16_to_cpu(daddr6->sin6_port));
700 if (ipv6_addr_is_multicast(&rt->rt6i_dst.addr)) {
701 pr_info("multi-cast route %pI6 port %u, dev %s.\n",
702 daddr6->sin6_addr.s6_addr,
703 ntohs(daddr6->sin6_port), ndev->name);
708 cdev = cxgbi_device_find_by_netdev(ndev, &port);
710 cdev = cxgbi_device_find_by_mac(ndev, &port);
712 pr_info("dst %pI6 %s, NOT cxgbi device.\n",
713 daddr6->sin6_addr.s6_addr, ndev->name);
717 log_debug(1 << CXGBI_DBG_SOCK,
718 "route to %pI6 :%u, ndev p#%d,%s, cdev 0x%p.\n",
719 daddr6->sin6_addr.s6_addr, ntohs(daddr6->sin6_port), port,
722 csk = cxgbi_sock_create(cdev);
732 if (ipv6_addr_any(&rt->rt6i_prefsrc.addr)) {
733 struct inet6_dev *idev = ip6_dst_idev((struct dst_entry *)rt);
735 err = ipv6_dev_get_saddr(&init_net, idev ? idev->dev : NULL,
736 &daddr6->sin6_addr, 0, &pref_saddr);
738 pr_info("failed to get source address to reach %pI6\n",
743 pref_saddr = rt->rt6i_prefsrc.addr;
746 csk->csk_family = AF_INET6;
747 csk->daddr6.sin6_addr = daddr6->sin6_addr;
748 csk->daddr6.sin6_port = daddr6->sin6_port;
749 csk->daddr6.sin6_family = daddr6->sin6_family;
750 csk->saddr6.sin6_addr = pref_saddr;
761 cxgbi_sock_closed(csk);
765 #endif /* IS_ENABLED(CONFIG_IPV6) */
767 void cxgbi_sock_established(struct cxgbi_sock *csk, unsigned int snd_isn,
770 csk->write_seq = csk->snd_nxt = csk->snd_una = snd_isn;
771 dst_confirm(csk->dst);
773 cxgbi_sock_set_state(csk, CTP_ESTABLISHED);
775 EXPORT_SYMBOL_GPL(cxgbi_sock_established);
777 static void cxgbi_inform_iscsi_conn_closing(struct cxgbi_sock *csk)
779 log_debug(1 << CXGBI_DBG_SOCK,
780 "csk 0x%p, state %u, flags 0x%lx, conn 0x%p.\n",
781 csk, csk->state, csk->flags, csk->user_data);
783 if (csk->state != CTP_ESTABLISHED) {
784 read_lock_bh(&csk->callback_lock);
786 iscsi_conn_failure(csk->user_data,
787 ISCSI_ERR_CONN_FAILED);
788 read_unlock_bh(&csk->callback_lock);
792 void cxgbi_sock_closed(struct cxgbi_sock *csk)
794 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
795 csk, (csk)->state, (csk)->flags, (csk)->tid);
796 cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED);
797 if (csk->state == CTP_ACTIVE_OPEN || csk->state == CTP_CLOSED)
799 if (csk->saddr.sin_port)
802 dst_release(csk->dst);
803 csk->cdev->csk_release_offload_resources(csk);
804 cxgbi_sock_set_state(csk, CTP_CLOSED);
805 cxgbi_inform_iscsi_conn_closing(csk);
808 EXPORT_SYMBOL_GPL(cxgbi_sock_closed);
810 static void need_active_close(struct cxgbi_sock *csk)
815 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
816 csk, (csk)->state, (csk)->flags, (csk)->tid);
817 spin_lock_bh(&csk->lock);
818 dst_confirm(csk->dst);
819 data_lost = skb_queue_len(&csk->receive_queue);
820 __skb_queue_purge(&csk->receive_queue);
822 if (csk->state == CTP_ACTIVE_OPEN)
823 cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED);
824 else if (csk->state == CTP_ESTABLISHED) {
826 cxgbi_sock_set_state(csk, CTP_ACTIVE_CLOSE);
827 } else if (csk->state == CTP_PASSIVE_CLOSE) {
829 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2);
834 csk->cdev->csk_send_abort_req(csk);
836 csk->cdev->csk_send_close_req(csk);
839 spin_unlock_bh(&csk->lock);
842 void cxgbi_sock_fail_act_open(struct cxgbi_sock *csk, int errno)
844 pr_info("csk 0x%p,%u,%lx, %pI4:%u-%pI4:%u, err %d.\n",
845 csk, csk->state, csk->flags,
846 &csk->saddr.sin_addr.s_addr, csk->saddr.sin_port,
847 &csk->daddr.sin_addr.s_addr, csk->daddr.sin_port,
850 cxgbi_sock_set_state(csk, CTP_CONNECTING);
852 cxgbi_sock_closed(csk);
854 EXPORT_SYMBOL_GPL(cxgbi_sock_fail_act_open);
856 void cxgbi_sock_act_open_req_arp_failure(void *handle, struct sk_buff *skb)
858 struct cxgbi_sock *csk = (struct cxgbi_sock *)skb->sk;
860 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
861 csk, (csk)->state, (csk)->flags, (csk)->tid);
863 spin_lock_bh(&csk->lock);
864 if (csk->state == CTP_ACTIVE_OPEN)
865 cxgbi_sock_fail_act_open(csk, -EHOSTUNREACH);
866 spin_unlock_bh(&csk->lock);
870 EXPORT_SYMBOL_GPL(cxgbi_sock_act_open_req_arp_failure);
872 void cxgbi_sock_rcv_abort_rpl(struct cxgbi_sock *csk)
875 spin_lock_bh(&csk->lock);
876 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
877 if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_RCVD))
878 cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_RCVD);
880 cxgbi_sock_clear_flag(csk, CTPF_ABORT_RPL_RCVD);
881 cxgbi_sock_clear_flag(csk, CTPF_ABORT_RPL_PENDING);
882 if (cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD))
883 pr_err("csk 0x%p,%u,0x%lx,%u,ABT_RPL_RSS.\n",
884 csk, csk->state, csk->flags, csk->tid);
885 cxgbi_sock_closed(csk);
888 spin_unlock_bh(&csk->lock);
891 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_abort_rpl);
893 void cxgbi_sock_rcv_peer_close(struct cxgbi_sock *csk)
895 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
896 csk, (csk)->state, (csk)->flags, (csk)->tid);
898 spin_lock_bh(&csk->lock);
900 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING))
903 switch (csk->state) {
904 case CTP_ESTABLISHED:
905 cxgbi_sock_set_state(csk, CTP_PASSIVE_CLOSE);
907 case CTP_ACTIVE_CLOSE:
908 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2);
910 case CTP_CLOSE_WAIT_1:
911 cxgbi_sock_closed(csk);
916 pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n",
917 csk, csk->state, csk->flags, csk->tid);
919 cxgbi_inform_iscsi_conn_closing(csk);
921 spin_unlock_bh(&csk->lock);
924 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_peer_close);
926 void cxgbi_sock_rcv_close_conn_rpl(struct cxgbi_sock *csk, u32 snd_nxt)
928 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n",
929 csk, (csk)->state, (csk)->flags, (csk)->tid);
931 spin_lock_bh(&csk->lock);
933 csk->snd_una = snd_nxt - 1;
934 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING))
937 switch (csk->state) {
938 case CTP_ACTIVE_CLOSE:
939 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_1);
941 case CTP_CLOSE_WAIT_1:
942 case CTP_CLOSE_WAIT_2:
943 cxgbi_sock_closed(csk);
948 pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n",
949 csk, csk->state, csk->flags, csk->tid);
952 spin_unlock_bh(&csk->lock);
955 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_close_conn_rpl);
957 void cxgbi_sock_rcv_wr_ack(struct cxgbi_sock *csk, unsigned int credits,
958 unsigned int snd_una, int seq_chk)
960 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
961 "csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, snd_una %u,%d.\n",
962 csk, csk->state, csk->flags, csk->tid, credits,
963 csk->wr_cred, csk->wr_una_cred, snd_una, seq_chk);
965 spin_lock_bh(&csk->lock);
967 csk->wr_cred += credits;
968 if (csk->wr_una_cred > csk->wr_max_cred - csk->wr_cred)
969 csk->wr_una_cred = csk->wr_max_cred - csk->wr_cred;
972 struct sk_buff *p = cxgbi_sock_peek_wr(csk);
975 pr_err("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, empty.\n",
976 csk, csk->state, csk->flags, csk->tid, credits,
977 csk->wr_cred, csk->wr_una_cred);
981 if (unlikely(credits < p->csum)) {
982 pr_warn("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, < %u.\n",
983 csk, csk->state, csk->flags, csk->tid,
984 credits, csk->wr_cred, csk->wr_una_cred,
989 cxgbi_sock_dequeue_wr(csk);
995 cxgbi_sock_check_wr_invariants(csk);
998 if (unlikely(before(snd_una, csk->snd_una))) {
999 pr_warn("csk 0x%p,%u,0x%lx,%u, snd_una %u/%u.",
1000 csk, csk->state, csk->flags, csk->tid, snd_una,
1005 if (csk->snd_una != snd_una) {
1006 csk->snd_una = snd_una;
1007 dst_confirm(csk->dst);
1011 if (skb_queue_len(&csk->write_queue)) {
1012 if (csk->cdev->csk_push_tx_frames(csk, 0))
1013 cxgbi_conn_tx_open(csk);
1015 cxgbi_conn_tx_open(csk);
1017 spin_unlock_bh(&csk->lock);
1019 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_wr_ack);
1021 static unsigned int cxgbi_sock_find_best_mtu(struct cxgbi_sock *csk,
1026 while (i < csk->cdev->nmtus - 1 && csk->cdev->mtus[i + 1] <= mtu)
1032 unsigned int cxgbi_sock_select_mss(struct cxgbi_sock *csk, unsigned int pmtu)
1035 struct dst_entry *dst = csk->dst;
1037 csk->advmss = dst_metric_advmss(dst);
1039 if (csk->advmss > pmtu - 40)
1040 csk->advmss = pmtu - 40;
1041 if (csk->advmss < csk->cdev->mtus[0] - 40)
1042 csk->advmss = csk->cdev->mtus[0] - 40;
1043 idx = cxgbi_sock_find_best_mtu(csk, csk->advmss + 40);
1047 EXPORT_SYMBOL_GPL(cxgbi_sock_select_mss);
1049 void cxgbi_sock_skb_entail(struct cxgbi_sock *csk, struct sk_buff *skb)
1051 cxgbi_skcb_tcp_seq(skb) = csk->write_seq;
1052 __skb_queue_tail(&csk->write_queue, skb);
1054 EXPORT_SYMBOL_GPL(cxgbi_sock_skb_entail);
1056 void cxgbi_sock_purge_wr_queue(struct cxgbi_sock *csk)
1058 struct sk_buff *skb;
1060 while ((skb = cxgbi_sock_dequeue_wr(csk)) != NULL)
1063 EXPORT_SYMBOL_GPL(cxgbi_sock_purge_wr_queue);
1065 void cxgbi_sock_check_wr_invariants(const struct cxgbi_sock *csk)
1067 int pending = cxgbi_sock_count_pending_wrs(csk);
1069 if (unlikely(csk->wr_cred + pending != csk->wr_max_cred))
1070 pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n",
1071 csk, csk->tid, csk->wr_cred, pending, csk->wr_max_cred);
1073 EXPORT_SYMBOL_GPL(cxgbi_sock_check_wr_invariants);
1075 static int cxgbi_sock_send_pdus(struct cxgbi_sock *csk, struct sk_buff *skb)
1077 struct cxgbi_device *cdev = csk->cdev;
1078 struct sk_buff *next;
1079 int err, copied = 0;
1081 spin_lock_bh(&csk->lock);
1083 if (csk->state != CTP_ESTABLISHED) {
1084 log_debug(1 << CXGBI_DBG_PDU_TX,
1085 "csk 0x%p,%u,0x%lx,%u, EAGAIN.\n",
1086 csk, csk->state, csk->flags, csk->tid);
1092 log_debug(1 << CXGBI_DBG_PDU_TX,
1093 "csk 0x%p,%u,0x%lx,%u, EPIPE %d.\n",
1094 csk, csk->state, csk->flags, csk->tid, csk->err);
1099 if (csk->write_seq - csk->snd_una >= cdev->snd_win) {
1100 log_debug(1 << CXGBI_DBG_PDU_TX,
1101 "csk 0x%p,%u,0x%lx,%u, FULL %u-%u >= %u.\n",
1102 csk, csk->state, csk->flags, csk->tid, csk->write_seq,
1103 csk->snd_una, cdev->snd_win);
1109 int frags = skb_shinfo(skb)->nr_frags +
1110 (skb->len != skb->data_len);
1112 if (unlikely(skb_headroom(skb) < cdev->skb_tx_rsvd)) {
1113 pr_err("csk 0x%p, skb head %u < %u.\n",
1114 csk, skb_headroom(skb), cdev->skb_tx_rsvd);
1119 if (frags >= SKB_WR_LIST_SIZE) {
1120 pr_err("csk 0x%p, frags %d, %u,%u >%u.\n",
1121 csk, skb_shinfo(skb)->nr_frags, skb->len,
1122 skb->data_len, (uint)(SKB_WR_LIST_SIZE));
1129 cxgbi_skcb_set_flag(skb, SKCBF_TX_NEED_HDR);
1130 cxgbi_sock_skb_entail(csk, skb);
1132 csk->write_seq += skb->len +
1133 cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));
1137 if (likely(skb_queue_len(&csk->write_queue)))
1138 cdev->csk_push_tx_frames(csk, 1);
1139 spin_unlock_bh(&csk->lock);
1143 if (copied == 0 && err == -EPIPE)
1144 copied = csk->err ? csk->err : -EPIPE;
1151 * Direct Data Placement -
1152 * Directly place the iSCSI Data-In or Data-Out PDU's payload into pre-posted
1153 * final destination host-memory buffers based on the Initiator Task Tag (ITT)
1154 * in Data-In or Target Task Tag (TTT) in Data-Out PDUs.
1155 * The host memory address is programmed into h/w in the format of pagepod
1157 * The location of the pagepod entry is encoded into ddp tag which is used as
1158 * the base for ITT/TTT.
1161 static unsigned char ddp_page_order[DDP_PGIDX_MAX] = {0, 1, 2, 4};
1162 static unsigned char ddp_page_shift[DDP_PGIDX_MAX] = {12, 13, 14, 16};
1163 static unsigned char page_idx = DDP_PGIDX_MAX;
1165 static unsigned char sw_tag_idx_bits;
1166 static unsigned char sw_tag_age_bits;
1169 * Direct-Data Placement page size adjustment
1171 static int ddp_adjust_page_table(void)
1174 unsigned int base_order, order;
1176 if (PAGE_SIZE < (1UL << ddp_page_shift[0])) {
1177 pr_info("PAGE_SIZE 0x%lx too small, min 0x%lx\n",
1178 PAGE_SIZE, 1UL << ddp_page_shift[0]);
1182 base_order = get_order(1UL << ddp_page_shift[0]);
1183 order = get_order(1UL << PAGE_SHIFT);
1185 for (i = 0; i < DDP_PGIDX_MAX; i++) {
1186 /* first is the kernel page size, then just doubling */
1187 ddp_page_order[i] = order - base_order + i;
1188 ddp_page_shift[i] = PAGE_SHIFT + i;
1193 static int ddp_find_page_index(unsigned long pgsz)
1197 for (i = 0; i < DDP_PGIDX_MAX; i++) {
1198 if (pgsz == (1UL << ddp_page_shift[i]))
1201 pr_info("ddp page size %lu not supported.\n", pgsz);
1202 return DDP_PGIDX_MAX;
1205 static void ddp_setup_host_page_size(void)
1207 if (page_idx == DDP_PGIDX_MAX) {
1208 page_idx = ddp_find_page_index(PAGE_SIZE);
1210 if (page_idx == DDP_PGIDX_MAX) {
1211 pr_info("system PAGE %lu, update hw.\n", PAGE_SIZE);
1212 if (ddp_adjust_page_table() < 0) {
1213 pr_info("PAGE %lu, disable ddp.\n", PAGE_SIZE);
1216 page_idx = ddp_find_page_index(PAGE_SIZE);
1218 pr_info("system PAGE %lu, ddp idx %u.\n", PAGE_SIZE, page_idx);
1222 void cxgbi_ddp_page_size_factor(int *pgsz_factor)
1226 for (i = 0; i < DDP_PGIDX_MAX; i++)
1227 pgsz_factor[i] = ddp_page_order[i];
1229 EXPORT_SYMBOL_GPL(cxgbi_ddp_page_size_factor);
1232 * DDP setup & teardown
1235 void cxgbi_ddp_ppod_set(struct cxgbi_pagepod *ppod,
1236 struct cxgbi_pagepod_hdr *hdr,
1237 struct cxgbi_gather_list *gl, unsigned int gidx)
1241 memcpy(ppod, hdr, sizeof(*hdr));
1242 for (i = 0; i < (PPOD_PAGES_MAX + 1); i++, gidx++) {
1243 ppod->addr[i] = gidx < gl->nelem ?
1244 cpu_to_be64(gl->phys_addr[gidx]) : 0ULL;
1247 EXPORT_SYMBOL_GPL(cxgbi_ddp_ppod_set);
1249 void cxgbi_ddp_ppod_clear(struct cxgbi_pagepod *ppod)
1251 memset(ppod, 0, sizeof(*ppod));
1253 EXPORT_SYMBOL_GPL(cxgbi_ddp_ppod_clear);
1255 static inline int ddp_find_unused_entries(struct cxgbi_ddp_info *ddp,
1256 unsigned int start, unsigned int max,
1258 struct cxgbi_gather_list *gl)
1260 unsigned int i, j, k;
1262 /* not enough entries */
1263 if ((max - start) < count) {
1264 log_debug(1 << CXGBI_DBG_DDP,
1265 "NOT enough entries %u+%u < %u.\n", start, count, max);
1270 spin_lock(&ddp->map_lock);
1271 for (i = start; i < max;) {
1272 for (j = 0, k = i; j < count; j++, k++) {
1277 for (j = 0, k = i; j < count; j++, k++)
1278 ddp->gl_map[k] = gl;
1279 spin_unlock(&ddp->map_lock);
1284 spin_unlock(&ddp->map_lock);
1285 log_debug(1 << CXGBI_DBG_DDP,
1286 "NO suitable entries %u available.\n", count);
1290 static inline void ddp_unmark_entries(struct cxgbi_ddp_info *ddp,
1291 int start, int count)
1293 spin_lock(&ddp->map_lock);
1294 memset(&ddp->gl_map[start], 0,
1295 count * sizeof(struct cxgbi_gather_list *));
1296 spin_unlock(&ddp->map_lock);
1299 static inline void ddp_gl_unmap(struct pci_dev *pdev,
1300 struct cxgbi_gather_list *gl)
1304 for (i = 0; i < gl->nelem; i++)
1305 dma_unmap_page(&pdev->dev, gl->phys_addr[i], PAGE_SIZE,
1306 PCI_DMA_FROMDEVICE);
1309 static inline int ddp_gl_map(struct pci_dev *pdev,
1310 struct cxgbi_gather_list *gl)
1314 for (i = 0; i < gl->nelem; i++) {
1315 gl->phys_addr[i] = dma_map_page(&pdev->dev, gl->pages[i], 0,
1317 PCI_DMA_FROMDEVICE);
1318 if (unlikely(dma_mapping_error(&pdev->dev, gl->phys_addr[i]))) {
1319 log_debug(1 << CXGBI_DBG_DDP,
1320 "page %d 0x%p, 0x%p dma mapping err.\n",
1321 i, gl->pages[i], pdev);
1328 unsigned int nelem = gl->nelem;
1331 ddp_gl_unmap(pdev, gl);
1337 static void ddp_release_gl(struct cxgbi_gather_list *gl,
1338 struct pci_dev *pdev)
1340 ddp_gl_unmap(pdev, gl);
1344 static struct cxgbi_gather_list *ddp_make_gl(unsigned int xferlen,
1345 struct scatterlist *sgl,
1347 struct pci_dev *pdev,
1350 struct cxgbi_gather_list *gl;
1351 struct scatterlist *sg = sgl;
1352 struct page *sgpage = sg_page(sg);
1353 unsigned int sglen = sg->length;
1354 unsigned int sgoffset = sg->offset;
1355 unsigned int npages = (xferlen + sgoffset + PAGE_SIZE - 1) >>
1359 if (xferlen < DDP_THRESHOLD) {
1360 log_debug(1 << CXGBI_DBG_DDP,
1361 "xfer %u < threshold %u, no ddp.\n",
1362 xferlen, DDP_THRESHOLD);
1366 gl = kzalloc(sizeof(struct cxgbi_gather_list) +
1367 npages * (sizeof(dma_addr_t) +
1368 sizeof(struct page *)), gfp);
1370 log_debug(1 << CXGBI_DBG_DDP,
1371 "xfer %u, %u pages, OOM.\n", xferlen, npages);
1375 log_debug(1 << CXGBI_DBG_DDP,
1376 "xfer %u, sgl %u, gl max %u.\n", xferlen, sgcnt, npages);
1378 gl->pages = (struct page **)&gl->phys_addr[npages];
1380 gl->length = xferlen;
1381 gl->offset = sgoffset;
1382 gl->pages[0] = sgpage;
1384 for (i = 1, sg = sg_next(sgl), j = 0; i < sgcnt;
1385 i++, sg = sg_next(sg)) {
1386 struct page *page = sg_page(sg);
1388 if (sgpage == page && sg->offset == sgoffset + sglen)
1389 sglen += sg->length;
1391 /* make sure the sgl is fit for ddp:
1392 * each has the same page size, and
1393 * all of the middle pages are used completely
1395 if ((j && sgoffset) || ((i != sgcnt - 1) &&
1396 ((sglen + sgoffset) & ~PAGE_MASK))) {
1397 log_debug(1 << CXGBI_DBG_DDP,
1398 "page %d/%u, %u + %u.\n",
1399 i, sgcnt, sgoffset, sglen);
1404 if (j == gl->nelem || sg->offset) {
1405 log_debug(1 << CXGBI_DBG_DDP,
1406 "page %d/%u, offset %u.\n",
1407 j, gl->nelem, sg->offset);
1410 gl->pages[j] = page;
1412 sgoffset = sg->offset;
1418 if (ddp_gl_map(pdev, gl) < 0)
1428 static void ddp_tag_release(struct cxgbi_hba *chba, u32 tag)
1430 struct cxgbi_device *cdev = chba->cdev;
1431 struct cxgbi_ddp_info *ddp = cdev->ddp;
1434 idx = (tag >> PPOD_IDX_SHIFT) & ddp->idx_mask;
1435 if (idx < ddp->nppods) {
1436 struct cxgbi_gather_list *gl = ddp->gl_map[idx];
1439 if (!gl || !gl->nelem) {
1440 pr_warn("tag 0x%x, idx %u, gl 0x%p, %u.\n",
1441 tag, idx, gl, gl ? gl->nelem : 0);
1444 npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
1445 log_debug(1 << CXGBI_DBG_DDP,
1446 "tag 0x%x, release idx %u, npods %u.\n",
1448 cdev->csk_ddp_clear(chba, tag, idx, npods);
1449 ddp_unmark_entries(ddp, idx, npods);
1450 ddp_release_gl(gl, ddp->pdev);
1452 pr_warn("tag 0x%x, idx %u > max %u.\n", tag, idx, ddp->nppods);
1455 static int ddp_tag_reserve(struct cxgbi_sock *csk, unsigned int tid,
1456 u32 sw_tag, u32 *tagp, struct cxgbi_gather_list *gl,
1459 struct cxgbi_device *cdev = csk->cdev;
1460 struct cxgbi_ddp_info *ddp = cdev->ddp;
1461 struct cxgbi_tag_format *tformat = &cdev->tag_format;
1462 struct cxgbi_pagepod_hdr hdr;
1468 npods = (gl->nelem + PPOD_PAGES_MAX - 1) >> PPOD_PAGES_SHIFT;
1469 if (ddp->idx_last == ddp->nppods)
1470 idx = ddp_find_unused_entries(ddp, 0, ddp->nppods,
1473 idx = ddp_find_unused_entries(ddp, ddp->idx_last + 1,
1476 if (idx < 0 && ddp->idx_last >= npods) {
1477 idx = ddp_find_unused_entries(ddp, 0,
1478 min(ddp->idx_last + npods, ddp->nppods),
1483 log_debug(1 << CXGBI_DBG_DDP,
1484 "xferlen %u, gl %u, npods %u NO DDP.\n",
1485 gl->length, gl->nelem, npods);
1489 tag = cxgbi_ddp_tag_base(tformat, sw_tag);
1490 tag |= idx << PPOD_IDX_SHIFT;
1493 hdr.vld_tid = htonl(PPOD_VALID_FLAG | PPOD_TID(tid));
1494 hdr.pgsz_tag_clr = htonl(tag & ddp->rsvd_tag_mask);
1495 hdr.max_offset = htonl(gl->length);
1496 hdr.page_offset = htonl(gl->offset);
1498 err = cdev->csk_ddp_set(csk, &hdr, idx, npods, gl);
1500 goto unmark_entries;
1502 ddp->idx_last = idx;
1503 log_debug(1 << CXGBI_DBG_DDP,
1504 "xfer %u, gl %u,%u, tid 0x%x, tag 0x%x->0x%x(%u,%u).\n",
1505 gl->length, gl->nelem, gl->offset, tid, sw_tag, tag, idx,
1511 ddp_unmark_entries(ddp, idx, npods);
1515 int cxgbi_ddp_reserve(struct cxgbi_sock *csk, unsigned int *tagp,
1516 unsigned int sw_tag, unsigned int xferlen,
1517 struct scatterlist *sgl, unsigned int sgcnt, gfp_t gfp)
1519 struct cxgbi_device *cdev = csk->cdev;
1520 struct cxgbi_tag_format *tformat = &cdev->tag_format;
1521 struct cxgbi_gather_list *gl;
1524 if (page_idx >= DDP_PGIDX_MAX || !cdev->ddp ||
1525 xferlen < DDP_THRESHOLD) {
1526 log_debug(1 << CXGBI_DBG_DDP,
1527 "pgidx %u, xfer %u, NO ddp.\n", page_idx, xferlen);
1531 if (!cxgbi_sw_tag_usable(tformat, sw_tag)) {
1532 log_debug(1 << CXGBI_DBG_DDP,
1533 "sw_tag 0x%x NOT usable.\n", sw_tag);
1537 gl = ddp_make_gl(xferlen, sgl, sgcnt, cdev->pdev, gfp);
1541 err = ddp_tag_reserve(csk, csk->tid, sw_tag, tagp, gl, gfp);
1543 ddp_release_gl(gl, cdev->pdev);
1548 static void ddp_destroy(struct kref *kref)
1550 struct cxgbi_ddp_info *ddp = container_of(kref,
1551 struct cxgbi_ddp_info,
1553 struct cxgbi_device *cdev = ddp->cdev;
1556 pr_info("kref 0, destroy ddp 0x%p, cdev 0x%p.\n", ddp, cdev);
1558 while (i < ddp->nppods) {
1559 struct cxgbi_gather_list *gl = ddp->gl_map[i];
1562 int npods = (gl->nelem + PPOD_PAGES_MAX - 1)
1563 >> PPOD_PAGES_SHIFT;
1564 pr_info("cdev 0x%p, ddp %d + %d.\n", cdev, i, npods);
1570 cxgbi_free_big_mem(ddp);
1573 int cxgbi_ddp_cleanup(struct cxgbi_device *cdev)
1575 struct cxgbi_ddp_info *ddp = cdev->ddp;
1577 log_debug(1 << CXGBI_DBG_DDP,
1578 "cdev 0x%p, release ddp 0x%p.\n", cdev, ddp);
1581 return kref_put(&ddp->refcnt, ddp_destroy);
1584 EXPORT_SYMBOL_GPL(cxgbi_ddp_cleanup);
1586 int cxgbi_ddp_init(struct cxgbi_device *cdev,
1587 unsigned int llimit, unsigned int ulimit,
1588 unsigned int max_txsz, unsigned int max_rxsz)
1590 struct cxgbi_ddp_info *ddp;
1591 unsigned int ppmax, bits;
1593 ppmax = (ulimit - llimit + 1) >> PPOD_SIZE_SHIFT;
1594 bits = __ilog2_u32(ppmax) + 1;
1595 if (bits > PPOD_IDX_MAX_SIZE)
1596 bits = PPOD_IDX_MAX_SIZE;
1597 ppmax = (1 << (bits - 1)) - 1;
1599 ddp = cxgbi_alloc_big_mem(sizeof(struct cxgbi_ddp_info) +
1600 ppmax * (sizeof(struct cxgbi_gather_list *) +
1601 sizeof(struct sk_buff *)),
1604 pr_warn("cdev 0x%p, ddp ppmax %u OOM.\n", cdev, ppmax);
1607 ddp->gl_map = (struct cxgbi_gather_list **)(ddp + 1);
1610 spin_lock_init(&ddp->map_lock);
1611 kref_init(&ddp->refcnt);
1614 ddp->pdev = cdev->pdev;
1615 ddp->llimit = llimit;
1616 ddp->ulimit = ulimit;
1617 ddp->max_txsz = min_t(unsigned int, max_txsz, ULP2_MAX_PKT_SIZE);
1618 ddp->max_rxsz = min_t(unsigned int, max_rxsz, ULP2_MAX_PKT_SIZE);
1619 ddp->nppods = ppmax;
1620 ddp->idx_last = ppmax;
1621 ddp->idx_bits = bits;
1622 ddp->idx_mask = (1 << bits) - 1;
1623 ddp->rsvd_tag_mask = (1 << (bits + PPOD_IDX_SHIFT)) - 1;
1625 cdev->tag_format.sw_bits = sw_tag_idx_bits + sw_tag_age_bits;
1626 cdev->tag_format.rsvd_bits = ddp->idx_bits;
1627 cdev->tag_format.rsvd_shift = PPOD_IDX_SHIFT;
1628 cdev->tag_format.rsvd_mask = (1 << cdev->tag_format.rsvd_bits) - 1;
1630 pr_info("%s tag format, sw %u, rsvd %u,%u, mask 0x%x.\n",
1631 cdev->ports[0]->name, cdev->tag_format.sw_bits,
1632 cdev->tag_format.rsvd_bits, cdev->tag_format.rsvd_shift,
1633 cdev->tag_format.rsvd_mask);
1635 cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
1636 ddp->max_txsz - ISCSI_PDU_NONPAYLOAD_LEN);
1637 cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD,
1638 ddp->max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN);
1640 log_debug(1 << CXGBI_DBG_DDP,
1641 "%s max payload size: %u/%u, %u/%u.\n",
1642 cdev->ports[0]->name, cdev->tx_max_size, ddp->max_txsz,
1643 cdev->rx_max_size, ddp->max_rxsz);
1646 EXPORT_SYMBOL_GPL(cxgbi_ddp_init);
1649 * APIs interacting with open-iscsi libraries
1652 static unsigned char padding[4];
1654 static void task_release_itt(struct iscsi_task *task, itt_t hdr_itt)
1656 struct scsi_cmnd *sc = task->sc;
1657 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
1658 struct cxgbi_conn *cconn = tcp_conn->dd_data;
1659 struct cxgbi_hba *chba = cconn->chba;
1660 struct cxgbi_tag_format *tformat = &chba->cdev->tag_format;
1661 u32 tag = ntohl((__force u32)hdr_itt);
1663 log_debug(1 << CXGBI_DBG_DDP,
1664 "cdev 0x%p, release tag 0x%x.\n", chba->cdev, tag);
1666 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE) &&
1667 cxgbi_is_ddp_tag(tformat, tag))
1668 ddp_tag_release(chba, tag);
1671 static int task_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt)
1673 struct scsi_cmnd *sc = task->sc;
1674 struct iscsi_conn *conn = task->conn;
1675 struct iscsi_session *sess = conn->session;
1676 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1677 struct cxgbi_conn *cconn = tcp_conn->dd_data;
1678 struct cxgbi_hba *chba = cconn->chba;
1679 struct cxgbi_tag_format *tformat = &chba->cdev->tag_format;
1680 u32 sw_tag = (sess->age << cconn->task_idx_bits) | task->itt;
1685 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_FROM_DEVICE)) {
1686 err = cxgbi_ddp_reserve(cconn->cep->csk, &tag, sw_tag,
1687 scsi_in(sc)->length,
1688 scsi_in(sc)->table.sgl,
1689 scsi_in(sc)->table.nents,
1692 log_debug(1 << CXGBI_DBG_DDP,
1693 "csk 0x%p, R task 0x%p, %u,%u, no ddp.\n",
1694 cconn->cep->csk, task, scsi_in(sc)->length,
1695 scsi_in(sc)->table.nents);
1699 tag = cxgbi_set_non_ddp_tag(tformat, sw_tag);
1700 /* the itt need to sent in big-endian order */
1701 *hdr_itt = (__force itt_t)htonl(tag);
1703 log_debug(1 << CXGBI_DBG_DDP,
1704 "cdev 0x%p, task 0x%p, 0x%x(0x%x,0x%x)->0x%x/0x%x.\n",
1705 chba->cdev, task, sw_tag, task->itt, sess->age, tag, *hdr_itt);
1709 void cxgbi_parse_pdu_itt(struct iscsi_conn *conn, itt_t itt, int *idx, int *age)
1711 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1712 struct cxgbi_conn *cconn = tcp_conn->dd_data;
1713 struct cxgbi_device *cdev = cconn->chba->cdev;
1714 u32 tag = ntohl((__force u32) itt);
1717 sw_bits = cxgbi_tag_nonrsvd_bits(&cdev->tag_format, tag);
1719 *idx = sw_bits & ((1 << cconn->task_idx_bits) - 1);
1721 *age = (sw_bits >> cconn->task_idx_bits) & ISCSI_AGE_MASK;
1723 log_debug(1 << CXGBI_DBG_DDP,
1724 "cdev 0x%p, tag 0x%x/0x%x, -> 0x%x(0x%x,0x%x).\n",
1725 cdev, tag, itt, sw_bits, idx ? *idx : 0xFFFFF,
1728 EXPORT_SYMBOL_GPL(cxgbi_parse_pdu_itt);
1730 void cxgbi_conn_tx_open(struct cxgbi_sock *csk)
1732 struct iscsi_conn *conn = csk->user_data;
1735 log_debug(1 << CXGBI_DBG_SOCK,
1736 "csk 0x%p, cid %d.\n", csk, conn->id);
1737 iscsi_conn_queue_work(conn);
1740 EXPORT_SYMBOL_GPL(cxgbi_conn_tx_open);
1743 * pdu receive, interact with libiscsi_tcp
1745 static inline int read_pdu_skb(struct iscsi_conn *conn,
1746 struct sk_buff *skb,
1747 unsigned int offset,
1753 bytes_read = iscsi_tcp_recv_skb(conn, skb, offset, offloaded, &status);
1755 case ISCSI_TCP_CONN_ERR:
1756 pr_info("skb 0x%p, off %u, %d, TCP_ERR.\n",
1757 skb, offset, offloaded);
1759 case ISCSI_TCP_SUSPENDED:
1760 log_debug(1 << CXGBI_DBG_PDU_RX,
1761 "skb 0x%p, off %u, %d, TCP_SUSPEND, rc %d.\n",
1762 skb, offset, offloaded, bytes_read);
1763 /* no transfer - just have caller flush queue */
1765 case ISCSI_TCP_SKB_DONE:
1766 pr_info("skb 0x%p, off %u, %d, TCP_SKB_DONE.\n",
1767 skb, offset, offloaded);
1769 * pdus should always fit in the skb and we should get
1770 * segment done notifcation.
1772 iscsi_conn_printk(KERN_ERR, conn, "Invalid pdu or skb.");
1774 case ISCSI_TCP_SEGMENT_DONE:
1775 log_debug(1 << CXGBI_DBG_PDU_RX,
1776 "skb 0x%p, off %u, %d, TCP_SEG_DONE, rc %d.\n",
1777 skb, offset, offloaded, bytes_read);
1780 pr_info("skb 0x%p, off %u, %d, invalid status %d.\n",
1781 skb, offset, offloaded, status);
1786 static int skb_read_pdu_bhs(struct iscsi_conn *conn, struct sk_buff *skb)
1788 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1790 log_debug(1 << CXGBI_DBG_PDU_RX,
1791 "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n",
1792 conn, skb, skb->len, cxgbi_skcb_flags(skb));
1794 if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn)) {
1795 pr_info("conn 0x%p, skb 0x%p, not hdr.\n", conn, skb);
1796 iscsi_conn_failure(conn, ISCSI_ERR_PROTO);
1800 if (conn->hdrdgst_en &&
1801 cxgbi_skcb_test_flag(skb, SKCBF_RX_HCRC_ERR)) {
1802 pr_info("conn 0x%p, skb 0x%p, hcrc.\n", conn, skb);
1803 iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST);
1807 return read_pdu_skb(conn, skb, 0, 0);
1810 static int skb_read_pdu_data(struct iscsi_conn *conn, struct sk_buff *lskb,
1811 struct sk_buff *skb, unsigned int offset)
1813 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1815 int opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK;
1817 log_debug(1 << CXGBI_DBG_PDU_RX,
1818 "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n",
1819 conn, skb, skb->len, cxgbi_skcb_flags(skb));
1821 if (conn->datadgst_en &&
1822 cxgbi_skcb_test_flag(lskb, SKCBF_RX_DCRC_ERR)) {
1823 pr_info("conn 0x%p, skb 0x%p, dcrc 0x%lx.\n",
1824 conn, lskb, cxgbi_skcb_flags(lskb));
1825 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
1829 if (iscsi_tcp_recv_segment_is_hdr(tcp_conn))
1832 /* coalesced, add header digest length */
1833 if (lskb == skb && conn->hdrdgst_en)
1834 offset += ISCSI_DIGEST_SIZE;
1836 if (cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA_DDPD))
1839 if (opcode == ISCSI_OP_SCSI_DATA_IN)
1840 log_debug(1 << CXGBI_DBG_PDU_RX,
1841 "skb 0x%p, op 0x%x, itt 0x%x, %u %s ddp'ed.\n",
1842 skb, opcode, ntohl(tcp_conn->in.hdr->itt),
1843 tcp_conn->in.datalen, offloaded ? "is" : "not");
1845 return read_pdu_skb(conn, skb, offset, offloaded);
1848 static void csk_return_rx_credits(struct cxgbi_sock *csk, int copied)
1850 struct cxgbi_device *cdev = csk->cdev;
1854 log_debug(1 << CXGBI_DBG_PDU_RX,
1855 "csk 0x%p,%u,0x%lx,%u, seq %u, wup %u, thre %u, %u.\n",
1856 csk, csk->state, csk->flags, csk->tid, csk->copied_seq,
1857 csk->rcv_wup, cdev->rx_credit_thres,
1860 if (csk->state != CTP_ESTABLISHED)
1863 credits = csk->copied_seq - csk->rcv_wup;
1864 if (unlikely(!credits))
1866 if (unlikely(cdev->rx_credit_thres == 0))
1869 must_send = credits + 16384 >= cdev->rcv_win;
1870 if (must_send || credits >= cdev->rx_credit_thres)
1871 csk->rcv_wup += cdev->csk_send_rx_credits(csk, credits);
1874 void cxgbi_conn_pdu_ready(struct cxgbi_sock *csk)
1876 struct cxgbi_device *cdev = csk->cdev;
1877 struct iscsi_conn *conn = csk->user_data;
1878 struct sk_buff *skb;
1879 unsigned int read = 0;
1882 log_debug(1 << CXGBI_DBG_PDU_RX,
1883 "csk 0x%p, conn 0x%p.\n", csk, conn);
1885 if (unlikely(!conn || conn->suspend_rx)) {
1886 log_debug(1 << CXGBI_DBG_PDU_RX,
1887 "csk 0x%p, conn 0x%p, id %d, suspend_rx %lu!\n",
1888 csk, conn, conn ? conn->id : 0xFF,
1889 conn ? conn->suspend_rx : 0xFF);
1894 skb = skb_peek(&csk->receive_queue);
1896 !(cxgbi_skcb_test_flag(skb, SKCBF_RX_STATUS))) {
1898 log_debug(1 << CXGBI_DBG_PDU_RX,
1899 "skb 0x%p, NOT ready 0x%lx.\n",
1900 skb, cxgbi_skcb_flags(skb));
1903 __skb_unlink(skb, &csk->receive_queue);
1905 read += cxgbi_skcb_rx_pdulen(skb);
1906 log_debug(1 << CXGBI_DBG_PDU_RX,
1907 "csk 0x%p, skb 0x%p,%u,f 0x%lx, pdu len %u.\n",
1908 csk, skb, skb->len, cxgbi_skcb_flags(skb),
1909 cxgbi_skcb_rx_pdulen(skb));
1911 if (cxgbi_skcb_test_flag(skb, SKCBF_RX_COALESCED)) {
1912 err = skb_read_pdu_bhs(conn, skb);
1914 pr_err("coalesced bhs, csk 0x%p, skb 0x%p,%u, "
1915 "f 0x%lx, plen %u.\n",
1917 cxgbi_skcb_flags(skb),
1918 cxgbi_skcb_rx_pdulen(skb));
1921 err = skb_read_pdu_data(conn, skb, skb,
1922 err + cdev->skb_rx_extra);
1924 pr_err("coalesced data, csk 0x%p, skb 0x%p,%u, "
1925 "f 0x%lx, plen %u.\n",
1927 cxgbi_skcb_flags(skb),
1928 cxgbi_skcb_rx_pdulen(skb));
1930 err = skb_read_pdu_bhs(conn, skb);
1932 pr_err("bhs, csk 0x%p, skb 0x%p,%u, "
1933 "f 0x%lx, plen %u.\n",
1935 cxgbi_skcb_flags(skb),
1936 cxgbi_skcb_rx_pdulen(skb));
1940 if (cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA)) {
1941 struct sk_buff *dskb;
1943 dskb = skb_peek(&csk->receive_queue);
1945 pr_err("csk 0x%p, skb 0x%p,%u, f 0x%lx,"
1946 " plen %u, NO data.\n",
1948 cxgbi_skcb_flags(skb),
1949 cxgbi_skcb_rx_pdulen(skb));
1953 __skb_unlink(dskb, &csk->receive_queue);
1955 err = skb_read_pdu_data(conn, skb, dskb, 0);
1957 pr_err("data, csk 0x%p, skb 0x%p,%u, "
1958 "f 0x%lx, plen %u, dskb 0x%p,"
1961 cxgbi_skcb_flags(skb),
1962 cxgbi_skcb_rx_pdulen(skb),
1966 err = skb_read_pdu_data(conn, skb, skb, 0);
1975 log_debug(1 << CXGBI_DBG_PDU_RX, "csk 0x%p, read %u.\n", csk, read);
1977 csk->copied_seq += read;
1978 csk_return_rx_credits(csk, read);
1979 conn->rxdata_octets += read;
1983 pr_info("csk 0x%p, 0x%p, rx failed %d, read %u.\n",
1984 csk, conn, err, read);
1985 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1988 EXPORT_SYMBOL_GPL(cxgbi_conn_pdu_ready);
1990 static int sgl_seek_offset(struct scatterlist *sgl, unsigned int sgcnt,
1991 unsigned int offset, unsigned int *off,
1992 struct scatterlist **sgp)
1995 struct scatterlist *sg;
1997 for_each_sg(sgl, sg, sgcnt, i) {
1998 if (offset < sg->length) {
2003 offset -= sg->length;
2008 static int sgl_read_to_frags(struct scatterlist *sg, unsigned int sgoffset,
2009 unsigned int dlen, struct page_frag *frags,
2012 unsigned int datalen = dlen;
2013 unsigned int sglen = sg->length - sgoffset;
2014 struct page *page = sg_page(sg);
2024 pr_warn("sg %d NULL, len %u/%u.\n",
2033 copy = min(datalen, sglen);
2034 if (i && page == frags[i - 1].page &&
2035 sgoffset + sg->offset ==
2036 frags[i - 1].offset + frags[i - 1].size) {
2037 frags[i - 1].size += copy;
2039 if (i >= frag_max) {
2040 pr_warn("too many pages %u, dlen %u.\n",
2045 frags[i].page = page;
2046 frags[i].offset = sg->offset + sgoffset;
2047 frags[i].size = copy;
2058 int cxgbi_conn_alloc_pdu(struct iscsi_task *task, u8 opcode)
2060 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
2061 struct cxgbi_conn *cconn = tcp_conn->dd_data;
2062 struct cxgbi_device *cdev = cconn->chba->cdev;
2063 struct iscsi_conn *conn = task->conn;
2064 struct iscsi_tcp_task *tcp_task = task->dd_data;
2065 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
2066 struct scsi_cmnd *sc = task->sc;
2067 int headroom = SKB_TX_ISCSI_PDU_HEADER_MAX;
2069 tcp_task->dd_data = tdata;
2072 if (SKB_MAX_HEAD(cdev->skb_tx_rsvd) > (512 * MAX_SKB_FRAGS) &&
2073 (opcode == ISCSI_OP_SCSI_DATA_OUT ||
2074 (opcode == ISCSI_OP_SCSI_CMD &&
2075 (scsi_bidi_cmnd(sc) || sc->sc_data_direction == DMA_TO_DEVICE))))
2076 /* data could goes into skb head */
2077 headroom += min_t(unsigned int,
2078 SKB_MAX_HEAD(cdev->skb_tx_rsvd),
2079 conn->max_xmit_dlength);
2081 tdata->skb = alloc_skb(cdev->skb_tx_rsvd + headroom, GFP_ATOMIC);
2083 struct cxgbi_sock *csk = cconn->cep->csk;
2084 struct net_device *ndev = cdev->ports[csk->port_id];
2085 ndev->stats.tx_dropped++;
2089 skb_reserve(tdata->skb, cdev->skb_tx_rsvd);
2090 task->hdr = (struct iscsi_hdr *)tdata->skb->data;
2091 task->hdr_max = SKB_TX_ISCSI_PDU_HEADER_MAX; /* BHS + AHS */
2093 /* data_out uses scsi_cmd's itt */
2094 if (opcode != ISCSI_OP_SCSI_DATA_OUT)
2095 task_reserve_itt(task, &task->hdr->itt);
2097 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2098 "task 0x%p, op 0x%x, skb 0x%p,%u+%u/%u, itt 0x%x.\n",
2099 task, opcode, tdata->skb, cdev->skb_tx_rsvd, headroom,
2100 conn->max_xmit_dlength, ntohl(task->hdr->itt));
2104 EXPORT_SYMBOL_GPL(cxgbi_conn_alloc_pdu);
2106 static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc)
2115 cxgbi_skcb_ulp_mode(skb) = (ULP2_MODE_ISCSI << 4) | submode;
2117 cxgbi_skcb_ulp_mode(skb) = 0;
2120 int cxgbi_conn_init_pdu(struct iscsi_task *task, unsigned int offset,
2123 struct iscsi_conn *conn = task->conn;
2124 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
2125 struct sk_buff *skb = tdata->skb;
2126 unsigned int datalen = count;
2127 int i, padlen = iscsi_padding(count);
2130 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2131 "task 0x%p,0x%p, skb 0x%p, 0x%x,0x%x,0x%x, %u+%u.\n",
2132 task, task->sc, skb, (*skb->data) & ISCSI_OPCODE_MASK,
2133 ntohl(task->cmdsn), ntohl(task->hdr->itt), offset, count);
2135 skb_put(skb, task->hdr_len);
2136 tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
2141 struct scsi_data_buffer *sdb = scsi_out(task->sc);
2142 struct scatterlist *sg = NULL;
2145 tdata->offset = offset;
2146 tdata->count = count;
2147 err = sgl_seek_offset(
2148 sdb->table.sgl, sdb->table.nents,
2149 tdata->offset, &tdata->sgoffset, &sg);
2151 pr_warn("tpdu, sgl %u, bad offset %u/%u.\n",
2152 sdb->table.nents, tdata->offset, sdb->length);
2155 err = sgl_read_to_frags(sg, tdata->sgoffset, tdata->count,
2156 tdata->frags, MAX_PDU_FRAGS);
2158 pr_warn("tpdu, sgl %u, bad offset %u + %u.\n",
2159 sdb->table.nents, tdata->offset, tdata->count);
2162 tdata->nr_frags = err;
2164 if (tdata->nr_frags > MAX_SKB_FRAGS ||
2165 (padlen && tdata->nr_frags == MAX_SKB_FRAGS)) {
2166 char *dst = skb->data + task->hdr_len;
2167 struct page_frag *frag = tdata->frags;
2169 /* data fits in the skb's headroom */
2170 for (i = 0; i < tdata->nr_frags; i++, frag++) {
2171 char *src = kmap_atomic(frag->page);
2173 memcpy(dst, src+frag->offset, frag->size);
2178 memset(dst, 0, padlen);
2181 skb_put(skb, count + padlen);
2183 /* data fit into frag_list */
2184 for (i = 0; i < tdata->nr_frags; i++) {
2185 __skb_fill_page_desc(skb, i,
2186 tdata->frags[i].page,
2187 tdata->frags[i].offset,
2188 tdata->frags[i].size);
2189 skb_frag_ref(skb, i);
2191 skb_shinfo(skb)->nr_frags = tdata->nr_frags;
2193 skb->data_len += count;
2194 skb->truesize += count;
2198 pg = virt_to_page(task->data);
2201 skb_fill_page_desc(skb, 0, pg, offset_in_page(task->data),
2204 skb->data_len += count;
2205 skb->truesize += count;
2209 i = skb_shinfo(skb)->nr_frags;
2210 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
2211 virt_to_page(padding), offset_in_page(padding),
2214 skb->data_len += padlen;
2215 skb->truesize += padlen;
2221 EXPORT_SYMBOL_GPL(cxgbi_conn_init_pdu);
2223 int cxgbi_conn_xmit_pdu(struct iscsi_task *task)
2225 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
2226 struct cxgbi_conn *cconn = tcp_conn->dd_data;
2227 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
2228 struct sk_buff *skb = tdata->skb;
2229 unsigned int datalen;
2233 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2234 "task 0x%p, skb NULL.\n", task);
2238 datalen = skb->data_len;
2240 err = cxgbi_sock_send_pdus(cconn->cep->csk, skb);
2244 log_debug(1 << CXGBI_DBG_PDU_TX,
2245 "task 0x%p,0x%p, skb 0x%p, len %u/%u, rv %d.\n",
2246 task, task->sc, skb, skb->len, skb->data_len, err);
2248 if (task->conn->hdrdgst_en)
2249 pdulen += ISCSI_DIGEST_SIZE;
2251 if (datalen && task->conn->datadgst_en)
2252 pdulen += ISCSI_DIGEST_SIZE;
2254 task->conn->txdata_octets += pdulen;
2258 if (err == -EAGAIN || err == -ENOBUFS) {
2259 log_debug(1 << CXGBI_DBG_PDU_TX,
2260 "task 0x%p, skb 0x%p, len %u/%u, %d EAGAIN.\n",
2261 task, skb, skb->len, skb->data_len, err);
2262 /* reset skb to send when we are called again */
2268 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,
2269 "itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n",
2270 task->itt, skb, skb->len, skb->data_len, err);
2271 iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err);
2272 iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED);
2275 EXPORT_SYMBOL_GPL(cxgbi_conn_xmit_pdu);
2277 void cxgbi_cleanup_task(struct iscsi_task *task)
2279 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task);
2281 log_debug(1 << CXGBI_DBG_ISCSI,
2282 "task 0x%p, skb 0x%p, itt 0x%x.\n",
2283 task, tdata->skb, task->hdr_itt);
2285 /* never reached the xmit task callout */
2287 __kfree_skb(tdata->skb);
2288 memset(tdata, 0, sizeof(*tdata));
2290 task_release_itt(task, task->hdr_itt);
2291 iscsi_tcp_cleanup_task(task);
2293 EXPORT_SYMBOL_GPL(cxgbi_cleanup_task);
2295 void cxgbi_get_conn_stats(struct iscsi_cls_conn *cls_conn,
2296 struct iscsi_stats *stats)
2298 struct iscsi_conn *conn = cls_conn->dd_data;
2300 stats->txdata_octets = conn->txdata_octets;
2301 stats->rxdata_octets = conn->rxdata_octets;
2302 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
2303 stats->dataout_pdus = conn->dataout_pdus_cnt;
2304 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
2305 stats->datain_pdus = conn->datain_pdus_cnt;
2306 stats->r2t_pdus = conn->r2t_pdus_cnt;
2307 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
2308 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
2309 stats->digest_err = 0;
2310 stats->timeout_err = 0;
2311 stats->custom_length = 1;
2312 strcpy(stats->custom[0].desc, "eh_abort_cnt");
2313 stats->custom[0].value = conn->eh_abort_cnt;
2315 EXPORT_SYMBOL_GPL(cxgbi_get_conn_stats);
2317 static int cxgbi_conn_max_xmit_dlength(struct iscsi_conn *conn)
2319 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2320 struct cxgbi_conn *cconn = tcp_conn->dd_data;
2321 struct cxgbi_device *cdev = cconn->chba->cdev;
2322 unsigned int headroom = SKB_MAX_HEAD(cdev->skb_tx_rsvd);
2323 unsigned int max_def = 512 * MAX_SKB_FRAGS;
2324 unsigned int max = max(max_def, headroom);
2326 max = min(cconn->chba->cdev->tx_max_size, max);
2327 if (conn->max_xmit_dlength)
2328 conn->max_xmit_dlength = min(conn->max_xmit_dlength, max);
2330 conn->max_xmit_dlength = max;
2331 cxgbi_align_pdu_size(conn->max_xmit_dlength);
2336 static int cxgbi_conn_max_recv_dlength(struct iscsi_conn *conn)
2338 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2339 struct cxgbi_conn *cconn = tcp_conn->dd_data;
2340 unsigned int max = cconn->chba->cdev->rx_max_size;
2342 cxgbi_align_pdu_size(max);
2344 if (conn->max_recv_dlength) {
2345 if (conn->max_recv_dlength > max) {
2346 pr_err("MaxRecvDataSegmentLength %u > %u.\n",
2347 conn->max_recv_dlength, max);
2350 conn->max_recv_dlength = min(conn->max_recv_dlength, max);
2351 cxgbi_align_pdu_size(conn->max_recv_dlength);
2353 conn->max_recv_dlength = max;
2358 int cxgbi_set_conn_param(struct iscsi_cls_conn *cls_conn,
2359 enum iscsi_param param, char *buf, int buflen)
2361 struct iscsi_conn *conn = cls_conn->dd_data;
2362 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2363 struct cxgbi_conn *cconn = tcp_conn->dd_data;
2364 struct cxgbi_sock *csk = cconn->cep->csk;
2367 log_debug(1 << CXGBI_DBG_ISCSI,
2368 "cls_conn 0x%p, param %d, buf(%d) %s.\n",
2369 cls_conn, param, buflen, buf);
2372 case ISCSI_PARAM_HDRDGST_EN:
2373 err = iscsi_set_param(cls_conn, param, buf, buflen);
2374 if (!err && conn->hdrdgst_en)
2375 err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid,
2377 conn->datadgst_en, 0);
2379 case ISCSI_PARAM_DATADGST_EN:
2380 err = iscsi_set_param(cls_conn, param, buf, buflen);
2381 if (!err && conn->datadgst_en)
2382 err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid,
2384 conn->datadgst_en, 0);
2386 case ISCSI_PARAM_MAX_R2T:
2387 return iscsi_tcp_set_max_r2t(conn, buf);
2388 case ISCSI_PARAM_MAX_RECV_DLENGTH:
2389 err = iscsi_set_param(cls_conn, param, buf, buflen);
2391 err = cxgbi_conn_max_recv_dlength(conn);
2393 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
2394 err = iscsi_set_param(cls_conn, param, buf, buflen);
2396 err = cxgbi_conn_max_xmit_dlength(conn);
2399 return iscsi_set_param(cls_conn, param, buf, buflen);
2403 EXPORT_SYMBOL_GPL(cxgbi_set_conn_param);
2405 static inline int csk_print_port(struct cxgbi_sock *csk, char *buf)
2409 cxgbi_sock_get(csk);
2410 len = sprintf(buf, "%hu\n", ntohs(csk->daddr.sin_port));
2411 cxgbi_sock_put(csk);
2416 static inline int csk_print_ip(struct cxgbi_sock *csk, char *buf)
2420 cxgbi_sock_get(csk);
2421 if (csk->csk_family == AF_INET)
2422 len = sprintf(buf, "%pI4",
2423 &csk->daddr.sin_addr.s_addr);
2425 len = sprintf(buf, "%pI6",
2426 &csk->daddr6.sin6_addr);
2428 cxgbi_sock_put(csk);
2433 int cxgbi_get_ep_param(struct iscsi_endpoint *ep, enum iscsi_param param,
2436 struct cxgbi_endpoint *cep = ep->dd_data;
2437 struct cxgbi_sock *csk;
2440 log_debug(1 << CXGBI_DBG_ISCSI,
2441 "cls_conn 0x%p, param %d.\n", ep, param);
2444 case ISCSI_PARAM_CONN_PORT:
2445 case ISCSI_PARAM_CONN_ADDRESS:
2453 return iscsi_conn_get_addr_param((struct sockaddr_storage *)
2454 &csk->daddr, param, buf);
2460 EXPORT_SYMBOL_GPL(cxgbi_get_ep_param);
2462 struct iscsi_cls_conn *
2463 cxgbi_create_conn(struct iscsi_cls_session *cls_session, u32 cid)
2465 struct iscsi_cls_conn *cls_conn;
2466 struct iscsi_conn *conn;
2467 struct iscsi_tcp_conn *tcp_conn;
2468 struct cxgbi_conn *cconn;
2470 cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*cconn), cid);
2474 conn = cls_conn->dd_data;
2475 tcp_conn = conn->dd_data;
2476 cconn = tcp_conn->dd_data;
2477 cconn->iconn = conn;
2479 log_debug(1 << CXGBI_DBG_ISCSI,
2480 "cid %u(0x%x), cls 0x%p,0x%p, conn 0x%p,0x%p,0x%p.\n",
2481 cid, cid, cls_session, cls_conn, conn, tcp_conn, cconn);
2485 EXPORT_SYMBOL_GPL(cxgbi_create_conn);
2487 int cxgbi_bind_conn(struct iscsi_cls_session *cls_session,
2488 struct iscsi_cls_conn *cls_conn,
2489 u64 transport_eph, int is_leading)
2491 struct iscsi_conn *conn = cls_conn->dd_data;
2492 struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2493 struct cxgbi_conn *cconn = tcp_conn->dd_data;
2494 struct iscsi_endpoint *ep;
2495 struct cxgbi_endpoint *cep;
2496 struct cxgbi_sock *csk;
2499 ep = iscsi_lookup_endpoint(transport_eph);
2503 /* setup ddp pagesize */
2506 err = csk->cdev->csk_ddp_setup_pgidx(csk, csk->tid, page_idx, 0);
2510 err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
2514 /* calculate the tag idx bits needed for this conn based on cmds_max */
2515 cconn->task_idx_bits = (__ilog2_u32(conn->session->cmds_max - 1)) + 1;
2517 write_lock_bh(&csk->callback_lock);
2518 csk->user_data = conn;
2519 cconn->chba = cep->chba;
2522 write_unlock_bh(&csk->callback_lock);
2524 cxgbi_conn_max_xmit_dlength(conn);
2525 cxgbi_conn_max_recv_dlength(conn);
2527 log_debug(1 << CXGBI_DBG_ISCSI,
2528 "cls 0x%p,0x%p, ep 0x%p, cconn 0x%p, csk 0x%p.\n",
2529 cls_session, cls_conn, ep, cconn, csk);
2530 /* init recv engine */
2531 iscsi_tcp_hdr_recv_prep(tcp_conn);
2535 EXPORT_SYMBOL_GPL(cxgbi_bind_conn);
2537 struct iscsi_cls_session *cxgbi_create_session(struct iscsi_endpoint *ep,
2538 u16 cmds_max, u16 qdepth,
2541 struct cxgbi_endpoint *cep;
2542 struct cxgbi_hba *chba;
2543 struct Scsi_Host *shost;
2544 struct iscsi_cls_session *cls_session;
2545 struct iscsi_session *session;
2548 pr_err("missing endpoint.\n");
2554 shost = chba->shost;
2556 BUG_ON(chba != iscsi_host_priv(shost));
2558 cls_session = iscsi_session_setup(chba->cdev->itp, shost,
2560 sizeof(struct iscsi_tcp_task) +
2561 sizeof(struct cxgbi_task_data),
2562 initial_cmdsn, ISCSI_MAX_TARGET);
2566 session = cls_session->dd_data;
2567 if (iscsi_tcp_r2tpool_alloc(session))
2568 goto remove_session;
2570 log_debug(1 << CXGBI_DBG_ISCSI,
2571 "ep 0x%p, cls sess 0x%p.\n", ep, cls_session);
2575 iscsi_session_teardown(cls_session);
2578 EXPORT_SYMBOL_GPL(cxgbi_create_session);
2580 void cxgbi_destroy_session(struct iscsi_cls_session *cls_session)
2582 log_debug(1 << CXGBI_DBG_ISCSI,
2583 "cls sess 0x%p.\n", cls_session);
2585 iscsi_tcp_r2tpool_free(cls_session->dd_data);
2586 iscsi_session_teardown(cls_session);
2588 EXPORT_SYMBOL_GPL(cxgbi_destroy_session);
2590 int cxgbi_set_host_param(struct Scsi_Host *shost, enum iscsi_host_param param,
2591 char *buf, int buflen)
2593 struct cxgbi_hba *chba = iscsi_host_priv(shost);
2596 shost_printk(KERN_ERR, shost, "Could not get host param. "
2597 "netdev for host not set.\n");
2601 log_debug(1 << CXGBI_DBG_ISCSI,
2602 "shost 0x%p, hba 0x%p,%s, param %d, buf(%d) %s.\n",
2603 shost, chba, chba->ndev->name, param, buflen, buf);
2606 case ISCSI_HOST_PARAM_IPADDRESS:
2608 __be32 addr = in_aton(buf);
2609 log_debug(1 << CXGBI_DBG_ISCSI,
2610 "hba %s, req. ipv4 %pI4.\n", chba->ndev->name, &addr);
2611 cxgbi_set_iscsi_ipv4(chba, addr);
2614 case ISCSI_HOST_PARAM_HWADDRESS:
2615 case ISCSI_HOST_PARAM_NETDEV_NAME:
2618 return iscsi_host_set_param(shost, param, buf, buflen);
2621 EXPORT_SYMBOL_GPL(cxgbi_set_host_param);
2623 int cxgbi_get_host_param(struct Scsi_Host *shost, enum iscsi_host_param param,
2626 struct cxgbi_hba *chba = iscsi_host_priv(shost);
2630 shost_printk(KERN_ERR, shost, "Could not get host param. "
2631 "netdev for host not set.\n");
2635 log_debug(1 << CXGBI_DBG_ISCSI,
2636 "shost 0x%p, hba 0x%p,%s, param %d.\n",
2637 shost, chba, chba->ndev->name, param);
2640 case ISCSI_HOST_PARAM_HWADDRESS:
2641 len = sysfs_format_mac(buf, chba->ndev->dev_addr, 6);
2643 case ISCSI_HOST_PARAM_NETDEV_NAME:
2644 len = sprintf(buf, "%s\n", chba->ndev->name);
2646 case ISCSI_HOST_PARAM_IPADDRESS:
2650 addr = cxgbi_get_iscsi_ipv4(chba);
2651 len = sprintf(buf, "%pI4", &addr);
2652 log_debug(1 << CXGBI_DBG_ISCSI,
2653 "hba %s, ipv4 %pI4.\n", chba->ndev->name, &addr);
2657 return iscsi_host_get_param(shost, param, buf);
2662 EXPORT_SYMBOL_GPL(cxgbi_get_host_param);
2664 struct iscsi_endpoint *cxgbi_ep_connect(struct Scsi_Host *shost,
2665 struct sockaddr *dst_addr,
2668 struct iscsi_endpoint *ep;
2669 struct cxgbi_endpoint *cep;
2670 struct cxgbi_hba *hba = NULL;
2671 struct cxgbi_sock *csk;
2674 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
2675 "shost 0x%p, non_blocking %d, dst_addr 0x%p.\n",
2676 shost, non_blocking, dst_addr);
2679 hba = iscsi_host_priv(shost);
2681 pr_info("shost 0x%p, priv NULL.\n", shost);
2686 if (dst_addr->sa_family == AF_INET) {
2687 csk = cxgbi_check_route(dst_addr);
2688 #if IS_ENABLED(CONFIG_IPV6)
2689 } else if (dst_addr->sa_family == AF_INET6) {
2690 csk = cxgbi_check_route6(dst_addr);
2693 pr_info("address family 0x%x NOT supported.\n",
2694 dst_addr->sa_family);
2695 err = -EAFNOSUPPORT;
2696 return (struct iscsi_endpoint *)ERR_PTR(err);
2700 return (struct iscsi_endpoint *)csk;
2701 cxgbi_sock_get(csk);
2704 hba = csk->cdev->hbas[csk->port_id];
2705 else if (hba != csk->cdev->hbas[csk->port_id]) {
2706 pr_info("Could not connect through requested host %u"
2707 "hba 0x%p != 0x%p (%u).\n",
2708 shost->host_no, hba,
2709 csk->cdev->hbas[csk->port_id], csk->port_id);
2714 err = sock_get_port(csk);
2718 cxgbi_sock_set_state(csk, CTP_CONNECTING);
2719 err = csk->cdev->csk_init_act_open(csk);
2723 if (cxgbi_sock_is_closing(csk)) {
2725 pr_info("csk 0x%p is closing.\n", csk);
2729 ep = iscsi_create_endpoint(sizeof(*cep));
2732 pr_info("iscsi alloc ep, OOM.\n");
2740 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
2741 "ep 0x%p, cep 0x%p, csk 0x%p, hba 0x%p,%s.\n",
2742 ep, cep, csk, hba, hba->ndev->name);
2746 cxgbi_sock_put(csk);
2747 cxgbi_sock_closed(csk);
2749 return ERR_PTR(err);
2751 EXPORT_SYMBOL_GPL(cxgbi_ep_connect);
2753 int cxgbi_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
2755 struct cxgbi_endpoint *cep = ep->dd_data;
2756 struct cxgbi_sock *csk = cep->csk;
2758 if (!cxgbi_sock_is_established(csk))
2762 EXPORT_SYMBOL_GPL(cxgbi_ep_poll);
2764 void cxgbi_ep_disconnect(struct iscsi_endpoint *ep)
2766 struct cxgbi_endpoint *cep = ep->dd_data;
2767 struct cxgbi_conn *cconn = cep->cconn;
2768 struct cxgbi_sock *csk = cep->csk;
2770 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK,
2771 "ep 0x%p, cep 0x%p, cconn 0x%p, csk 0x%p,%u,0x%lx.\n",
2772 ep, cep, cconn, csk, csk->state, csk->flags);
2774 if (cconn && cconn->iconn) {
2775 iscsi_suspend_tx(cconn->iconn);
2776 write_lock_bh(&csk->callback_lock);
2777 cep->csk->user_data = NULL;
2779 write_unlock_bh(&csk->callback_lock);
2781 iscsi_destroy_endpoint(ep);
2783 if (likely(csk->state >= CTP_ESTABLISHED))
2784 need_active_close(csk);
2786 cxgbi_sock_closed(csk);
2788 cxgbi_sock_put(csk);
2790 EXPORT_SYMBOL_GPL(cxgbi_ep_disconnect);
2792 int cxgbi_iscsi_init(struct iscsi_transport *itp,
2793 struct scsi_transport_template **stt)
2795 *stt = iscsi_register_transport(itp);
2797 pr_err("unable to register %s transport 0x%p.\n",
2801 log_debug(1 << CXGBI_DBG_ISCSI,
2802 "%s, registered iscsi transport 0x%p.\n",
2806 EXPORT_SYMBOL_GPL(cxgbi_iscsi_init);
2808 void cxgbi_iscsi_cleanup(struct iscsi_transport *itp,
2809 struct scsi_transport_template **stt)
2812 log_debug(1 << CXGBI_DBG_ISCSI,
2813 "de-register transport 0x%p, %s, stt 0x%p.\n",
2814 itp, itp->name, *stt);
2816 iscsi_unregister_transport(itp);
2819 EXPORT_SYMBOL_GPL(cxgbi_iscsi_cleanup);
2821 umode_t cxgbi_attr_is_visible(int param_type, int param)
2823 switch (param_type) {
2824 case ISCSI_HOST_PARAM:
2826 case ISCSI_HOST_PARAM_NETDEV_NAME:
2827 case ISCSI_HOST_PARAM_HWADDRESS:
2828 case ISCSI_HOST_PARAM_IPADDRESS:
2829 case ISCSI_HOST_PARAM_INITIATOR_NAME:
2836 case ISCSI_PARAM_MAX_RECV_DLENGTH:
2837 case ISCSI_PARAM_MAX_XMIT_DLENGTH:
2838 case ISCSI_PARAM_HDRDGST_EN:
2839 case ISCSI_PARAM_DATADGST_EN:
2840 case ISCSI_PARAM_CONN_ADDRESS:
2841 case ISCSI_PARAM_CONN_PORT:
2842 case ISCSI_PARAM_EXP_STATSN:
2843 case ISCSI_PARAM_PERSISTENT_ADDRESS:
2844 case ISCSI_PARAM_PERSISTENT_PORT:
2845 case ISCSI_PARAM_PING_TMO:
2846 case ISCSI_PARAM_RECV_TMO:
2847 case ISCSI_PARAM_INITIAL_R2T_EN:
2848 case ISCSI_PARAM_MAX_R2T:
2849 case ISCSI_PARAM_IMM_DATA_EN:
2850 case ISCSI_PARAM_FIRST_BURST:
2851 case ISCSI_PARAM_MAX_BURST:
2852 case ISCSI_PARAM_PDU_INORDER_EN:
2853 case ISCSI_PARAM_DATASEQ_INORDER_EN:
2854 case ISCSI_PARAM_ERL:
2855 case ISCSI_PARAM_TARGET_NAME:
2856 case ISCSI_PARAM_TPGT:
2857 case ISCSI_PARAM_USERNAME:
2858 case ISCSI_PARAM_PASSWORD:
2859 case ISCSI_PARAM_USERNAME_IN:
2860 case ISCSI_PARAM_PASSWORD_IN:
2861 case ISCSI_PARAM_FAST_ABORT:
2862 case ISCSI_PARAM_ABORT_TMO:
2863 case ISCSI_PARAM_LU_RESET_TMO:
2864 case ISCSI_PARAM_TGT_RESET_TMO:
2865 case ISCSI_PARAM_IFACE_NAME:
2866 case ISCSI_PARAM_INITIATOR_NAME:
2875 EXPORT_SYMBOL_GPL(cxgbi_attr_is_visible);
2877 static int __init libcxgbi_init_module(void)
2879 sw_tag_idx_bits = (__ilog2_u32(ISCSI_ITT_MASK)) + 1;
2880 sw_tag_age_bits = (__ilog2_u32(ISCSI_AGE_MASK)) + 1;
2882 pr_info("tag itt 0x%x, %u bits, age 0x%x, %u bits.\n",
2883 ISCSI_ITT_MASK, sw_tag_idx_bits,
2884 ISCSI_AGE_MASK, sw_tag_age_bits);
2886 ddp_setup_host_page_size();
2890 static void __exit libcxgbi_exit_module(void)
2892 cxgbi_device_unregister_all(0xFF);
2896 module_init(libcxgbi_init_module);
2897 module_exit(libcxgbi_exit_module);